Mercurial > illumos > s390-betelgeuse
changeset 10880:76b7c881fd34
Branch merge
| author | tide@localhost |
|---|---|
| date | Mon, 21 Sep 2009 11:26:40 -0400 |
| parents | 790152ecf13b (current diff) 8513e8d83d7c (diff) |
| children | 70f7defc280d |
| files | usr/src/cmd/ssh/include/daemon.h usr/src/cmd/ssh/libopenbsd-compat/common/daemon.c usr/src/uts/common/io/tpm/tpm.c usr/src/uts/common/syscall/uadmin.c usr/src/uts/intel/amd_iommu/Makefile usr/src/uts/intel/io/amd_iommu/amd_iommu.c usr/src/uts/intel/io/amd_iommu/amd_iommu.conf usr/src/uts/intel/io/amd_iommu/amd_iommu_acpi.c usr/src/uts/intel/io/amd_iommu/amd_iommu_acpi.h usr/src/uts/intel/io/amd_iommu/amd_iommu_cmd.c usr/src/uts/intel/io/amd_iommu/amd_iommu_impl.c usr/src/uts/intel/io/amd_iommu/amd_iommu_impl.h usr/src/uts/intel/io/amd_iommu/amd_iommu_log.c usr/src/uts/intel/io/amd_iommu/amd_iommu_log.h usr/src/uts/intel/io/amd_iommu/amd_iommu_page_tables.c usr/src/uts/intel/io/amd_iommu/amd_iommu_page_tables.h usr/src/uts/intel/sys/amd_iommu.h |
| diffstat | 179 files changed, 12587 insertions(+), 9473 deletions(-) [+] |
line wrap: on
line diff
--- a/usr/src/cmd/auditd/auditd.xml Mon Sep 21 11:25:09 2009 -0400 +++ b/usr/src/cmd/auditd/auditd.xml Mon Sep 21 11:26:40 2009 -0400 @@ -1,15 +1,14 @@ <?xml version="1.0"?> <!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1"> <!-- - Copyright 2005 Sun Microsystems, Inc. All rights reserved. + Copyright 2009 Sun Microsystems, Inc. All rights reserved. Use is subject to license terms. CDDL HEADER START The contents of this file are subject to the terms of the - Common Development and Distribution License, Version 1.0 only - (the "License"). You may not use this file except in compliance - with the License. + Common Development and Distribution License (the "License"). + You may not use this file except in compliance with the License. You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE or http://www.opensolaris.org/os/licensing. @@ -24,8 +23,6 @@ CDDL HEADER END - ident "%Z%%M% %I% %E% SMI" - NOTE: This service manifest is not editable; its contents will be overwritten by package or patch operations, including operating system upgrade. Make customizations in a different @@ -60,7 +57,7 @@ <dependency name='syslog' type='service' - grouping='require_all' + grouping='optional_all' restart_on='none'> <service_fmri value='svc:/system/system-log' /> </dependency>
--- a/usr/src/cmd/autopush/autopush.c Mon Sep 21 11:25:09 2009 -0400 +++ b/usr/src/cmd/autopush/autopush.c Mon Sep 21 11:26:40 2009 -0400 @@ -2,9 +2,8 @@ * CDDL HEADER START * * The contents of this file are subject to the terms of the - * Common Development and Distribution License, Version 1.0 only - * (the "License"). You may not use this file except in compliance - * with the License. + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. @@ -20,7 +19,7 @@ * CDDL HEADER END */ /* - * Copyright 1999-2003 Sun Microsystems, Inc. All rights reserved. + * Copyright 2009 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. */ @@ -28,8 +27,6 @@ /* All Rights Reserved */ -#pragma ident "%Z%%M% %I% %E% SMI" - /* * autopush(1) is the command interface to the STREAMS autopush * mechanism. The autopush command can be used to configure autopush @@ -74,6 +71,7 @@ #include <string.h> #include <locale.h> #include <sys/stat.h> +#include <zone.h> #define OPTIONS "M:f:gm:r" /* command line options for getopt(3C) */ #define COMMENT '#' @@ -207,6 +205,13 @@ usage(); exit(1); } + + if (getzoneid() != GLOBAL_ZONEID) { + (void) fprintf(stderr, gettext("autopush " + "can only be run from the global zone.\n")); + exit(1); + } + if (fflag) exitcode = set_info(filenamep); else if (rflag)
--- a/usr/src/cmd/boot/filelist/i386/filelist.ramdisk Mon Sep 21 11:25:09 2009 -0400 +++ b/usr/src/cmd/boot/filelist/i386/filelist.ramdisk Mon Sep 21 11:26:40 2009 -0400 @@ -7,6 +7,7 @@ etc/devices/mdi_ib_cache etc/devices/mdi_scsi_vhci_cache etc/devices/retire_store +etc/devices/pci_unitaddr_persistent etc/driver_aliases etc/driver_classes etc/mach
--- a/usr/src/cmd/boot/installgrub/installgrub.c Mon Sep 21 11:25:09 2009 -0400 +++ b/usr/src/cmd/boot/installgrub/installgrub.c Mon Sep 21 11:26:40 2009 -0400 @@ -228,11 +228,13 @@ get_start_sector(int fd) { static unsigned int start_sect = 0; - uint32_t secnum, numsec; - int i, pno, rval, ext_sol_part_found = 0; + uint32_t secnum = 0, numsec = 0; + int i, pno, rval, log_part = 0; struct mboot *mboot; struct ipart *part; ext_part_t *epp; + struct part_info dkpi; + struct extpart_info edkpi; if (start_sect) return (start_sect); @@ -241,24 +243,84 @@ for (i = 0; i < FD_NUMPART; i++) { part = (struct ipart *)mboot->parts + i; if (is_bootpar) { - if (part->systid == 0xbe) - break; + if (part->systid == 0xbe) { + start_sect = part->relsect; + partition = i; + goto found_part; + } + } + } + + /* + * We will not support x86 boot partition on extended partitions + */ + if (is_bootpar) { + (void) fprintf(stderr, NOBOOTPAR); + exit(-1); + } + + /* + * Not an x86 boot partition. Search for Solaris fdisk partition + * Get the solaris partition information from the device + * and compare the offset of S2 with offset of solaris partition + * from fdisk partition table. + */ + if (ioctl(fd, DKIOCEXTPARTINFO, &edkpi) < 0) { + if (ioctl(fd, DKIOCPARTINFO, &dkpi) < 0) { + (void) fprintf(stderr, PART_FAIL); + exit(-1); + } else { + edkpi.p_start = dkpi.p_start; } } - /* Read extended partition to find a solaris partition */ + for (i = 0; i < FD_NUMPART; i++) { + part = (struct ipart *)mboot->parts + i; + + if (part->relsect == 0) { + (void) fprintf(stderr, BAD_PART, i); + exit(-1); + } + + if (edkpi.p_start >= part->relsect && + edkpi.p_start < (part->relsect + part->numsect)) { + /* Found the partition */ + break; + } + } + + if (i == FD_NUMPART) { + /* No solaris fdisk partitions (primary or logical) */ + (void) fprintf(stderr, NOSOLPAR); + exit(-1); + } + + /* + * We have found a Solaris fdisk partition (primary or extended) + * Handle the simple case first: Solaris in a primary partition + */ + if (!fdisk_is_dos_extended(part->systid)) { + start_sect = part->relsect; + partition = i; + goto found_part; + } + + /* + * Solaris in a logical partition. Find that partition in the + * extended part. + */ if ((rval = libfdisk_init(&epp, device_p0, NULL, FDISK_READ_DISK)) != FDISK_SUCCESS) { switch (rval) { /* - * FDISK_EBADLOGDRIVE and FDISK_ENOLOGDRIVE can - * be considered as soft errors and hence - * we do not exit + * The first 2 cases are not an error per-se, just that + * there is no Solaris logical partition */ case FDISK_EBADLOGDRIVE: - break; case FDISK_ENOLOGDRIVE: - break; + (void) fprintf(stderr, NOSOLPAR); + exit(-1); + /*NOTREACHED*/ case FDISK_ENOVGEOM: (void) fprintf(stderr, NO_VIRT_GEOM); exit(1); @@ -279,54 +341,18 @@ } rval = fdisk_get_solaris_part(epp, &pno, &secnum, &numsec); - if (rval == FDISK_SUCCESS) { - ext_sol_part_found = 1; + if (rval != FDISK_SUCCESS) { + /* No solaris logical partition */ + (void) fprintf(stderr, NOSOLPAR); + exit(-1); } libfdisk_fini(&epp); - /* - * If there is no boot partition, find the solaris partition - */ - - if (i == FD_NUMPART) { - struct part_info dkpi; - struct extpart_info edkpi; - - /* - * Get the solaris partition information from the device - * and compare the offset of S2 with offset of solaris partition - * from fdisk partition table. - */ - if (ioctl(fd, DKIOCEXTPARTINFO, &edkpi) < 0) { - if (ioctl(fd, DKIOCPARTINFO, &dkpi) < 0) { - (void) fprintf(stderr, PART_FAIL); - exit(-1); - } else { - edkpi.p_start = dkpi.p_start; - } - } + start_sect = secnum; + partition = pno - 1; + log_part = 1; - for (i = 0; i < FD_NUMPART; i++) { - part = (struct ipart *)mboot->parts + i; - - if (part->relsect == 0) { - (void) fprintf(stderr, BAD_PART, i); - exit(-1); - } - - if (edkpi.p_start >= part->relsect && - edkpi.p_start < (part->relsect + part->numsect)) { - /* Found the partition */ - break; - } - } - } - - if ((i == FD_NUMPART) && (!ext_sol_part_found)) { - (void) fprintf(stderr, BOOTPAR); - exit(-1); - } - +found_part: /* get confirmation for -m */ if (write_mboot && !force_mboot) { (void) fprintf(stdout, MBOOT_PROMPT); @@ -336,16 +362,21 @@ } } - if (fdisk_is_dos_extended(part->systid)) { - start_sect = secnum; - partition = pno; - } else { - start_sect = part->relsect; - partition = i; + /* + * Currently if Solaris is in an extended partition we need to + * write GRUB to the MBR. Check for this. + */ + if (log_part && !write_mboot) { + (void) fprintf(stderr, EXTSOLPAR); + exit(-1); } - if (part->bootid != 128 && write_mboot == 0) { - (void) fprintf(stdout, BOOTPAR_INACTIVE, i + 1); + /* + * warn, if Solaris in primary partition and GRUB not in MBR and + * partition is not active + */ + if (!log_part && part->bootid != 128 && !write_mboot) { + (void) fprintf(stdout, SOLPAR_INACTIVE, partition + 1); } return (start_sect);
--- a/usr/src/cmd/boot/installgrub/message.h Mon Sep 21 11:25:09 2009 -0400 +++ b/usr/src/cmd/boot/installgrub/message.h Mon Sep 21 11:26:40 2009 -0400 @@ -34,13 +34,20 @@ #define DRY_RUN gettext("dry run--nothing will be written to disk\n") -#define BOOTPAR gettext("Solaris partition not found. Abort operation.\n") +#define NOSOLPAR \ + gettext("Solaris partition not found. Aborting operation.\n") -#define BOOTPAR_INACTIVE gettext("Solaris boot partition inactive.\n") +#define NOBOOTPAR \ + gettext("Solaris x86 boot partition not found. Aborting operation.\n") + +#define SOLPAR_INACTIVE gettext("Solaris fdisk partition is inactive.\n") #define BOOTPAR_NOTFOUND \ gettext("Solaris boot partition not found on %s\n") +#define EXTSOLPAR \ + gettext("Solaris in extended partition. -m (MBR) option required\n") + #define NOT_RAW_DEVICE gettext("device %s is not a char special device\n") #define NOT_ROOT_SLICE gettext("raw device must be a root slice (not s2)\n") @@ -112,6 +119,8 @@ #define LIBFDISK_INIT_FAIL gettext("Failed to initialize libfdisk.\n") + + #ifdef __cplusplus } #endif
--- a/usr/src/cmd/cmd-inet/usr.lib/wpad/wpa_supplicant.c Mon Sep 21 11:25:09 2009 -0400 +++ b/usr/src/cmd/cmd-inet/usr.lib/wpad/wpa_supplicant.c Mon Sep 21 11:26:40 2009 -0400 @@ -1,5 +1,5 @@ /* - * Copyright 2008 Sun Microsystems, Inc. All rights reserved. + * Copyright 2009 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. */ @@ -762,34 +762,6 @@ free(config); } -static int -daemon(boolean_t nochdir, boolean_t noclose) -{ - int retv; - - if ((retv = fork()) == -1) - return (-1); - if (retv != 0) - _exit(EXIT_SUCCESS); - if (setsid() == -1) - return (-1); - - if (!nochdir && chdir("/") == -1) - return (-1); - - if (!noclose) { - (void) close(0); - (void) close(1); - (void) close(2); - if ((retv = open("/dev/null", O_RDWR)) != -1) { - (void) dup2(retv, 1); - (void) dup2(retv, 2); - } - } - - return (0); -} - /* * make sure wpad is running under SMF context. */
--- a/usr/src/cmd/cmd-inet/usr.sbin/in.routed/main.c Mon Sep 21 11:25:09 2009 -0400 +++ b/usr/src/cmd/cmd-inet/usr.sbin/in.routed/main.c Mon Sep 21 11:26:40 2009 -0400 @@ -1,5 +1,5 @@ /* - * Copyright 2006 Sun Microsystems, Inc. All rights reserved. + * Copyright 2009 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. * * Copyright (c) 1983, 1988, 1993 @@ -38,8 +38,6 @@ * " The Regents of the University of California. All rights reserved.\n"; */ -#pragma ident "%Z%%M% %I% %E% SMI" - #include "defs.h" #include "pathnames.h" #include <signal.h> @@ -104,35 +102,6 @@ static void sigalrm(int); static void sigterm(int); -static int -daemon(boolean_t nochdir, boolean_t noclose) -{ - int retv; - - if ((retv = fork()) == -1) - return (-1); - if (retv != 0) - _exit(EXIT_SUCCESS); - if (setsid() == -1) - return (-1); - if ((retv = fork()) == -1) - return (-1); - if (retv != 0) - _exit(EXIT_SUCCESS); - if (!nochdir && chdir("/") == -1) - return (-1); - if (!noclose) { - (void) close(0); - (void) close(1); - (void) close(2); - if ((retv = open("/dev/null", O_RDWR)) != -1) { - (void) dup2(retv, 1); - (void) dup2(retv, 2); - } - } - return (0); -} - int main(int argc, char *argv[]) { @@ -310,9 +279,9 @@ goto usage; if (argc != 0) { usage: - (void) fprintf(stderr, - gettext("usage: in.routed [-AdghmnqsStVvz] " - "[-T <tracefile>]\n")); + (void) fprintf(stderr, gettext( + "usage: in.routed [-AdghmnqsStVvz] " + "[-T <tracefile>]\n")); (void) fprintf(stderr, gettext("\t[-F <net>[/<mask>][,<metric>]] [-P <parms>]\n")); logbad(_B_FALSE, gettext("excess arguments")); @@ -358,7 +327,7 @@ msglog("signal: %s", rip_strerror(sigerr)); /* get into the background */ - if (background && daemon(_B_FALSE, _B_FALSE) < 0) + if (background && daemon(0, 0) < 0) BADERR(_B_FALSE, "daemon()"); /* Store our process id, blow away any existing file if it exists. */
--- a/usr/src/cmd/devfsadm/audio_link.c Mon Sep 21 11:25:09 2009 -0400 +++ b/usr/src/cmd/devfsadm/audio_link.c Mon Sep 21 11:26:40 2009 -0400 @@ -68,33 +68,32 @@ */ /* /dev/audio, /dev/audioctl, /dev/dsp */ - { "audio", "^(audio|audioctl|dsp)$", - RM_POST|RM_HOT|RM_ALWAYS, ILEVEL_0, devfsadm_rm_link + { "audio", "^audio$", + RM_POST|RM_HOT|RM_ALWAYS, ILEVEL_0, devfsadm_rm_all }, - /* /dev/mixer0, /dev/dsp0 */ - { "audio", "^(mixer|dsp)[0-9]+$", - RM_POST|RM_HOT|RM_ALWAYS, ILEVEL_0, devfsadm_rm_link - }, - /* /dev/sound/0, 0ctl */ - { "audio", "^sound/[0-9]+(ctl)?$", - RM_POST|RM_HOT|RM_ALWAYS, ILEVEL_0, devfsadm_rm_link + { "audio", "^audioctl$", + RM_POST|RM_HOT|RM_ALWAYS, ILEVEL_0, devfsadm_rm_all }, - /* /dev/mixer */ - { "pseudo", "^(mixer)$", - RM_POST|RM_HOT|RM_ALWAYS, ILEVEL_0, devfsadm_rm_link + { "audio", "^dsp$", + RM_POST|RM_HOT|RM_ALWAYS, ILEVEL_0, devfsadm_rm_all }, - - /* - * Primary links. - */ - - /* /dev/sndstat */ - { "pseudo", "^sndstat$", + { "audio", "^mixer", + RM_POST|RM_HOT|RM_ALWAYS, ILEVEL_0, devfsadm_rm_all + }, + { "audio", "^sndstat$", RM_PRE|RM_HOT|RM_ALWAYS, ILEVEL_0, devfsadm_rm_all }, - /* /dev/sound/audio810:0, 0ctl, etc */ - { "audio", "^sound/.*:[0-9]+(ctl|dsp|mixer)?$", - RM_PRE|RM_HOT|RM_ALWAYS, ILEVEL_0, devfsadm_rm_all + { "audio", "^mixer[0-9]+$", + RM_POST|RM_HOT|RM_ALWAYS, ILEVEL_0, devfsadm_rm_all + }, + { "audio", "^dsp[0-9]+$", + RM_POST|RM_HOT|RM_ALWAYS, ILEVEL_0, devfsadm_rm_all + }, + { "audio", "^sound/[0-9]+$", + RM_POST|RM_HOT|RM_ALWAYS, ILEVEL_0, devfsadm_rm_all + }, + { "audio", "^sound/[0-9]+ctl$", + RM_POST|RM_HOT|RM_ALWAYS, ILEVEL_0, devfsadm_rm_all }, };
--- a/usr/src/cmd/devfsadm/devfsadm.c Mon Sep 21 11:25:09 2009 -0400 +++ b/usr/src/cmd/devfsadm/devfsadm.c Mon Sep 21 11:26:40 2009 -0400 @@ -42,6 +42,7 @@ #include <utime.h> #include <sys/param.h> #include <bsm/libbsm.h> +#include <zone.h> #include "devfsadm_impl.h" /* externs from devalloc.c */ @@ -267,6 +268,11 @@ /*NOTREACHED*/ } + if (getzoneid() != GLOBAL_ZONEID) { + err_print(MUST_BE_GLOBAL_ZONE); + devfsadm_exit(1); + } + /* * Close all files except stdin/stdout/stderr */
--- a/usr/src/cmd/devfsadm/message.h Mon Sep 21 11:25:09 2009 -0400 +++ b/usr/src/cmd/devfsadm/message.h Mon Sep 21 11:26:40 2009 -0400 @@ -255,6 +255,9 @@ #define ZONE_PATHCHECK \ gettext("cannot manage root path '%s': path is part of zone '%s'\n") +#define MUST_BE_GLOBAL_ZONE \ + gettext("can only be run from the global zone\n") + #define DEVNAME_CONTACT_FAILED \ gettext("cannot talk to devname fs %s: %s\n")
--- a/usr/src/cmd/fm/fmd/common/fmd_xprt.c Mon Sep 21 11:25:09 2009 -0400 +++ b/usr/src/cmd/fm/fmd/common/fmd_xprt.c Mon Sep 21 11:26:40 2009 -0400 @@ -1173,6 +1173,7 @@ uint8_t *proxy_asru = NULL; int got_proxy_asru = 0; int got_hc_rsrc = 0; + int got_hc_asru = 0; int got_present_rsrc = 0; uint8_t *diag_asru = NULL; char *scheme; @@ -1230,7 +1231,7 @@ nvlist_lookup_string(asru, FM_FMRI_SCHEME, &scheme) == 0 && strcmp(scheme, FM_FMRI_SCHEME_HC) == 0) { - got_hc_rsrc = 1; + got_hc_asru = 1; if (xip->xi_flags & FMD_XPRT_EXTERNAL) continue; if (topo_fmri_present(thp, asru, &err) != 0) @@ -1266,7 +1267,8 @@ * If we're set up only to report hc-scheme faults, and * there aren't any, then just drop the event. */ - if (got_hc_rsrc == 0 && (xip->xi_flags & FMD_XPRT_HCONLY)) { + if (got_hc_rsrc == 0 && got_hc_asru == 0 && + (xip->xi_flags & FMD_XPRT_HCONLY)) { if (nelem > 0) { fmd_free(proxy_asru, sizeof (uint8_t) * nelem); fmd_free(diag_asru, sizeof (uint8_t) * nelem); @@ -1333,10 +1335,13 @@ (void) nvlist_add_nvlist(flt_copy, FM_FAULT_ASRU, asrua[i]); nvlist_free(asrua[i]); - } else if (nvlist_lookup_nvlist(flt_copy, FM_FAULT_ASRU, + } else if (got_hc_asru == 0 && + nvlist_lookup_nvlist(flt_copy, FM_FAULT_ASRU, &asru) == 0 && asru != NULL) { /* - * keep asru from diag side, but but mark as no retire + * If we have an asru from diag side, but it's not + * in hc scheme, then we can't be sure what it + * represents, so mark as no retire. */ (void) nvlist_add_boolean_value(flt_copy, FM_SUSPECT_RETIRE, B_FALSE);
--- a/usr/src/cmd/fm/modules/common/zfs-diagnosis/zfs_de.c Mon Sep 21 11:25:09 2009 -0400 +++ b/usr/src/cmd/fm/modules/common/zfs-diagnosis/zfs_de.c Mon Sep 21 11:26:40 2009 -0400 @@ -19,12 +19,10 @@ * CDDL HEADER END */ /* - * Copyright 2008 Sun Microsystems, Inc. All rights reserved. + * Copyright 2009 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. */ -#pragma ident "%Z%%M% %I% %E% SMI" - #include <assert.h> #include <stddef.h> #include <strings.h> @@ -710,12 +708,11 @@ }; static const fmd_prop_t fmd_props[] = { - { "case_timeout", FMD_TYPE_TIME, "5sec" }, { "checksum_N", FMD_TYPE_UINT32, "10" }, { "checksum_T", FMD_TYPE_TIME, "10min" }, { "io_N", FMD_TYPE_UINT32, "10" }, { "io_T", FMD_TYPE_TIME, "10min" }, - { "remove_timeout", FMD_TYPE_TIME, "5sec" }, + { "remove_timeout", FMD_TYPE_TIME, "15sec" }, { NULL, 0, NULL } };
--- a/usr/src/cmd/fm/modules/sun4v/cpumem-diagnosis/cmd_memerr_arch.c Mon Sep 21 11:25:09 2009 -0400 +++ b/usr/src/cmd/fm/modules/sun4v/cpumem-diagnosis/cmd_memerr_arch.c Mon Sep 21 11:26:40 2009 -0400 @@ -19,7 +19,7 @@ * CDDL HEADER END */ /* - * Copyright 2008 Sun Microsystems, Inc. All rights reserved. + * Copyright 2009 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. */ @@ -292,8 +292,9 @@ cmd_ce(fmd_hdl_t *hdl, fmd_event_t *ep, nvlist_t *nvl, const char *class, cmd_errcl_t clcode) { - if (strcmp(class, "ereport.cpu.ultraSPARC-T2plus.dsc") == 0) - return (CMD_EVD_UNUSED); /* drop VF dsc's */ + if ((strcmp(class, "ereport.cpu.ultraSPARC-T2plus.dsc") == 0) || + (strcmp(class, "ereport.cpu.ultraSPARC-T2.dsc") == 0)) + return (CMD_EVD_UNUSED); /* drop T2/T2+ dsc's */ else return (xe_common(hdl, ep, nvl, class, clcode, cmd_ce_common)); }
--- a/usr/src/cmd/halt/halt.c Mon Sep 21 11:25:09 2009 -0400 +++ b/usr/src/cmd/halt/halt.c Mon Sep 21 11:26:40 2009 -0400 @@ -381,6 +381,42 @@ (void) sigsend(P_CTID, next->ctid, SIGCONT); } +#define FMRI_GDM "svc:/application/graphical-login/gdm:default" + +/* + * If gdm is running, try to stop gdm. + * Returns 0 on success, -1 on failure. + */ +static int +stop_gdm() +{ + char *gdm_state = NULL; + int retry = 0; + + /* + * If gdm is running, try to stop gdm. + */ + while ((gdm_state = smf_get_state(FMRI_GDM)) != NULL && + strcmp(gdm_state, SCF_STATE_STRING_ONLINE) == 0 && retry++ < 5) { + if (smf_disable_instance(FMRI_GDM, SMF_TEMPORARY) != 0) { + (void) fprintf(stderr, + gettext("%s: Failed to stop %s: %s.\n"), + cmdname, FMRI_GDM, scf_strerror(scf_error())); + return (-1); + } + (void) sleep(1); + } + + if (retry >= 5) { + (void) fprintf(stderr, gettext("%s: Failed to stop %s.\n"), + cmdname, FMRI_GDM); + return (-1); + } + + return (0); +} + + static void stop_restarters() { @@ -1482,7 +1518,29 @@ (void) fprintf(stderr, gettext("%s: could not create %s.\n"), cmdname, resetting); + } + /* + * Make sure we don't get stopped by a jobcontrol shell + * once we start killing everybody. + */ + (void) signal(SIGTSTP, SIG_IGN); + (void) signal(SIGTTIN, SIG_IGN); + (void) signal(SIGTTOU, SIG_IGN); + (void) signal(SIGPIPE, SIG_IGN); + (void) signal(SIGTERM, SIG_IGN); + + /* + * Try to stop gdm so X has a chance to return the screen and + * keyboard to a sane state. + */ + if (fast_reboot && stop_gdm() != 0) { + (void) fprintf(stderr, + gettext("%s: Falling back to regular reboot.\n"), cmdname); + fast_reboot = 0; + } + + if (cmd != A_DUMP) { /* * Stop all restarters so they do not try to restart services * that are terminated. @@ -1502,16 +1560,6 @@ } /* - * Make sure we don't get stopped by a jobcontrol shell - * once we start killing everybody. - */ - (void) signal(SIGTSTP, SIG_IGN); - (void) signal(SIGTTIN, SIG_IGN); - (void) signal(SIGTTOU, SIG_IGN); - (void) signal(SIGPIPE, SIG_IGN); - (void) signal(SIGTERM, SIG_IGN); - - /* * If we're not forcing a crash dump, give everyone 5 seconds to * handle a SIGTERM and clean up properly. */
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/usr/src/cmd/mdb/common/modules/rootnex/intel_iommu.c Mon Sep 21 11:26:40 2009 -0400 @@ -0,0 +1,883 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright 2009 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +/* + * Copyright (c) 2009, Intel Corporation. + * All rights reserved. + */ +#include <sys/mdb_modapi.h> +#include <sys/list.h> +#include <sys/note.h> +#include <sys/dditypes.h> +#include <sys/ddi_impldefs.h> +#include <sys/intel_iommu.h> +#include <sys/iommulib.h> +#include <stddef.h> + +/* + * Does Intel IOMMU works on this system? + */ +static boolean_t iommu_support = B_FALSE; + +static void +iomuvtop_help(void) +{ + mdb_printf("print physical mapping of IO virtual address\n\n" + "Usage:\n\n" + " address::iomuvtop <iova>\n\n" + "Where, \"address\" is the address of the devinfo node, " + "while \"iova\" is the DMA virtual address.\n"); +} + +static boolean_t +iommu_supported(void) +{ + if (iommu_support == B_FALSE) + mdb_printf("No Intel IOMMU active on this system\n"); + return (iommu_support); +} + +/* + * print_device_scope_cb() + * call back for print_device_scope() + */ +static int +print_device_scope_cb(uintptr_t addr, pci_dev_scope_t *devs, void *cbdata) +{ + _NOTE(ARGUNUSED(addr)) + + mdb_printf((char *)cbdata); + mdb_printf("BDF[%x:%x:%x],type[%x]\n", + devs->pds_bus, + devs->pds_dev, + devs->pds_func, + devs->pds_type); + + return (WALK_NEXT); +} + +/* + * print_device_scope() + * a common function to print device scope of a drhd or rmrr + */ +static void +print_device_scope(const char *pre, uintptr_t addr) +{ + mdb_pwalk("list", + (mdb_walk_cb_t)print_device_scope_cb, (void *)pre, addr); +} + +/* + * parse_hw_capa() + * parse_hw_excapa() + * + * Given the capability and extension capability register contents, + * parse and print supported features in <output> + * + * Please refer to chapter 10.4.2/3 in "Intel virutalization technology + * for direct IO specification" for register details + */ +static void +parse_hw_capa(uint64_t capa) +{ + char string[128]; + size_t len; + + strcpy(string, " Hardware Capability:\t\t"); + if (IOMMU_CAP_GET_DRD(capa)) + strcat(string, "DRD "); + if (IOMMU_CAP_GET_DWD(capa)) + strcat(string, "DWD "); + if (IOMMU_CAP_GET_PSI(capa)) + strcat(string, "PSI "); + if (IOMMU_CAP_GET_ISOCH(capa)) + strcat(string, "ISOCH "); + if (IOMMU_CAP_GET_ZLR(capa)) + strcat(string, "ZLR "); + if (IOMMU_CAP_GET_CM(capa)) + strcat(string, "CM "); + if (IOMMU_CAP_GET_PHMR(capa)) + strcat(string, "PHMR "); + if (IOMMU_CAP_GET_PLMR(capa)) + strcat(string, "PLMR "); + if (IOMMU_CAP_GET_RWBF(capa)) + strcat(string, "RWBF "); + if (IOMMU_CAP_GET_AFL(capa)) + strcat(string, "AFL "); + + len = strlen(string); + if ((len > 1) && + (string[len - 1] == ' ')) + string[len - 1] = 0; + + strcat(string, "\n"); + mdb_printf(string); +} + +static void +parse_hw_excapa(uint64_t excapa) +{ + char string[128]; + size_t len; + + strcpy(string, " Hardware Ex-Capability:\t"); + if (IOMMU_ECAP_GET_SC(excapa)) + strcat(string, "SC "); + if (IOMMU_ECAP_GET_PT(excapa)) + strcat(string, "PT "); + if (IOMMU_ECAP_GET_CH(excapa)) + strcat(string, "CH "); + if (IOMMU_ECAP_GET_EIM(excapa)) + strcat(string, "EIM "); + if (IOMMU_ECAP_GET_IR(excapa)) + strcat(string, "IR "); + if (IOMMU_ECAP_GET_DI(excapa)) + strcat(string, "DI "); + if (IOMMU_ECAP_GET_QI(excapa)) + strcat(string, "QI "); + if (IOMMU_ECAP_GET_C(excapa)) + strcat(string, "C "); + + len = strlen(string); + if ((len > 1) && + (string[len - 1] == ' ')) + string[len - 1] = 0; + + strcat(string, "\n"); + mdb_printf(string); +} + +typedef enum { + ERROR_SCOPE, + INCLUDE_ALL_SCOPE, + DEV_SCOPE +} iomu_scope_t; + +/* + * print_iommu_state() + * Given an iommu_state structure, parse and print iommu information + * + * Returns: + * INCLUDE_ALL_SCOPE if include all is set + * DEV_SCOPE if not set + * ERROR_SCOPE on error. + */ +static iomu_scope_t +print_iommu_state(intel_iommu_state_t *iommu, drhd_info_t *drhd) +{ + if ((iommu == NULL) || (drhd == NULL)) { + mdb_warn("Internal error - NULL iommu state pointer passed\n"); + return (ERROR_SCOPE); + } + + mdb_printf("Intel DMA remapping unit\n"); + mdb_printf(" IOMMU Status:\t\t\t%s\n", + (iommu->iu_enabled & DMAR_ENABLE) ? "Enabled" : "Disabled"); + mdb_printf(" Queued Invalid:\t\t%s\n", + (iommu->iu_enabled & QINV_ENABLE) ? "Enabled" : "Disabled"); + mdb_printf(" Interrupt remapping:\t\t%s\n", + (iommu->iu_enabled & INTRR_ENABLE) ? "Enabled" : "Disabled"); + mdb_printf(" Register Physical Address:\t%p\n", + (uintptr_t)drhd->di_reg_base); + mdb_printf(" Register Virtual Address:\t%p\n", + (uintptr_t)iommu->iu_reg_address); + parse_hw_capa(iommu->iu_capability); + parse_hw_excapa(iommu->iu_excapability); + mdb_printf(" Root Entry Table:\t\t%p\n", + (uintptr_t)iommu->iu_root_entry_paddr); + mdb_printf(" Guest Address Width:\t\t%d\n", iommu->iu_gaw); + mdb_printf(" Adjust Guest Address Width:\t%d\n", iommu->iu_agaw); + mdb_printf(" Page Table Level:\t\t%d\n", iommu->iu_level); + mdb_printf(" Max Domain Supported:\t\t%d\n", iommu->iu_max_domain); + mdb_printf(" System Coherence:\t\t%s\n", + iommu->iu_coherency ? "Yes" : "No"); + mdb_printf(" Include All unit:\t\t%s\n", + drhd->di_include_all ? "Yes" : "No"); + mdb_printf(" Devinfo Node:\t\t\t%p\n", + (intptr_t)drhd->di_dip); + + if (iommu->iu_enabled & QINV_ENABLE) { + struct inv_queue_state qi_state; + if (iommu->iu_inv_queue && + mdb_vread(&qi_state, sizeof (qi_state), + (intptr_t)iommu->iu_inv_queue) == sizeof (qi_state)) { + mdb_printf(" Qinv Table:\t\t\tpaddr:%p, " + "vaddr:%p, size:%x\n", + (uintptr_t)qi_state.iq_table.paddr, + (uintptr_t)qi_state.iq_table.vaddr, + qi_state.iq_table.size); + mdb_printf(" Sync Table:\t\t\tpaddr:%p, " + "vaddr:%p, size:%x\n", + (uintptr_t)qi_state.iq_sync.paddr, + (uintptr_t)qi_state.iq_sync.vaddr, + qi_state.iq_sync.size); + } else { + mdb_warn("failed to read iommu invalidation " + "queue state at %p\n", + (uintptr_t)iommu->iu_inv_queue); + return (ERROR_SCOPE); + } + } + + return (drhd->di_include_all ? INCLUDE_ALL_SCOPE : DEV_SCOPE); +} + +/* + * dcmd: iomuprt + */ +static int +iomuprt(uintptr_t addr, uint_t flags, int argc, const mdb_arg_t *argv) +{ + _NOTE(ARGUNUSED(argv)) + intel_iommu_state_t iommu; + drhd_info_t drhd; + + if (iommu_supported() == B_FALSE) + return (DCMD_OK); + + if ((argc != 0) || !(flags & DCMD_ADDRSPEC)) + return (DCMD_USAGE); + + if (!DCMD_HDRSPEC(flags)) + mdb_printf("\n"); + + if ((mdb_vread(&iommu, sizeof (iommu), addr) == sizeof (iommu)) && + (iommu.iu_drhd != NULL) && + (mdb_vread(&drhd, sizeof (drhd), + (intptr_t)iommu.iu_drhd) == sizeof (drhd))) { + switch (print_iommu_state(&iommu, &drhd)) { + case DEV_SCOPE: + /* + * Use actual address of list_t in kernel for walker + */ + print_device_scope(" Device Scope:\t\t\t", + (uintptr_t)((char *)iommu.iu_drhd + + offsetof(drhd_info_t, di_dev_list))); + break; + case ERROR_SCOPE: + return (DCMD_ERR); + default: + break; + } + } else { + mdb_warn("failed to read iommu state at %p\n", addr); + return (DCMD_ERR); + } + + return (DCMD_OK); +} + +/* + * print_iommu_addr() + * callback to print addresses of IOMMU unit software structures + */ +static int +print_iommu_addr(uintptr_t addr, intel_iommu_state_t *ip, void *cbdata) +{ + _NOTE(ARGUNUSED(cbdata)) + _NOTE(ARGUNUSED(ip)) + intel_iommu_state_t iommu; + + if (mdb_vread(&iommu, sizeof (iommu), addr) != sizeof (iommu)) { + mdb_warn("failed to read IOMMU structure at %p\n", addr); + return (WALK_ERR); + } + + mdb_printf("%p\n", addr); + + return (WALK_NEXT); +} + +/* + * dcmd: iomunits + */ +static int +iomunits(uintptr_t addr, uint_t flags, int argc, const mdb_arg_t *argv) +{ + _NOTE(ARGUNUSED(addr)) + _NOTE(ARGUNUSED(argv)) + GElf_Sym sym; + + if (iommu_supported() == B_FALSE) + return (DCMD_OK); + + if ((flags & DCMD_ADDRSPEC) || (argc != 0)) { + return (DCMD_USAGE); + } + + if (mdb_lookup_by_name("iommu_states", &sym) == -1) { + mdb_warn("failed to find symbol iommu_states\n"); + return (DCMD_ERR); + } + + addr = (uintptr_t)sym.st_value; + if (mdb_pwalk("list", (mdb_walk_cb_t)print_iommu_addr, NULL, addr)) { + mdb_warn("couldn't walk IOMMU state structures\n"); + return (DCMD_ERR); + } + return (DCMD_OK); +} + + + +/* + * print_domain_state() + * Given an device domain structure, parse and print information + */ +static void +print_domain_state(dmar_domain_state_t *domain) +{ + if (domain == NULL) { + mdb_warn("Internal error: NULL domain pointer passed\n"); + return; + } + + mdb_printf("IOMMU device domain:\n"); + mdb_printf("Domain ID:\t\t%d\n", domain->dm_domain_id); + mdb_printf("Bind IOMMU:\t\t%p\n", (uintptr_t)domain->dm_iommu); + mdb_printf("DVMA vmem:\t\t%p\n", + (uintptr_t)domain->dm_dvma_map); + mdb_printf("Top Level Page Table:\t%p\n", + (uintptr_t)domain->dm_page_table_paddr); + mdb_printf("Identity Mapping:\t\t%s\n", + domain->dm_identity ? "YES" : "NO"); +} + +/* + * dcmd: iomudomprt + */ +static int +iomudomprt(uintptr_t addr, uint_t flags, int argc, const mdb_arg_t *argv) +{ + _NOTE(ARGUNUSED(argv)) + dmar_domain_state_t domain; + + if (iommu_supported() == B_FALSE) + return (DCMD_OK); + + if ((argc != 0) || !(flags & DCMD_ADDRSPEC)) + return (DCMD_USAGE); + + if (!DCMD_HDRSPEC(flags)) + mdb_printf("\n"); + + if (mdb_vread(&domain, sizeof (domain), addr) == sizeof (domain)) { + print_domain_state(&domain); + } else { + mdb_warn("failed to read domain at %p\n", addr); + return (DCMD_ERR); + } + + return (DCMD_OK); +} + +/* + * print_domain_addr() + */ +static int +print_domain_addr(uintptr_t addr, dmar_domain_state_t *domp, void *cbdata) +{ + _NOTE(ARGUNUSED(domp)) + _NOTE(ARGUNUSED(cbdata)) + dmar_domain_state_t domain; + + if (iommu_supported() == B_FALSE) + return (WALK_NEXT); + + if (mdb_vread(&domain, sizeof (domain), addr) != sizeof (domain)) { + mdb_warn("failed to read domain at %p\n", addr); + return (WALK_ERR); + } + + mdb_printf("%p\n", addr); + + return (WALK_NEXT); +} + +/* + * dcmd: iomudoms + */ +static int +iomudoms(uintptr_t addr, uint_t flags, int argc, const mdb_arg_t *argv) +{ + _NOTE(ARGUNUSED(addr)) + _NOTE(ARGUNUSED(argv)) + GElf_Sym sym; + + if (iommu_supported() == B_FALSE) + return (DCMD_OK); + + if ((flags & DCMD_ADDRSPEC) || (argc != 0)) { + return (DCMD_USAGE); + } + + if (mdb_lookup_by_name("domain_states", &sym) == -1) { + mdb_warn("failed to find symbol domain_states\n"); + return (DCMD_ERR); + } + + addr = (uintptr_t)sym.st_value; + if (mdb_pwalk("list", (mdb_walk_cb_t)print_domain_addr, NULL, addr)) + return (DCMD_ERR); + return (DCMD_OK); +} + +/* + * print_rmrr_info() + */ +static void +print_rmrr_info(rmrr_info_t *rmrr) +{ + mdb_printf("Reserved Memory Region Reporting:\n"); + mdb_printf(" Segment:\t%d\n", rmrr->ri_segment); + mdb_printf(" BaseAddr:\t%p\n", (uintptr_t)rmrr->ri_baseaddr); + mdb_printf(" LimiAddr:\t%p\n", (uintptr_t)rmrr->ri_limiaddr); +} + +/* + * print_rmrr_addr() + * list walk callback for list_rmrr + */ +static int +print_rmrr_addr(uintptr_t addr, rmrr_info_t *rp, void *cbdata) +{ + _NOTE(ARGUNUSED(rp)) + _NOTE(ARGUNUSED(cbdata)) + rmrr_info_t rmrr; + + if (mdb_vread(&rmrr, sizeof (rmrr), addr) != sizeof (rmrr)) { + mdb_warn("failed to read RMRR structure at %p\n", addr); + return (WALK_ERR); + } + + mdb_printf("%p\n", addr); + + return (WALK_NEXT); +} + +/* + * dcmd: iomurmrrs + */ +static int +iomurmrrs(uintptr_t addr, uint_t flags, int argc, const mdb_arg_t *argv) +{ + _NOTE(ARGUNUSED(addr)) + _NOTE(ARGUNUSED(argv)) + GElf_Sym sym; + + if (iommu_supported() == B_FALSE) + return (DCMD_OK); + + if ((flags & DCMD_ADDRSPEC) || (argc != 0)) { + return (DCMD_USAGE); + } + + if (mdb_lookup_by_name("rmrr_states", &sym) == -1) { + mdb_warn("failed to find symbol rmrr_states\n"); + return (DCMD_ERR); + } + + addr = (uintptr_t)sym.st_value; + if (mdb_pwalk("list", (mdb_walk_cb_t)print_rmrr_addr, NULL, addr)) + return (DCMD_ERR); + return (DCMD_OK); +} + +/* + * dcmd: iomurmrrprt: Given an RMRR address print the RMRR. + */ +static int +iomurmrrprt(uintptr_t addr, uint_t flags, int argc, const mdb_arg_t *argv) +{ + _NOTE(ARGUNUSED(argv)) + uintptr_t dev_list_addr; + rmrr_info_t rmrr; + + if (iommu_supported() == B_FALSE) + return (DCMD_OK); + + if (!(flags & DCMD_ADDRSPEC) || (argc != 0)) { + return (DCMD_USAGE); + } + + if (mdb_vread(&rmrr, sizeof (rmrr), addr) != sizeof (rmrr)) { + mdb_warn("failed to read RMRR structure at %p\n", addr); + return (DCMD_ERR); + } + + dev_list_addr = addr + offsetof(rmrr_info_t, ri_dev_list); + print_rmrr_info(&rmrr); + print_device_scope(" DevScope:\t", dev_list_addr); + + return (DCMD_OK); +} + +/* + * iova_level_to_offset() + * Given an iova and page table level, return the corresponding offset + */ +static int +iova_level_to_offset(uintptr_t iova, int level) +{ + int start, offset; + + start = (level - 1) * IOMMU_LEVEL_STRIDE + IOMMU_PAGE_SHIFT; + offset = (iova >> start) & IOMMU_LEVEL_OFFSET; + + return (offset); +} + +/* + * iovtp_read_table_entry() + */ +static int +iovtp_read_table_entry(uint64_t ptaddr, size_t offset, + void *ent_buf, size_t ent_size) +{ + if (mdb_pread(ent_buf, ent_size, ptaddr + offset * ent_size) + != ent_size) { + return (B_FALSE); + } else { + return (B_TRUE); + } +} + +/* + * dcmd: iomuvtop + */ +static int +iomuvtop(uintptr_t addr, uint_t flags, int argc, const mdb_arg_t *argv) +{ + iommu_private_t private; + dmar_domain_state_t domain; + struct dev_info dinfo; + intel_iommu_state_t iommu; + int i, level, offset; + uintptr_t iova; + uint64_t ptaddr, ptentr; + int bus, devfn; + + struct root_context_entry { + uint64_t asr; + uint64_t pro; + } rc_entry; + + if (iommu_supported() == B_FALSE) + return (DCMD_OK); + + if (!(flags & DCMD_ADDRSPEC) || (argc != 1)) { + return (DCMD_USAGE); + } + + iova = (argv[0].a_type == MDB_TYPE_IMMEDIATE) ? + (uintptr_t)argv[0].a_un.a_val : + (uintptr_t)mdb_strtoull(argv->a_un.a_str); + + /* read iommu private */ + if ((mdb_vread(&dinfo, sizeof (dinfo), addr) != sizeof (dinfo)) || + (dinfo.devi_iommu_private == NULL) || + (mdb_vread(&private, sizeof (private), + (uintptr_t)dinfo.devi_iommu_private) != sizeof (private))) { + mdb_warn("failed to read iommu private structure for " + "devinfo node at address %p\n", addr); + return (DCMD_ERR); + } + + bus = private.idp_bus; + devfn = private.idp_devfn; + + /* read domain */ + if (private.idp_intel_domain == NULL) { + mdb_printf("IOMMU domain for this device has not yet been " + "allocated.\nNo mapped physical address for this vaddr\n"); + return (DCMD_OK); + } + + if (mdb_vread(&domain, sizeof (domain), + (uintptr_t)private.idp_intel_domain) + != sizeof (domain)) { + mdb_warn("failed to read domain structure at %p\n", + (uintptr_t)private.idp_intel_domain); + return (DCMD_ERR); + } + + /* read iommu */ + if (mdb_vread(&iommu, sizeof (iommu), (uintptr_t)domain.dm_iommu) + != sizeof (iommu)) { + mdb_warn("failed to read iommu structure at %p\n", + (uintptr_t)domain.dm_iommu); + return (DCMD_ERR); + } + + mdb_printf("Level\tPageTableAddress\tOffset\tPageTableEntry\n"); + + /* walk and print root context tabls */ + ptaddr = iommu.iu_root_entry_paddr; + if (iovtp_read_table_entry(ptaddr, bus, &rc_entry, sizeof (rc_entry)) + == B_FALSE) { + mdb_warn("failed to read root table entry for bus %x " + "at %p\n", bus, (uintptr_t)ptaddr); + return (DCMD_ERR); + } + mdb_printf("Root\t%p\t\t%x\tlow :%p\n", (uintptr_t)ptaddr, + bus, (uintptr_t)rc_entry.asr); + mdb_printf("Root\t%p\t\t%x\thigh:%p\n", (uintptr_t)ptaddr, + bus, (uintptr_t)rc_entry.pro); + + ptaddr = rc_entry.asr & IOMMU_PAGE_MASK; + if (iovtp_read_table_entry(ptaddr, devfn, &rc_entry, sizeof (rc_entry)) + == B_FALSE) { + mdb_warn("failed to read context table entry for " + "device-function %x at %p\n", devfn, (uintptr_t)ptaddr); + return (DCMD_ERR); + } + mdb_printf("Context\t%p\t\t%x\tlow :%p\n", (uintptr_t)ptaddr, + devfn, (uintptr_t)rc_entry.asr); + mdb_printf("Context\t%p\t\t%x\thigh:%p\n", (uintptr_t)ptaddr, + devfn, (uintptr_t)rc_entry.pro); + + /* walk and print page tables */ + ptaddr = rc_entry.asr & IOMMU_PAGE_MASK; + + /* + * Toppest level page table address should be the same + * as that stored in domain structure + */ + if (ptaddr != domain.dm_page_table_paddr) { + mdb_warn("The top level page table retrieved from context" + " table doesn't match that from the domain structure." + " Aborting PA lookup.\n"); + return (DCMD_ERR); + } + + level = iommu.iu_level; + for (i = level; i > 0; i--) { + if (!ptaddr) { + mdb_printf("\nNULL page table entry encountered at " + " page table level %d. Aborting PA lookup.\n", i); + return (DCMD_OK); + } + offset = iova_level_to_offset(iova, i); + if (iovtp_read_table_entry(ptaddr, offset, &ptentr, + sizeof (ptentr)) == B_FALSE) { + mdb_warn("failed to read page table entry " + "(level %d) at %p\n", i, (uintptr_t)ptaddr); + return (DCMD_ERR); + } + mdb_printf("%x\t%p\t\t%x\t%p\n", i, (uintptr_t)ptaddr, + offset, (uintptr_t)ptentr); + ptaddr = ptentr & IOMMU_PAGE_MASK; + } + + return (DCMD_OK); +} + +typedef struct bdf_cb_data { + int dc_seg; + int dc_bus; + int dc_devfunc; + int dc_match; +} bdf_cb_data_t; + +/* + * match_bdf() + * call back function that matches BDF + */ +static int +match_bdf(uintptr_t addr, struct dev_info *dev, bdf_cb_data_t *cbdata) +{ + _NOTE(ARGUNUSED(addr)) + /* if there is iommu private, get it */ + if (dev->devi_iommu_private != NULL) { + iommu_private_t private; + if (mdb_vread((void*)&private, sizeof (private), + (uintptr_t)dev->devi_iommu_private) != sizeof (private)) { + mdb_warn("failed to read iommu private at %p\n", + (uintptr_t)dev->devi_iommu_private); + return (WALK_ERR); + } + + if (private.idp_seg == cbdata->dc_seg && + private.idp_bus == cbdata->dc_bus && + private.idp_devfn == cbdata->dc_devfunc) { + if (cbdata->dc_match == 0) { + mdb_printf("%p\n", addr); + cbdata->dc_match = 1; + } else { + mdb_warn("More than one devinfo node matches " + "a single pci device. Aborting devinfo " + "lookup\n"); + return (WALK_ERR); + } + } + } + + return (WALK_NEXT); +} + +/* + * dcmd: bdf2devinfo + */ +static int +bdf2devinfo(uintptr_t addr, uint_t flags, int argc, const mdb_arg_t *argv) +{ + _NOTE(ARGUNUSED(addr)) + bdf_cb_data_t cbdata; + uint_t i, bdf[4]; + + if (iommu_supported() == B_FALSE) + return (DCMD_OK); + + if ((flags & DCMD_ADDRSPEC) || (argc != 4)) { + return (DCMD_USAGE); + } + + for (i = 0; i < 4; i++) { + bdf[i] = (argv[i].a_type == MDB_TYPE_IMMEDIATE) ? + (int)argv[i].a_un.a_val : + (int)mdb_strtoull(argv[i].a_un.a_str); + } + + if ((bdf[0] != 0) || (bdf[1] > 255) || (bdf[2] > 31) || (bdf[3] > 7)) { + mdb_warn("invalid pci segment, bus, device, function" + "tuple (%x, %x, %x, %x)\n", bdf[0], bdf[1], bdf[2], bdf[3]); + return (DCMD_USAGE); + } + + + cbdata.dc_seg = bdf[0]; + cbdata.dc_bus = bdf[1]; + cbdata.dc_devfunc = bdf[2] << 3 | bdf[3]; + cbdata.dc_match = 0; + + if (mdb_readvar(&addr, "top_devinfo") == -1) { + mdb_warn("failed to read 'top_devinfo'\n"); + return (DCMD_ERR); + } + + if (mdb_pwalk("devinfo", + (mdb_walk_cb_t)match_bdf, &cbdata, addr)) { + mdb_warn("couldn't walk devinfo tree\n"); + return (DCMD_ERR); + } + + if (cbdata.dc_match == 0) + mdb_printf("No devinfo node found for %x:%x:%x:%x\n", + bdf[0], bdf[1], bdf[2], bdf[3]); + + return (DCMD_OK); +} + +/* + * dcmd: iomudip2dom + */ +static int +iomudip2dom(uintptr_t addr, uint_t flags, int argc, const mdb_arg_t *argv) +{ + _NOTE(ARGUNUSED(argv)) + struct dev_info dinfo; + iommu_private_t private; + + if (iommu_supported() == B_FALSE) + return (DCMD_OK); + + if (!(flags & DCMD_ADDRSPEC) || (argc != 0)) { + return (DCMD_USAGE); + } + + /* read iommu private */ + if ((mdb_vread(&dinfo, sizeof (dinfo), addr) != sizeof (dinfo)) || + (dinfo.devi_iommu_private == NULL) || + (mdb_vread(&private, sizeof (private), + (uintptr_t)dinfo.devi_iommu_private) != sizeof (private))) { + mdb_warn("failed to read iommu private structure for " + "devinfo node at %p\n", addr); + return (DCMD_ERR); + } + + /* read domain */ + if (private.idp_intel_domain != NULL) { + mdb_printf("%p\n", (uintptr_t)private.idp_intel_domain); + } else { + mdb_printf("No domain dedicated for this device\n"); + } + + return (DCMD_OK); +} + +static const mdb_dcmd_t dcmds[] = { + { "iomunits", NULL, + "list addresses of software state structure for all IOMMUs", + iomunits }, + { "iomuprt", "?", + "given an IOMMU's state structure address, print its contents", + iomuprt}, + { "iomudoms", NULL, + "list addresses of all IOMMU domain software structures", + iomudoms }, + { "iomudomprt", "?", + "given an IOMMU's domain struct address, print its contents", + iomudomprt }, + { "iomurmrrs", NULL, + "list addresses of all Intel IOMMU RMRR software structures", + iomurmrrs }, + { "iomurmrrprt", NULL, + "given an IOMMU RMRR structure address, print its contents", + iomurmrrprt }, + { "iomuvtop", "?<iova>", + "print physical address of an IO virtual address", + iomuvtop, iomuvtop_help }, + { "bdf2devinfo", "[segment] [bus] [dev] [func]", + "given its pci segment/bus/dev/func, print the devinfo node", + bdf2devinfo }, + { "iomudip2dom", "?", + "given a devinfo node, print the address of its IOMMU domain", + iomudip2dom }, + { NULL } +}; + +static const mdb_walker_t walkers[] = { + { NULL } +}; + +static const mdb_modinfo_t modinfo = { + MDB_API_VERSION, dcmds, walkers +}; + +const mdb_modinfo_t * +_mdb_init(void) +{ + GElf_Sym sym; + + /* check to see if kernel supports iommu */ + if (mdb_lookup_by_name("intel_iommu_support", &sym) != -1) { + if (mdb_vread(&iommu_support, sizeof (boolean_t), + (uintptr_t)sym.st_value) != sizeof (boolean_t)) { + iommu_support = B_FALSE; + } + } + + return (&modinfo); +}
--- a/usr/src/cmd/mdb/intel/amd64/Makefile Mon Sep 21 11:25:09 2009 -0400 +++ b/usr/src/cmd/mdb/intel/amd64/Makefile Mon Sep 21 11:26:40 2009 -0400 @@ -2,9 +2,8 @@ # CDDL HEADER START # # The contents of this file are subject to the terms of the -# Common Development and Distribution License, Version 1.0 only -# (the "License"). You may not use this file except in compliance -# with the License. +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. # # You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE # or http://www.opensolaris.org/os/licensing. @@ -20,14 +19,13 @@ # CDDL HEADER END # # -# Copyright 2005 Sun Microsystems, Inc. All rights reserved. +# Copyright 2009 Sun Microsystems, Inc. All rights reserved. # Use is subject to license terms. # -#ident "%Z%%M% %I% %E% SMI" include ../../Makefile.common -MODULES = $(COMMON_MODULES_PROC) $(COMMON_MODULES_KVM) uhci +MODULES = $(COMMON_MODULES_PROC) $(COMMON_MODULES_KVM) uhci rootnex $(CLOSED_BUILD)MODULES += \ $(CLOSED_COMMON_MODULES_KVM:%=$(CLOSED)/cmd/mdb/intel/amd64/%)
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/usr/src/cmd/mdb/intel/amd64/rootnex/Makefile Mon Sep 21 11:26:40 2009 -0400 @@ -0,0 +1,38 @@ +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# +# +# Copyright 2009 Sun Microsystems, Inc. All rights reserved. +# Use is subject to license terms. + +# Copyright (c) 2009, Intel Corporation. +# All rights reserved. + +MODULE = rootnex.so +MDBTGT = kvm + +MODSRCS = intel_iommu.c + +include ../../../../Makefile.cmd +include ../../../../Makefile.cmd.64 +include ../../Makefile.amd64 +include ../../../Makefile.module + +CPPFLAGS += -I$(SRC)/uts/i86pc
--- a/usr/src/cmd/mdb/intel/ia32/Makefile Mon Sep 21 11:25:09 2009 -0400 +++ b/usr/src/cmd/mdb/intel/ia32/Makefile Mon Sep 21 11:26:40 2009 -0400 @@ -19,15 +19,14 @@ # CDDL HEADER END # # -# Copyright 2007 Sun Microsystems, Inc. All rights reserved. +# Copyright 2009 Sun Microsystems, Inc. All rights reserved. # Use is subject to license terms. # -#ident "%Z%%M% %I% %E% SMI" include ../../Makefile.common MODULES = $(COMMON_MODULES_PROC) $(COMMON_MODULES_PROC_32BIT) \ - $(COMMON_MODULES_KVM) uhci + $(COMMON_MODULES_KVM) uhci rootnex $(CLOSED_BUILD)MODULES += \ $(CLOSED_COMMON_MODULES_KVM:%=$(CLOSED)/cmd/mdb/intel/ia32/%)
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/usr/src/cmd/mdb/intel/ia32/rootnex/Makefile Mon Sep 21 11:26:40 2009 -0400 @@ -0,0 +1,37 @@ +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# +# +# Copyright 2009 Sun Microsystems, Inc. All rights reserved. +# Use is subject to license terms. + +# Copyright (c) 2009, Intel Corporation. +# All rights reserved. + +MODULE = rootnex.so +MDBTGT = kvm + +MODSRCS = intel_iommu.c + +include ../../../../Makefile.cmd +include ../../Makefile.ia32 +include ../../../Makefile.module + +CPPFLAGS += -I$(SRC)/uts/i86pc
--- a/usr/src/cmd/modload/add_drv.c Mon Sep 21 11:25:09 2009 -0400 +++ b/usr/src/cmd/modload/add_drv.c Mon Sep 21 11:26:40 2009 -0400 @@ -42,6 +42,7 @@ #include <libdevinfo.h> #include <sys/sysmacros.h> #include <fcntl.h> +#include <zone.h> #include "addrem.h" #include "errmsg.h" #include "plcysubr.h" @@ -200,6 +201,11 @@ exit(1); } + if (getzoneid() != GLOBAL_ZONEID) { + (void) fprintf(stderr, gettext(ERR_NOT_GLOBAL_ZONE)); + exit(1); + } + /* * Fail if add_drv was invoked with a pathname prepended to the * driver_name argument.
--- a/usr/src/cmd/modload/errmsg.h Mon Sep 21 11:25:09 2009 -0400 +++ b/usr/src/cmd/modload/errmsg.h Mon Sep 21 11:26:40 2009 -0400 @@ -63,6 +63,8 @@ #define ERR_FORK_FAIL "Fork failed; cannot exec : %s\n" #define ERR_PROG_IN_USE "add_drv/rem_drv currently busy; try later\n" #define ERR_NOT_ROOT "You must be root to run this program.\n" +#define ERR_NOT_GLOBAL_ZONE \ +"add_drv/rem_drv can only be run from the global zone.\n" #define ERR_BAD_LINE "Bad line in file %s : %s\n" #define ERR_CANNOT_OPEN "Cannot open (%s): %s.\n" #define ERR_MIS_TOK "Option (%s) : missing token: (%s)\n"
--- a/usr/src/cmd/modload/modload.c Mon Sep 21 11:25:09 2009 -0400 +++ b/usr/src/cmd/modload/modload.c Mon Sep 21 11:26:40 2009 -0400 @@ -2,9 +2,8 @@ * CDDL HEADER START * * The contents of this file are subject to the terms of the - * Common Development and Distribution License, Version 1.0 only - * (the "License"). You may not use this file except in compliance - * with the License. + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. @@ -20,12 +19,10 @@ * CDDL HEADER END */ /* - * Copyright 2004 Sun Microsystems, Inc. All rights reserved. + * Copyright 2009 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. */ -#pragma ident "%Z%%M% %I% %E% SMI" - #include <sys/types.h> #include <sys/wait.h> #include <sys/param.h> @@ -37,6 +34,7 @@ #include <string.h> #include <fcntl.h> #include <errno.h> +#include <zone.h> void l_exec_userfile(char *execfile, int id, char **envp); void l_usage(); @@ -75,6 +73,11 @@ l_usage(); } } + + if (getzoneid() != GLOBAL_ZONEID) { + fatal("modload can only be run from the global zone\n"); + } + modpath = argv[optind]; if (modpath == NULL) {
--- a/usr/src/cmd/modload/modunload.c Mon Sep 21 11:25:09 2009 -0400 +++ b/usr/src/cmd/modload/modunload.c Mon Sep 21 11:26:40 2009 -0400 @@ -2,9 +2,8 @@ * CDDL HEADER START * * The contents of this file are subject to the terms of the - * Common Development and Distribution License, Version 1.0 only - * (the "License"). You may not use this file except in compliance - * with the License. + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. @@ -20,12 +19,10 @@ * CDDL HEADER END */ /* - * Copyright 2003 Sun Microsystems, Inc. All rights reserved. + * Copyright 2009 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. */ -#pragma ident "%Z%%M% %I% %E% SMI" - #include <sys/types.h> #include <sys/mman.h> #include <sys/stat.h> @@ -36,6 +33,7 @@ #include <stdio.h> #include <stdlib.h> #include <sys/modctl.h> +#include <zone.h> void usage(); void exec_userfile(char *execfile, int id, char **envp); @@ -70,6 +68,10 @@ } } + if (getzoneid() != GLOBAL_ZONEID) { + fatal("modunload can only be run from the global zone\n"); + } + if (execfile) { child = fork(); if (child == -1) @@ -90,10 +92,10 @@ * Unload the module. */ if (modctl(MODUNLOAD, id) < 0) { - if (errno == EPERM) - fatal("Insufficient privileges to unload a module\n"); - else if (id != 0) - error("can't unload the module"); + if (errno == EPERM) + fatal("Insufficient privileges to unload a module\n"); + else if (id != 0) + error("can't unload the module"); } return (0); /* success */
--- a/usr/src/cmd/modload/rem_drv.c Mon Sep 21 11:25:09 2009 -0400 +++ b/usr/src/cmd/modload/rem_drv.c Mon Sep 21 11:26:40 2009 -0400 @@ -42,6 +42,7 @@ #include <sys/modctl.h> #include <sys/instance.h> #include <libdevinfo.h> +#include <zone.h> #include "addrem.h" #include "errmsg.h" @@ -120,6 +121,11 @@ exit(1); } + if (getzoneid() != GLOBAL_ZONEID) { + (void) fprintf(stderr, gettext(ERR_NOT_GLOBAL_ZONE)); + exit(1); + } + /* set up add_drv filenames */ if ((build_filenames(basedir)) == ERROR) { exit(1);
--- a/usr/src/cmd/print/bsd-sysv-commands/lpstat.c Mon Sep 21 11:25:09 2009 -0400 +++ b/usr/src/cmd/print/bsd-sysv-commands/lpstat.c Mon Sep 21 11:26:40 2009 -0400 @@ -403,7 +403,7 @@ switch (pstat) { case 0x03: /* idle */ - printf(gettext("idle. enabled")); + printf(gettext("is idle. enabled")); break; case 0x04: /* processing */ status = papiPrinterListJobs(svc, name, NULL,
--- a/usr/src/cmd/rmvolmgr/rmvolmgr.c Mon Sep 21 11:25:09 2009 -0400 +++ b/usr/src/cmd/rmvolmgr/rmvolmgr.c Mon Sep 21 11:26:40 2009 -0400 @@ -74,7 +74,6 @@ static boolean_t rmm_prop_eject_button = B_TRUE; static void get_smf_properties(); -static int daemon(int nochdir, int noclose); static void rmm_device_added(LibHalContext *ctx, const char *udi); static void rmm_device_removed(LibHalContext *ctx, const char *udi); static void rmm_property_modified(LibHalContext *ctx, const char *udi, @@ -594,50 +593,6 @@ } } -static int -daemon(int nochdir, int noclose) -{ - int fd; - - switch (fork()) { - case -1: - return (-1); - case 0: - break; - default: - exit(0); - } - - if (setsid() == -1) - return (-1); - - if (!nochdir) - (void) chdir("/"); - - if (!noclose) { - struct stat64 st; - - if (((fd = open("/dev/null", O_RDWR, 0)) != -1) && - (fstat64(fd, &st) == 0)) { - if (S_ISCHR(st.st_mode) != 0) { - (void) dup2(fd, STDIN_FILENO); - (void) dup2(fd, STDOUT_FILENO); - (void) dup2(fd, STDERR_FILENO); - if (fd > 2) - (void) close(fd); - } else { - (void) close(fd); - (void) __set_errno(ENODEV); - return (-1); - } - } else { - (void) close(fd); - return (-1); - } - } - return (0); -} - int main(int argc, char **argv) {
--- a/usr/src/cmd/sgs/include/debug.h Mon Sep 21 11:25:09 2009 -0400 +++ b/usr/src/cmd/sgs/include/debug.h Mon Sep 21 11:26:40 2009 -0400 @@ -424,6 +424,7 @@ #define Dbg_syms_spec_title Dbg64_syms_spec_title #define Dbg_syms_updated Dbg64_syms_updated #define Dbg_syms_up_title Dbg64_syms_up_title +#define Dbg_syms_wrap Dbg64_syms_wrap #define Dbg_util_call_array Dbg64_util_call_array #define Dbg_util_call_fini Dbg64_util_call_fini @@ -632,6 +633,7 @@ #define Dbg_syms_spec_title Dbg32_syms_spec_title #define Dbg_syms_updated Dbg32_syms_updated #define Dbg_syms_up_title Dbg32_syms_up_title +#define Dbg_syms_wrap Dbg32_syms_wrap #define Dbg_util_call_array Dbg32_util_call_array #define Dbg_util_call_fini Dbg32_util_call_fini @@ -896,6 +898,7 @@ extern void Dbg_syms_spec_title(Lm_list *); extern void Dbg_syms_updated(Ofl_desc *, Sym_desc *, const char *); extern void Dbg_syms_up_title(Lm_list *); +extern void Dbg_syms_wrap(Lm_list *, Word, const char *, const char *); extern void Dbg_tls_modactivity(Lm_list *, void *, uint_t); extern void Dbg_tls_static_block(Lm_list *, void *, ulong_t, ulong_t);
--- a/usr/src/cmd/sgs/include/libld.h Mon Sep 21 11:25:09 2009 -0400 +++ b/usr/src/cmd/sgs/include/libld.h Mon Sep 21 11:26:40 2009 -0400 @@ -150,6 +150,20 @@ } Rlxrel_cache; /* + * Nodes in an ofl_wrap AVL tree + * + * wsn_name is the name of the symbol to be wrapped. wsn_wrapname is used + * when we need to refer to the wrap symbol, and consists of the symbol + * name with a __wrap_ prefix. + */ +typedef struct wrap_sym_node { + avl_node_t wsn_avlnode; /* AVL book-keeping */ + const char *wsn_name; /* Symbol name: XXX */ + const char *wsn_wrapname; /* Wrap symbol name: __wrap_XXX */ +} WrapSymNode; + + +/* * Output file processing structure */ typedef Lword ofl_flag_t; @@ -304,6 +318,7 @@ /* sloppy_comdat_reloc() */ APlist *ofl_maptext; /* mapfile added text sections */ APlist *ofl_mapdata; /* mapfile added data sections */ + avl_tree_t *ofl_wrap; /* -z wrap symbols */ }; #define FLG_OF_DYNAMIC 0x00000001 /* generate dynamic output module */
--- a/usr/src/cmd/sgs/libld/Makefile.com Mon Sep 21 11:25:09 2009 -0400 +++ b/usr/src/cmd/sgs/libld/Makefile.com Mon Sep 21 11:26:40 2009 -0400 @@ -33,14 +33,14 @@ libs32.o files32.o map32.o order32.o \ outfile32.o place32.o relocate32.o resolve32.o \ sections32.o sunwmove32.o support32.o syms32.o \ - update32.o unwind32.o version32.o + update32.o unwind32.o version32.o wrap32.o COMOBJS64 = args64.o entry64.o exit64.o groups64.o \ ldentry64.o ldlibs64.o ldmachdep64.o ldmain64.o \ libs64.o files64.o map64.o order64.o \ outfile64.o place64.o relocate64.o resolve64.o \ sections64.o sunwmove64.o support64.o syms64.o \ - update64.o unwind64.o version64.o + update64.o unwind64.o version64.o wrap64.o TOOLOBJS = alist.o assfail.o findprime.o string_table.o \ strhash.o
--- a/usr/src/cmd/sgs/libld/common/_libld.h Mon Sep 21 11:25:09 2009 -0400 +++ b/usr/src/cmd/sgs/libld/common/_libld.h Mon Sep 21 11:26:40 2009 -0400 @@ -617,6 +617,7 @@ #define ld_vers_promote ld64_vers_promote #define ld_vers_sym_process ld64_vers_sym_process #define ld_vers_verify ld64_vers_verify +#define ld_wrap_enter ld64_wrap_enter #else @@ -705,6 +706,7 @@ #define ld_vers_promote ld32_vers_promote #define ld_vers_sym_process ld32_vers_sym_process #define ld_vers_verify ld32_vers_verify +#define ld_wrap_enter ld32_wrap_enter #endif @@ -834,6 +836,7 @@ Ofl_desc *); extern int ld_vers_sym_process(Lm_list *, Is_desc *, Ifl_desc *); extern int ld_vers_verify(Ofl_desc *); +extern WrapSymNode *ld_wrap_enter(Ofl_desc *, const char *); extern uintptr_t add_regsym(Sym_desc *, Ofl_desc *); extern Word hashbkts(Word);
--- a/usr/src/cmd/sgs/libld/common/args.c Mon Sep 21 11:25:09 2009 -0400 +++ b/usr/src/cmd/sgs/libld/common/args.c Mon Sep 21 11:26:40 2009 -0400 @@ -55,6 +55,21 @@ * -z nosighandler suppress the registration of the signal handler * used to manage SIGBUS. */ + +/* + * The following flags are committed, and will not be removed, but are + * not publically documented, either because they are obsolete, or because + * they exist to work around defects in other software and are not of + * sufficient interest otherwise. + * + * OPTION MEANING + * + * -Wl,... compiler drivers and configuration tools + * have been known to pass this compiler option + * to ld(1). Strip off the "-Wl," prefix and + * process the remainder (...) as a normal option. + */ + #include <sys/link.h> #include <stdio.h> #include <fcntl.h> @@ -198,6 +213,7 @@ (void) fprintf(stderr, MSG_INTL(MSG_ARG_DETAIL_ZT)); (void) fprintf(stderr, MSG_INTL(MSG_ARG_DETAIL_ZTO)); (void) fprintf(stderr, MSG_INTL(MSG_ARG_DETAIL_ZTW)); + (void) fprintf(stderr, MSG_INTL(MSG_ARG_DETAIL_ZWRAP)); (void) fprintf(stderr, MSG_INTL(MSG_ARG_DETAIL_ZV)); } @@ -1299,6 +1315,17 @@ /* Don't report cascading errors */ ofl->ofl_ars_gsandx = -1; } + + /* + * If -z wrap is seen, enter the symbol to be wrapped + * into the wrap AVL tree. + */ + } else if (strncmp(optarg, MSG_ORIG(MSG_ARG_WRAP), + MSG_ARG_WRAP_SIZE) == 0) { + if (ld_wrap_enter(ofl, + optarg + MSG_ARG_WRAP_SIZE) == NULL) + return (S_ERROR); + /* * The following options just need validation as they * are interpreted on the second pass through the
--- a/usr/src/cmd/sgs/libld/common/ldmain.c Mon Sep 21 11:25:09 2009 -0400 +++ b/usr/src/cmd/sgs/libld/common/ldmain.c Mon Sep 21 11:26:40 2009 -0400 @@ -408,9 +408,8 @@ * our memory consumption and freeing are doing. We should be able to * free all the memory that has been allocated as part of the link-edit * process. - * - * ofl_cleanup(ofl); */ + /* ld_ofl_cleanup(ofl); */ return (0); }
--- a/usr/src/cmd/sgs/libld/common/libld.msg Mon Sep 21 11:25:09 2009 -0400 +++ b/usr/src/cmd/sgs/libld/common/libld.msg Mon Sep 21 11:26:40 2009 -0400 @@ -228,6 +228,8 @@ text\n" @ MSG_ARG_DETAIL_ZTW "\t[-z textwarn]\twarn if there are relocations \ against text\n" +@ MSG_ARG_DETAIL_ZWRAP "\t[-z wrap=symbol], [-wrap=symbol], [--wrap=symbol]\n\ + \t\t\twrap symbol references\n" @ MSG_ARG_DETAIL_ZV "\t[-z verbose]\t\ generate warnings for suspicious processings\n" @@ -652,11 +654,14 @@ @ MSG_STR_ISALIST "$ISALIST" @ MSG_STR_OSNAME "$OSNAME" @ MSG_STR_OSREL "$OSREL" +@ MSG_STR_UU_REAL_U "__real_" +@ MSG_STR_UU_WRAP_U "__wrap_" @ MSG_FMT_ARMEM "%s(%s)" @ MSG_FMT_COLPATH "%s:%s" @ MSG_FMT_SYMNAM "`%s'" @ MSG_FMT_NULLSYMNAM "%s[%d]" +@ MSG_FMT_STRCAT "%s%s" @ MSG_PTH_RTLD "/usr/lib/ld.so.1" @@ -1211,6 +1216,7 @@ @ MSG_ARG_NOSIGHANDLER "nosighandler" @ MSG_ARG_GLOBAUDIT "globalaudit" @ MSG_ARG_TARGET "target=" +@ MSG_ARG_WRAP "wrap=" @ MSG_ARG_HELP "help" @ MSG_ARG_GROUP "group" @ MSG_ARG_REDUCE "reduce" @@ -1243,6 +1249,7 @@ @ MSG_ARG_T_UNDEF "-undefined" @ MSG_ARG_T_VERSION "-version" @ MSG_ARG_T_WHOLEARC "-whole-archive" +@ MSG_ARG_T_WRAP "-wrap" @ MSG_ARG_T_OPAR "(" @ MSG_ARG_T_CPAR ")"
--- a/usr/src/cmd/sgs/libld/common/syms.c Mon Sep 21 11:25:09 2009 -0400 +++ b/usr/src/cmd/sgs/libld/common/syms.c Mon Sep 21 11:26:40 2009 -0400 @@ -2312,6 +2312,48 @@ } /* + * The '-z wrap=XXX' option emulates the GNU ld --wrap=XXX + * option. When XXX is the symbol to be wrapped: + * + * - An undefined reference to XXX is converted to __wrap_XXX + * - An undefined reference to __real_XXX is converted to XXX + * + * The idea is that the user can supply a wrapper function + * __wrap_XXX that does some work, and then uses the name + * __real_XXX to pass the call on to the real function. The + * wrapper objects are linked with the original unmodified + * objects to produce a wrapped version of the output object. + */ + if (ofl->ofl_wrap && name[0] && (shndx == SHN_UNDEF)) { + WrapSymNode wsn, *wsnp; + + /* + * If this is the __real_XXX form, advance the + * pointer to reference the wrapped name. + */ + wsn.wsn_name = name; + if ((*name == '_') && + (strncmp(name, MSG_ORIG(MSG_STR_UU_REAL_U), + MSG_STR_UU_REAL_U_SIZE) == 0)) + wsn.wsn_name += MSG_STR_UU_REAL_U_SIZE; + + /* + * Is this symbol in the wrap AVL tree? If so, map + * XXX to __wrap_XXX, and __real_XXX to XXX. Note that + * wsn.wsn_name will equal the current value of name + * if the __real_ prefix is not present. + */ + if ((wsnp = avl_find(ofl->ofl_wrap, &wsn, 0)) != NULL) { + const char *old_name = name; + + name = (wsn.wsn_name == name) ? + wsnp->wsn_wrapname : wsn.wsn_name; + DBG_CALL(Dbg_syms_wrap(ofl->ofl_lml, ndx, + old_name, name)); + } + } + + /* * Determine and validate the symbols binding. */ bind = ELF_ST_BIND(sym->st_info);
--- a/usr/src/cmd/sgs/libld/common/util.c Mon Sep 21 11:25:09 2009 -0400 +++ b/usr/src/cmd/sgs/libld/common/util.c Mon Sep 21 11:26:40 2009 -0400 @@ -279,13 +279,48 @@ } /* + * The GNU ld '-wrap=XXX' and '--wrap=XXX' options correspond to our + * '-z wrap=XXX'. When str2chr() does this conversion, we end up with + * the return character set to 'z' and optarg set to 'XXX'. This callback + * changes optarg to include the missing wrap= prefix. + * + * exit: + * Returns c on success, or '?' on error. + */ +static int +str2chr_wrap_cb(int c) +{ + char *str; + size_t len = MSG_ARG_WRAP_SIZE + strlen(optarg) + 1; + + if ((str = libld_malloc(len)) == NULL) + return ('?'); + (void) snprintf(str, len, MSG_ORIG(MSG_FMT_STRCAT), + MSG_ORIG(MSG_ARG_WRAP), optarg); + optarg = str; + return (c); +} + +/* * Determine whether this string, possibly with an associated option, should be * translated to an option character. If so, update the optind and optarg * as described for short options in getopt(3c). + * + * entry: + * lml - Link map list for debug messages + * ndx - Starting optind for current item + * argc, argv - Command line arguments + * arg - Option to be examined + * c, opt - Option character (c) and corresponding long name (opt) + * optsz - 0 if option does not accept a value. If option does + * accept a value, strlen(opt), giving the offset to the + * value if the option and value are combined in one string. + * cbfunc - NULL, or pointer to function to call if a translation is + * successful. */ static int str2chr(Lm_list *lml, int ndx, int argc, char **argv, char *arg, int c, - const char *opt, size_t optsz) + const char *opt, size_t optsz, int cbfunc(int)) { if (optsz == 0) { /* @@ -331,6 +366,10 @@ return ('?'); } } + + if (cbfunc != NULL) + c = (*cbfunc)(c); + return (c); } return (0); @@ -362,20 +401,28 @@ /* Translate -rpath <optarg> to -R <optarg> */ if ((c = str2chr(lml, ndx, argc, argv, arg, 'R', MSG_ORIG(MSG_ARG_T_RPATH), - MSG_ARG_T_RPATH_SIZE)) != 0) { + MSG_ARG_T_RPATH_SIZE, NULL)) != 0) { return (c); } break; case 's': /* Translate -shared to -G */ if ((c = str2chr(lml, ndx, argc, argv, arg, 'G', - MSG_ORIG(MSG_ARG_T_SHARED), 0)) != 0) { + MSG_ORIG(MSG_ARG_T_SHARED), 0, NULL)) != 0) { return (c); /* Translate -soname <optarg> to -h <optarg> */ } else if ((c = str2chr(lml, ndx, argc, argv, arg, 'h', MSG_ORIG(MSG_ARG_T_SONAME), - MSG_ARG_T_SONAME_SIZE)) != 0) { + MSG_ARG_T_SONAME_SIZE, NULL)) != 0) { + return (c); + } + break; + case 'w': + /* Translate -wrap to -z wrap= */ + if ((c = str2chr(lml, ndx, argc, argv, arg, 'z', + MSG_ORIG(MSG_ARG_T_WRAP) + 1, + MSG_ARG_T_WRAP_SIZE - 1, str2chr_wrap_cb)) != 0) { return (c); } break; @@ -384,7 +431,8 @@ * Translate -( to -z rescan-start */ if ((c = str2chr(lml, ndx, argc, argv, - arg, 'z', MSG_ORIG(MSG_ARG_T_OPAR), 0)) != 0) { + arg, 'z', MSG_ORIG(MSG_ARG_T_OPAR), 0, NULL)) != + 0) { optarg = (char *)MSG_ORIG(MSG_ARG_RESCAN_START); return (c); } @@ -394,7 +442,8 @@ * Translate -) to -z rescan-end */ if ((c = str2chr(lml, ndx, argc, argv, - arg, 'z', MSG_ORIG(MSG_ARG_T_CPAR), 0)) != 0) { + arg, 'z', MSG_ORIG(MSG_ARG_T_CPAR), 0, NULL)) != + 0) { optarg = (char *)MSG_ORIG(MSG_ARG_RESCAN_END); return (c); } @@ -407,7 +456,8 @@ * -zmuldefs */ if ((c = str2chr(lml, ndx, argc, argv, arg, 'z', - MSG_ORIG(MSG_ARG_T_MULDEFS), 0)) != 0) { + MSG_ORIG(MSG_ARG_T_MULDEFS), 0, NULL)) != + 0) { optarg = (char *)MSG_ORIG(MSG_ARG_MULDEFS); return (c); @@ -418,7 +468,7 @@ */ } else if ((c = str2chr(lml, argc, ndx, argv, arg, 'f', MSG_ORIG(MSG_ARG_T_AUXFLTR), - MSG_ARG_T_AUXFLTR_SIZE)) != 0) { + MSG_ARG_T_AUXFLTR_SIZE, NULL)) != 0) { return (c); } break; @@ -429,7 +479,7 @@ */ if ((c = str2chr(lml, ndx, argc, argv, arg, 'I', MSG_ORIG(MSG_ARG_T_INTERP), - MSG_ARG_T_INTERP_SIZE)) != 0) { + MSG_ARG_T_INTERP_SIZE, NULL)) != 0) { return (c); } break; @@ -437,15 +487,15 @@ /* Translate --entry <optarg> to -e <optarg> */ if ((c = str2chr(lml, ndx, argc, argv, arg, 'e', MSG_ORIG(MSG_ARG_T_ENTRY), - MSG_ARG_T_ENTRY_SIZE)) != 0) { + MSG_ARG_T_ENTRY_SIZE, NULL)) != 0) { return (c); } /* * Translate --end-group to -z rescan-end */ if ((c = str2chr(lml, ndx, argc, argv, - arg, 'z', - MSG_ORIG(MSG_ARG_T_ENDGROUP), 0)) != 0) { + arg, 'z', MSG_ORIG(MSG_ARG_T_ENDGROUP), + 0, NULL)) != 0) { optarg = (char *) MSG_ORIG(MSG_ARG_RESCAN_END); return (c); @@ -455,14 +505,15 @@ /* Translate --filter <optarg> to -F <optarg> */ if ((c = str2chr(lml, ndx, argc, argv, arg, 'F', MSG_ORIG(MSG_ARG_T_STDFLTR), - MSG_ARG_T_STDFLTR_SIZE)) != 0) { + MSG_ARG_T_STDFLTR_SIZE, NULL)) != 0) { return (c); } break; case 'h': /* Translate --help to -zhelp */ if ((c = str2chr(lml, ndx, argc, argv, arg, 'z', - MSG_ORIG(MSG_ARG_T_HELP), 0)) != 0) { + MSG_ORIG(MSG_ARG_T_HELP), 0, NULL)) != + 0) { optarg = (char *)MSG_ORIG(MSG_ARG_HELP); return (c); } @@ -473,7 +524,7 @@ */ if ((c = str2chr(lml, ndx, argc, argv, arg, 'l', MSG_ORIG(MSG_ARG_T_LIBRARY), - MSG_ARG_T_LIBRARY_SIZE)) != 0) { + MSG_ARG_T_LIBRARY_SIZE, NULL)) != 0) { return (c); /* @@ -482,14 +533,15 @@ */ } else if ((c = str2chr(lml, ndx, argc, argv, arg, 'L', MSG_ORIG(MSG_ARG_T_LIBPATH), - MSG_ARG_T_LIBPATH_SIZE)) != 0) { + MSG_ARG_T_LIBPATH_SIZE, NULL)) != 0) { return (c); } break; case 'n': /* Translate --no-undefined to -zdefs */ if ((c = str2chr(lml, ndx, argc, argv, arg, 'z', - MSG_ORIG(MSG_ARG_T_NOUNDEF), 0)) != 0) { + MSG_ORIG(MSG_ARG_T_NOUNDEF), 0, NULL)) != + 0) { optarg = (char *)MSG_ORIG(MSG_ARG_DEFS); return (c); @@ -498,8 +550,8 @@ * -z defaultextract */ } else if ((c = str2chr(lml, ndx, argc, argv, - arg, 'z', - MSG_ORIG(MSG_ARG_T_NOWHOLEARC), 0)) != 0) { + arg, 'z', MSG_ORIG(MSG_ARG_T_NOWHOLEARC), + 0, NULL)) != 0) { optarg = (char *)MSG_ORIG(MSG_ARG_DFLEXTRT); return (c); @@ -509,29 +561,31 @@ /* Translate --output <optarg> to -o <optarg> */ if ((c = str2chr(lml, ndx, argc, argv, arg, 'o', MSG_ORIG(MSG_ARG_T_OUTPUT), - MSG_ARG_T_OUTPUT_SIZE)) != 0) { + MSG_ARG_T_OUTPUT_SIZE, NULL)) != 0) { return (c); } break; case 'r': /* Translate --relocatable to -r */ if ((c = str2chr(lml, ndx, argc, argv, arg, 'r', - MSG_ORIG(MSG_ARG_T_RELOCATABLE), 0)) != 0) { + MSG_ORIG(MSG_ARG_T_RELOCATABLE), 0, + NULL)) != 0) { return (c); } break; case 's': /* Translate --strip-all to -s */ if ((c = str2chr(lml, ndx, argc, argv, arg, 's', - MSG_ORIG(MSG_ARG_T_STRIP), 0)) != 0) { + MSG_ORIG(MSG_ARG_T_STRIP), 0, NULL)) != + 0) { return (c); } /* * Translate --start-group to -z rescan-start */ if ((c = str2chr(lml, ndx, argc, argv, - arg, 'z', - MSG_ORIG(MSG_ARG_T_STARTGROUP), 0)) != 0) { + arg, 'z', MSG_ORIG(MSG_ARG_T_STARTGROUP), + 0, NULL)) != 0) { optarg = (char *) MSG_ORIG(MSG_ARG_RESCAN_START); return (c); @@ -544,14 +598,15 @@ */ if ((c = str2chr(lml, ndx, argc, argv, arg, 'u', MSG_ORIG(MSG_ARG_T_UNDEF), - MSG_ARG_T_UNDEF_SIZE)) != 0) { + MSG_ARG_T_UNDEF_SIZE, NULL)) != 0) { return (c); } break; case 'v': /* Translate --version to -V */ if ((c = str2chr(lml, ndx, argc, argv, arg, 'V', - MSG_ORIG(MSG_ARG_T_VERSION), 0)) != 0) { + MSG_ORIG(MSG_ARG_T_VERSION), 0, NULL)) != + 0) { return (c); } break; @@ -560,17 +615,27 @@ * Translate --whole-archive to -z alltextract */ if ((c = str2chr(lml, ndx, argc, argv, - arg, 'z', - MSG_ORIG(MSG_ARG_T_WHOLEARC), 0)) != 0) { + arg, 'z', MSG_ORIG(MSG_ARG_T_WHOLEARC), + 0, NULL)) != 0) { optarg = (char *)MSG_ORIG(MSG_ARG_ALLEXTRT); return (c); } + /* + * Translate --wrap to -z wrap= + */ + if ((c = str2chr(lml, ndx, argc, argv, + arg, 'z', MSG_ORIG(MSG_ARG_T_WRAP), + MSG_ARG_T_WRAP_SIZE, str2chr_wrap_cb)) != + 0) { + return (c); + } break; } break; } } + if ((c = getopt(argc, argv, MSG_ORIG(MSG_STR_OPTIONS))) != -1) { /* * It is possible that a "-Wl," argument has been used to
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/usr/src/cmd/sgs/libld/common/wrap.c Mon Sep 21 11:26:40 2009 -0400 @@ -0,0 +1,131 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ + +/* + * Copyright 2009 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +#include <unistd.h> +#include <stdio.h> +#include <stdarg.h> +#include "msg.h" +#include "_libld.h" + +/* + * GNU ld --wrap support, also known as -z wrap. + * + * We maintain an AVL tree of wrapped symbol names. Every undefined + * symbol is tested against this tree, and those that match have + * their names modified to produce the wrapping effect: + * + * - An undefined reference to XXX is converted to __wrap_XXX + * - An undefined reference to __real_XXX is converted to XXX + * + * This operation has a cost, but that is mitigated by two factors: + * + * - This is a test feature, not used for production code, so somewhat + * longer link times are tolerable. + * - The cost of this feature is only paid when it is used. Otherwise, + * the sole overhead is the cost of testing the NULL AVL tree pointer + * during symbol processing. + */ + + +/* + * AVL comparison function for WrapSymNode items. + * + * entry: + * n1, n2 - pointers to nodes to be compared + * + * exit: + * Returns -1 if (n1 < n2), 0 if they are equal, and 1 if (n1 > n2) + */ +static int +wrap_cmp(const void *n1, const void *n2) +{ + int rc; + + rc = strcmp(((WrapSymNode *)n1)->wsn_name, + ((WrapSymNode *)n2)->wsn_name); + + if (rc > 0) + return (1); + if (rc < 0) + return (-1); + return (0); +} + +/* + * Enter a -z wrap symbol into the ofl_wrap AVL tree + * + * entry: + * ofl - Output file descriptor + * name - Name of symbol to be entered. Caller must ensure that + * memory used to hold name remains available for the life + * of the link-edit process. + * + * exit: + * On success, updates ofl->wrap_cache with a pointer to the + * resulting WrapSymNode, and returns that pointer. On failure, + * returns NULL. + */ +WrapSymNode * +ld_wrap_enter(Ofl_desc *ofl, const char *name) +{ + WrapSymNode *wsnp, wsn; + avl_index_t where; + size_t name_len, wrapname_len; + char *tmpname; + + /* If this is the first wrap symbol, create the AVL tree */ + if (ofl->ofl_wrap == NULL) { + ofl->ofl_wrap = libld_calloc(1, sizeof (*ofl->ofl_wrap)); + if (ofl->ofl_wrap == NULL) + return (NULL); + avl_create(ofl->ofl_wrap, wrap_cmp, sizeof (WrapSymNode), + SGSOFFSETOF(WrapSymNode, wsn_avlnode)); + } + + /* Have we already entered this one? */ + wsn.wsn_name = name; + if ((wsnp = avl_find(ofl->ofl_wrap, &wsn, &where)) != NULL) + return (wsnp); + + /* + * Allocate a new node, along with room for the wrapped name. + * Since strings have byte alignment, we can allocate it immediately + * following the AVL node without the need for alignment padding. + */ + name_len = strlen(wsn.wsn_name); + wrapname_len = MSG_STR_UU_WRAP_U_SIZE + name_len + 1; + if ((wsnp = libld_calloc(1, sizeof (*wsnp) + wrapname_len)) == NULL) + return (NULL); + wsnp->wsn_name = name; + + wsnp->wsn_wrapname = tmpname = (char *)(wsnp + 1); + (void) snprintf(tmpname, wrapname_len, MSG_ORIG(MSG_FMT_STRCAT), + MSG_ORIG(MSG_STR_UU_WRAP_U), name); + + /* Insert the new node */ + avl_insert(ofl->ofl_wrap, wsnp, where); + return (wsnp); +}
--- a/usr/src/cmd/sgs/liblddbg/common/liblddbg.msg Mon Sep 21 11:25:09 2009 -0400 +++ b/usr/src/cmd/sgs/liblddbg/common/liblddbg.msg Mon Sep 21 11:26:40 2009 -0400 @@ -953,6 +953,7 @@ discarded file=%s" @ MSG_SYM_DISCARD_DUP "symbol[%d]=%s; discarded duplicate: originates from \ file=%s" +@ MSG_SYM_WRAP "symbol[%d]=%s renamed to %s (-z wrap)" @ MSG_SYM_AOUT "symbol=%s; (original AOUT name)" @ MSG_SYM_LOOKUP "symbol=%s; lookup in file=%s [ %s ]"
--- a/usr/src/cmd/sgs/liblddbg/common/llib-llddbg Mon Sep 21 11:25:09 2009 -0400 +++ b/usr/src/cmd/sgs/liblddbg/common/llib-llddbg Mon Sep 21 11:26:40 2009 -0400 @@ -404,6 +404,8 @@ void Dbg64_syms_updated(Ofl_desc *, Sym_desc *, const char *); void Dbg32_syms_up_title(Lm_list *); void Dbg64_syms_up_title(Lm_list *); +void Dbg32_syms_wrap(Lm_list *, Elf32_Word, const char *, const char *); +void Dbg64_syms_wrap(Lm_list *, Elf64_Word, const char *, const char *); void Dbg32_util_broadcast(Rt_map *); void Dbg64_util_broadcast(Rt_map *);
--- a/usr/src/cmd/sgs/liblddbg/common/mapfile-vers Mon Sep 21 11:25:09 2009 -0400 +++ b/usr/src/cmd/sgs/liblddbg/common/mapfile-vers Mon Sep 21 11:26:40 2009 -0400 @@ -41,7 +41,7 @@ # MAPFILE HEADER END # -SUNWprivate_4.73 { +SUNWprivate_4.74 { global: dbg_desc = NODIRECT; # interposed - ld.so.1(1) dbg_print = NODIRECT; # interposed - ld(1) and ld.so.1(1) @@ -399,6 +399,8 @@ Dbg64_syms_updated; Dbg32_syms_up_title; Dbg64_syms_up_title; + Dbg32_syms_wrap; + Dbg64_syms_wrap; Dbg_tls_modactivity; Dbg_tls_static_block;
--- a/usr/src/cmd/sgs/liblddbg/common/syms.c Mon Sep 21 11:25:09 2009 -0400 +++ b/usr/src/cmd/sgs/liblddbg/common/syms.c Mon Sep 21 11:26:40 2009 -0400 @@ -284,6 +284,16 @@ } void +Dbg_syms_wrap(Lm_list *lml, Word ndx, const char *orig_name, const char *name) +{ + if (DBG_NOTCLASS(DBG_C_SYMBOLS)) + return; + + dbg_print(lml, MSG_INTL(MSG_SYM_WRAP), EC_WORD(ndx), + Dbg_demangle_name(orig_name), Dbg_demangle_name(name)); +} + +void Dbg_syms_sec_title(Lm_list *lml) { if (DBG_NOTCLASS(DBG_C_SYMBOLS))
--- a/usr/src/cmd/sgs/packages/common/SUNWonld-README Mon Sep 21 11:25:09 2009 -0400 +++ b/usr/src/cmd/sgs/packages/common/SUNWonld-README Mon Sep 21 11:26:40 2009 -0400 @@ -1531,3 +1531,5 @@ 6834197 ld pukes when given an empty plate 6516644 per-symbol filtering shouldn't be allowed in executables 6878605 ld should accept '%' syntax when matching input SHT_PROGBITS sections +6850768 ld option to autogenerate wrappers/interposers similar to GNU ld --wrap + PSARC/2009/493 ld -z wrap option
--- a/usr/src/cmd/ssh/include/config.h Mon Sep 21 11:25:09 2009 -0400 +++ b/usr/src/cmd/ssh/include/config.h Mon Sep 21 11:26:40 2009 -0400 @@ -279,9 +279,6 @@ /* Define if your libraries define login() */ /* #undef HAVE_LOGIN */ -/* Define if your libraries define daemon() */ -/* #undef HAVE_DAEMON */ - /* Define if your libraries define getpagesize() */ #define HAVE_GETPAGESIZE 1
--- a/usr/src/cmd/ssh/include/daemon.h Mon Sep 21 11:25:09 2009 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,21 +0,0 @@ -/* $Id: daemon.h,v 1.2 2001/02/09 01:55:36 djm Exp $ */ - -#ifndef _DAEMON_H -#define _DAEMON_H - -#pragma ident "%Z%%M% %I% %E% SMI" - -#ifdef __cplusplus -extern "C" { -#endif - -#include "config.h" -#ifndef HAVE_DAEMON -int daemon(int nochdir, int noclose); -#endif /* !HAVE_DAEMON */ - -#ifdef __cplusplus -} -#endif - -#endif /* _DAEMON_H */
--- a/usr/src/cmd/ssh/include/openbsd-compat.h Mon Sep 21 11:25:09 2009 -0400 +++ b/usr/src/cmd/ssh/include/openbsd-compat.h Mon Sep 21 11:26:40 2009 -0400 @@ -44,7 +44,6 @@ #include "strlcat.h" #include "strmode.h" #include "mktemp.h" -#include "daemon.h" #include "dirname.h" #include "base64.h" #include "sigact.h"
--- a/usr/src/cmd/ssh/libopenbsd-compat/Makefile.com Mon Sep 21 11:25:09 2009 -0400 +++ b/usr/src/cmd/ssh/libopenbsd-compat/Makefile.com Mon Sep 21 11:26:40 2009 -0400 @@ -18,7 +18,7 @@ # # CDDL HEADER END # -# Copyright 2008 Sun Microsystems, Inc. All rights reserved. +# Copyright 2009 Sun Microsystems, Inc. All rights reserved. # Use is subject to license terms. # # cmd/ssh/libopenbsd-compat/Makefile.com @@ -41,7 +41,6 @@ xmmap.o \ base64.o \ bindresvport.o \ - daemon.o \ dirname.o \ getcwd.o \ getgrouplist.o \
--- a/usr/src/cmd/ssh/libopenbsd-compat/common/daemon.c Mon Sep 21 11:25:09 2009 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,86 +0,0 @@ -/*- - * Copyright (c) 1990, 1993 - * The Regents of the University of California. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * 3. All advertising materials mentioning features or use of this software - * must display the following acknowledgement: - * This product includes software developed by the University of - * California, Berkeley and its contributors. - * 4. Neither the name of the University nor the names of its contributors - * may be used to endorse or promote products derived from this software - * without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ - -#include "includes.h" - -#ifndef HAVE_DAEMON - -#if defined(LIBC_SCCS) && !defined(lint) -static char rcsid[] = "$OpenBSD: daemon.c,v 1.2 1996/08/19 08:22:13 tholo Exp $"; -#endif /* LIBC_SCCS and not lint */ - -int -daemon(nochdir, noclose) - int nochdir, noclose; -{ - int fd; - - switch (fork()) { - case -1: - return (-1); - case 0: -#ifdef HAVE_CYGWIN - register_9x_service(); -#endif - break; - default: -#ifdef HAVE_CYGWIN - /* - * This sleep avoids a race condition which kills the - * child process if parent is started by a NT/W2K service. - */ - sleep(1); -#endif - _exit(0); - } - - if (setsid() == -1) - return (-1); - - if (!nochdir) - (void)chdir("/"); - - if (!noclose && (fd = open(_PATH_DEVNULL, O_RDWR, 0)) != -1) { - (void)dup2(fd, STDIN_FILENO); - (void)dup2(fd, STDOUT_FILENO); - (void)dup2(fd, STDERR_FILENO); - if (fd > 2) - (void)close (fd); - } - return (0); -} - -#endif /* !HAVE_DAEMON */ - - -#pragma ident "%Z%%M% %I% %E% SMI"
--- a/usr/src/cmd/ssh/libopenbsd-compat/common/llib-lopenbsd-compat Mon Sep 21 11:25:09 2009 -0400 +++ b/usr/src/cmd/ssh/libopenbsd-compat/common/llib-lopenbsd-compat Mon Sep 21 11:26:40 2009 -0400 @@ -21,7 +21,7 @@ * * CDDL HEADER END * - * Copyright 2008 Sun Microsystems, Inc. All rights reserved. + * Copyright 2009 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. */ @@ -38,7 +38,6 @@ #include <bsd-waitpid.h> #include <config.h> #include <crc32.h> -#include <daemon.h> #include <deattack.h> #include <defines.h> #include <dirname.h>
--- a/usr/src/cmd/ssh/libssh/common/llib-lssh Mon Sep 21 11:25:09 2009 -0400 +++ b/usr/src/cmd/ssh/libssh/common/llib-lssh Mon Sep 21 11:26:40 2009 -0400 @@ -21,7 +21,7 @@ * * CDDL HEADER END * - * Copyright 2008 Sun Microsystems, Inc. All rights reserved. + * Copyright 2009 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. */ @@ -53,7 +53,6 @@ #include <compress.h> #include <config.h> #include <crc32.h> -#include <daemon.h> #include <deattack.h> #include <defines.h> #include <dh.h>
--- a/usr/src/cmd/uadmin/uadmin.c Mon Sep 21 11:25:09 2009 -0400 +++ b/usr/src/cmd/uadmin/uadmin.c Mon Sep 21 11:26:40 2009 -0400 @@ -67,10 +67,6 @@ adt_event_data_t *event = NULL; /* event to be generated */ au_event_t event_id; enum adt_uadmin_fcn fcn_id; -#ifdef __i386 - uint8_t boot_config = 0; -#endif /* __i386 */ - if (argc < 3 || argc > 4) { (void) fprintf(stderr, Usage, argv[0]); @@ -208,10 +204,16 @@ } #ifdef __i386 } else if (cmd == A_CONFIG) { + uint8_t boot_config = 0; + uint8_t boot_config_ovr = 0; + switch (fcn) { case AD_UPDATE_BOOT_CONFIG: fcn_id = ADT_UADMIN_FCN_AD_UPDATE_BOOT_CONFIG; scf_get_boot_config(&boot_config); + boot_config_ovr = boot_config; + scf_get_boot_config_ovr(&boot_config_ovr); + boot_config &= boot_config_ovr; mdep = (uintptr_t)(&boot_config); break; }
--- a/usr/src/common/zfs/zfs_prop.c Mon Sep 21 11:25:09 2009 -0400 +++ b/usr/src/common/zfs/zfs_prop.c Mon Sep 21 11:26:40 2009 -0400 @@ -335,6 +335,10 @@ ZFS_TYPE_DATASET, "GUID"); register_hidden(ZFS_PROP_USERACCOUNTING, "useraccounting", PROP_TYPE_NUMBER, PROP_READONLY, ZFS_TYPE_DATASET, NULL); + register_hidden(ZFS_PROP_UNIQUE, "unique", PROP_TYPE_NUMBER, + PROP_READONLY, ZFS_TYPE_DATASET, NULL); + register_hidden(ZFS_PROP_OBJSETID, "objsetid", PROP_TYPE_NUMBER, + PROP_READONLY, ZFS_TYPE_DATASET, "OBJSETID"); /* oddball properties */ register_impl(ZFS_PROP_CREATION, "creation", PROP_TYPE_NUMBER, 0, NULL,
--- a/usr/src/head/stdlib.h Mon Sep 21 11:25:09 2009 -0400 +++ b/usr/src/head/stdlib.h Mon Sep 21 11:26:40 2009 -0400 @@ -20,7 +20,7 @@ */ /* - * Copyright 2008 Sun Microsystems, Inc. All rights reserved. + * Copyright 2009 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. */ @@ -30,8 +30,6 @@ #ifndef _STDLIB_H #define _STDLIB_H -#pragma ident "%Z%%M% %I% %E% SMI" /* SVr4.0 1.22 */ - #include <iso/stdlib_iso.h> #include <iso/stdlib_c99.h> @@ -215,6 +213,7 @@ #if defined(__EXTENSIONS__) || \ (!defined(_STRICT_STDC) && !defined(__XOPEN_OR_POSIX)) extern void closefrom(int); +extern int daemon(int, int); extern int dup2(int, int); extern int fdwalk(int (*)(void *, int), void *); extern char *qecvt(long double, int, int *, int *); @@ -317,6 +316,7 @@ #if defined(__EXTENSIONS__) || !defined(__XOPEN_OR_POSIX) extern void closefrom(); +extern int daemon(); extern int dup2(); extern int fdwalk(); extern char *qecvt();
--- a/usr/src/lib/libc/amd64/Makefile Mon Sep 21 11:25:09 2009 -0400 +++ b/usr/src/lib/libc/amd64/Makefile Mon Sep 21 11:26:40 2009 -0400 @@ -378,6 +378,7 @@ csetlen.o \ ctime.o \ ctime_r.o \ + daemon.o \ deflt.o \ directio.o \ dirname.o \
--- a/usr/src/lib/libc/amd64/gen/proc64_id.c Mon Sep 21 11:25:09 2009 -0400 +++ b/usr/src/lib/libc/amd64/gen/proc64_id.c Mon Sep 21 11:26:40 2009 -0400 @@ -20,7 +20,7 @@ */ /* - * Copyright (c) 2008, Intel Corporation. + * Copyright (c) 2009, Intel Corporation. * All rights reserved. */ @@ -226,6 +226,7 @@ if (cpuid_info.edx & CPUID_INTC_EDX_SSE2) { use_sse |= USE_SSE2; } + use_sse |= USE_BSF; __intel_set_memops_method(use_sse); } else { __set_cache_sizes(INTEL_DFLT_L1_CACHE_SIZE,
--- a/usr/src/lib/libc/amd64/gen/proc64_id.h Mon Sep 21 11:25:09 2009 -0400 +++ b/usr/src/lib/libc/amd64/gen/proc64_id.h Mon Sep 21 11:26:40 2009 -0400 @@ -20,7 +20,7 @@ */ /* - * Copyright (c) 2008, Intel Corporation + * Copyright (c) 2009, Intel Corporation * All rights reserved. */ @@ -38,7 +38,7 @@ #endif /* - * Defines to determine what SSE instructions can be used for memops or strops. + * Defines to determine what SSE instructions can be used for memops or strops */ #define NO_SSE 0x00 /* Default -- Don't use SSE instructions */ #define USE_SSE2 0x01 /* SSE2 */ @@ -46,6 +46,7 @@ #define USE_SSSE3 0x04 /* Supplemental SSE3 */ #define USE_SSE4_1 0x08 /* SSE 4.1 */ #define USE_SSE4_2 0x10 /* SSE 4.2 */ +#define USE_BSF 0x20 /* USE BSF class of instructions */ /* * Cache size defaults for Core 2 Duo
--- a/usr/src/lib/libc/amd64/gen/proc64_support.s Mon Sep 21 11:25:09 2009 -0400 +++ b/usr/src/lib/libc/amd64/gen/proc64_support.s Mon Sep 21 11:26:40 2009 -0400 @@ -25,7 +25,7 @@ */ /* - * Copyright (c) 2008, Intel Corporation + * Copyright (c) 2009, Intel Corporation * All rights reserved. */ @@ -38,8 +38,6 @@ * cache size information. Cache information used by memset, strcpy, etc.. */ - .file "proc64_support.s" - #include <sys/asm_linkage.h> #include "proc64_id.h"
--- a/usr/src/lib/libc/amd64/gen/strcmp.s Mon Sep 21 11:25:09 2009 -0400 +++ b/usr/src/lib/libc/amd64/gen/strcmp.s Mon Sep 21 11:26:40 2009 -0400 @@ -1,540 +1,2049 @@ /* - * Copyright 2008 Sun Microsystems, Inc. All rights reserved. - * Use is subject to license terms. + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END */ /* - * Copyright (c) 2002 Advanced Micro Devices, Inc. - * + * Copyright (c) 2009, Intel Corporation * All rights reserved. - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the - * following conditions are met: - * - * + Redistributions of source code must retain the above - * copyright notice, this list of conditions and the - * following disclaimer. - * - * + Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the - * following disclaimer in the documentation and/or other - * materials provided with the distribution. - * - * + Neither the name of Advanced Micro Devices, Inc. nor the - * names of its contributors may be used to endorse or - * promote products derived from this software without - * specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND - * CONTRIBUTORS AS IS AND ANY EXPRESS OR IMPLIED WARRANTIES, - * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF - * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL ADVANCED MICRO DEVICES, - * INC. OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, - * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE - * GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR - * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF - * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT - * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - * - * It is licensee's responsibility to comply with any export - * regulations applicable in licensee's jurisdiction. */ - .file "strcmp.s" +/* + * str[n]cmp - compare chars between two string + */ #include "SYS.h" -#include "cache.h" +#include "proc64_id.h" #define LABEL(s) .strcmp/**/s #ifdef USE_AS_STRNCMP + /* + * Since the counter, %r11, is unsigned, we branch to strcmp_exitz + * if the new counter > the old one or is 0. + */ +#define UPDATE_STRNCMP_COUNTER \ + /* calculate left number to compare */ \ + lea -16(%rcx, %r11), %r9; \ + cmp %r9, %r11; \ + jb LABEL(strcmp_exitz); \ + test %r9, %r9; \ + je LABEL(strcmp_exitz); \ + mov %r9, %r11 +#else +#define UPDATE_STRNCMP_COUNTER +#endif + + /* + * This implementation uses SSE to compare up to 16 bytes at a time. + */ +#ifdef USE_AS_STRNCMP ENTRY(strncmp) + test %rdx, %rdx + je LABEL(strcmp_exitz) + mov %rdx, %r11 #else ENTRY(strcmp) /* (const char *, const char *) */ #endif - xor %ecx, %ecx + mov %esi, %ecx + mov %edi, %eax + and $0x3f, %rcx /* rsi alignment in cache line */ + and $0x3f, %rax /* rdi alignment in cache line */ + cmp $0x30, %ecx + ja LABEL(crosscache) /* rsi: 16-byte load will cross cache line */ + cmp $0x30, %eax + ja LABEL(crosscache) /* rdi: 16-byte load will cross cache line */ + movlpd (%rdi), %xmm1 + movlpd (%rsi), %xmm2 + movhpd 8(%rdi), %xmm1 + movhpd 8(%rsi), %xmm2 + pxor %xmm0, %xmm0 /* clear %xmm0 for null char checks */ + pcmpeqb %xmm1, %xmm0 /* Any null chars? */ + pcmpeqb %xmm2, %xmm1 /* compare first 16 bytes for equality */ + psubb %xmm0, %xmm1 /* packed sub of comparison results*/ + pmovmskb %xmm1, %edx + sub $0xffff, %edx /* if first 16 bytes are same, edx == 0xffff */ + jnz LABEL(less16bytes) /* If not, found mismatch or null char */ +#ifdef USE_AS_STRNCMP + sub $16, %r11 + jbe LABEL(strcmp_exitz) /* finish comparision */ +#endif + add $16, %rsi /* prepare to search next 16 bytes */ + add $16, %rdi /* prepare to search next 16 bytes */ + + /* + * Determine rdi and rsi string offsets from 16-byte alignment. + * Use relative offset difference between the two to determine which case + * below to use. + */ + .p2align 4 +LABEL(crosscache): + and $0xfffffffffffffff0, %rsi /* force %rsi to be 16 byte aligned */ + and $0xfffffffffffffff0, %rdi /* force %rdi to be 16 byte aligned */ + mov $0xffff, %edx /* for equivalent offset */ + xor %r8d, %r8d + and $0xf, %ecx /* offset of rsi */ + and $0xf, %eax /* offset of rdi */ + cmp %eax, %ecx + je LABEL(ashr_0) /* both strings have the same alignment */ + ja LABEL(bigger) + mov %edx, %r8d /* r8d is offset flag for exit tail */ + xchg %ecx, %eax + xchg %rsi, %rdi +LABEL(bigger): + mov %rcx, %r9 + sub %rax, %r9 + lea LABEL(unaligned_table)(%rip), %r10 + movslq (%r10, %r9, 4), %r9 + lea (%r10, %r9), %r10 + jmp *%r10 /* jump to corresponding case */ + +/* + * ashr_0 handles the following cases: + * str1 offset = str2 offset + */ + .p2align 4 +LABEL(ashr_0): + movdqa (%rsi), %xmm1 + pxor %xmm0, %xmm0 /* clear %xmm0 for null char check */ + pcmpeqb %xmm1, %xmm0 /* Any null chars? */ + pcmpeqb (%rdi), %xmm1 /* compare 16 bytes for equality */ + psubb %xmm0, %xmm1 /* packed sub of comparison results*/ + pmovmskb %xmm1, %r9d + shr %cl, %edx /* adjust 0xffff for offset */ + shr %cl, %r9d /* adjust for 16-byte offset */ + sub %r9d, %edx + /* + * edx must be the same with r9d if in left byte (16-rcx) is equal to + * the start from (16-rax) and no null char was seen. + */ + jne LABEL(less32bytes) /* mismatch or null char */ + UPDATE_STRNCMP_COUNTER + mov $16, %rcx + mov $16, %r9 + pxor %xmm0, %xmm0 /* clear xmm0, may have changed above */ + + /* + * Now both strings are aligned at 16-byte boundary. Loop over strings + * checking 32-bytes per iteration. + */ + .p2align 4 +LABEL(loop_ashr_0): + movdqa (%rsi, %rcx), %xmm1 + movdqa (%rdi, %rcx), %xmm2 + + pcmpeqb %xmm1, %xmm0 + pcmpeqb %xmm2, %xmm1 + psubb %xmm0, %xmm1 + pmovmskb %xmm1, %edx + sub $0xffff, %edx + jnz LABEL(exit) /* mismatch or null char seen */ #ifdef USE_AS_STRNCMP - test %rdx, %rdx /* (const char *, const char *, size_t) */ - mov %r14, -8 (%rsp) - mov %rdx, %r14 - mov %edx, %eax - jz LABEL(exitz) /* early exit */ + sub $16, %r11 + jbe LABEL(strcmp_exitz) #endif + add $16, %rcx + movdqa (%rsi, %rcx), %xmm1 + movdqa (%rdi, %rcx), %xmm2 + + pcmpeqb %xmm1, %xmm0 + pcmpeqb %xmm2, %xmm1 + psubb %xmm0, %xmm1 + pmovmskb %xmm1, %edx + sub $0xffff, %edx + jnz LABEL(exit) +#ifdef USE_AS_STRNCMP + sub $16, %r11 + jbe LABEL(strcmp_exitz) +#endif + add $16, %rcx + jmp LABEL(loop_ashr_0) -LABEL(aligntry): - mov %rsi, %r8 /* align by "source" */ - and $8 - 1, %r8 /* between 0 and 8 characters compared */ - jz LABEL(alignafter) +/* + * ashr_1 handles the following cases: + * abs(str1 offset - str2 offset) = 15 + */ + .p2align 4 +LABEL(ashr_1): + pxor %xmm0, %xmm0 + movdqa (%rdi), %xmm2 + movdqa (%rsi), %xmm1 + pcmpeqb %xmm1, %xmm0 /* Any null chars? */ + pslldq $15, %xmm2 /* shift first string to align with second */ + pcmpeqb %xmm1, %xmm2 /* compare 16 bytes for equality */ + psubb %xmm0, %xmm2 /* packed sub of comparison results*/ + pmovmskb %xmm2, %r9d + shr %cl, %edx /* adjust 0xffff for offset */ + shr %cl, %r9d /* adjust for 16-byte offset */ + sub %r9d, %edx + jnz LABEL(less32bytes) /* mismatch or null char seen */ + movdqa (%rdi), %xmm3 + UPDATE_STRNCMP_COUNTER -LABEL(align): - sub $8, %r8 + pxor %xmm0, %xmm0 + mov $16, %rcx /* index for loads */ + mov $1, %r9d /* rdi bytes already examined. Used in exit code */ + /* + * Setup %r10 value allows us to detect crossing a page boundary. + * When %r10 goes positive we are crossing a page boundary and + * need to do a nibble. + */ + lea 1(%rdi), %r10 + and $0xfff, %r10 /* offset into 4K page */ + sub $0x1000, %r10 /* subtract 4K pagesize */ + movdqa %xmm3, %xmm4 - .p2align 4 + .p2align 4 +LABEL(loop_ashr_1): + add $16, %r10 + jg LABEL(nibble_ashr_1) /* cross page boundary */ + +LABEL(gobble_ashr_1): + movdqa (%rsi, %rcx), %xmm1 + movdqa (%rdi, %rcx), %xmm2 + movdqa %xmm2, %xmm4 /* store for next cycle */ -LABEL(alignloop): - mov (%rsi, %rcx), %al - mov (%rdi, %rcx), %dl + psrldq $1, %xmm3 + pslldq $15, %xmm2 + por %xmm3, %xmm2 /* merge into one 16byte value */ + + pcmpeqb %xmm1, %xmm0 + pcmpeqb %xmm2, %xmm1 + psubb %xmm0, %xmm1 + pmovmskb %xmm1, %edx + sub $0xffff, %edx + jnz LABEL(exit) #ifdef USE_AS_STRNCMP - dec %r14 - jl LABEL(exitafter) + sub $16, %r11 + jbe LABEL(strcmp_exitz) #endif + add $16, %rcx + movdqa %xmm4, %xmm3 + + add $16, %r10 + jg LABEL(nibble_ashr_1) /* cross page boundary */ - cmp %dl, %al /* check if same character */ - jne LABEL(exitafter) - test %al, %al /* check if character a NUL */ - jz LABEL(exitafter) + movdqa (%rsi, %rcx), %xmm1 + movdqa (%rdi, %rcx), %xmm2 + movdqa %xmm2, %xmm4 /* store for next cycle */ - inc %ecx + psrldq $1, %xmm3 + pslldq $15, %xmm2 + por %xmm3, %xmm2 /* merge into one 16byte value */ - inc %r8 - jnz LABEL(alignloop) + pcmpeqb %xmm1, %xmm0 + pcmpeqb %xmm2, %xmm1 + psubb %xmm0, %xmm1 + pmovmskb %xmm1, %edx + sub $0xffff, %edx + jnz LABEL(exit) #ifdef USE_AS_STRNCMP - test %r14, %r14 - jz LABEL(exitafter) + sub $16, %r11 + jbe LABEL(strcmp_exitz) +#endif + add $16, %rcx + movdqa %xmm4, %xmm3 + jmp LABEL(loop_ashr_1) + + /* + * Nibble avoids loads across page boundary. This is to avoid a potential + * access into unmapped memory. + */ + .p2align 4 +LABEL(nibble_ashr_1): + psrldq $1, %xmm4 + movdqa (%rsi, %rcx), %xmm1 + pcmpeqb %xmm1, %xmm0 + pcmpeqb %xmm4, %xmm1 + psubb %xmm0, %xmm1 + pmovmskb %xmm1, %edx + sub $0x7fff, %edx + jnz LABEL(exit) +#ifdef USE_AS_STRNCMP + cmp $15, %r11 + jbe LABEL(strcmp_exitz) +#endif + pxor %xmm0, %xmm0 + sub $0x1000, %r10 /* subtract 4K from %r10 */ + jmp LABEL(gobble_ashr_1) + +/* + * ashr_2 handles the following cases: + * abs(str1 offset - str2 offset) = 14 + */ + .p2align 4 +LABEL(ashr_2): + pxor %xmm0, %xmm0 + movdqa (%rdi), %xmm2 + movdqa (%rsi), %xmm1 + pcmpeqb %xmm1, %xmm0 + pslldq $14, %xmm2 + pcmpeqb %xmm1, %xmm2 + psubb %xmm0, %xmm2 + pmovmskb %xmm2, %r9d + shr %cl, %edx + shr %cl, %r9d + sub %r9d, %edx + jnz LABEL(less32bytes) + movdqa (%rdi), %xmm3 + UPDATE_STRNCMP_COUNTER + + pxor %xmm0, %xmm0 + mov $16, %rcx /* index for loads */ + mov $2, %r9d /* rdi bytes already examined. Used in exit code */ + /* + * Setup %r10 value allows us to detect crossing a page boundary. + * When %r10 goes positive we are crossing a page boundary and + * need to do a nibble. + */ + lea 2(%rdi), %r10 + and $0xfff, %r10 /* offset into 4K page */ + sub $0x1000, %r10 /* subtract 4K pagesize */ + movdqa %xmm3, %xmm4 + + .p2align 4 +LABEL(loop_ashr_2): + add $16, %r10 + jg LABEL(nibble_ashr_2) + +LABEL(gobble_ashr_2): + movdqa (%rsi, %rcx), %xmm1 + movdqa (%rdi, %rcx), %xmm2 + movdqa %xmm2, %xmm4 + + psrldq $2, %xmm3 + pslldq $14, %xmm2 + por %xmm3, %xmm2 + + pcmpeqb %xmm1, %xmm0 + pcmpeqb %xmm2, %xmm1 + psubb %xmm0, %xmm1 + pmovmskb %xmm1, %edx + sub $0xffff, %edx + jnz LABEL(exit) + +#ifdef USE_AS_STRNCMP + sub $16, %r11 + jbe LABEL(strcmp_exitz) #endif - .p2align 4 - -LABEL(alignafter): + add $16, %rcx + movdqa %xmm4, %xmm3 - mov %r15, -32 (%rsp) - mov %rbp, -24 (%rsp) - mov %rbx, -16 (%rsp) + add $16, %r10 + jg LABEL(nibble_ashr_2) /* cross page boundary */ + + movdqa (%rsi, %rcx), %xmm1 + movdqa (%rdi, %rcx), %xmm2 + movdqa %xmm2, %xmm4 -LABEL(pagealigntry): /* page align by "destination" */ - lea (%rdi, %rcx), %ebp - mov $AMD64PAGESIZE, %r15d - and $AMD64PAGEMASK, %ebp - sub %r15d, %ebp - /* - * When we go to 64gobble, %ebp was adjusted at the top of 64loop. - * When we go to 64nibble(crossing page boundary), we'll compare - * 128 byte since we'll fall through to 64gobble. Therefore, %ebp - * needs to be re-adjusted(add 64) when we fall into 64nibble. - * It can be done by adjusting %r15 since %r15 is only used to - * rewind %ebp when crossing page boundary. - */ - sub $64, %r15d + psrldq $2, %xmm3 + pslldq $14, %xmm2 + por %xmm3, %xmm2 -LABEL(64): /* 64-byte */ - mov $0xfefefefefefefeff, %rbx /* magic number */ - - .p2align 4 - -LABEL(64loop): - add $64, %ebp /* check if "destination" crosses a page unevenly */ - jle LABEL(64gobble) - - sub %r15d, %ebp - lea 64 (%rcx), %r8 - - .p2align 4 - -LABEL(64nibble): - mov (%rsi, %rcx), %al - mov (%rdi, %rcx), %dl + pcmpeqb %xmm1, %xmm0 + pcmpeqb %xmm2, %xmm1 + psubb %xmm0, %xmm1 + pmovmskb %xmm1, %edx + sub $0xffff, %edx + jnz LABEL(exit) #ifdef USE_AS_STRNCMP - dec %r14 - jle LABEL(exit) + sub $16, %r11 + jbe LABEL(strcmp_exitz) #endif - cmp %dl, %al /* check if same character */ - jne LABEL(exit) - test %al, %al /* check if character a NUL */ - jz LABEL(exit) + add $16, %rcx + movdqa %xmm4, %xmm3 + jmp LABEL(loop_ashr_2) - inc %ecx + .p2align 4 +LABEL(nibble_ashr_2): + psrldq $2, %xmm4 + movdqa (%rsi, %rcx), %xmm1 + pcmpeqb %xmm1, %xmm0 + pcmpeqb %xmm4, %xmm1 + psubb %xmm0, %xmm1 + pmovmskb %xmm1, %edx + sub $0x3fff, %edx + jnz LABEL(exit) +#ifdef USE_AS_STRNCMP + cmp $14, %r11 + jbe LABEL(strcmp_exitz) +#endif + pxor %xmm0, %xmm0 + sub $0x1000, %r10 /* subtract 4K from %r10 */ + jmp LABEL(gobble_ashr_2) - cmp %ecx, %r8d - ja LABEL(64nibble) +/* + * ashr_3 handles the following cases: + * abs(str1 offset - str2 offset) = 13 + */ + .p2align 4 +LABEL(ashr_3): + pxor %xmm0, %xmm0 + movdqa (%rdi), %xmm2 + movdqa (%rsi), %xmm1 + pcmpeqb %xmm1, %xmm0 + pslldq $13, %xmm2 + pcmpeqb %xmm1, %xmm2 + psubb %xmm0, %xmm2 + pmovmskb %xmm2, %r9d + shr %cl, %edx + shr %cl, %r9d + sub %r9d, %edx + jnz LABEL(less32bytes) + movdqa (%rdi), %xmm3 + + UPDATE_STRNCMP_COUNTER - .p2align 4 + pxor %xmm0, %xmm0 + mov $16, %rcx /* index for loads */ + mov $3, %r9d /* rdi bytes already examined. Used in exit code */ + /* + * Setup %r10 value allows us to detect crossing a page boundary. + * When %r10 goes positive we are crossing a page boundary and + * need to do a nibble. + */ + lea 3(%rdi), %r10 + and $0xfff, %r10 /* offset into 4K page */ + sub $0x1000, %r10 /* subtract 4K pagesize */ + movdqa %xmm3, %xmm4 -LABEL(64gobble): - mov (%rsi, %rcx), %rax - mov (%rdi, %rcx), %rdx + .p2align 4 +LABEL(loop_ashr_3): + add $16, %r10 + jg LABEL(nibble_ashr_3) + +LABEL(gobble_ashr_3): + movdqa (%rsi, %rcx), %xmm1 + movdqa (%rdi, %rcx), %xmm2 + movdqa %xmm2, %xmm4 + + psrldq $3, %xmm3 + pslldq $13, %xmm2 + por %xmm3, %xmm2 + + pcmpeqb %xmm1, %xmm0 + pcmpeqb %xmm2, %xmm1 + psubb %xmm0, %xmm1 + pmovmskb %xmm1, %edx + sub $0xffff, %edx + jnz LABEL(exit) #ifdef USE_AS_STRNCMP - sub $8, %r14 - jle LABEL(tail) + sub $16, %r11 + jbe LABEL(strcmp_exitz) +#endif + + add $16, %rcx + movdqa %xmm4, %xmm3 + + add $16, %r10 + jg LABEL(nibble_ashr_3) /* cross page boundary */ + + movdqa (%rsi, %rcx), %xmm1 + movdqa (%rdi, %rcx), %xmm2 + movdqa %xmm2, %xmm4 + + psrldq $3, %xmm3 + pslldq $13, %xmm2 + por %xmm3, %xmm2 + + pcmpeqb %xmm1, %xmm0 + pcmpeqb %xmm2, %xmm1 + psubb %xmm0, %xmm1 + pmovmskb %xmm1, %edx + sub $0xffff, %edx + jnz LABEL(exit) + +#ifdef USE_AS_STRNCMP + sub $16, %r11 + jbe LABEL(strcmp_exitz) #endif - mov %rbx, %r8 - add %rax, %r8 - sbb %r10, %r10 + add $16, %rcx + movdqa %xmm4, %xmm3 + jmp LABEL(loop_ashr_3) + + .p2align 4 +LABEL(nibble_ashr_3): + psrldq $3, %xmm4 + movdqa (%rsi, %rcx), %xmm1 + pcmpeqb %xmm1, %xmm0 + pcmpeqb %xmm4, %xmm1 + psubb %xmm0, %xmm1 + pmovmskb %xmm1, %edx + sub $0x1fff, %edx + jnz LABEL(exit) +#ifdef USE_AS_STRNCMP + cmp $13, %r11 + jbe LABEL(strcmp_exitz) +#endif + pxor %xmm0, %xmm0 + sub $0x1000, %r10 /* subtract 4K from %r10 */ + jmp LABEL(gobble_ashr_3) - mov %rbx, %r9 - add %rdx, %r9 - sbb %r11, %r11 +/* + * ashr_4 handles the following cases: + * abs(str1 offset - str2 offset) = 12 + */ + .p2align 4 +LABEL(ashr_4): + pxor %xmm0, %xmm0 + movdqa (%rdi), %xmm2 + movdqa (%rsi), %xmm1 + pcmpeqb %xmm1, %xmm0 + pslldq $12, %xmm2 + pcmpeqb %xmm1, %xmm2 + psubb %xmm0, %xmm2 + pmovmskb %xmm2, %r9d + shr %cl, %edx + shr %cl, %r9d + sub %r9d, %edx + jnz LABEL(less32bytes) + movdqa (%rdi), %xmm3 + + UPDATE_STRNCMP_COUNTER - xor %rax, %r8 - or %rbx, %r8 - sub %r10, %r8 - jnz LABEL(tail) + pxor %xmm0, %xmm0 + mov $16, %rcx /* index for loads */ + mov $4, %r9d /* rdi bytes already examined. Used in exit code */ + /* + * Setup %r10 value allows us to detect crossing a page boundary. + * When %r10 goes positive we are crossing a page boundary and + * need to do a nibble. + */ + lea 4(%rdi), %r10 + and $0xfff, %r10 /* offset into 4K page */ + sub $0x1000, %r10 /* subtract 4K pagesize */ + movdqa %xmm3, %xmm4 + + .p2align 4 +LABEL(loop_ashr_4): + add $16, %r10 + jg LABEL(nibble_ashr_4) + +LABEL(gobble_ashr_4): + movdqa (%rsi, %rcx), %xmm1 + movdqa (%rdi, %rcx), %xmm2 + movdqa %xmm2, %xmm4 + + psrldq $4, %xmm3 + pslldq $12, %xmm2 + por %xmm3, %xmm2 - xor %rdx, %r9 - or %rbx, %r9 - sub %r11, %r9 - jnz LABEL(tail) + pcmpeqb %xmm1, %xmm0 + pcmpeqb %xmm2, %xmm1 + psubb %xmm0, %xmm1 + pmovmskb %xmm1, %edx + sub $0xffff, %edx + jnz LABEL(exit) + +#ifdef USE_AS_STRNCMP + sub $16, %r11 + jbe LABEL(strcmp_exitz) +#endif + + add $16, %rcx + movdqa %xmm4, %xmm3 - cmp %rdx, %rax - jne LABEL(tail) + add $16, %r10 + jg LABEL(nibble_ashr_4) /* cross page boundary */ + + movdqa (%rsi, %rcx), %xmm1 + movdqa (%rdi, %rcx), %xmm2 + movdqa %xmm2, %xmm4 - mov 8 (%rsi, %rcx), %rax - mov 8 (%rdi, %rcx), %rdx - add $8, %ecx + psrldq $4, %xmm3 + pslldq $12, %xmm2 + por %xmm3, %xmm2 + + pcmpeqb %xmm1, %xmm0 + pcmpeqb %xmm2, %xmm1 + psubb %xmm0, %xmm1 + pmovmskb %xmm1, %edx + sub $0xffff, %edx + jnz LABEL(exit) #ifdef USE_AS_STRNCMP - sub $8, %r14 - jle LABEL(tail) + sub $16, %r11 + jbe LABEL(strcmp_exitz) #endif - mov %rbx, %r8 - add %rax, %r8 - sbb %r10, %r10 + add $16, %rcx + movdqa %xmm4, %xmm3 + jmp LABEL(loop_ashr_4) + + .p2align 4 +LABEL(nibble_ashr_4): + psrldq $4, %xmm4 + movdqa (%rsi, %rcx), %xmm1 + pcmpeqb %xmm1, %xmm0 + pcmpeqb %xmm4, %xmm1 + psubb %xmm0, %xmm1 + pmovmskb %xmm1, %edx + sub $0x0fff, %edx + jnz LABEL(exit) +#ifdef USE_AS_STRNCMP + cmp $12, %r11 + jbe LABEL(strcmp_exitz) +#endif + pxor %xmm0, %xmm0 + sub $0x1000, %r10 /* subtract 4K from %r10 */ + jmp LABEL(gobble_ashr_4) - mov %rbx, %r9 - add %rdx, %r9 - sbb %r11, %r11 +/* + * ashr_5 handles the following cases: + * abs(str1 offset - str2 offset) = 11 + */ + .p2align 4 +LABEL(ashr_5): + pxor %xmm0, %xmm0 + movdqa (%rdi), %xmm2 + movdqa (%rsi), %xmm1 + pcmpeqb %xmm1, %xmm0 + pslldq $11, %xmm2 + pcmpeqb %xmm1, %xmm2 + psubb %xmm0, %xmm2 + pmovmskb %xmm2, %r9d + shr %cl, %edx + shr %cl, %r9d + sub %r9d, %edx + jnz LABEL(less32bytes) + movdqa (%rdi), %xmm3 + + UPDATE_STRNCMP_COUNTER - xor %rax, %r8 - or %rbx, %r8 - sub %r10, %r8 - jnz LABEL(tail) + pxor %xmm0, %xmm0 + mov $16, %rcx /* index for loads */ + mov $5, %r9d /* rdi bytes already examined. Used in exit code */ + /* + * Setup %r10 value allows us to detect crossing a page boundary. + * When %r10 goes positive we are crossing a page boundary and + * need to do a nibble. + */ + lea 5(%rdi), %r10 + and $0xfff, %r10 /* offset into 4K page */ + sub $0x1000, %r10 /* subtract 4K pagesize */ + movdqa %xmm3, %xmm4 + + .p2align 4 +LABEL(loop_ashr_5): + add $16, %r10 + jg LABEL(nibble_ashr_5) + +LABEL(gobble_ashr_5): + movdqa (%rsi, %rcx), %xmm1 + movdqa (%rdi, %rcx), %xmm2 + movdqa %xmm2, %xmm4 + + psrldq $5, %xmm3 + pslldq $11, %xmm2 + por %xmm3, %xmm2 - xor %rdx, %r9 - or %rbx, %r9 - sub %r11, %r9 - jnz LABEL(tail) + pcmpeqb %xmm1, %xmm0 + pcmpeqb %xmm2, %xmm1 + psubb %xmm0, %xmm1 + pmovmskb %xmm1, %edx + sub $0xffff, %edx + jnz LABEL(exit) + +#ifdef USE_AS_STRNCMP + sub $16, %r11 + jbe LABEL(strcmp_exitz) +#endif + + add $16, %rcx + movdqa %xmm4, %xmm3 - cmp %rdx, %rax - jne LABEL(tail) + add $16, %r10 + jg LABEL(nibble_ashr_5) /* cross page boundary */ + + movdqa (%rsi, %rcx), %xmm1 + movdqa (%rdi, %rcx), %xmm2 + movdqa %xmm2, %xmm4 - mov 8 (%rsi, %rcx), %rax - mov 8 (%rdi, %rcx), %rdx - add $8, %ecx + psrldq $5, %xmm3 + pslldq $11, %xmm2 + por %xmm3, %xmm2 + + pcmpeqb %xmm1, %xmm0 + pcmpeqb %xmm2, %xmm1 + psubb %xmm0, %xmm1 + pmovmskb %xmm1, %edx + sub $0xffff, %edx + jnz LABEL(exit) #ifdef USE_AS_STRNCMP - sub $8, %r14 - jle LABEL(tail) + sub $16, %r11 + jbe LABEL(strcmp_exitz) #endif - mov %rbx, %r8 - add %rax, %r8 - sbb %r10, %r10 + add $16, %rcx + movdqa %xmm4, %xmm3 + jmp LABEL(loop_ashr_5) + + .p2align 4 +LABEL(nibble_ashr_5): + psrldq $5, %xmm4 + movdqa (%rsi, %rcx), %xmm1 + pcmpeqb %xmm1, %xmm0 + pcmpeqb %xmm4, %xmm1 + psubb %xmm0, %xmm1 + pmovmskb %xmm1, %edx + sub $0x07ff, %edx + jnz LABEL(exit) +#ifdef USE_AS_STRNCMP + cmp $11, %r11 + jbe LABEL(strcmp_exitz) +#endif + pxor %xmm0, %xmm0 + sub $0x1000, %r10 /* subtract 4K from %r10 */ + jmp LABEL(gobble_ashr_5) - mov %rbx, %r9 - add %rdx, %r9 - sbb %r11, %r11 +/* + * ashr_6 handles the following cases: + * abs(str1 offset - str2 offset) = 10 + */ + .p2align 4 +LABEL(ashr_6): + pxor %xmm0, %xmm0 + movdqa (%rdi), %xmm2 + movdqa (%rsi), %xmm1 + pcmpeqb %xmm1, %xmm0 + pslldq $10, %xmm2 + pcmpeqb %xmm1, %xmm2 + psubb %xmm0, %xmm2 + pmovmskb %xmm2, %r9d + shr %cl, %edx + shr %cl, %r9d + sub %r9d, %edx + jnz LABEL(less32bytes) + movdqa (%rdi), %xmm3 + + UPDATE_STRNCMP_COUNTER - xor %rax, %r8 - or %rbx, %r8 - sub %r10, %r8 - jnz LABEL(tail) + pxor %xmm0, %xmm0 + mov $16, %rcx /* index for loads */ + mov $6, %r9d /* rdi bytes already examined. Used in exit code */ + /* + * Setup %r10 value allows us to detect crossing a page boundary. + * When %r10 goes positive we are crossing a page boundary and + * need to do a nibble. + */ + lea 6(%rdi), %r10 + and $0xfff, %r10 /* offset into 4K page */ + sub $0x1000, %r10 /* subtract 4K pagesize */ + movdqa %xmm3, %xmm4 + + .p2align 4 +LABEL(loop_ashr_6): + add $16, %r10 + jg LABEL(nibble_ashr_6) + +LABEL(gobble_ashr_6): + movdqa (%rsi, %rcx), %xmm1 + movdqa (%rdi, %rcx), %xmm2 + movdqa %xmm2, %xmm4 + + psrldq $6, %xmm3 + pslldq $10, %xmm2 + por %xmm3, %xmm2 - xor %rdx, %r9 - or %rbx, %r9 - sub %r11, %r9 - jnz LABEL(tail) + pcmpeqb %xmm1, %xmm0 + pcmpeqb %xmm2, %xmm1 + psubb %xmm0, %xmm1 + pmovmskb %xmm1, %edx + sub $0xffff, %edx + jnz LABEL(exit) + +#ifdef USE_AS_STRNCMP + sub $16, %r11 + jbe LABEL(strcmp_exitz) +#endif + + add $16, %rcx + movdqa %xmm4, %xmm3 - cmp %rdx, %rax - jne LABEL(tail) + add $16, %r10 + jg LABEL(nibble_ashr_6) /* cross page boundary */ + + movdqa (%rsi, %rcx), %xmm1 + movdqa (%rdi, %rcx), %xmm2 + movdqa %xmm2, %xmm4 - mov 8 (%rsi, %rcx), %rax - mov 8 (%rdi, %rcx), %rdx - add $8, %ecx + psrldq $6, %xmm3 + pslldq $10, %xmm2 + por %xmm3, %xmm2 + + pcmpeqb %xmm1, %xmm0 + pcmpeqb %xmm2, %xmm1 + psubb %xmm0, %xmm1 + pmovmskb %xmm1, %edx + sub $0xffff, %edx + jnz LABEL(exit) #ifdef USE_AS_STRNCMP - sub $8, %r14 - jle LABEL(tail) + sub $16, %r11 + jbe LABEL(strcmp_exitz) +#endif + + add $16, %rcx + movdqa %xmm4, %xmm3 + jmp LABEL(loop_ashr_6) + + .p2align 4 +LABEL(nibble_ashr_6): + psrldq $6, %xmm4 + movdqa (%rsi, %rcx), %xmm1 + pcmpeqb %xmm1, %xmm0 + pcmpeqb %xmm4, %xmm1 + psubb %xmm0, %xmm1 + pmovmskb %xmm1, %edx + sub $0x03ff, %edx + jnz LABEL(exit) +#ifdef USE_AS_STRNCMP + cmp $10, %r11 + jbe LABEL(strcmp_exitz) +#endif + pxor %xmm0, %xmm0 + sub $0x1000, %r10 /* subtract 4K from %r10 */ + jmp LABEL(gobble_ashr_6) + +/* + * ashr_7 handles the following cases: + * abs(str1 offset - str2 offset) = 9 + */ + .p2align 4 +LABEL(ashr_7): + pxor %xmm0, %xmm0 + movdqa (%rdi), %xmm2 + movdqa (%rsi), %xmm1 + pcmpeqb %xmm1, %xmm0 + pslldq $9, %xmm2 + pcmpeqb %xmm1, %xmm2 + psubb %xmm0, %xmm2 + pmovmskb %xmm2, %r9d + shr %cl, %edx + shr %cl, %r9d + sub %r9d, %edx + jnz LABEL(less32bytes) + movdqa (%rdi), %xmm3 + + UPDATE_STRNCMP_COUNTER + + pxor %xmm0, %xmm0 + mov $16, %rcx /* index for loads */ + mov $7, %r9d /* rdi bytes already examined. Used in exit code */ + /* + * Setup %r10 value allows us to detect crossing a page boundary. + * When %r10 goes positive we are crossing a page boundary and + * need to do a nibble. + */ + lea 7(%rdi), %r10 + and $0xfff, %r10 /* offset into 4K page */ + sub $0x1000, %r10 /* subtract 4K pagesize */ + movdqa %xmm3, %xmm4 + + .p2align 4 +LABEL(loop_ashr_7): + add $16, %r10 + jg LABEL(nibble_ashr_7) + +LABEL(gobble_ashr_7): + movdqa (%rsi, %rcx), %xmm1 + movdqa (%rdi, %rcx), %xmm2 + movdqa %xmm2, %xmm4 + + psrldq $7, %xmm3 + pslldq $9, %xmm2 + por %xmm3, %xmm2 + + pcmpeqb %xmm1, %xmm0 + pcmpeqb %xmm2, %xmm1 + psubb %xmm0, %xmm1 + pmovmskb %xmm1, %edx + sub $0xffff, %edx + jnz LABEL(exit) + +#ifdef USE_AS_STRNCMP + sub $16, %r11 + jbe LABEL(strcmp_exitz) #endif - mov %rbx, %r8 - add %rax, %r8 - sbb %r10, %r10 + add $16, %rcx + movdqa %xmm4, %xmm3 - mov %rbx, %r9 - add %rdx, %r9 - sbb %r11, %r11 + add $16, %r10 + jg LABEL(nibble_ashr_7) /* cross page boundary */ + + movdqa (%rsi, %rcx), %xmm1 + movdqa (%rdi, %rcx), %xmm2 + movdqa %xmm2, %xmm4 - xor %rax, %r8 - or %rbx, %r8 - sub %r10, %r8 - jnz LABEL(tail) + psrldq $7, %xmm3 + pslldq $9, %xmm2 + por %xmm3, %xmm2 - xor %rdx, %r9 - or %rbx, %r9 - sub %r11, %r9 - jnz LABEL(tail) - - cmp %rdx, %rax - jne LABEL(tail) - - mov 8 (%rsi, %rcx), %rax - mov 8 (%rdi, %rcx), %rdx - add $8, %ecx + pcmpeqb %xmm1, %xmm0 + pcmpeqb %xmm2, %xmm1 + psubb %xmm0, %xmm1 + pmovmskb %xmm1, %edx + sub $0xffff, %edx + jnz LABEL(exit) #ifdef USE_AS_STRNCMP - sub $8, %r14 - jle LABEL(tail) + sub $16, %r11 + jbe LABEL(strcmp_exitz) +#endif + + add $16, %rcx + movdqa %xmm4, %xmm3 + jmp LABEL(loop_ashr_7) + + .p2align 4 +LABEL(nibble_ashr_7): + psrldq $7, %xmm4 + movdqa (%rsi, %rcx), %xmm1 + pcmpeqb %xmm1, %xmm0 + pcmpeqb %xmm4, %xmm1 + psubb %xmm0, %xmm1 + pmovmskb %xmm1, %edx + sub $0x01ff, %edx + jnz LABEL(exit) +#ifdef USE_AS_STRNCMP + cmp $9, %r11 + jbe LABEL(strcmp_exitz) +#endif + pxor %xmm0, %xmm0 + sub $0x1000, %r10 /* subtract 4K from %r10 */ + jmp LABEL(gobble_ashr_7) + +/* + * ashr_8 handles the following cases: + * abs(str1 offset - str2 offset) = 8 + */ + .p2align 4 +LABEL(ashr_8): + pxor %xmm0, %xmm0 + movdqa (%rdi), %xmm2 + movdqa (%rsi), %xmm1 + pcmpeqb %xmm1, %xmm0 + pslldq $8, %xmm2 + pcmpeqb %xmm1, %xmm2 + psubb %xmm0, %xmm2 + pmovmskb %xmm2, %r9d + shr %cl, %edx + shr %cl, %r9d + sub %r9d, %edx + jnz LABEL(less32bytes) + movdqa (%rdi), %xmm3 + + UPDATE_STRNCMP_COUNTER + + pxor %xmm0, %xmm0 + mov $16, %rcx /* index for loads */ + mov $8, %r9d /* rdi bytes already examined. Used in exit code */ + /* + * Setup %r10 value allows us to detect crossing a page boundary. + * When %r10 goes positive we are crossing a page boundary and + * need to do a nibble. + */ + lea 8(%rdi), %r10 + and $0xfff, %r10 /* offset into 4K page */ + sub $0x1000, %r10 /* subtract 4K pagesize */ + movdqa %xmm3, %xmm4 + + .p2align 4 +LABEL(loop_ashr_8): + add $16, %r10 + jg LABEL(nibble_ashr_8) + +LABEL(gobble_ashr_8): + movdqa (%rsi, %rcx), %xmm1 + movdqa (%rdi, %rcx), %xmm2 + movdqa %xmm2, %xmm4 + + psrldq $8, %xmm3 + pslldq $8, %xmm2 + por %xmm3, %xmm2 + + pcmpeqb %xmm1, %xmm0 + pcmpeqb %xmm2, %xmm1 + psubb %xmm0, %xmm1 + pmovmskb %xmm1, %edx + sub $0xffff, %edx + jnz LABEL(exit) + +#ifdef USE_AS_STRNCMP + sub $16, %r11 + jbe LABEL(strcmp_exitz) +#endif + + add $16, %rcx + movdqa %xmm4, %xmm3 + + add $16, %r10 + jg LABEL(nibble_ashr_8) /* cross page boundary */ + + movdqa (%rsi, %rcx), %xmm1 + movdqa (%rdi, %rcx), %xmm2 + movdqa %xmm2, %xmm4 + + psrldq $8, %xmm3 + pslldq $8, %xmm2 + por %xmm3, %xmm2 + + pcmpeqb %xmm1, %xmm0 + pcmpeqb %xmm2, %xmm1 + psubb %xmm0, %xmm1 + pmovmskb %xmm1, %edx + sub $0xffff, %edx + jnz LABEL(exit) + +#ifdef USE_AS_STRNCMP + sub $16, %r11 + jbe LABEL(strcmp_exitz) #endif - mov %rbx, %r8 - add %rax, %r8 - sbb %r10, %r10 + add $16, %rcx + movdqa %xmm4, %xmm3 + jmp LABEL(loop_ashr_8) - mov %rbx, %r9 - add %rdx, %r9 - sbb %r11, %r11 + .p2align 4 +LABEL(nibble_ashr_8): + psrldq $8, %xmm4 + movdqa (%rsi, %rcx), %xmm1 + pcmpeqb %xmm1, %xmm0 + pcmpeqb %xmm4, %xmm1 + psubb %xmm0, %xmm1 + pmovmskb %xmm1, %edx + sub $0x00ff, %edx + jnz LABEL(exit) +#ifdef USE_AS_STRNCMP + cmp $8, %r11 + jbe LABEL(strcmp_exitz) +#endif + pxor %xmm0, %xmm0 + sub $0x1000, %r10 /* subtract 4K from %r10 */ + jmp LABEL(gobble_ashr_8) - xor %rax, %r8 - or %rbx, %r8 - sub %r10, %r8 - jnz LABEL(tail) +/* + * ashr_9 handles the following cases: + * abs(str1 offset - str2 offset) = 7 + */ + .p2align 4 +LABEL(ashr_9): + pxor %xmm0, %xmm0 + movdqa (%rdi), %xmm2 + movdqa (%rsi), %xmm1 + pcmpeqb %xmm1, %xmm0 + pslldq $7, %xmm2 + pcmpeqb %xmm1, %xmm2 + psubb %xmm0, %xmm2 + pmovmskb %xmm2, %r9d + shr %cl, %edx + shr %cl, %r9d + sub %r9d, %edx + jnz LABEL(less32bytes) + movdqa (%rdi), %xmm3 + + UPDATE_STRNCMP_COUNTER - xor %rdx, %r9 - or %rbx, %r9 - sub %r11, %r9 - jnz LABEL(tail) + pxor %xmm0, %xmm0 + mov $16, %rcx /* index for loads */ + mov $9, %r9d /* rdi bytes already examined. Used in exit code */ + /* + * Setup %r10 value allows us to detect crossing a page boundary. + * When %r10 goes positive we are crossing a page boundary and + * need to do a nibble. + */ + lea 9(%rdi), %r10 + and $0xfff, %r10 /* offset into 4K page */ + sub $0x1000, %r10 /* subtract 4K pagesize */ + movdqa %xmm3, %xmm4 - cmp %rdx, %rax - jne LABEL(tail) + .p2align 4 +LABEL(loop_ashr_9): + add $16, %r10 + jg LABEL(nibble_ashr_9) + +LABEL(gobble_ashr_9): + movdqa (%rsi, %rcx), %xmm1 + movdqa (%rdi, %rcx), %xmm2 + movdqa %xmm2, %xmm4 - mov 8 (%rsi, %rcx), %rax - mov 8 (%rdi, %rcx), %rdx - add $8, %ecx + psrldq $9, %xmm3 + pslldq $7, %xmm2 + por %xmm3, %xmm2 + + pcmpeqb %xmm1, %xmm0 + pcmpeqb %xmm2, %xmm1 + psubb %xmm0, %xmm1 + pmovmskb %xmm1, %edx + sub $0xffff, %edx + jnz LABEL(exit) #ifdef USE_AS_STRNCMP - sub $8, %r14 - jle LABEL(tail) + sub $16, %r11 + jbe LABEL(strcmp_exitz) +#endif + + add $16, %rcx + movdqa %xmm4, %xmm3 + + add $16, %r10 + jg LABEL(nibble_ashr_9) /* cross page boundary */ + + movdqa (%rsi, %rcx), %xmm1 + movdqa (%rdi, %rcx), %xmm2 + movdqa %xmm2, %xmm4 + + psrldq $9, %xmm3 + pslldq $7, %xmm2 + por %xmm3, %xmm2 + + pcmpeqb %xmm1, %xmm0 + pcmpeqb %xmm2, %xmm1 + psubb %xmm0, %xmm1 + pmovmskb %xmm1, %edx + sub $0xffff, %edx + jnz LABEL(exit) + +#ifdef USE_AS_STRNCMP + sub $16, %r11 + jbe LABEL(strcmp_exitz) +#endif + + add $16, %rcx + movdqa %xmm4, %xmm3 /* store for next cycle */ + jmp LABEL(loop_ashr_9) + + .p2align 4 +LABEL(nibble_ashr_9): + psrldq $9, %xmm4 + movdqa (%rsi, %rcx), %xmm1 + pcmpeqb %xmm1, %xmm0 + pcmpeqb %xmm4, %xmm1 + psubb %xmm0, %xmm1 + pmovmskb %xmm1, %edx + sub $0x007f, %edx + jnz LABEL(exit) +#ifdef USE_AS_STRNCMP + cmp $7, %r11 + jbe LABEL(strcmp_exitz) +#endif + pxor %xmm0, %xmm0 + sub $0x1000, %r10 /* subtract 4K from %r10 */ + jmp LABEL(gobble_ashr_9) + +/* + * ashr_10 handles the following cases: + * abs(str1 offset - str2 offset) = 6 + */ + .p2align 4 +LABEL(ashr_10): + pxor %xmm0, %xmm0 + movdqa (%rdi), %xmm2 + movdqa (%rsi), %xmm1 + pcmpeqb %xmm1, %xmm0 + pslldq $6, %xmm2 + pcmpeqb %xmm1, %xmm2 + psubb %xmm0, %xmm2 + pmovmskb %xmm2, %r9d + shr %cl, %edx + shr %cl, %r9d + sub %r9d, %edx + jnz LABEL(less32bytes) + movdqa (%rdi), %xmm3 + + UPDATE_STRNCMP_COUNTER + + pxor %xmm0, %xmm0 + mov $16, %rcx /* index for loads */ + mov $10, %r9d /* rdi bytes already examined. Used in exit code */ + /* + * Setup %r10 value allows us to detect crossing a page boundary. + * When %r10 goes positive we are crossing a page boundary and + * need to do a nibble. + */ + lea 10(%rdi), %r10 + and $0xfff, %r10 /* offset into 4K page */ + sub $0x1000, %r10 /* subtract 4K pagesize */ + movdqa %xmm3, %xmm4 + + .p2align 4 +LABEL(loop_ashr_10): + add $16, %r10 + jg LABEL(nibble_ashr_10) + +LABEL(gobble_ashr_10): + movdqa (%rsi, %rcx), %xmm1 + movdqa (%rdi, %rcx), %xmm2 + movdqa %xmm2, %xmm4 + + psrldq $10, %xmm3 + pslldq $6, %xmm2 + por %xmm3, %xmm2 + + pcmpeqb %xmm1, %xmm0 + pcmpeqb %xmm2, %xmm1 + psubb %xmm0, %xmm1 + pmovmskb %xmm1, %edx + sub $0xffff, %edx + jnz LABEL(exit) + +#ifdef USE_AS_STRNCMP + sub $16, %r11 + jbe LABEL(strcmp_exitz) #endif - mov %rbx, %r8 - add %rax, %r8 - sbb %r10, %r10 + add $16, %rcx + movdqa %xmm4, %xmm3 + + add $16, %r10 + jg LABEL(nibble_ashr_10) /* cross page boundary */ + + movdqa (%rsi, %rcx), %xmm1 + movdqa (%rdi, %rcx), %xmm2 + movdqa %xmm2, %xmm4 + + psrldq $10, %xmm3 + pslldq $6, %xmm2 + por %xmm3, %xmm2 + + pcmpeqb %xmm1, %xmm0 + pcmpeqb %xmm2, %xmm1 + psubb %xmm0, %xmm1 + pmovmskb %xmm1, %edx + sub $0xffff, %edx + jnz LABEL(exit) - mov %rbx, %r9 - add %rdx, %r9 - sbb %r11, %r11 +#ifdef USE_AS_STRNCMP + sub $16, %r11 + jbe LABEL(strcmp_exitz) +#endif + + add $16, %rcx + movdqa %xmm4, %xmm3 + jmp LABEL(loop_ashr_10) + + .p2align 4 +LABEL(nibble_ashr_10): + psrldq $10, %xmm4 + movdqa (%rsi, %rcx), %xmm1 + pcmpeqb %xmm1, %xmm0 + pcmpeqb %xmm4, %xmm1 + psubb %xmm0, %xmm1 + pmovmskb %xmm1, %edx + sub $0x003f, %edx + jnz LABEL(exit) +#ifdef USE_AS_STRNCMP + cmp $6, %r11 + jbe LABEL(strcmp_exitz) +#endif + pxor %xmm0, %xmm0 + sub $0x1000, %r10 /* subtract 4K from %r10 */ + jmp LABEL(gobble_ashr_10) - xor %rax, %r8 - or %rbx, %r8 - sub %r10, %r8 - jnz LABEL(tail) +/* + * ashr_11 handles the following cases: + * abs(str1 offset - str2 offset) = 5 + */ + .p2align 4 +LABEL(ashr_11): + pxor %xmm0, %xmm0 + movdqa (%rdi), %xmm2 + movdqa (%rsi), %xmm1 + pcmpeqb %xmm1, %xmm0 + pslldq $5, %xmm2 + pcmpeqb %xmm1, %xmm2 + psubb %xmm0, %xmm2 + pmovmskb %xmm2, %r9d + shr %cl, %edx + shr %cl, %r9d + sub %r9d, %edx + jnz LABEL(less32bytes) + movdqa (%rdi), %xmm3 + + UPDATE_STRNCMP_COUNTER - xor %rdx, %r9 - or %rbx, %r9 - sub %r11, %r9 - jnz LABEL(tail) + pxor %xmm0, %xmm0 + mov $16, %rcx /* index for loads */ + mov $11, %r9d /* rdi bytes already examined. Used in exit code */ + /* + * Setup %r10 value allows us to detect crossing a page boundary. + * When %r10 goes positive we are crossing a page boundary and + * need to do a nibble. + */ + lea 11(%rdi), %r10 + and $0xfff, %r10 /* offset into 4K page */ + sub $0x1000, %r10 /* subtract 4K pagesize */ + movdqa %xmm3, %xmm4 - cmp %rdx, %rax - jne LABEL(tail) + .p2align 4 +LABEL(loop_ashr_11): + add $16, %r10 + jg LABEL(nibble_ashr_11) + +LABEL(gobble_ashr_11): + movdqa (%rsi, %rcx), %xmm1 + movdqa (%rdi, %rcx), %xmm2 + movdqa %xmm2, %xmm4 - mov 8 (%rsi, %rcx), %rax - mov 8 (%rdi, %rcx), %rdx - add $8, %ecx + psrldq $11, %xmm3 + pslldq $5, %xmm2 + por %xmm3, %xmm2 + + pcmpeqb %xmm1, %xmm0 + pcmpeqb %xmm2, %xmm1 + psubb %xmm0, %xmm1 + pmovmskb %xmm1, %edx + sub $0xffff, %edx + jnz LABEL(exit) #ifdef USE_AS_STRNCMP - sub $8, %r14 - jle LABEL(tail) + sub $16, %r11 + jbe LABEL(strcmp_exitz) +#endif + + add $16, %rcx + movdqa %xmm4, %xmm3 + + add $16, %r10 + jg LABEL(nibble_ashr_11) /* cross page boundary */ + + movdqa (%rsi, %rcx), %xmm1 + movdqa (%rdi, %rcx), %xmm2 + movdqa %xmm2, %xmm4 + + psrldq $11, %xmm3 + pslldq $5, %xmm2 + por %xmm3, %xmm2 + + pcmpeqb %xmm1, %xmm0 + pcmpeqb %xmm2, %xmm1 + psubb %xmm0, %xmm1 + pmovmskb %xmm1, %edx + sub $0xffff, %edx + jnz LABEL(exit) + +#ifdef USE_AS_STRNCMP + sub $16, %r11 + jbe LABEL(strcmp_exitz) #endif - mov %rbx, %r8 - add %rax, %r8 - sbb %r10, %r10 + add $16, %rcx + movdqa %xmm4, %xmm3 + jmp LABEL(loop_ashr_11) - mov %rbx, %r9 - add %rdx, %r9 - sbb %r11, %r11 + .p2align 4 +LABEL(nibble_ashr_11): + psrldq $11, %xmm4 + movdqa (%rsi, %rcx), %xmm1 + pcmpeqb %xmm1, %xmm0 + pcmpeqb %xmm4, %xmm1 + psubb %xmm0, %xmm1 + pmovmskb %xmm1, %edx + sub $0x001f, %edx + jnz LABEL(exit) +#ifdef USE_AS_STRNCMP + cmp $5, %r11 + jbe LABEL(strcmp_exitz) +#endif + pxor %xmm0, %xmm0 + sub $0x1000, %r10 /* subtract 4K from %r10 */ + jmp LABEL(gobble_ashr_11) - xor %rax, %r8 - or %rbx, %r8 - sub %r10, %r8 - jnz LABEL(tail) +/* + * ashr_12 handles the following cases: + * abs(str1 offset - str2 offset) = 4 + */ + .p2align 4 +LABEL(ashr_12): + pxor %xmm0, %xmm0 + movdqa (%rdi), %xmm2 + movdqa (%rsi), %xmm1 + pcmpeqb %xmm1, %xmm0 + pslldq $4, %xmm2 + pcmpeqb %xmm1, %xmm2 + psubb %xmm0, %xmm2 + pmovmskb %xmm2, %r9d + shr %cl, %edx + shr %cl, %r9d + sub %r9d, %edx + jnz LABEL(less32bytes) + movdqa (%rdi), %xmm3 + + UPDATE_STRNCMP_COUNTER - xor %rdx, %r9 - or %rbx, %r9 - sub %r11, %r9 - jnz LABEL(tail) + pxor %xmm0, %xmm0 + mov $16, %rcx /* index for loads */ + mov $12, %r9d /* rdi bytes already examined. Used in exit code */ + /* + * Setup %r10 value allows us to detect crossing a page boundary. + * When %r10 goes positive we are crossing a page boundary and + * need to do a nibble. + */ + lea 12(%rdi), %r10 + and $0xfff, %r10 /* offset into 4K page */ + sub $0x1000, %r10 /* subtract 4K pagesize */ + movdqa %xmm3, %xmm4 - cmp %rdx, %rax - jne LABEL(tail) + .p2align 4 +LABEL(loop_ashr_12): + add $16, %r10 + jg LABEL(nibble_ashr_12) + +LABEL(gobble_ashr_12): + movdqa (%rsi, %rcx), %xmm1 + movdqa (%rdi, %rcx), %xmm2 + movdqa %xmm2, %xmm4 - mov 8 (%rsi, %rcx), %rax - mov 8 (%rdi, %rcx), %rdx - add $8, %ecx + psrldq $12, %xmm3 + pslldq $4, %xmm2 + por %xmm3, %xmm2 + + pcmpeqb %xmm1, %xmm0 + pcmpeqb %xmm2, %xmm1 + psubb %xmm0, %xmm1 + pmovmskb %xmm1, %edx + sub $0xffff, %edx + jnz LABEL(exit) #ifdef USE_AS_STRNCMP - sub $8, %r14 - jle LABEL(tail) + sub $16, %r11 + jbe LABEL(strcmp_exitz) +#endif + + add $16, %rcx + movdqa %xmm4, %xmm3 + + add $16, %r10 + jg LABEL(nibble_ashr_12) /* cross page boundary */ + + movdqa (%rsi, %rcx), %xmm1 + movdqa (%rdi, %rcx), %xmm2 + movdqa %xmm2, %xmm4 + + psrldq $12, %xmm3 + pslldq $4, %xmm2 + por %xmm3, %xmm2 + + pcmpeqb %xmm1, %xmm0 + pcmpeqb %xmm2, %xmm1 + psubb %xmm0, %xmm1 + pmovmskb %xmm1, %edx + sub $0xffff, %edx + jnz LABEL(exit) + +#ifdef USE_AS_STRNCMP + sub $16, %r11 + jbe LABEL(strcmp_exitz) #endif - mov %rbx, %r8 - add %rax, %r8 - sbb %r10, %r10 + add $16, %rcx + movdqa %xmm4, %xmm3 + jmp LABEL(loop_ashr_12) - mov %rbx, %r9 - add %rdx, %r9 - sbb %r11, %r11 - - xor %rax, %r8 - or %rbx, %r8 - sub %r10, %r8 - jnz LABEL(tail) + .p2align 4 +LABEL(nibble_ashr_12): + psrldq $12, %xmm4 + movdqa (%rsi, %rcx), %xmm1 + pcmpeqb %xmm1, %xmm0 + pcmpeqb %xmm4, %xmm1 + psubb %xmm0, %xmm1 + pmovmskb %xmm1, %edx + sub $0x000f, %edx + jnz LABEL(exit) +#ifdef USE_AS_STRNCMP + cmp $4, %r11 + jbe LABEL(strcmp_exitz) +#endif + pxor %xmm0, %xmm0 + sub $0x1000, %r10 /* subtract 4K from %r10 */ + jmp LABEL(gobble_ashr_12) - xor %rdx, %r9 - or %rbx, %r9 - sub %r11, %r9 - jnz LABEL(tail) +/* + * ashr_13 handles the following cases: + * abs(str1 offset - str2 offset) = 3 + */ + .p2align 4 +LABEL(ashr_13): + pxor %xmm0, %xmm0 + movdqa (%rdi), %xmm2 + movdqa (%rsi), %xmm1 + pcmpeqb %xmm1, %xmm0 + pslldq $3, %xmm2 + pcmpeqb %xmm1, %xmm2 + psubb %xmm0, %xmm2 + pmovmskb %xmm2, %r9d + shr %cl, %edx + shr %cl, %r9d + sub %r9d, %edx + jnz LABEL(less32bytes) + movdqa (%rdi), %xmm3 - cmp %rdx, %rax - jne LABEL(tail) - - add $8, %ecx + UPDATE_STRNCMP_COUNTER - jmp LABEL(64loop) + pxor %xmm0, %xmm0 + mov $16, %rcx /* index for loads */ + mov $13, %r9d /* rdi bytes already examined. Used in exit code */ + /* + * Setup %r10 value allows us to detect crossing a page boundary. + * When %r10 goes positive we are crossing a page boundary and + * need to do a nibble. + */ + lea 13(%rdi), %r10 + and $0xfff, %r10 /* offset into 4K page */ + sub $0x1000, %r10 /* subtract 4K pagesize */ + movdqa %xmm3, %xmm4 -LABEL(64after): + .p2align 4 +LABEL(loop_ashr_13): + add $16, %r10 + jg LABEL(nibble_ashr_13) -LABEL(tailtry): +LABEL(gobble_ashr_13): + movdqa (%rsi, %rcx), %xmm1 + movdqa (%rdi, %rcx), %xmm2 + movdqa %xmm2, %xmm4 + + psrldq $13, %xmm3 + pslldq $3, %xmm2 + por %xmm3, %xmm2 -LABEL(tail): /* byte tail */ + pcmpeqb %xmm1, %xmm0 + pcmpeqb %xmm2, %xmm1 + psubb %xmm0, %xmm1 + pmovmskb %xmm1, %edx + sub $0xffff, %edx + jnz LABEL(exit) + #ifdef USE_AS_STRNCMP - add $7, %r14 + sub $16, %r11 + jbe LABEL(strcmp_exitz) #endif - cmp %dl, %al /* check if same character */ - jne LABEL(exit) - test %al, %al /* check if character a NUL */ - jz LABEL(exit) + add $16, %rcx + movdqa %xmm4, %xmm3 + + add $16, %r10 + jg LABEL(nibble_ashr_13) /* cross page boundary */ + + movdqa (%rsi, %rcx), %xmm1 + movdqa (%rdi, %rcx), %xmm2 + movdqa %xmm2, %xmm4 - shr $8, %rax - shr $8, %rdx + psrldq $13, %xmm3 + pslldq $3, %xmm2 + por %xmm3, %xmm2 + + pcmpeqb %xmm1, %xmm0 + pcmpeqb %xmm2, %xmm1 + psubb %xmm0, %xmm1 + pmovmskb %xmm1, %edx + sub $0xffff, %edx + jnz LABEL(exit) #ifdef USE_AS_STRNCMP - dec %r14 - jl LABEL(exit) + sub $16, %r11 + jbe LABEL(strcmp_exitz) #endif - cmp %dl, %al - jne LABEL(exit) - test %al, %al - jz LABEL(exit) + add $16, %rcx + movdqa %xmm4, %xmm3 + jmp LABEL(loop_ashr_13) + + .p2align 4 +LABEL(nibble_ashr_13): + psrldq $13, %xmm4 + movdqa (%rsi, %rcx), %xmm1 + pcmpeqb %xmm1, %xmm0 + pcmpeqb %xmm4, %xmm1 + psubb %xmm0, %xmm1 + pmovmskb %xmm1, %edx + sub $0x0007, %edx + jnz LABEL(exit) +#ifdef USE_AS_STRNCMP + cmp $3, %r11 + jbe LABEL(strcmp_exitz) +#endif + pxor %xmm0, %xmm0 + sub $0x1000, %r10 /* subtract 4K from %r10 */ + jmp LABEL(gobble_ashr_13) - shr $8, %rax - shr $8, %rdx +/* + * ashr_14 handles the following cases: + * abs(str1 offset - str2 offset) = 2 + */ + .p2align 4 +LABEL(ashr_14): + pxor %xmm0, %xmm0 + movdqa (%rdi), %xmm2 + movdqa (%rsi), %xmm1 + pcmpeqb %xmm1, %xmm0 + pslldq $2, %xmm2 + pcmpeqb %xmm1, %xmm2 + psubb %xmm0, %xmm2 + pmovmskb %xmm2, %r9d + shr %cl, %edx + shr %cl, %r9d + sub %r9d, %edx + jnz LABEL(less32bytes) + movdqa (%rdi), %xmm3 + + UPDATE_STRNCMP_COUNTER + + pxor %xmm0, %xmm0 + mov $16, %rcx /* index for loads */ + mov $14, %r9d /* rdi bytes already examined. Used in exit code */ + /* + * Setup %r10 value allows us to detect crossing a page boundary. + * When %r10 goes positive we are crossing a page boundary and + * need to do a nibble. + */ + lea 14(%rdi), %r10 + and $0xfff, %r10 /* offset into 4K page */ + sub $0x1000, %r10 /* subtract 4K pagesize */ + movdqa %xmm3, %xmm4 + + .p2align 4 +LABEL(loop_ashr_14): + add $16, %r10 + jg LABEL(nibble_ashr_14) + +LABEL(gobble_ashr_14): + movdqa (%rsi, %rcx), %xmm1 + movdqa (%rdi, %rcx), %xmm2 + movdqa %xmm2, %xmm4 + + psrldq $14, %xmm3 + pslldq $2, %xmm2 + por %xmm3, %xmm2 + + pcmpeqb %xmm1, %xmm0 + pcmpeqb %xmm2, %xmm1 + psubb %xmm0, %xmm1 + pmovmskb %xmm1, %edx + sub $0xffff, %edx + jnz LABEL(exit) #ifdef USE_AS_STRNCMP - dec %r14 - jl LABEL(exit) + sub $16, %r11 + jbe LABEL(strcmp_exitz) #endif - cmp %dl, %al - jne LABEL(exit) - test %al, %al - jz LABEL(exit) + add $16, %rcx + movdqa %xmm4, %xmm3 + + add $16, %r10 + jg LABEL(nibble_ashr_14) /* cross page boundary */ + + movdqa (%rsi, %rcx), %xmm1 + movdqa (%rdi, %rcx), %xmm2 + movdqa %xmm2, %xmm4 - shr $8, %rax - shr $8, %rdx + psrldq $14, %xmm3 + pslldq $2, %xmm2 + por %xmm3, %xmm2 + + pcmpeqb %xmm1, %xmm0 + pcmpeqb %xmm2, %xmm1 + psubb %xmm0, %xmm1 + pmovmskb %xmm1, %edx + sub $0xffff, %edx + jnz LABEL(exit) #ifdef USE_AS_STRNCMP - dec %r14 - jl LABEL(exit) + sub $16, %r11 + jbe LABEL(strcmp_exitz) #endif - cmp %dl, %al - jne LABEL(exit) - test %al, %al - jz LABEL(exit) + add $16, %rcx + movdqa %xmm4, %xmm3 + jmp LABEL(loop_ashr_14) + + .p2align 4 +LABEL(nibble_ashr_14): + psrldq $14, %xmm4 + movdqa (%rsi, %rcx), %xmm1 + pcmpeqb %xmm1, %xmm0 + pcmpeqb %xmm4, %xmm1 + psubb %xmm0, %xmm1 + pmovmskb %xmm1, %edx + sub $0x0003, %edx + jnz LABEL(exit) +#ifdef USE_AS_STRNCMP + cmp $2, %r11 + jbe LABEL(strcmp_exitz) +#endif + pxor %xmm0, %xmm0 + sub $0x1000, %r10 /* subtract 4K from %r10 */ + jmp LABEL(gobble_ashr_14) - shr $8, %rax - shr $8, %rdx +/* + * ashr_15 handles the following cases: + * abs(str1 offset - str2 offset) = 1 + */ + .p2align 4 +LABEL(ashr_15): + pxor %xmm0, %xmm0 + movdqa (%rdi), %xmm2 + movdqa (%rsi), %xmm1 + pcmpeqb %xmm1, %xmm0 + pslldq $1, %xmm2 + pcmpeqb %xmm1, %xmm2 + psubb %xmm0, %xmm2 + pmovmskb %xmm2, %r9d + shr %cl, %edx + shr %cl, %r9d + sub %r9d, %edx + jnz LABEL(less32bytes) + + movdqa (%rdi), %xmm3 + + UPDATE_STRNCMP_COUNTER + + pxor %xmm0, %xmm0 + mov $16, %rcx /* index for loads */ + mov $15, %r9d /* rdi bytes already examined. Used in exit code */ + /* + * Setup %r10 value allows us to detect crossing a page boundary. + * When %r10 goes positive we are crossing a page boundary and + * need to do a nibble. + */ + lea 15(%rdi), %r10 + and $0xfff, %r10 /* offset into 4K page */ + sub $0x1000, %r10 /* subtract 4K pagesize */ + movdqa %xmm3, %xmm4 + + .p2align 4 +LABEL(loop_ashr_15): + add $16, %r10 + jg LABEL(nibble_ashr_15) + +LABEL(gobble_ashr_15): + movdqa (%rsi, %rcx), %xmm1 + movdqa (%rdi, %rcx), %xmm2 + movdqa %xmm2, %xmm4 + + psrldq $15, %xmm3 + pslldq $1, %xmm2 + por %xmm3, %xmm2 + + pcmpeqb %xmm1, %xmm0 + pcmpeqb %xmm2, %xmm1 + psubb %xmm0, %xmm1 + pmovmskb %xmm1, %edx + sub $0xffff, %edx + jnz LABEL(exit) #ifdef USE_AS_STRNCMP - dec %r14 - jl LABEL(exit) + sub $16, %r11 + jbe LABEL(strcmp_exitz) #endif - cmp %dl, %al - jne LABEL(exit) - test %al, %al - jz LABEL(exit) + add $16, %rcx + movdqa %xmm4, %xmm3 + + add $16, %r10 + jg LABEL(nibble_ashr_15) /* cross page boundary */ + + movdqa (%rsi, %rcx), %xmm1 + movdqa (%rdi, %rcx), %xmm2 + movdqa %xmm2, %xmm4 - shr $8, %eax - shr $8, %edx + psrldq $15, %xmm3 + pslldq $1, %xmm2 + por %xmm3, %xmm2 + + pcmpeqb %xmm1, %xmm0 + pcmpeqb %xmm2, %xmm1 + psubb %xmm0, %xmm1 + pmovmskb %xmm1, %edx + sub $0xffff, %edx + jnz LABEL(exit) #ifdef USE_AS_STRNCMP - dec %r14 - jl LABEL(exit) + sub $16, %r11 + jbe LABEL(strcmp_exitz) #endif - cmp %dl, %al - jne LABEL(exit) - test %al, %al - jz LABEL(exit) + add $16, %rcx + movdqa %xmm4, %xmm3 + jmp LABEL(loop_ashr_15) - shr $8, %eax - shr $8, %edx + .p2align 4 +LABEL(nibble_ashr_15): + psrldq $15, %xmm4 + movdqa (%rsi, %rcx), %xmm1 + pcmpeqb %xmm1, %xmm0 + pcmpeqb %xmm4, %xmm1 + psubb %xmm0, %xmm1 + pmovmskb %xmm1, %edx + sub $0x0001, %edx + jnz LABEL(exit) +#ifdef USE_AS_STRNCMP + cmp $1, %r11 + jbe LABEL(strcmp_exitz) +#endif + pxor %xmm0, %xmm0 + sub $0x1000, %r10 /* subtract 4K from %r10 */ + jmp LABEL(gobble_ashr_15) + + .p2align 4 +LABEL(exit): + lea -16(%r9, %rcx), %rax /* locate the exact offset for rdi */ +LABEL(less32bytes): + lea (%rdi, %rax), %rdi /* locate the exact address for first operand(rdi) */ + lea (%rsi, %rcx), %rsi /* locate the exact address for second operand(rsi) */ + test %r8d, %r8d + jz LABEL(ret) + xchg %rsi, %rdi /* recover original order according to flag(%r8d) */ + + .p2align 4 +LABEL(ret): +LABEL(less16bytes): + /* + * Check to see if BSF is fast on this processor. If not, use a different + * exit tail. + */ + testl $USE_BSF,.memops_method(%rip) + jz LABEL(AMD_exit) + bsf %rdx, %rdx /* find and store bit index in %rdx */ #ifdef USE_AS_STRNCMP - dec %r14 - jl LABEL(exit) + sub %rdx, %r11 + jbe LABEL(strcmp_exitz) +#endif + xor %ecx, %ecx /* clear %ecx */ + xor %eax, %eax /* clear %eax */ + + movb (%rsi, %rdx), %cl + movb (%rdi, %rdx), %al + + sub %ecx, %eax + ret + +#ifdef USE_AS_STRNCMP +LABEL(strcmp_exitz): + xor %eax, %eax + ret #endif - cmp %dl, %al - jne LABEL(exit) - test %al, %al - jz LABEL(exit) + /* + * This exit tail does not use the bsf instruction. + */ + .p2align 4 +LABEL(AMD_exit): + test %dl, %dl + jz LABEL(next_8_bytes) + + test $0x01, %dl + jnz LABEL(Byte0) + + test $0x02, %dl + jnz LABEL(Byte1) + + test $0x04, %dl + jnz LABEL(Byte2) + + test $0x08, %dl + jnz LABEL(Byte3) + + test $0x10, %dl + jnz LABEL(Byte4) + + test $0x20, %dl + jnz LABEL(Byte5) + + test $0x40, %dl + jnz LABEL(Byte6) + +#ifdef USE_AS_STRNCMP + sub $7, %r11 + jbe LABEL(strcmp_exitz) +#endif + movzx 7(%rsi), %ecx + movzx 7(%rdi), %eax + + sub %ecx, %eax + ret - shr $8, %eax - shr $8, %edx + .p2align 4 +LABEL(Byte0): + /* + * never need to handle byte 0 for strncmpy +#ifdef USE_AS_STRNCMP + sub $0, %r11 + jbe LABEL(strcmp_exitz) +#endif + */ + movzx (%rsi), %ecx + movzx (%rdi), %eax + + sub %ecx, %eax + ret + + .p2align 4 +LABEL(Byte1): + +#ifdef USE_AS_STRNCMP + sub $1, %r11 + jbe LABEL(strcmp_exitz) +#endif + movzx 1(%rsi), %ecx + movzx 1(%rdi), %eax + + sub %ecx, %eax + ret + + .p2align 4 +LABEL(Byte2): + +#ifdef USE_AS_STRNCMP + sub $2, %r11 + jbe LABEL(strcmp_exitz) +#endif + movzx 2(%rsi), %ecx + movzx 2(%rdi), %eax + + sub %ecx, %eax + ret + + .p2align 4 +LABEL(Byte3): #ifdef USE_AS_STRNCMP - dec %r14 - jl LABEL(exit) + sub $3, %r11 + jbe LABEL(strcmp_exitz) #endif + movzx 3(%rsi), %ecx + movzx 3(%rdi), %eax + + sub %ecx, %eax + ret - cmp %dl, %al - jne LABEL(exit) + .p2align 4 +LABEL(Byte4): - .p2align 4,, 15 - -LABEL(tailafter): +#ifdef USE_AS_STRNCMP + sub $4, %r11 + jbe LABEL(strcmp_exitz) +#endif + movzx 4(%rsi), %ecx + movzx 4(%rdi), %eax -LABEL(exit): - mov -32 (%rsp), %r15 - mov -24 (%rsp), %rbp - mov -16 (%rsp), %rbx + sub %ecx, %eax + ret - .p2align 4,, 3 + .p2align 4 +LABEL(Byte5): -LABEL(exitafter): #ifdef USE_AS_STRNCMP - test %r14, %r14 - cmovl %edx, %eax + sub $5, %r11 + jbe LABEL(strcmp_exitz) #endif + movzx 5(%rsi), %ecx + movzx 5(%rdi), %eax - movzx %al, %eax - movzx %dl, %edx - sub %eax, %edx - xchg %edx, %eax + sub %ecx, %eax + ret + + .p2align 4 +LABEL(Byte6): #ifdef USE_AS_STRNCMP -LABEL(exitz): - mov -8 (%rsp), %r14 + sub $6, %r11 + jbe LABEL(strcmp_exitz) +#endif + movzx 6(%rsi), %ecx + movzx 6(%rdi), %eax + + sub %ecx, %eax + ret + + .p2align 4 +LABEL(next_8_bytes): + add $8, %rdi + add $8, %rsi +#ifdef USE_AS_STRNCMP + sub $8, %r11 + jbe LABEL(strcmp_exitz) #endif - ret + test $0x01, %dh + jnz LABEL(Byte0) + + test $0x02, %dh + jnz LABEL(Byte1) + + test $0x04, %dh + jnz LABEL(Byte2) + + test $0x08, %dh + jnz LABEL(Byte3) + + test $0x10, %dh + jnz LABEL(Byte4) + + test $0x20, %dh + jnz LABEL(Byte5) + + test $0x40, %dh + jnz LABEL(Byte6) #ifdef USE_AS_STRNCMP + sub $7, %r11 + jbe LABEL(strcmp_exitz) +#endif + movzx 7(%rsi), %ecx + movzx 7(%rdi), %eax + + sub %ecx, %eax + ret + + .pushsection .rodata + .p2align 4 +LABEL(unaligned_table): + .int LABEL(ashr_0) - LABEL(unaligned_table) + .int LABEL(ashr_15) - LABEL(unaligned_table) + .int LABEL(ashr_14) - LABEL(unaligned_table) + .int LABEL(ashr_13) - LABEL(unaligned_table) + .int LABEL(ashr_12) - LABEL(unaligned_table) + .int LABEL(ashr_11) - LABEL(unaligned_table) + .int LABEL(ashr_10) - LABEL(unaligned_table) + .int LABEL(ashr_9) - LABEL(unaligned_table) + .int LABEL(ashr_8) - LABEL(unaligned_table) + .int LABEL(ashr_7) - LABEL(unaligned_table) + .int LABEL(ashr_6) - LABEL(unaligned_table) + .int LABEL(ashr_5) - LABEL(unaligned_table) + .int LABEL(ashr_4) - LABEL(unaligned_table) + .int LABEL(ashr_3) - LABEL(unaligned_table) + .int LABEL(ashr_2) - LABEL(unaligned_table) + .int LABEL(ashr_1) - LABEL(unaligned_table) + .popsection +#ifdef USE_AS_STRNCMP SET_SIZE(strncmp) #else SET_SIZE(strcmp) /* (const char *, const char *) */
--- a/usr/src/lib/libc/amd64/gen/strcpy.s Mon Sep 21 11:25:09 2009 -0400 +++ b/usr/src/lib/libc/amd64/gen/strcpy.s Mon Sep 21 11:26:40 2009 -0400 @@ -1,862 +1,2582 @@ /* - * Copyright 2008 Sun Microsystems, Inc. All rights reserved. - * Use is subject to license terms. + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END */ /* - * Copyright (c) 2002 Advanced Micro Devices, Inc. - * + * Copyright (c) 2009, Intel Corporation * All rights reserved. - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the - * following conditions are met: - * - * + Redistributions of source code must retain the above - * copyright notice, this list of conditions and the - * following disclaimer. - * - * + Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the - * following disclaimer in the documentation and/or other - * materials provided with the distribution. - * - * + Neither the name of Advanced Micro Devices, Inc. nor the - * names of its contributors may be used to endorse or - * promote products derived from this software without - * specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND - * CONTRIBUTORS AS IS AND ANY EXPRESS OR IMPLIED WARRANTIES, - * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF - * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL ADVANCED MICRO DEVICES, - * INC. OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, - * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE - * GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR - * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF - * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT - * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - * - * It is licensee's responsibility to comply with any export - * regulations applicable in licensee's jurisdiction. */ - .file "strcpy.s" - +/* + * str[n]cpy - copy [n] chars from second operand into first operand + */ #include "SYS.h" -#include "cache.h" +#include "proc64_id.h" #define LABEL(s) .strcpy/**/s #ifdef USE_AS_STRNCPY ENTRY(strncpy) + test %edx, %edx + jz LABEL(strncpy_exitz) + mov %rdx, %r8 #else - ENTRY(strcpy) /* (char *, const char *) */ + ENTRY(strcpy) /* (char *, const char *) */ + xor %rdx, %rdx +#endif + mov %esi, %ecx + and $0xfffffffffffffff0, %rsi /* force rsi 16 byte align */ + and $0xf, %rcx + mov %rdi, %rax /* save destination address for return value */ + + + pxor %xmm0, %xmm0 /* clear %xmm0 for null char checks */ + pcmpeqb (%rsi), %xmm0 /* check 16 bytes in src for null */ + pmovmskb %xmm0, %edx + shr %cl, %edx /* adjust for offset from 16byte boundary */ + test %edx, %edx /* edx will be 0 if chars are non-null */ + jnz LABEL(less16bytes) /* null char found in first 16 bytes examined */ +#ifdef USE_AS_STRNCPY + /* + * Check if the count is satisfied in first 16 bytes examined. + */ + lea -16(%r8, %rcx), %r11 + cmp $0, %r11 + jle LABEL(less16bytes) +#endif + mov %rcx, %r9 /* rsi alignment offset */ + or %edi, %ecx + and $0xf, %ecx + lea -16(%r9), %r10 + jz LABEL(ashr_0) /* src and dest are both 16 byte aligned */ + + neg %r10 /* max src bytes remaining in current dqword */ + + pxor %xmm0, %xmm0 /* clear %xmm0, may be polluted by unaligned operation */ + pcmpeqb 16(%rsi), %xmm0 /* check next 16 bytes in src for a null */ + pmovmskb %xmm0, %edx + test %edx, %edx + jnz LABEL(less32bytes) /* null char found in first 32 bytes examined */ + +#ifdef USE_AS_STRNCPY + /* + * If strncpy count <= 16 go to exit case + */ + sub $16, %r8 + jbe LABEL(less32bytes_strncpy_truncation) +#endif + /* + * At least 16 bytes to copy to destination string. Move them now. + * Don't worry about alignment. + */ + mov (%rsi, %r9), %rdx + mov %rdx, (%rdi) + mov 8(%rsi, %r9), %rdx + mov %rdx, 8(%rdi) + + /* + * so far destination rdi may be aligned by 16, re-calculate rsi and + * jump to corresponding src/dest relative offset case. + * rcx is offset of rsi + * rdx is offset of rdi + */ + and $0xfffffffffffffff0, %rdi /* force rdi 16 byte align */ + mov %rax, %rdx /* rax contains orignal rdi */ + xor %rdi, %rdx /* same effect as "and $0xf, %rdx" */ +#ifdef USE_AS_STRNCPY + /* + * Will now do 16 byte aligned stores. Stores may overlap some bytes + * (ie store twice) if destination was unaligned. Compensate here. + */ + add %rdx, %r8 /* compensate for overlap */ +#endif + + add $16, %rdi /* next 16 bytes for dest */ + + /* + * align src to 16-byte boundary. Could be up or down depending on + * whether src offset - dest offset > 0 (up) or + * src offset - dest offset < 0 (down). + */ + sub %rdx, %r9 /* src offset - dest offset */ + + lea 16(%r9, %rsi), %rsi + mov %esi, %ecx /* for new src offset */ + and $0xfffffffffffffff0, %rsi /* force rsi 16 byte align */ + + and $0xf, %ecx /* new src offset is 0 if rsi/rdi have same alignment */ + jz LABEL(ashr_0) + +#ifdef USE_AS_STRNCPY + xor %edx, %edx /* In case unaligned_exit is taken */ +#endif + /* + * Jump to case corresponding to source/dest string relative offsets + * Index = (16 + (src offset - dest offset)) % 16 + */ + lea -16(%rcx), %r10 + mov %rcx, %r9 + neg %r10 /* max src bytes remaining in current dqword */ + lea LABEL(unaligned_table)(%rip), %r11 + movslq (%r11, %rcx, 4), %rcx + lea (%r11, %rcx), %rcx + jmp *%rcx + +/* + * ashr_0 handles the following cases: + * src alignment offset = dest alignment offset + */ + .p2align 5 +LABEL(ashr_0): +#ifdef USE_AS_STRNCPY + sub $16, %r8 + jbe LABEL(strncpy_truncation_aligned) +#endif + movdqa (%rsi), %xmm1 /* fetch 16 bytes from src string */ + movdqa %xmm1, (%rdi) /* store 16 bytes into dest string */ + add $16, %rsi + add $16, %rdi + pcmpeqb (%rsi), %xmm0 /* check 16 bytes in src for a null */ + pmovmskb %xmm0, %edx + + test %edx, %edx /* edx will be 0 if chars are non-null */ + jnz LABEL(aligned_16bytes) /* exit tail */ + +LABEL(ashr_0_loop): +#ifdef USE_AS_STRNCPY + sub $16, %r8 + jbe LABEL(strncpy_truncation_aligned) +#endif + movdqa (%rsi, %rcx), %xmm1 + movdqa %xmm1, (%rdi, %rcx) + add $16, %rcx + pcmpeqb (%rsi, %rcx), %xmm0 + pmovmskb %xmm0, %edx + test %edx, %edx + jnz LABEL(aligned_exit) + +#ifdef USE_AS_STRNCPY + sub $16, %r8 + jbe LABEL(strncpy_truncation_aligned) +#endif + movdqa (%rsi, %rcx), %xmm1 + movdqa %xmm1, (%rdi, %rcx) + add $16, %rcx + pcmpeqb (%rsi, %rcx), %xmm0 + pmovmskb %xmm0, %edx + test %edx, %edx + jnz LABEL(aligned_exit) + +#ifdef USE_AS_STRNCPY + sub $16, %r8 + jbe LABEL(strncpy_truncation_aligned) +#endif + movdqa (%rsi, %rcx), %xmm1 + movdqa %xmm1, (%rdi, %rcx) + + add $16, %rcx + pcmpeqb (%rsi, %rcx), %xmm0 + pmovmskb %xmm0, %edx + test %edx, %edx + jnz LABEL(aligned_exit) + +#ifdef USE_AS_STRNCPY + sub $16, %r8 + jbe LABEL(strncpy_truncation_aligned) +#endif + movdqa (%rsi, %rcx), %xmm1 + movdqa %xmm1, (%rdi, %rcx) + add $16, %rcx + pcmpeqb (%rsi, %rcx), %xmm0 + pmovmskb %xmm0, %edx + test %edx, %edx + jz LABEL(ashr_0_loop) + jmp LABEL(aligned_exit) + + +/* + * ashr_15 handles the following cases: + * (16 + (src offset - dest offset)) % 16 = 15 + * + * Based on above operation, start from (%r9 + rsi) to the left of this cache + * bank, there is no null byte. + */ + .p2align 4 +LABEL(ashr_15): + xor %ecx, %ecx /* clear index */ +#ifdef USE_AS_STRNCPY + cmp %r10, %r8 + jbe LABEL(unaligned_exit) +#endif + testl $USE_SSSE3, .memops_method(%rip) /* use sse2 or ssse3? */ + jz LABEL(ashr_15_use_sse2) + + .p2align 4 +LABEL(ashr_15_use_ssse3): + movdqa 16(%rsi, %rcx), %xmm3 + pcmpeqb %xmm3, %xmm0 + pmovmskb %xmm0, %edx + test %edx, %edx + jnz LABEL(unaligned_exit) +#ifdef USE_AS_STRNCPY + sub $16, %r8 + jbe LABEL(strncpy_truncation_unaligned) +#endif + + #palignr $15, (%rsi, %rcx), %xmm3 + .byte 0x66, 0x0F, 0x3A ,0x0F + .byte 0x1c, 0x0e, 0x0f + + movdqa %xmm3, (%rdi, %rcx) + add $16, %rcx + +#ifdef USE_AS_STRNCPY + cmp %r10, %r8 + jbe LABEL(unaligned_exit) +#endif + movdqa 16(%rsi, %rcx), %xmm3 + pcmpeqb %xmm3, %xmm0 + pmovmskb %xmm0, %edx + test %edx, %edx + jnz LABEL(unaligned_exit) +#ifdef USE_AS_STRNCPY + sub $16, %r8 + jbe LABEL(strncpy_truncation_unaligned) +#endif + + #palignr $15, (%rsi, %rcx), %xmm3 + .byte 0x66, 0x0F, 0x3A ,0x0F + .byte 0x1c, 0x0e, 0x0f + + movdqa %xmm3, (%rdi, %rcx) + add $16, %rcx + +#ifdef USE_AS_STRNCPY + cmp %r10, %r8 + jbe LABEL(unaligned_exit) +#endif + jmp LABEL(ashr_15_use_ssse3) + + .p2align 4 +LABEL(ashr_15_use_sse2): + pcmpeqb 16(%rsi, %rcx), %xmm0 + pmovmskb %xmm0, %edx + test %edx, %edx + jnz LABEL(unaligned_exit) +#ifdef USE_AS_STRNCPY + sub $16, %r8 + jbe LABEL(strncpy_truncation_unaligned) +#endif + + movdqa 16(%rsi, %rcx), %xmm3 + movdqa (%rsi, %rcx), %xmm2 + + psrldq $15, %xmm2 + pslldq $1, %xmm3 + por %xmm2, %xmm3 + + movdqa %xmm3, (%rdi, %rcx) + add $16, %rcx +#ifdef USE_AS_STRNCPY + cmp %r10, %r8 + jbe LABEL(unaligned_exit) +#endif + pcmpeqb 16(%rsi, %rcx), %xmm0 + pmovmskb %xmm0, %edx + test %edx, %edx + jnz LABEL(unaligned_exit) +#ifdef USE_AS_STRNCPY + sub $16, %r8 + jbe LABEL(strncpy_truncation_unaligned) +#endif + + movdqa 16(%rsi, %rcx), %xmm3 + movdqa (%rsi, %rcx), %xmm2 + + psrldq $15, %xmm2 + pslldq $1, %xmm3 + por %xmm2, %xmm3 + + movdqa %xmm3, (%rdi, %rcx) + add $16, %rcx +#ifdef USE_AS_STRNCPY + cmp %r10, %r8 + jbe LABEL(unaligned_exit) +#endif + jmp LABEL(ashr_15_use_sse2) + + +/* + * ashr_14 handles the following cases: + * (16 + (src offset - dest offset)) % 16 = 14 + * + * Based on above operation, start from (%r9 + rsi) to the left of this cache + * bank, there is no null byte. + */ + .p2align 4 +LABEL(ashr_14): + xor %ecx, %ecx /* clear index */ +#ifdef USE_AS_STRNCPY + cmp %r10, %r8 + jbe LABEL(unaligned_exit) +#endif + testl $USE_SSSE3, .memops_method(%rip) /* use sse2 or ssse3? */ + jz LABEL(ashr_14_use_sse2) + + .p2align 4 +LABEL(ashr_14_use_ssse3): + movdqa 16(%rsi, %rcx), %xmm3 + pcmpeqb %xmm3, %xmm0 + pmovmskb %xmm0, %edx + test %edx, %edx + jnz LABEL(unaligned_exit) +#ifdef USE_AS_STRNCPY + sub $16, %r8 + jbe LABEL(strncpy_truncation_unaligned) +#endif + + #palignr $14, (%rsi, %rcx), %xmm3 + .byte 0x66, 0x0F, 0x3A ,0x0F + .byte 0x1c, 0x0e, 0x0e + + movdqa %xmm3, (%rdi, %rcx) + add $16, %rcx + +#ifdef USE_AS_STRNCPY + cmp %r10, %r8 + jbe LABEL(unaligned_exit) +#endif + movdqa 16(%rsi, %rcx), %xmm3 + pcmpeqb %xmm3, %xmm0 + pmovmskb %xmm0, %edx + test %edx, %edx + jnz LABEL(unaligned_exit) +#ifdef USE_AS_STRNCPY + sub $16, %r8 + jbe LABEL(strncpy_truncation_unaligned) +#endif + + #palignr $14, (%rsi, %rcx), %xmm3 + .byte 0x66, 0x0F, 0x3A ,0x0F + .byte 0x1c, 0x0e, 0x0e + + movdqa %xmm3, (%rdi, %rcx) + add $16, %rcx +#ifdef USE_AS_STRNCPY + cmp %r10, %r8 + jbe LABEL(unaligned_exit) +#endif + jmp LABEL(ashr_14_use_ssse3) + + .p2align 4 +LABEL(ashr_14_use_sse2): + pcmpeqb 16(%rsi, %rcx), %xmm0 + pmovmskb %xmm0, %edx + test %edx, %edx + jnz LABEL(unaligned_exit) +#ifdef USE_AS_STRNCPY + sub $16, %r8 + jbe LABEL(strncpy_truncation_unaligned) +#endif + + movdqa 16(%rsi, %rcx), %xmm3 + movdqa (%rsi, %rcx), %xmm2 + + psrldq $14, %xmm2 + pslldq $2, %xmm3 + por %xmm2, %xmm3 + + movdqa %xmm3, (%rdi, %rcx) + add $16, %rcx + +#ifdef USE_AS_STRNCPY + cmp %r10, %r8 + jbe LABEL(unaligned_exit) +#endif + pcmpeqb 16(%rsi, %rcx), %xmm0 + pmovmskb %xmm0, %edx + test %edx, %edx + jnz LABEL(unaligned_exit) +#ifdef USE_AS_STRNCPY + sub $16, %r8 + jbe LABEL(strncpy_truncation_unaligned) +#endif + + movdqa 16(%rsi, %rcx), %xmm3 + movdqa (%rsi, %rcx), %xmm2 + + psrldq $14, %xmm2 + pslldq $2, %xmm3 + por %xmm2, %xmm3 + + movdqa %xmm3, (%rdi, %rcx) + add $16, %rcx +#ifdef USE_AS_STRNCPY + cmp %r10, %r8 + jbe LABEL(unaligned_exit) +#endif + jmp LABEL(ashr_14_use_sse2) + + +/* + * ashr_13 handles the following cases: + * (16 + (src offset - dest offset)) % 16 = 13 + * + * Based on above operation, start from (%r9 + rsi) to the left of this cache + * bank, there is no null byte. + */ + .p2align 4 +LABEL(ashr_13): + xor %ecx, %ecx /* clear index */ +#ifdef USE_AS_STRNCPY + cmp %r10, %r8 + jbe LABEL(unaligned_exit) +#endif + testl $USE_SSSE3, .memops_method(%rip) /* use sse2 or ssse3? */ + jz LABEL(ashr_13_use_sse2) + + .p2align 4 +LABEL(ashr_13_use_ssse3): + movdqa 16(%rsi, %rcx), %xmm3 + pcmpeqb %xmm3, %xmm0 + pmovmskb %xmm0, %edx + test %edx, %edx + jnz LABEL(unaligned_exit) +#ifdef USE_AS_STRNCPY + sub $16, %r8 + jbe LABEL(strncpy_truncation_unaligned) +#endif + + #palignr $13, (%rsi, %rcx), %xmm3 + .byte 0x66, 0x0F, 0x3A ,0x0F + .byte 0x1c, 0x0e, 0x0d + + movdqa %xmm3, (%rdi, %rcx) + add $16, %rcx + +#ifdef USE_AS_STRNCPY + cmp %r10, %r8 + jbe LABEL(unaligned_exit) +#endif + movdqa 16(%rsi, %rcx), %xmm3 + pcmpeqb %xmm3, %xmm0 + pmovmskb %xmm0, %edx + test %edx, %edx + jnz LABEL(unaligned_exit) +#ifdef USE_AS_STRNCPY + sub $16, %r8 + jbe LABEL(strncpy_truncation_unaligned) #endif + #palignr $13, (%rsi, %rcx), %xmm3 + .byte 0x66, 0x0F, 0x3A ,0x0F + .byte 0x1c, 0x0e, 0x0d + + movdqa %xmm3, (%rdi, %rcx) + add $16, %rcx #ifdef USE_AS_STRNCPY - test %rdx, %rdx /* (char *, const char *, size_t) */ - mov %rdx, %r11 - jz LABEL(exitn) /* early exit */ + cmp %r10, %r8 + jbe LABEL(unaligned_exit) +#endif + jmp LABEL(ashr_13_use_ssse3) + + .p2align 4 +LABEL(ashr_13_use_sse2): + pcmpeqb 16(%rsi, %rcx), %xmm0 + pmovmskb %xmm0, %edx + test %edx, %edx + jnz LABEL(unaligned_exit) +#ifdef USE_AS_STRNCPY + sub $16, %r8 + jbe LABEL(strncpy_truncation_unaligned) +#endif + + movdqa 16(%rsi, %rcx), %xmm3 + movdqa (%rsi, %rcx), %xmm2 + + psrldq $13, %xmm2 + pslldq $3, %xmm3 + por %xmm2, %xmm3 + + movdqa %xmm3, (%rdi, %rcx) + add $16, %rcx + +#ifdef USE_AS_STRNCPY + cmp %r10, %r8 + jbe LABEL(unaligned_exit) +#endif + pcmpeqb 16(%rsi, %rcx), %xmm0 + pmovmskb %xmm0, %edx + test %edx, %edx + jnz LABEL(unaligned_exit) +#ifdef USE_AS_STRNCPY + sub $16, %r8 + jbe LABEL(strncpy_truncation_unaligned) +#endif + + movdqa 16(%rsi, %rcx), %xmm3 + movdqa (%rsi, %rcx), %xmm2 + + psrldq $13, %xmm2 + pslldq $3, %xmm3 + por %xmm2, %xmm3 + + movdqa %xmm3, (%rdi, %rcx) + add $16, %rcx +#ifdef USE_AS_STRNCPY + cmp %r10, %r8 + jbe LABEL(unaligned_exit) +#endif + jmp LABEL(ashr_13_use_sse2) + + +/* + * ashr_12 handles the following cases: + * (16 + (src offset - dest offset)) % 16 = 12 + * + * Based on above operation, start from (%r9 + rsi) to the left of this cache + * bank, there is no null byte. + */ + .p2align 4 +LABEL(ashr_12): + xor %ecx, %ecx /* clear index */ +#ifdef USE_AS_STRNCPY + cmp %r10, %r8 + jbe LABEL(unaligned_exit) +#endif + testl $USE_SSSE3, .memops_method(%rip) /* use sse2 or ssse3? */ + jz LABEL(ashr_12_use_sse2) + + .p2align 4 +LABEL(ashr_12_use_ssse3): + movdqa 16(%rsi, %rcx), %xmm3 + pcmpeqb %xmm3, %xmm0 + pmovmskb %xmm0, %edx + test %edx, %edx + jnz LABEL(unaligned_exit) +#ifdef USE_AS_STRNCPY + sub $16, %r8 + jbe LABEL(strncpy_truncation_unaligned) +#endif + + #palignr $12, (%rsi, %rcx), %xmm3 + .byte 0x66, 0x0F, 0x3A ,0x0F + .byte 0x1c, 0x0e, 0x0c + + movdqa %xmm3, (%rdi, %rcx) + add $16, %rcx + +#ifdef USE_AS_STRNCPY + cmp %r10, %r8 + jbe LABEL(unaligned_exit) +#endif + movdqa 16(%rsi, %rcx), %xmm3 + pcmpeqb %xmm3, %xmm0 + pmovmskb %xmm0, %edx + test %edx, %edx + jnz LABEL(unaligned_exit) +#ifdef USE_AS_STRNCPY + sub $16, %r8 + jbe LABEL(strncpy_truncation_unaligned) +#endif + + #palignr $12, (%rsi, %rcx), %xmm3 + .byte 0x66, 0x0F, 0x3A ,0x0F + .byte 0x1c, 0x0e, 0x0c + + movdqa %xmm3, (%rdi, %rcx) + add $16, %rcx +#ifdef USE_AS_STRNCPY + cmp %r10, %r8 + jbe LABEL(unaligned_exit) +#endif + jmp LABEL(ashr_12_use_ssse3) + + .p2align 4 +LABEL(ashr_12_use_sse2): + pcmpeqb 16(%rsi, %rcx), %xmm0 + pmovmskb %xmm0, %edx + test %edx, %edx + jnz LABEL(unaligned_exit) +#ifdef USE_AS_STRNCPY + sub $16, %r8 + jbe LABEL(strncpy_truncation_unaligned) +#endif + + movdqa 16(%rsi, %rcx), %xmm3 + movdqa (%rsi, %rcx), %xmm2 + + psrldq $12, %xmm2 + pslldq $4, %xmm3 + por %xmm2, %xmm3 + + movdqa %xmm3, (%rdi, %rcx) + add $16, %rcx + +#ifdef USE_AS_STRNCPY + cmp %r10, %r8 + jbe LABEL(unaligned_exit) +#endif + pcmpeqb 16(%rsi, %rcx), %xmm0 + pmovmskb %xmm0, %edx + test %edx, %edx + jnz LABEL(unaligned_exit) +#ifdef USE_AS_STRNCPY + sub $16, %r8 + jbe LABEL(strncpy_truncation_unaligned) +#endif + + movdqa 16(%rsi, %rcx), %xmm3 + movdqa (%rsi, %rcx), %xmm2 + + psrldq $12, %xmm2 + pslldq $4, %xmm3 + por %xmm2, %xmm3 + + movdqa %xmm3, (%rdi, %rcx) + add $16, %rcx +#ifdef USE_AS_STRNCPY + cmp %r10, %r8 + jbe LABEL(unaligned_exit) +#endif + jmp LABEL(ashr_12_use_sse2) + + +/* + * ashr_11 handles the following cases: + * (16 + (src offset - dest offset)) % 16 = 11 + * + * Based on above operation, start from (%r9 + rsi) to the left of this cache + * bank, there is no null byte. + */ + .p2align 4 +LABEL(ashr_11): + xor %ecx, %ecx /* clear index */ +#ifdef USE_AS_STRNCPY + cmp %r10, %r8 + jbe LABEL(unaligned_exit) +#endif + testl $USE_SSSE3, .memops_method(%rip) /* use sse2 or ssse3? */ + jz LABEL(ashr_11_use_sse2) + + .p2align 4 +LABEL(ashr_11_use_ssse3): + movdqa 16(%rsi, %rcx), %xmm3 + pcmpeqb %xmm3, %xmm0 + pmovmskb %xmm0, %edx + test %edx, %edx + jnz LABEL(unaligned_exit) +#ifdef USE_AS_STRNCPY + sub $16, %r8 + jbe LABEL(strncpy_truncation_unaligned) +#endif + + #palignr $11, (%rsi, %rcx), %xmm3 + .byte 0x66, 0x0F, 0x3A ,0x0F + .byte 0x1c, 0x0e, 0x0b + + movdqa %xmm3, (%rdi, %rcx) + add $16, %rcx + +#ifdef USE_AS_STRNCPY + cmp %r10, %r8 + jbe LABEL(unaligned_exit) +#endif + movdqa 16(%rsi, %rcx), %xmm3 + pcmpeqb %xmm3, %xmm0 + pmovmskb %xmm0, %edx + test %edx, %edx + jnz LABEL(unaligned_exit) +#ifdef USE_AS_STRNCPY + sub $16, %r8 + jbe LABEL(strncpy_truncation_unaligned) +#endif + + #palignr $11, (%rsi, %rcx), %xmm3 + .byte 0x66, 0x0F, 0x3A ,0x0F + .byte 0x1c, 0x0e, 0x0b + + movdqa %xmm3, (%rdi, %rcx) + add $16, %rcx +#ifdef USE_AS_STRNCPY + cmp %r10, %r8 + jbe LABEL(unaligned_exit) +#endif + jmp LABEL(ashr_11_use_ssse3) + + .p2align 4 +LABEL(ashr_11_use_sse2): + pcmpeqb 16(%rsi, %rcx), %xmm0 + pmovmskb %xmm0, %edx + test %edx, %edx + jnz LABEL(unaligned_exit) +#ifdef USE_AS_STRNCPY + sub $16, %r8 + jbe LABEL(strncpy_truncation_unaligned) +#endif + + movdqa 16(%rsi, %rcx), %xmm3 + movdqa (%rsi, %rcx), %xmm2 + + psrldq $11, %xmm2 + pslldq $5, %xmm3 + por %xmm2, %xmm3 + + movdqa %xmm3, (%rdi, %rcx) + add $16, %rcx + +#ifdef USE_AS_STRNCPY + cmp %r10, %r8 + jbe LABEL(unaligned_exit) +#endif + pcmpeqb 16(%rsi, %rcx), %xmm0 + pmovmskb %xmm0, %edx + test %edx, %edx + jnz LABEL(unaligned_exit) +#ifdef USE_AS_STRNCPY + sub $16, %r8 + jbe LABEL(strncpy_truncation_unaligned) +#endif + + movdqa 16(%rsi, %rcx), %xmm3 + movdqa (%rsi, %rcx), %xmm2 + + psrldq $11, %xmm2 + pslldq $5, %xmm3 + por %xmm2, %xmm3 + + movdqa %xmm3, (%rdi, %rcx) + add $16, %rcx +#ifdef USE_AS_STRNCPY + cmp %r10, %r8 + jbe LABEL(unaligned_exit) +#endif + jmp LABEL(ashr_11_use_sse2) + + +/* + * ashr_10 handles the following cases: + * (16 + (src offset - dest offset)) % 16 = 10 + * + * Based on above operation, start from (%r9 + rsi) to the left of this cache + * bank, there is no null byte. + */ + .p2align 4 +LABEL(ashr_10): + xor %ecx, %ecx /* clear index */ +#ifdef USE_AS_STRNCPY + cmp %r10, %r8 + jbe LABEL(unaligned_exit) +#endif + testl $USE_SSSE3, .memops_method(%rip) /* use sse2 or ssse3? */ + jz LABEL(ashr_10_use_sse2) + + .p2align 4 +LABEL(ashr_10_use_ssse3): + movdqa 16(%rsi, %rcx), %xmm3 + pcmpeqb %xmm3, %xmm0 + pmovmskb %xmm0, %edx + test %edx, %edx + jnz LABEL(unaligned_exit) +#ifdef USE_AS_STRNCPY + sub $16, %r8 + jbe LABEL(strncpy_truncation_unaligned) +#endif + + #palignr $10, (%rsi, %rcx), %xmm3 + .byte 0x66, 0x0F, 0x3A ,0x0F + .byte 0x1c, 0x0e, 0x0a + + movdqa %xmm3, (%rdi, %rcx) + add $16, %rcx + +#ifdef USE_AS_STRNCPY + cmp %r10, %r8 + jbe LABEL(unaligned_exit) +#endif + movdqa 16(%rsi, %rcx), %xmm3 + pcmpeqb %xmm3, %xmm0 + pmovmskb %xmm0, %edx + test %edx, %edx + jnz LABEL(unaligned_exit) +#ifdef USE_AS_STRNCPY + sub $16, %r8 + jbe LABEL(strncpy_truncation_unaligned) +#endif + + #palignr $10, (%rsi, %rcx), %xmm3 + .byte 0x66, 0x0F, 0x3A ,0x0F + .byte 0x1c, 0x0e, 0x0a + + movdqa %xmm3, (%rdi, %rcx) + add $16, %rcx +#ifdef USE_AS_STRNCPY + cmp %r10, %r8 + jbe LABEL(unaligned_exit) +#endif + jmp LABEL(ashr_10_use_ssse3) + + .p2align 4 +LABEL(ashr_10_use_sse2): + pcmpeqb 16(%rsi, %rcx), %xmm0 + pmovmskb %xmm0, %edx + test %edx, %edx + jnz LABEL(unaligned_exit) +#ifdef USE_AS_STRNCPY + sub $16, %r8 + jbe LABEL(strncpy_truncation_unaligned) +#endif + + movdqa 16(%rsi, %rcx), %xmm3 + movdqa (%rsi, %rcx), %xmm2 + + psrldq $10, %xmm2 + pslldq $6, %xmm3 + por %xmm2, %xmm3 + + movdqa %xmm3, (%rdi, %rcx) + add $16, %rcx + +#ifdef USE_AS_STRNCPY + cmp %r10, %r8 + jbe LABEL(unaligned_exit) +#endif + pcmpeqb 16(%rsi, %rcx), %xmm0 + pmovmskb %xmm0, %edx + test %edx, %edx + jnz LABEL(unaligned_exit) +#ifdef USE_AS_STRNCPY + sub $16, %r8 + jbe LABEL(strncpy_truncation_unaligned) +#endif + + movdqa 16(%rsi, %rcx), %xmm3 + movdqa (%rsi, %rcx), %xmm2 + + psrldq $10, %xmm2 + pslldq $6, %xmm3 + por %xmm2, %xmm3 + + movdqa %xmm3, (%rdi, %rcx) + add $16, %rcx +#ifdef USE_AS_STRNCPY + cmp %r10, %r8 + jbe LABEL(unaligned_exit) +#endif + jmp LABEL(ashr_10_use_sse2) + + +/* + * ashr_9 handles the following cases: + * (16 + (src offset - dest offset)) % 16 = 9 + * + * Based on above operation, start from (%r9 + rsi) to the left of this cache + * bank, there is no null byte. + */ + .p2align 4 +LABEL(ashr_9): + xor %ecx, %ecx /* clear index */ +#ifdef USE_AS_STRNCPY + cmp %r10, %r8 + jbe LABEL(unaligned_exit) +#endif + testl $USE_SSSE3, .memops_method(%rip) /* use sse2 or ssse3? */ + jz LABEL(ashr_9_use_sse2) + + .p2align 4 +LABEL(ashr_9_use_ssse3): + movdqa 16(%rsi, %rcx), %xmm3 + pcmpeqb %xmm3, %xmm0 + pmovmskb %xmm0, %edx + test %edx, %edx + jnz LABEL(unaligned_exit) +#ifdef USE_AS_STRNCPY + sub $16, %r8 + jbe LABEL(strncpy_truncation_unaligned) +#endif + + #palignr $9, (%rsi, %rcx), %xmm3 + .byte 0x66, 0x0F, 0x3A ,0x0F + .byte 0x1c, 0x0e, 0x09 + + movdqa %xmm3, (%rdi, %rcx) + add $16, %rcx + +#ifdef USE_AS_STRNCPY + cmp %r10, %r8 + jbe LABEL(unaligned_exit) +#endif + movdqa 16(%rsi, %rcx), %xmm3 + pcmpeqb %xmm3, %xmm0 + pmovmskb %xmm0, %edx + test %edx, %edx + jnz LABEL(unaligned_exit) +#ifdef USE_AS_STRNCPY + sub $16, %r8 + jbe LABEL(strncpy_truncation_unaligned) +#endif + + #palignr $9, (%rsi, %rcx), %xmm3 + .byte 0x66, 0x0F, 0x3A ,0x0F + .byte 0x1c, 0x0e, 0x09 + + movdqa %xmm3, (%rdi, %rcx) + add $16, %rcx +#ifdef USE_AS_STRNCPY + cmp %r10, %r8 + jbe LABEL(unaligned_exit) +#endif + jmp LABEL(ashr_9_use_ssse3) + + .p2align 4 +LABEL(ashr_9_use_sse2): + pcmpeqb 16(%rsi, %rcx), %xmm0 + pmovmskb %xmm0, %edx + test %edx, %edx + jnz LABEL(unaligned_exit) +#ifdef USE_AS_STRNCPY + sub $16, %r8 + jbe LABEL(strncpy_truncation_unaligned) +#endif + + movdqa 16(%rsi, %rcx), %xmm3 + movdqa (%rsi, %rcx), %xmm2 + + psrldq $9, %xmm2 + pslldq $7, %xmm3 + por %xmm2, %xmm3 + + movdqa %xmm3, (%rdi, %rcx) + add $16, %rcx + +#ifdef USE_AS_STRNCPY + cmp %r10, %r8 + jbe LABEL(unaligned_exit) +#endif + pcmpeqb 16(%rsi, %rcx), %xmm0 + pmovmskb %xmm0, %edx + test %edx, %edx + jnz LABEL(unaligned_exit) +#ifdef USE_AS_STRNCPY + sub $16, %r8 + jbe LABEL(strncpy_truncation_unaligned) #endif - xor %edx, %edx + movdqa 16(%rsi, %rcx), %xmm3 + movdqa (%rsi, %rcx), %xmm2 + + psrldq $9, %xmm2 + pslldq $7, %xmm3 + por %xmm2, %xmm3 + + movdqa %xmm3, (%rdi, %rcx) + add $16, %rcx +#ifdef USE_AS_STRNCPY + cmp %r10, %r8 + jbe LABEL(unaligned_exit) +#endif + jmp LABEL(ashr_9_use_sse2) + + +/* + * ashr_8 handles the following cases: + * (16 + (src offset - dest offset)) % 16 = 8 + * + * Based on above operation, start from (%r9 + rsi) to the left of this cache + * bank, there is no null byte. + */ + .p2align 4 +LABEL(ashr_8): + xor %ecx, %ecx /* clear index */ +#ifdef USE_AS_STRNCPY + cmp %r10, %r8 + jbe LABEL(unaligned_exit) +#endif + testl $USE_SSSE3, .memops_method(%rip) /* use sse2 or ssse3? */ + jz LABEL(ashr_8_use_sse2) + + .p2align 4 +LABEL(ashr_8_use_ssse3): + movdqa 16(%rsi, %rcx), %xmm3 + pcmpeqb %xmm3, %xmm0 + pmovmskb %xmm0, %edx + test %edx, %edx + jnz LABEL(unaligned_exit) +#ifdef USE_AS_STRNCPY + sub $16, %r8 + jbe LABEL(strncpy_truncation_unaligned) +#endif + + #palignr $8, (%rsi, %rcx), %xmm3 + .byte 0x66, 0x0F, 0x3A ,0x0F + .byte 0x1c, 0x0e, 0x08 + + movdqa %xmm3, (%rdi, %rcx) + add $16, %rcx + +#ifdef USE_AS_STRNCPY + cmp %r10, %r8 + jbe LABEL(unaligned_exit) +#endif + movdqa 16(%rsi, %rcx), %xmm3 + pcmpeqb %xmm3, %xmm0 + pmovmskb %xmm0, %edx + test %edx, %edx + jnz LABEL(unaligned_exit) +#ifdef USE_AS_STRNCPY + sub $16, %r8 + jbe LABEL(strncpy_truncation_unaligned) +#endif + + #palignr $8, (%rsi, %rcx), %xmm3 + .byte 0x66, 0x0F, 0x3A ,0x0F + .byte 0x1c, 0x0e, 0x08 + + movdqa %xmm3, (%rdi, %rcx) + add $16, %rcx +#ifdef USE_AS_STRNCPY + cmp %r10, %r8 + jbe LABEL(unaligned_exit) +#endif + jmp LABEL(ashr_8_use_ssse3) + + .p2align 4 +LABEL(ashr_8_use_sse2): + pcmpeqb 16(%rsi, %rcx), %xmm0 + pmovmskb %xmm0, %edx + test %edx, %edx + jnz LABEL(unaligned_exit) +#ifdef USE_AS_STRNCPY + sub $16, %r8 + jbe LABEL(strncpy_truncation_unaligned) +#endif + + movdqa 16(%rsi, %rcx), %xmm3 + movdqa (%rsi, %rcx), %xmm2 + + psrldq $8, %xmm2 + pslldq $8, %xmm3 + por %xmm2, %xmm3 + + movdqa %xmm3, (%rdi, %rcx) + add $16, %rcx + +#ifdef USE_AS_STRNCPY + cmp %r10, %r8 + jbe LABEL(unaligned_exit) +#endif + pcmpeqb 16(%rsi, %rcx), %xmm0 + pmovmskb %xmm0, %edx + test %edx, %edx + jnz LABEL(unaligned_exit) +#ifdef USE_AS_STRNCPY + sub $16, %r8 + jbe LABEL(strncpy_truncation_unaligned) +#endif + + movdqa 16(%rsi, %rcx), %xmm3 + movdqa (%rsi, %rcx), %xmm2 + + psrldq $8, %xmm2 + pslldq $8, %xmm3 + por %xmm2, %xmm3 + + movdqa %xmm3, (%rdi, %rcx) + add $16, %rcx +#ifdef USE_AS_STRNCPY + cmp %r10, %r8 + jbe LABEL(unaligned_exit) +#endif + jmp LABEL(ashr_8_use_sse2) + + +/* + * ashr_7 handles the following cases: + * (16 + (src offset - dest offset)) % 16 = 7 + * + * Based on above operation, start from (%r9 + rsi) to the left of this cache + * bank, there is no null byte. + */ + .p2align 4 +LABEL(ashr_7): + xor %ecx, %ecx /* clear index */ +#ifdef USE_AS_STRNCPY + cmp %r10, %r8 + jbe LABEL(unaligned_exit) +#endif + testl $USE_SSSE3, .memops_method(%rip) /* use sse2 or ssse3? */ + jz LABEL(ashr_7_use_sse2) + + .p2align 4 +LABEL(ashr_7_use_ssse3): + movdqa 16(%rsi, %rcx), %xmm3 + pcmpeqb %xmm3, %xmm0 + pmovmskb %xmm0, %edx + test %edx, %edx + jnz LABEL(unaligned_exit) +#ifdef USE_AS_STRNCPY + sub $16, %r8 + jbe LABEL(strncpy_truncation_unaligned) +#endif + + #palignr $7, (%rsi, %rcx), %xmm3 + .byte 0x66, 0x0F, 0x3A ,0x0F + .byte 0x1c, 0x0e, 0x07 + + movdqa %xmm3, (%rdi, %rcx) + add $16, %rcx + +#ifdef USE_AS_STRNCPY + cmp %r10, %r8 + jbe LABEL(unaligned_exit) +#endif + movdqa 16(%rsi, %rcx), %xmm3 + pcmpeqb %xmm3, %xmm0 + pmovmskb %xmm0, %edx + test %edx, %edx + jnz LABEL(unaligned_exit) +#ifdef USE_AS_STRNCPY + sub $16, %r8 + jbe LABEL(strncpy_truncation_unaligned) +#endif + + #palignr $7, (%rsi, %rcx), %xmm3 + .byte 0x66, 0x0F, 0x3A ,0x0F + .byte 0x1c, 0x0e, 0x07 + + movdqa %xmm3, (%rdi, %rcx) + add $16, %rcx +#ifdef USE_AS_STRNCPY + cmp %r10, %r8 + jbe LABEL(unaligned_exit) +#endif + jmp LABEL(ashr_7_use_ssse3) + + .p2align 4 +LABEL(ashr_7_use_sse2): + pcmpeqb 16(%rsi, %rcx), %xmm0 + pmovmskb %xmm0, %edx + test %edx, %edx + jnz LABEL(unaligned_exit) +#ifdef USE_AS_STRNCPY + sub $16, %r8 + jbe LABEL(strncpy_truncation_unaligned) +#endif + + movdqa 16(%rsi, %rcx), %xmm3 + movdqa (%rsi, %rcx), %xmm2 + + psrldq $7, %xmm2 + pslldq $9, %xmm3 + por %xmm2, %xmm3 + + movdqa %xmm3, (%rdi, %rcx) + add $16, %rcx + +#ifdef USE_AS_STRNCPY + cmp %r10, %r8 + jbe LABEL(unaligned_exit) +#endif + pcmpeqb 16(%rsi, %rcx), %xmm0 + pmovmskb %xmm0, %edx + test %edx, %edx + jnz LABEL(unaligned_exit) +#ifdef USE_AS_STRNCPY + sub $16, %r8 + jbe LABEL(strncpy_truncation_unaligned) +#endif + + movdqa 16(%rsi, %rcx), %xmm3 + movdqa (%rsi, %rcx), %xmm2 + + psrldq $7, %xmm2 + pslldq $9, %xmm3 + por %xmm2, %xmm3 + + movdqa %xmm3, (%rdi, %rcx) + add $16, %rcx +#ifdef USE_AS_STRNCPY + cmp %r10, %r8 + jbe LABEL(unaligned_exit) +#endif + jmp LABEL(ashr_7_use_sse2) + -LABEL(aligntry): - mov %rsi, %r8 /* align by source */ - and $7, %r8 - jz LABEL(alignafter) +/* + * ashr_6 handles the following cases: + * (16 + (src offset - dest offset)) % 16 = 6 + * + * Based on above operation, start from (%r9 + rsi) to the left of this cache + * bank, there is no null byte. + */ + .p2align 4 +LABEL(ashr_6): + xor %ecx, %ecx /* clear index */ +#ifdef USE_AS_STRNCPY + cmp %r10, %r8 + jbe LABEL(unaligned_exit) +#endif + testl $USE_SSSE3, .memops_method(%rip) /* use sse2 or ssse3? */ + jz LABEL(ashr_6_use_sse2) + + .p2align 4 +LABEL(ashr_6_use_ssse3): + movdqa 16(%rsi, %rcx), %xmm3 + pcmpeqb %xmm3, %xmm0 + pmovmskb %xmm0, %edx + test %edx, %edx + jnz LABEL(unaligned_exit) +#ifdef USE_AS_STRNCPY + sub $16, %r8 + jbe LABEL(strncpy_truncation_unaligned) +#endif + + #palignr $6, (%rsi, %rcx), %xmm3 + .byte 0x66, 0x0F, 0x3A ,0x0F + .byte 0x1c, 0x0e, 0x06 + + movdqa %xmm3, (%rdi, %rcx) + add $16, %rcx + +#ifdef USE_AS_STRNCPY + cmp %r10, %r8 + jbe LABEL(unaligned_exit) +#endif + movdqa 16(%rsi, %rcx), %xmm3 + pcmpeqb %xmm3, %xmm0 + pmovmskb %xmm0, %edx + test %edx, %edx + jnz LABEL(unaligned_exit) +#ifdef USE_AS_STRNCPY + sub $16, %r8 + jbe LABEL(strncpy_truncation_unaligned) +#endif + + #palignr $6, (%rsi, %rcx), %xmm3 + .byte 0x66, 0x0F, 0x3A ,0x0F + .byte 0x1c, 0x0e, 0x06 + + movdqa %xmm3, (%rdi, %rcx) + add $16, %rcx +#ifdef USE_AS_STRNCPY + cmp %r10, %r8 + jbe LABEL(unaligned_exit) +#endif + jmp LABEL(ashr_6_use_ssse3) + + .p2align 4 +LABEL(ashr_6_use_sse2): + pcmpeqb 16(%rsi, %rcx), %xmm0 + pmovmskb %xmm0, %edx + test %edx, %edx + jnz LABEL(unaligned_exit) +#ifdef USE_AS_STRNCPY + sub $16, %r8 + jbe LABEL(strncpy_truncation_unaligned) +#endif + + movdqa 16(%rsi, %rcx), %xmm3 + movdqa (%rsi, %rcx), %xmm2 + + psrldq $6, %xmm2 + pslldq $10, %xmm3 + por %xmm2, %xmm3 + + movdqa %xmm3, (%rdi, %rcx) + add $16, %rcx + +#ifdef USE_AS_STRNCPY + cmp %r10, %r8 + jbe LABEL(unaligned_exit) +#endif + pcmpeqb 16(%rsi, %rcx), %xmm0 + pmovmskb %xmm0, %edx + test %edx, %edx + jnz LABEL(unaligned_exit) +#ifdef USE_AS_STRNCPY + sub $16, %r8 + jbe LABEL(strncpy_truncation_unaligned) +#endif + + movdqa 16(%rsi, %rcx), %xmm3 + movdqa (%rsi, %rcx), %xmm2 + + psrldq $6, %xmm2 + pslldq $10, %xmm3 + por %xmm2, %xmm3 + + movdqa %xmm3, (%rdi, %rcx) + add $16, %rcx +#ifdef USE_AS_STRNCPY + cmp %r10, %r8 + jbe LABEL(unaligned_exit) +#endif + jmp LABEL(ashr_6_use_sse2) + -LABEL(align): /* 8-byte align */ - sub $8, %r8 +/* + * ashr_5 handles the following cases: + * (16 + (src offset - dest offset)) % 16 = 5 + * + * Based on above operation, start from (%r9 + rsi) to the left of this cache + * bank, there is no null byte. + */ + .p2align 4 +LABEL(ashr_5): + xor %ecx, %ecx /* clear index */ +#ifdef USE_AS_STRNCPY + cmp %r10, %r8 + jbe LABEL(unaligned_exit) +#endif + testl $USE_SSSE3, .memops_method(%rip) /* use sse2 or ssse3? */ + jz LABEL(ashr_5_use_sse2) + + .p2align 4 +LABEL(ashr_5_use_ssse3): + movdqa 16(%rsi, %rcx), %xmm3 + pcmpeqb %xmm3, %xmm0 + pmovmskb %xmm0, %edx + test %edx, %edx + jnz LABEL(unaligned_exit) +#ifdef USE_AS_STRNCPY + sub $16, %r8 + jbe LABEL(strncpy_truncation_unaligned) +#endif + + #palignr $5, (%rsi, %rcx), %xmm3 + .byte 0x66, 0x0F, 0x3A ,0x0F + .byte 0x1c, 0x0e, 0x05 + + movdqa %xmm3, (%rdi, %rcx) + add $16, %rcx + +#ifdef USE_AS_STRNCPY + cmp %r10, %r8 + jbe LABEL(unaligned_exit) +#endif + movdqa 16(%rsi, %rcx), %xmm3 + pcmpeqb %xmm3, %xmm0 + pmovmskb %xmm0, %edx + test %edx, %edx + jnz LABEL(unaligned_exit) +#ifdef USE_AS_STRNCPY + sub $16, %r8 + jbe LABEL(strncpy_truncation_unaligned) +#endif + + #palignr $5, (%rsi, %rcx), %xmm3 + .byte 0x66, 0x0F, 0x3A ,0x0F + .byte 0x1c, 0x0e, 0x05 + + movdqa %xmm3, (%rdi, %rcx) + add $16, %rcx +#ifdef USE_AS_STRNCPY + cmp %r10, %r8 + jbe LABEL(unaligned_exit) +#endif + jmp LABEL(ashr_5_use_ssse3) + + .p2align 4 +LABEL(ashr_5_use_sse2): + pcmpeqb 16(%rsi, %rcx), %xmm0 + pmovmskb %xmm0, %edx + test %edx, %edx + jnz LABEL(unaligned_exit) +#ifdef USE_AS_STRNCPY + sub $16, %r8 + jbe LABEL(strncpy_truncation_unaligned) +#endif + + movdqa 16(%rsi, %rcx), %xmm3 + movdqa (%rsi, %rcx), %xmm2 + + psrldq $5, %xmm2 + pslldq $11, %xmm3 + por %xmm2, %xmm3 + + movdqa %xmm3, (%rdi, %rcx) + add $16, %rcx + +#ifdef USE_AS_STRNCPY + cmp %r10, %r8 + jbe LABEL(unaligned_exit) +#endif + pcmpeqb 16(%rsi, %rcx), %xmm0 + pmovmskb %xmm0, %edx + test %edx, %edx + jnz LABEL(unaligned_exit) +#ifdef USE_AS_STRNCPY + sub $16, %r8 + jbe LABEL(strncpy_truncation_unaligned) +#endif + + movdqa 16(%rsi, %rcx), %xmm3 + movdqa (%rsi, %rcx), %xmm2 + + psrldq $5, %xmm2 + pslldq $11, %xmm3 + por %xmm2, %xmm3 + + movdqa %xmm3, (%rdi, %rcx) + add $16, %rcx +#ifdef USE_AS_STRNCPY + cmp %r10, %r8 + jbe LABEL(unaligned_exit) +#endif + jmp LABEL(ashr_5_use_sse2) + + +/* + * ashr_4 handles the following cases: + * (16 + (src offset - dest offset)) % 16 = 4 + * + * Based on above operation, start from (%r9 + rsi) to the left of this cache + * bank, there is no null byte. + */ + .p2align 4 +LABEL(ashr_4): + xor %ecx, %ecx /* clear index */ +#ifdef USE_AS_STRNCPY + cmp %r10, %r8 + jbe LABEL(unaligned_exit) +#endif + testl $USE_SSSE3, .memops_method(%rip) /* use sse2 or ssse3? */ + jz LABEL(ashr_4_use_sse2) .p2align 4 +LABEL(ashr_4_use_ssse3): + movdqa 16(%rsi, %rcx), %xmm3 + pcmpeqb %xmm3, %xmm0 + pmovmskb %xmm0, %edx + test %edx, %edx + jnz LABEL(unaligned_exit) +#ifdef USE_AS_STRNCPY + sub $16, %r8 + jbe LABEL(strncpy_truncation_unaligned) +#endif -LABEL(alignloop): + #palignr $4, (%rsi, %rcx), %xmm3 + .byte 0x66, 0x0F, 0x3A ,0x0F + .byte 0x1c, 0x0e, 0x04 + + movdqa %xmm3, (%rdi, %rcx) + add $16, %rcx + +#ifdef USE_AS_STRNCPY + cmp %r10, %r8 + jbe LABEL(unaligned_exit) +#endif + movdqa 16(%rsi, %rcx), %xmm3 + pcmpeqb %xmm3, %xmm0 + pmovmskb %xmm0, %edx + test %edx, %edx + jnz LABEL(unaligned_exit) +#ifdef USE_AS_STRNCPY + sub $16, %r8 + jbe LABEL(strncpy_truncation_unaligned) +#endif + + #palignr $4, (%rsi, %rcx), %xmm3 + .byte 0x66, 0x0F, 0x3A ,0x0F + .byte 0x1c, 0x0e, 0x04 + + movdqa %xmm3, (%rdi, %rcx) + add $16, %rcx +#ifdef USE_AS_STRNCPY + cmp %r10, %r8 + jbe LABEL(unaligned_exit) +#endif + jmp LABEL(ashr_4_use_ssse3) + + .p2align 4 +LABEL(ashr_4_use_sse2): + pcmpeqb 16(%rsi, %rcx), %xmm0 + pmovmskb %xmm0, %edx + test %edx, %edx + jnz LABEL(unaligned_exit) +#ifdef USE_AS_STRNCPY + sub $16, %r8 + jbe LABEL(strncpy_truncation_unaligned) +#endif + + movdqa 16(%rsi, %rcx), %xmm3 + movdqa (%rsi, %rcx), %xmm2 + + psrldq $4, %xmm2 + pslldq $12, %xmm3 + por %xmm2, %xmm3 + + movdqa %xmm3, (%rdi, %rcx) + add $16, %rcx + +#ifdef USE_AS_STRNCPY + cmp %r10, %r8 + jbe LABEL(unaligned_exit) +#endif + pcmpeqb 16(%rsi, %rcx), %xmm0 + pmovmskb %xmm0, %edx + test %edx, %edx + jnz LABEL(unaligned_exit) +#ifdef USE_AS_STRNCPY + sub $16, %r8 + jbe LABEL(strncpy_truncation_unaligned) +#endif + + movdqa 16(%rsi, %rcx), %xmm3 + movdqa (%rsi, %rcx), %xmm2 + + psrldq $4, %xmm2 + pslldq $12, %xmm3 + por %xmm2, %xmm3 + + movdqa %xmm3, (%rdi, %rcx) + add $16, %rcx +#ifdef USE_AS_STRNCPY + cmp %r10, %r8 + jbe LABEL(unaligned_exit) +#endif + jmp LABEL(ashr_4_use_sse2) + + +/* + * ashr_3 handles the following cases: + * (16 + (src offset - dest offset)) % 16 = 3 + * + * Based on above operation, start from (%r9 + rsi) to the left of this cache + * bank, there is no null byte. + */ + .p2align 4 +LABEL(ashr_3): + xor %ecx, %ecx /* clear index */ #ifdef USE_AS_STRNCPY - dec %r11 - jl LABEL(exitn) + cmp %r10, %r8 + jbe LABEL(unaligned_exit) +#endif + testl $USE_SSSE3, .memops_method(%rip) /* use sse2 or ssse3? */ + jz LABEL(ashr_3_use_sse2) + + .p2align 4 +LABEL(ashr_3_use_ssse3): + movdqa 16(%rsi, %rcx), %xmm3 + pcmpeqb %xmm3, %xmm0 + pmovmskb %xmm0, %edx + test %edx, %edx + jnz LABEL(unaligned_exit) +#ifdef USE_AS_STRNCPY + sub $16, %r8 + jbe LABEL(strncpy_truncation_unaligned) +#endif + + #palignr $3, (%rsi, %rcx), %xmm3 + .byte 0x66, 0x0F, 0x3A ,0x0F + .byte 0x1c, 0x0e, 0x03 + + movdqa %xmm3, (%rdi, %rcx) + add $16, %rcx + +#ifdef USE_AS_STRNCPY + cmp %r10, %r8 + jbe LABEL(unaligned_exit) +#endif + movdqa 16(%rsi, %rcx), %xmm3 + pcmpeqb %xmm3, %xmm0 + pmovmskb %xmm0, %edx + test %edx, %edx + jnz LABEL(unaligned_exit) +#ifdef USE_AS_STRNCPY + sub $16, %r8 + jbe LABEL(strncpy_truncation_unaligned) +#endif + + #palignr $3, (%rsi, %rcx), %xmm3 + .byte 0x66, 0x0F, 0x3A ,0x0F + .byte 0x1c, 0x0e, 0x03 + + movdqa %xmm3, (%rdi, %rcx) + add $16, %rcx +#ifdef USE_AS_STRNCPY + cmp %r10, %r8 + jbe LABEL(unaligned_exit) +#endif + jmp LABEL(ashr_3_use_ssse3) + + .p2align 4 +LABEL(ashr_3_use_sse2): + pcmpeqb 16(%rsi, %rcx), %xmm0 + pmovmskb %xmm0, %edx + test %edx, %edx + jnz LABEL(unaligned_exit) +#ifdef USE_AS_STRNCPY + sub $16, %r8 + jbe LABEL(strncpy_truncation_unaligned) +#endif + + movdqa 16(%rsi, %rcx), %xmm3 + movdqa (%rsi, %rcx), %xmm2 + + psrldq $3, %xmm2 + pslldq $13, %xmm3 + por %xmm2, %xmm3 + + movdqa %xmm3, (%rdi, %rcx) + add $16, %rcx + +#ifdef USE_AS_STRNCPY + cmp %r10, %r8 + jbe LABEL(unaligned_exit) +#endif + pcmpeqb 16(%rsi, %rcx), %xmm0 + pmovmskb %xmm0, %edx + test %edx, %edx + jnz LABEL(unaligned_exit) +#ifdef USE_AS_STRNCPY + sub $16, %r8 + jbe LABEL(strncpy_truncation_unaligned) +#endif + + movdqa 16(%rsi, %rcx), %xmm3 + movdqa (%rsi, %rcx), %xmm2 + + psrldq $3, %xmm2 + pslldq $13, %xmm3 + por %xmm2, %xmm3 + + movdqa %xmm3, (%rdi, %rcx) + add $16, %rcx +#ifdef USE_AS_STRNCPY + cmp %r10, %r8 + jbe LABEL(unaligned_exit) +#endif + jmp LABEL(ashr_3_use_sse2) + + +/* + * ashr_2 handles the following cases: + * (16 + (src offset - dest offset)) % 16 = 2 + * + * Based on above operation, start from (%r9 + rsi) to the left of this cache + * bank, there is no null byte. + */ + .p2align 4 +LABEL(ashr_2): + xor %ecx, %ecx /* clear index */ +#ifdef USE_AS_STRNCPY + cmp %r10, %r8 + jbe LABEL(unaligned_exit) +#endif + testl $USE_SSSE3, .memops_method(%rip) /* use sse2 or ssse3? */ + jz LABEL(ashr_2_use_sse2) + + .p2align 4 +LABEL(ashr_2_use_ssse3): + movdqa 16(%rsi, %rcx), %xmm3 + pcmpeqb %xmm3, %xmm0 + pmovmskb %xmm0, %edx + test %edx, %edx + jnz LABEL(unaligned_exit) +#ifdef USE_AS_STRNCPY + sub $16, %r8 + jbe LABEL(strncpy_truncation_unaligned) #endif - mov (%rsi, %rdx), %al /* check if same character */ - test %al, %al /* check if character a NUL */ - mov %al, (%rdi, %rdx) - jz LABEL(exit) + #palignr $2, (%rsi, %rcx), %xmm3 + .byte 0x66, 0x0F, 0x3A ,0x0F + .byte 0x1c, 0x0e, 0x02 + + movdqa %xmm3, (%rdi, %rcx) + add $16, %rcx + +#ifdef USE_AS_STRNCPY + cmp %r10, %r8 + jbe LABEL(unaligned_exit) +#endif + movdqa 16(%rsi, %rcx), %xmm3 + pcmpeqb %xmm3, %xmm0 + pmovmskb %xmm0, %edx + test %edx, %edx + jnz LABEL(unaligned_exit) +#ifdef USE_AS_STRNCPY + sub $16, %r8 + jbe LABEL(strncpy_truncation_unaligned) +#endif + + #palignr $2, (%rsi, %rcx), %xmm3 + .byte 0x66, 0x0F, 0x3A ,0x0F + .byte 0x1c, 0x0e, 0x02 + + movdqa %xmm3, (%rdi, %rcx) + add $16, %rcx +#ifdef USE_AS_STRNCPY + cmp %r10, %r8 + jbe LABEL(unaligned_exit) +#endif + jmp LABEL(ashr_2_use_ssse3) + + .p2align 4 +LABEL(ashr_2_use_sse2): + pcmpeqb 16(%rsi, %rcx), %xmm0 + pmovmskb %xmm0, %edx + test %edx, %edx + jnz LABEL(unaligned_exit) +#ifdef USE_AS_STRNCPY + sub $16, %r8 + jbe LABEL(strncpy_truncation_unaligned) +#endif + + movdqa 16(%rsi, %rcx), %xmm3 + movdqa (%rsi, %rcx), %xmm2 + + psrldq $2, %xmm2 + pslldq $14, %xmm3 + por %xmm2, %xmm3 + + movdqa %xmm3, (%rdi, %rcx) + add $16, %rcx - inc %edx - inc %r8 - jnz LABEL(alignloop) +#ifdef USE_AS_STRNCPY + cmp %r10, %r8 + jbe LABEL(unaligned_exit) +#endif + pcmpeqb 16(%rsi, %rcx), %xmm0 + pmovmskb %xmm0, %edx + test %edx, %edx + jnz LABEL(unaligned_exit) +#ifdef USE_AS_STRNCPY + sub $16, %r8 + jbe LABEL(strncpy_truncation_unaligned) +#endif + + movdqa 16(%rsi, %rcx), %xmm3 + movdqa (%rsi, %rcx), %xmm2 + + psrldq $2, %xmm2 + pslldq $14, %xmm3 + por %xmm2, %xmm3 + + movdqa %xmm3, (%rdi, %rcx) + add $16, %rcx +#ifdef USE_AS_STRNCPY + cmp %r10, %r8 + jbe LABEL(unaligned_exit) +#endif + jmp LABEL(ashr_2_use_sse2) + + +/* + * ashr_1 handles the following cases: + * (16 + (src offset - dest offset)) % 16 = 1 + * + * Based on above operation, start from (%r9 + rsi) to the left of this cache + * bank, there is no null byte. + */ + .p2align 4 +LABEL(ashr_1): + xor %ecx, %ecx /* clear index */ +#ifdef USE_AS_STRNCPY + cmp %r10, %r8 + jbe LABEL(unaligned_exit) +#endif + testl $USE_SSSE3, .memops_method(%rip) /* use sse2 or ssse3? */ + jz LABEL(ashr_1_use_sse2) + + .p2align 4 +LABEL(ashr_1_use_ssse3): + movdqa 16(%rsi, %rcx), %xmm3 + pcmpeqb %xmm3, %xmm0 + pmovmskb %xmm0, %edx + test %edx, %edx + jnz LABEL(unaligned_exit) +#ifdef USE_AS_STRNCPY + sub $16, %r8 + jbe LABEL(strncpy_truncation_unaligned) +#endif + + #palignr $1, (%rsi, %rcx), %xmm3 + .byte 0x66, 0x0F, 0x3A ,0x0F + .byte 0x1c, 0x0e, 0x01 + + movdqa %xmm3, (%rdi, %rcx) + add $16, %rcx #ifdef USE_AS_STRNCPY - test %r11, %r11 /* must check remaining size */ - jz LABEL(exitn) /* If we've already done, exit */ + cmp %r10, %r8 + jbe LABEL(unaligned_exit) +#endif + movdqa 16(%rsi, %rcx), %xmm3 + pcmpeqb %xmm3, %xmm0 + pmovmskb %xmm0, %edx + test %edx, %edx + jnz LABEL(unaligned_exit) +#ifdef USE_AS_STRNCPY + sub $16, %r8 + jbe LABEL(strncpy_truncation_unaligned) +#endif + #palignr $1, (%rsi, %rcx), %xmm3 + .byte 0x66, 0x0F, 0x3A ,0x0F + .byte 0x1c, 0x0e, 0x01 + + movdqa %xmm3, (%rdi, %rcx) + add $16, %rcx +#ifdef USE_AS_STRNCPY + cmp %r10, %r8 + jbe LABEL(unaligned_exit) +#endif + jmp LABEL(ashr_1_use_ssse3) + + .p2align 4 +LABEL(ashr_1_use_sse2): + pcmpeqb 16(%rsi, %rcx), %xmm0 + pmovmskb %xmm0, %edx + test %edx, %edx + jnz LABEL(unaligned_exit) +#ifdef USE_AS_STRNCPY + sub $16, %r8 + jbe LABEL(strncpy_truncation_unaligned) +#endif + movdqa 16(%rsi, %rcx), %xmm3 + movdqa (%rsi, %rcx), %xmm2 + + psrldq $1, %xmm2 + pslldq $15, %xmm3 + por %xmm2, %xmm3 + + movdqa %xmm3, (%rdi, %rcx) + add $16, %rcx + +#ifdef USE_AS_STRNCPY + cmp %r10, %r8 + jbe LABEL(unaligned_exit) +#endif + pcmpeqb 16(%rsi, %rcx), %xmm0 + pmovmskb %xmm0, %edx + test %edx, %edx + jnz LABEL(unaligned_exit) +#ifdef USE_AS_STRNCPY + sub $16, %r8 + jbe LABEL(strncpy_truncation_unaligned) +#endif + + movdqa 16(%rsi, %rcx), %xmm3 + movdqa (%rsi, %rcx), %xmm2 + + psrldq $1, %xmm2 + pslldq $15, %xmm3 + por %xmm2, %xmm3 + + movdqa %xmm3, (%rdi, %rcx) + add $16, %rcx +#ifdef USE_AS_STRNCPY + cmp %r10, %r8 + jbe LABEL(unaligned_exit) +#endif + jmp LABEL(ashr_1_use_sse2) + + + /* + * Exit tail code: + * Up to 32 bytes are copied in the case of strcpy. + */ + .p2align 4 +LABEL(less32bytes): + xor %ecx, %ecx +LABEL(unaligned_exit): + add %r9, %rsi /* r9 holds offset of rsi */ + mov %rcx, %r9 + mov %r10, %rcx + shl %cl, %edx /* after shl, calculate the exact number to be filled */ + mov %r9, %rcx + .p2align 4 +LABEL(aligned_exit): + add %rcx, %rdi /* locate exact address for rdi */ +LABEL(less16bytes): + add %rcx, %rsi /* locate exact address for rsi */ +LABEL(aligned_16bytes): +#ifdef USE_AS_STRNCPY + /* + * Null found in 16bytes checked. Set bit in bitmask corresponding to + * the strncpy count argument. We will copy to the null (inclusive) + * or count whichever comes first. + */ + mov $1, %r9d + lea -1(%r8), %rcx + shl %cl, %r9d + cmp $32, %r8 + ja LABEL(strncpy_tail) + or %r9d, %edx +LABEL(strncpy_tail): +#endif + /* + * Check to see if BSF is fast on this processor. If not, use a + * different exit tail. + */ + testb $USE_BSF, .memops_method(%rip) + jz LABEL(AMD_exit) + bsf %rdx, %rcx /* Find byte with null char */ + lea LABEL(tail_table)(%rip), %r11 + movslq (%r11, %rcx, 4), %rcx + lea (%r11, %rcx), %rcx + jmp *%rcx + +#ifdef USE_AS_STRNCPY + /* + * Count reached before null found. + */ + .p2align 4 +LABEL(less32bytes_strncpy_truncation): + xor %ecx, %ecx +LABEL(strncpy_truncation_unaligned): + add %r9, %rsi /* next src char to copy */ +LABEL(strncpy_truncation_aligned): + add %rcx, %rdi + add %rcx, %rsi + add $16, %r8 /* compensation */ + lea -1(%r8), %rcx + lea LABEL(tail_table)(%rip), %r11 + movslq (%r11, %rcx, 4), %rcx + lea (%r11, %rcx), %rcx + jmp *%rcx + + .p2align 4 +LABEL(strncpy_exitz): + mov %rdi, %rax + ret #endif .p2align 4 - -LABEL(alignafter): - -LABEL(8try): - mov $0xfefefefefefefeff, %rcx - -LABEL(8): /* 8-byte */ - mov (%rsi, %rdx), %rax - -LABEL(8loop): -#ifdef USE_AS_STRNCPY - sub $8, %r11 - jle LABEL(tail) -#endif - - mov %rcx, %r8 - add %rax, %r8 - sbb %r10, %r10 - - xor %rax, %r8 - or %rcx, %r8 - sub %r10, %r8 - jnz LABEL(tail) - - mov %rax, (%rdi, %rdx) - mov 8 (%rsi, %rdx), %rax - add $8, %edx - -#ifdef USE_AS_STRNCPY - sub $8, %r11 - jle LABEL(tail) -#endif - - mov %rcx, %r8 - add %rax, %r8 - sbb %r10, %r10 - - xor %rax, %r8 - or %rcx, %r8 - sub %r10, %r8 - jnz LABEL(tail) - - mov %rax, (%rdi, %rdx) - mov 8 (%rsi, %rdx), %rax - add $8, %edx - -#ifdef USE_AS_STRNCPY - sub $8, %r11 - jle LABEL(tail) -#endif - - mov %rcx, %r8 - add %rax, %r8 - sbb %r10, %r10 - - xor %rax, %r8 - or %rcx, %r8 - sub %r10, %r8 - jnz LABEL(tail) - - mov %rax, (%rdi, %rdx) - mov 8 (%rsi, %rdx), %rax - add $8, %edx - -#ifdef USE_AS_STRNCPY - sub $8, %r11 - jle LABEL(tail) -#endif - - mov %rcx, %r8 - add %rax, %r8 - sbb %r10, %r10 - - xor %rax, %r8 - or %rcx, %r8 - sub %r10, %r8 - jnz LABEL(tail) - - mov %rax, (%rdi, %rdx) - mov 8 (%rsi, %rdx), %rax - add $8, %edx - -#ifdef USE_AS_STRNCPY - sub $8, %r11 - jle LABEL(tail) -#endif - - mov %rcx, %r8 - add %rax, %r8 - sbb %r10, %r10 - - xor %rax, %r8 - or %rcx, %r8 - sub %r10, %r8 - jnz LABEL(tail) - - mov %rax, (%rdi, %rdx) - mov 8 (%rsi, %rdx), %rax - add $8, %edx - -#ifdef USE_AS_STRNCPY - sub $8, %r11 - jle LABEL(tail) -#endif - - mov %rcx, %r8 - add %rax, %r8 - sbb %r10, %r10 - - xor %rax, %r8 - or %rcx, %r8 - sub %r10, %r8 - jnz LABEL(tail) - - mov %rax, (%rdi, %rdx) - mov 8 (%rsi, %rdx), %rax - add $8, %edx - -#ifdef USE_AS_STRNCPY - sub $8, %r11 - jle LABEL(tail) -#endif - - mov %rcx, %r8 - add %rax, %r8 - sbb %r10, %r10 - - xor %rax, %r8 - or %rcx, %r8 - sub %r10, %r8 - jnz LABEL(tail) - - mov %rax, (%rdi, %rdx) - mov 8 (%rsi, %rdx), %rax - add $8, %edx - -#ifdef USE_AS_STRNCPY - sub $8, %r11 - jle LABEL(tail) -#endif - - mov %rcx, %r8 - add %rax, %r8 - sbb %r10, %r10 - - xor %rax, %r8 - or %rcx, %r8 - sub %r10, %r8 - jnz LABEL(tail) - - mov %rax, (%rdi, %rdx) - mov 8 (%rsi, %rdx), %rax - add $8, %edx - -LABEL(8after): - -LABEL(64try): - mov _sref_(.amd64cache1half), %r9 - -LABEL(64): /* 64-byte */ - - .p2align 4 - -LABEL(64loop): -#ifdef USE_AS_STRNCPY - sub $8, %r11 - jle LABEL(tail) -#endif - - mov %rcx, %r8 - add %rax, %r8 - sbb %r10, %r10 - - xor %rax, %r8 - or %rcx, %r8 - sub %r10, %r8 - jnz LABEL(tail) - - mov %rax, (%rdi, %rdx) - mov 8 (%rsi, %rdx), %rax - add $8, %edx - -#ifdef USE_AS_STRNCPY - sub $8, %r11 - jle LABEL(tail) -#endif +LABEL(AMD_exit): + test %dl, %dl + jz LABEL(AMD_exit_more_8) + test $0x01, %dl + jnz LABEL(tail_0) + test $0x02, %dl + jnz LABEL(tail_1) + test $0x04, %dl + jnz LABEL(tail_2) + test $0x08, %dl + jnz LABEL(tail_3) + test $0x10, %dl + jnz LABEL(tail_4) + test $0x20, %dl + jnz LABEL(tail_5) + test $0x40, %dl + jnz LABEL(tail_6) - mov %rcx, %r8 - add %rax, %r8 - sbb %r10, %r10 - - xor %rax, %r8 - or %rcx, %r8 - sub %r10, %r8 - jnz LABEL(tail) - - mov %rax, (%rdi, %rdx) - mov 8 (%rsi, %rdx), %rax - add $8, %edx - -#ifdef USE_AS_STRNCPY - sub $8, %r11 - jle LABEL(tail) -#endif - - mov %rcx, %r8 - add %rax, %r8 - sbb %r10, %r10 - - xor %rax, %r8 - or %rcx, %r8 - sub %r10, %r8 - jnz LABEL(tail) - - mov %rax, (%rdi, %rdx) - mov 8 (%rsi, %rdx), %rax - add $8, %edx - + .p2align 4 +LABEL(tail_7): /* 8 bytes */ + mov (%rsi), %rcx + mov %rcx, (%rdi) #ifdef USE_AS_STRNCPY - sub $8, %r11 - jle LABEL(tail) -#endif - - mov %rcx, %r8 - add %rax, %r8 - sbb %r10, %r10 - - xor %rax, %r8 - or %rcx, %r8 - sub %r10, %r8 - jnz LABEL(tail) - - mov %rax, (%rdi, %rdx) - mov 8 (%rsi, %rdx), %rax - add $8, %edx - -#ifdef USE_AS_STRNCPY - sub $8, %r11 - jle LABEL(tail) -#endif - - mov %rcx, %r8 - add %rax, %r8 - sbb %r10, %r10 - - xor %rax, %r8 - or %rcx, %r8 - sub %r10, %r8 - jnz LABEL(tail) - - mov %rax, (%rdi, %rdx) - mov 8 (%rsi, %rdx), %rax - add $8, %edx - -#ifdef USE_AS_STRNCPY - sub $8, %r11 - jle LABEL(tail) -#endif - - mov %rcx, %r8 - add %rax, %r8 - sbb %r10, %r10 - - xor %rax, %r8 - or %rcx, %r8 - sub %r10, %r8 - jnz LABEL(tail) - - mov %rax, (%rdi, %rdx) - mov 8 (%rsi, %rdx), %rax - add $8, %edx - -#ifdef USE_AS_STRNCPY - sub $8, %r11 - jle LABEL(tail) + mov $8, %cl + sub $8, %r8 + jnz LABEL(strncpy_fill_tail) #endif - - mov %rcx, %r8 - add %rax, %r8 - sbb %r10, %r10 - - xor %rax, %r8 - or %rcx, %r8 - sub %r10, %r8 - jnz LABEL(tail) - - mov %rax, (%rdi, %rdx) - mov 8 (%rsi, %rdx), %rax - add $8, %edx - -#ifdef USE_AS_STRNCPY - sub $8, %r11 - jle LABEL(tail) -#endif - - mov %rcx, %r8 - add %rax, %r8 - sbb %r10, %r10 - - xor %rax, %r8 - or %rcx, %r8 - sub %r10, %r8 - jnz LABEL(tail) - - cmp %r9, %rdx - - mov %rax, (%rdi, %rdx) - mov 8 (%rsi, %rdx), %rax - lea 8 (%rdx), %rdx - - jbe LABEL(64loop) - -LABEL(64after): - -LABEL(pretry): - mov _sref_(.amd64cache2half), %r9 - -LABEL(pre): /* 64-byte prefetch */ - - .p2align 4 - -LABEL(preloop): -#ifdef USE_AS_STRNCPY - sub $8, %r11 - jle LABEL(tail) -#endif - - mov %rcx, %r8 - add %rax, %r8 - sbb %r10, %r10 - - xor %rax, %r8 - or %rcx, %r8 - sub %r10, %r8 - jnz LABEL(tail) - - mov %rax, (%rdi, %rdx) - mov 8 (%rsi, %rdx), %rax - add $8, %edx - -#ifdef USE_AS_STRNCPY - sub $8, %r11 - jle LABEL(tail) -#endif - - mov %rcx, %r8 - add %rax, %r8 - sbb %r10, %r10 - - xor %rax, %r8 - or %rcx, %r8 - sub %r10, %r8 - jnz LABEL(tail) - - mov %rax, (%rdi, %rdx) - mov 8 (%rsi, %rdx), %rax - add $8, %edx - -#ifdef USE_AS_STRNCPY - sub $8, %r11 - jle LABEL(tail) -#endif - - mov %rcx, %r8 - add %rax, %r8 - sbb %r10, %r10 - - xor %rax, %r8 - or %rcx, %r8 - sub %r10, %r8 - jnz LABEL(tail) - - mov %rax, (%rdi, %rdx) - mov 8 (%rsi, %rdx), %rax - add $8, %edx + ret #ifdef USE_AS_STRNCPY - sub $8, %r11 - jle LABEL(tail) -#endif - - mov %rcx, %r8 - add %rax, %r8 - sbb %r10, %r10 - - xor %rax, %r8 - or %rcx, %r8 - sub %r10, %r8 - jnz LABEL(tail) - - mov %rax, (%rdi, %rdx) - mov 8 (%rsi, %rdx), %rax - add $8, %edx - -#ifdef USE_AS_STRNCPY - sub $8, %r11 - jle LABEL(tail) -#endif - - mov %rcx, %r8 - add %rax, %r8 - sbb %r10, %r10 - - xor %rax, %r8 - or %rcx, %r8 - sub %r10, %r8 - jnz LABEL(tail) - - mov %rax, (%rdi, %rdx) - mov 8 (%rsi, %rdx), %rax - add $8, %edx - -#ifdef USE_AS_STRNCPY - sub $8, %r11 - jle LABEL(tail) -#endif - - mov %rcx, %r8 - add %rax, %r8 - sbb %r10, %r10 - - xor %rax, %r8 - or %rcx, %r8 - sub %r10, %r8 - jnz LABEL(tail) - - mov %rax, (%rdi, %rdx) - mov 8 (%rsi, %rdx), %rax - add $8, %edx - -#ifdef USE_AS_STRNCPY - sub $8, %r11 - jle LABEL(tail) -#endif - - mov %rcx, %r8 - add %rax, %r8 - sbb %r10, %r10 - - xor %rax, %r8 - or %rcx, %r8 - sub %r10, %r8 - jnz LABEL(tail) - - mov %rax, (%rdi, %rdx) - mov 8 (%rsi, %rdx), %rax - add $8, %edx - -#ifdef USE_AS_STRNCPY - sub $8, %r11 - jle LABEL(tail) -#endif - - mov %rcx, %r8 - add %rax, %r8 - sbb %r10, %r10 - - xor %rax, %r8 - or %rcx, %r8 - sub %r10, %r8 - jnz LABEL(tail) - - cmp %r9, %rdx + /* + * Null terminated src string shorter than count. Fill the rest of the + * destination with null chars. + */ + .p2align 4 +LABEL(strncpy_fill_tail): + mov %rax, %rdx + movzx %cl, %rax + mov %r8, %rcx + add %rax, %rdi + xor %eax, %eax + shr $3, %ecx + jz LABEL(strncpy_fill_less_8) - mov %rax, (%rdi, %rdx) - prefetchnta 512 + 8 (%rdi, %rdx) /* 3DNow: use prefetchw */ - mov 8 (%rsi, %rdx), %rax - prefetchnta 512 + 8 (%rsi, %rdx) /* 3DNow: use prefetch */ - lea 8 (%rdx), %rdx - - jb LABEL(preloop) - - .p2align 4 - -LABEL(preafter): - -LABEL(NTtry): - mfence - -LABEL(NT): /* 64-byte NT */ - - .p2align 4 - -LABEL(NTloop): -#ifdef USE_AS_STRNCPY - sub $8, %r11 - jle LABEL(tail) -#endif - - mov %rcx, %r8 - add %rax, %r8 - sbb %r10, %r10 - - xor %rax, %r8 - or %rcx, %r8 - sub %r10, %r8 - jnz LABEL(NTtail) - - movnti %rax, (%rdi, %rdx) - mov 8 (%rsi, %rdx), %rax - add $8, %rdx - -#ifdef USE_AS_STRNCPY - sub $8, %r11 - jle LABEL(tail) -#endif - - mov %rcx, %r8 - add %rax, %r8 - sbb %r10, %r10 - - xor %rax, %r8 - or %rcx, %r8 - sub %r10, %r8 - jnz LABEL(NTtail) - - movnti %rax, (%rdi, %rdx) - mov 8 (%rsi, %rdx), %rax - add $8, %rdx - -#ifdef USE_AS_STRNCPY - sub $8, %r11 - jle LABEL(tail) -#endif - - mov %rcx, %r8 - add %rax, %r8 - sbb %r10, %r10 - - xor %rax, %r8 - or %rcx, %r8 - sub %r10, %r8 - jnz LABEL(NTtail) - - movnti %rax, (%rdi, %rdx) - mov 8 (%rsi, %rdx), %rax - add $8, %rdx - -#ifdef USE_AS_STRNCPY - sub $8, %r11 - jle LABEL(tail) -#endif - - mov %rcx, %r8 - add %rax, %r8 - sbb %r10, %r10 - - xor %rax, %r8 - or %rcx, %r8 - sub %r10, %r8 - jnz LABEL(NTtail) - - movnti %rax, (%rdi, %rdx) - mov 8 (%rsi, %rdx), %rax - add $8, %rdx - -#ifdef USE_AS_STRNCPY - sub $8, %r11 - jle LABEL(tail) + rep stosq +LABEL(strncpy_fill_less_8): + mov %r8, %rcx + and $7, %rcx + jz LABEL(strncpy_fill_return) +LABEL(strncpy_fill_less_7): + sub $1, %ecx + mov %al, (%rdi, %rcx) + jnz LABEL(strncpy_fill_less_7) +LABEL(strncpy_fill_return): + mov %rdx, %rax + ret #endif - mov %rcx, %r8 - add %rax, %r8 - sbb %r10, %r10 + .p2align 4 +LABEL(tail_0): /* 1 byte */ + mov (%rsi), %cl + mov %cl, (%rdi) +#ifdef USE_AS_STRNCPY + mov $1, %cl + sub $1, %r8 + jnz LABEL(strncpy_fill_tail) +#endif + ret - xor %rax, %r8 - or %rcx, %r8 - sub %r10, %r8 - jnz LABEL(NTtail) + .p2align 4 +LABEL(tail_1): /* 2 bytes */ + mov (%rsi), %cx + mov %cx, (%rdi) +#ifdef USE_AS_STRNCPY + mov $2, %cl + sub $2, %r8 + jnz LABEL(strncpy_fill_tail) +#endif + ret - movnti %rax, (%rdi, %rdx) - mov 8 (%rsi, %rdx), %rax - add $8, %rdx + .p2align 4 +LABEL(tail_2): /* 3 bytes */ + mov (%rsi), %cx + mov %cx, (%rdi) + mov 1(%rsi), %cx + mov %cx, 1(%rdi) +#ifdef USE_AS_STRNCPY + mov $3, %cl + sub $3, %r8 + jnz LABEL(strncpy_fill_tail) +#endif + ret + .p2align 4 +LABEL(tail_3): /* 4 bytes */ + mov (%rsi), %ecx + mov %ecx, (%rdi) #ifdef USE_AS_STRNCPY - sub $8, %r11 - jle LABEL(tail) + mov $4, %cl + sub $4, %r8 + jnz LABEL(strncpy_fill_tail) #endif + ret - mov %rcx, %r8 - add %rax, %r8 - sbb %r10, %r10 + .p2align 4 +LABEL(tail_4): /* 5 bytes */ + mov (%rsi), %ecx + mov %ecx, (%rdi) + mov 1(%rsi), %edx + mov %edx, 1(%rdi) +#ifdef USE_AS_STRNCPY + mov $5, %cl + sub $5, %r8 + jnz LABEL(strncpy_fill_tail) +#endif + ret - xor %rax, %r8 - or %rcx, %r8 - sub %r10, %r8 - jnz LABEL(NTtail) + .p2align 4 +LABEL(tail_5): /* 6 bytes */ + mov (%rsi), %ecx + mov %ecx, (%rdi) + mov 2(%rsi), %edx + mov %edx, 2(%rdi) +#ifdef USE_AS_STRNCPY + mov $6, %cl + sub $6, %r8 + jnz LABEL(strncpy_fill_tail) +#endif + ret - movnti %rax, (%rdi, %rdx) - mov 8 (%rsi, %rdx), %rax - add $8, %rdx - + .p2align 4 +LABEL(tail_6): /* 7 bytes */ + mov (%rsi), %ecx + mov %ecx, (%rdi) + mov 3(%rsi), %edx + mov %edx,3(%rdi) #ifdef USE_AS_STRNCPY - sub $8, %r11 - jle LABEL(tail) + mov $7, %cl + sub $7, %r8 + jnz LABEL(strncpy_fill_tail) #endif + ret - mov %rcx, %r8 - add %rax, %r8 - sbb %r10, %r10 + .p2align 4 +LABEL(tail_8): /* 9 bytes */ + mov (%rsi), %rcx + mov %rcx, (%rdi) + mov 5(%rsi), %edx + mov %edx, 5(%rdi) +#ifdef USE_AS_STRNCPY + mov $9, %cl + sub $9, %r8 + jnz LABEL(strncpy_fill_tail) +#endif + ret - xor %rax, %r8 - or %rcx, %r8 - sub %r10, %r8 - jnz LABEL(NTtail) + .p2align 4 +LABEL(AMD_exit_more_8): + test %dh, %dh + jz LABEL(AMD_exit_more_16) + test $0x01, %dh + jnz LABEL(tail_8) + test $0x02, %dh + jnz LABEL(tail_9) + test $0x04, %dh + jnz LABEL(tail_10) + test $0x08, %dh + jnz LABEL(tail_11) + test $0x10, %dh + jnz LABEL(tail_12) + test $0x20, %dh + jnz LABEL(tail_13) + test $0x40, %dh + jnz LABEL(tail_14) - movnti %rax, (%rdi, %rdx) - mov 8 (%rsi, %rdx), %rax - add $8, %rdx + .p2align 4 +LABEL(tail_15): /* 16 bytes */ + mov (%rsi), %rcx + mov %rcx, (%rdi) + mov 8(%rsi), %rdx + mov %rdx, 8(%rdi) +#ifdef USE_AS_STRNCPY + mov $16, %cl + sub $16, %r8 + jnz LABEL(strncpy_fill_tail) +#endif + ret + .p2align 4 +LABEL(tail_9): /* 10 bytes */ + mov (%rsi), %rcx + mov %rcx, (%rdi) + mov 6(%rsi), %edx + mov %edx, 6(%rdi) #ifdef USE_AS_STRNCPY - sub $8, %r11 - jle LABEL(tail) + mov $10, %cl + sub $10, %r8 + jnz LABEL(strncpy_fill_tail) #endif + ret - mov %rcx, %r8 - add %rax, %r8 - sbb %r10, %r10 + .p2align 4 +LABEL(tail_10): /* 11 bytes */ + mov (%rsi), %rcx + mov %rcx, (%rdi) + mov 7(%rsi), %edx + mov %edx, 7(%rdi) +#ifdef USE_AS_STRNCPY + mov $11, %cl + sub $11, %r8 + jnz LABEL(strncpy_fill_tail) +#endif + ret - xor %rax, %r8 - or %rcx, %r8 - sub %r10, %r8 - jnz LABEL(NTtail) + .p2align 4 +LABEL(tail_11): /* 12 bytes */ + mov (%rsi), %rcx + mov %rcx, (%rdi) + mov 8(%rsi), %edx + mov %edx, 8(%rdi) +#ifdef USE_AS_STRNCPY + mov $12, %cl + sub $12, %r8 + jnz LABEL(strncpy_fill_tail) +#endif + ret - movnti %rax, (%rdi, %rdx) - mov 8 (%rsi, %rdx), %rax - prefetchnta 768 + 8 (%rsi, %rdx) - add $8, %rdx + .p2align 4 +LABEL(tail_12): /* 13 bytes */ + mov (%rsi), %rcx + mov %rcx, (%rdi) + mov 5(%rsi), %rcx + mov %rcx, 5(%rdi) +#ifdef USE_AS_STRNCPY + mov $13, %cl + sub $13, %r8 + jnz LABEL(strncpy_fill_tail) +#endif + ret - jmp LABEL(NTloop) - - .p2align 4 + .p2align 4 +LABEL(tail_13): /* 14 bytes */ + mov (%rsi), %rcx + mov %rcx, (%rdi) + mov 6(%rsi), %rcx + mov %rcx, 6(%rdi) +#ifdef USE_AS_STRNCPY + mov $14, %cl + sub $14, %r8 + jnz LABEL(strncpy_fill_tail) +#endif + ret -LABEL(NTtail): - mfence - - .p2align 4 - -LABEL(NTafter): - -LABEL(tailtry): - -LABEL(tail): /* 1-byte tail */ + .p2align 4 +LABEL(tail_14): /* 15 bytes */ + mov (%rsi), %rcx + mov %rcx, (%rdi) + mov 7(%rsi), %rcx + mov %rcx, 7(%rdi) #ifdef USE_AS_STRNCPY - add $8, %r11 + mov $15, %cl + sub $15, %r8 + jnz LABEL(strncpy_fill_tail) #endif + ret - .p2align 4 + .p2align 4 +LABEL(AMD_exit_more_16): + shr $16, %edx + test %dl, %dl + jz LABEL(AMD_exit_more_24) + test $0x01, %dl + jnz LABEL(tail_16) + test $0x02, %dl + jnz LABEL(tail_17) + test $0x04, %dl + jnz LABEL(tail_18) + test $0x08, %dl + jnz LABEL(tail_19) + test $0x10, %dl + jnz LABEL(tail_20) + test $0x20, %dl + jnz LABEL(tail_21) + test $0x40, %dl + jnz LABEL(tail_22) -LABEL(tailloop): + .p2align 4 +LABEL(tail_23): /* 24 bytes */ + mov (%rsi), %rcx + mov %rcx, (%rdi) + mov 8(%rsi), %rdx + mov %rdx, 8(%rdi) + mov 16(%rsi), %rcx + mov %rcx, 16(%rdi) #ifdef USE_AS_STRNCPY - dec %r11 - jl LABEL(exitn) + mov $24, %cl + sub $24, %r8 + jnz LABEL(strncpy_fill_tail) #endif + ret - test %al, %al - mov %al, (%rdi, %rdx) - jz LABEL(exit) + .p2align 4 +LABEL(tail_16): /* 17 bytes */ + mov (%rsi), %rcx + mov %rcx, (%rdi) + mov 8(%rsi), %rdx + mov %rdx, 8(%rdi) + mov 16(%rsi), %cl + mov %cl, 16(%rdi) +#ifdef USE_AS_STRNCPY + mov $17, %cl + sub $17, %r8 + jnz LABEL(strncpy_fill_tail) +#endif + ret + + .p2align 4 +LABEL(tail_17): /* 18 bytes */ + mov (%rsi), %rcx + mov %rcx, (%rdi) + mov 8(%rsi), %rdx + mov %rdx, 8(%rdi) + mov 16(%rsi), %cx + mov %cx, 16(%rdi) +#ifdef USE_AS_STRNCPY + mov $18, %cl + sub $18, %r8 + jnz LABEL(strncpy_fill_tail) +#endif + ret - inc %rdx - + .p2align 4 +LABEL(tail_18): /* 19 bytes */ + mov (%rsi), %rcx + mov %rcx, (%rdi) + mov 8(%rsi), %rdx + mov %rdx, 8(%rdi) + mov 15(%rsi), %ecx + mov %ecx,15(%rdi) #ifdef USE_AS_STRNCPY - dec %r11 - jl LABEL(exitn) + mov $19, %cl + sub $19, %r8 + jnz LABEL(strncpy_fill_tail) +#endif + ret - mov %ah, %al + .p2align 4 +LABEL(tail_19): /* 20 bytes */ + mov (%rsi), %rcx + mov %rcx, (%rdi) + mov 8(%rsi), %rdx + mov %rdx, 8(%rdi) + mov 16(%rsi), %ecx + mov %ecx, 16(%rdi) +#ifdef USE_AS_STRNCPY + mov $20, %cl + sub $20, %r8 + jnz LABEL(strncpy_fill_tail) #endif + ret - test %ah, %ah - mov %ah, (%rdi, %rdx) - jz LABEL(exit) - - inc %rdx + .p2align 4 +LABEL(tail_20): /* 21 bytes */ + mov (%rsi), %rcx + mov %rcx, (%rdi) + mov 8(%rsi), %rdx + mov %rdx, 8(%rdi) + mov 13(%rsi), %rcx + mov %rcx, 13(%rdi) +#ifdef USE_AS_STRNCPY + mov $21, %cl + sub $21, %r8 + jnz LABEL(strncpy_fill_tail) +#endif + ret + .p2align 4 +LABEL(tail_21): /* 22 bytes */ + mov (%rsi), %rcx + mov %rcx, (%rdi) + mov 8(%rsi), %rdx + mov %rdx, 8(%rdi) + mov 14(%rsi), %rcx + mov %rcx, 14(%rdi) #ifdef USE_AS_STRNCPY - dec %r11 - jl LABEL(exitn) + mov $22, %cl + sub $22, %r8 + jnz LABEL(strncpy_fill_tail) #endif - - shr $16, %rax + ret - test %al, %al - mov %al, (%rdi, %rdx) - jz LABEL(exit) - - inc %rdx + .p2align 4 +LABEL(tail_22): /* 23 bytes */ + mov (%rsi), %rcx + mov %rcx, (%rdi) + mov 8(%rsi), %rdx + mov %rdx, 8(%rdi) + mov 15(%rsi), %rcx + mov %rcx, 15(%rdi) +#ifdef USE_AS_STRNCPY + mov $23, %cl + sub $23, %r8 + jnz LABEL(strncpy_fill_tail) +#endif + ret -#ifdef USE_AS_STRNCPY - dec %r11 - jl LABEL(exitn) - - mov %ah, %al -#endif + .p2align 4 +LABEL(AMD_exit_more_24): + test $0x01, %dh + jnz LABEL(tail_24) + test $0x02, %dh + jnz LABEL(tail_25) + test $0x04, %dh + jnz LABEL(tail_26) + test $0x08, %dh + jnz LABEL(tail_27) + test $0x10, %dh + jnz LABEL(tail_28) + test $0x20, %dh + jnz LABEL(tail_29) + test $0x40, %dh + jnz LABEL(tail_30) - test %ah, %ah - mov %ah, (%rdi, %rdx) - jz LABEL(exit) + .p2align 4 +LABEL(tail_31): /* 32 bytes */ + mov (%rsi), %rcx + mov %rcx, (%rdi) + mov 8(%rsi), %rdx + mov %rdx, 8(%rdi) + mov 16(%rsi), %rcx + mov %rcx, 16(%rdi) + mov 24(%rsi), %rdx + mov %rdx, 24(%rdi) +#ifdef USE_AS_STRNCPY + mov $32, %cl + sub $32, %r8 + jnz LABEL(strncpy_fill_tail) +#endif + ret - shr $16, %rax - inc %rdx - - jmp LABEL(tailloop) - - .p2align 4 - -LABEL(tailafter): + .p2align 4 +LABEL(tail_24): /* 25 bytes */ + mov (%rsi), %rcx + mov %rcx, (%rdi) + mov 8(%rsi), %rdx + mov %rdx, 8(%rdi) + mov 16(%rsi), %rcx + mov %rcx, 16(%rdi) + mov 21(%rsi), %edx + mov %edx, 21(%rdi) +#ifdef USE_AS_STRNCPY + mov $25, %cl + sub $25, %r8 + jnz LABEL(strncpy_fill_tail) +#endif + ret -LABEL(exit): + .p2align 4 +LABEL(tail_25): /* 26 bytes */ + mov (%rsi), %rcx + mov %rcx, (%rdi) + mov 8(%rsi), %rdx + mov %rdx, 8(%rdi) + mov 16(%rsi), %rcx + mov %rcx, 16(%rdi) + mov 22(%rsi), %edx + mov %edx, 22(%rdi) #ifdef USE_AS_STRNCPY - test %r11, %r11 - mov %r11, %rcx + mov $26, %cl + sub $26, %r8 + jnz LABEL(strncpy_fill_tail) +#endif + ret -#ifdef USE_AS_STPCPY - lea (%rdi, %rdx), %r8 -#else - mov %rdi, %r8 + .p2align 4 +LABEL(tail_26): /* 27 bytes */ + mov (%rsi), %rcx + mov %rcx, (%rdi) + mov 8(%rsi), %rdx + mov %rdx, 8(%rdi) + mov 16(%rsi), %rcx + mov %rcx, 16(%rdi) + mov 23(%rsi), %edx + mov %edx, 23(%rdi) +#ifdef USE_AS_STRNCPY + mov $27, %cl + sub $27, %r8 + jnz LABEL(strncpy_fill_tail) #endif + ret - jz 2f - - xor %eax, %eax /* bzero () would do too, but usually there are only a handfull of bytes left */ - shr $3, %rcx - lea 1 (%rdi, %rdx), %rdi - jz 1f + .p2align 4 +LABEL(tail_27): /* 28 bytes */ + mov (%rsi), %rcx + mov %rcx, (%rdi) + mov 8(%rsi), %rdx + mov %rdx, 8(%rdi) + mov 16(%rsi), %rcx + mov %rcx, 16(%rdi) + mov 24(%rsi), %edx + mov %edx, 24(%rdi) +#ifdef USE_AS_STRNCPY + mov $28, %cl + sub $28, %r8 + jnz LABEL(strncpy_fill_tail) +#endif + ret - rep stosq - -1: - mov %r11d, %ecx - and $7, %ecx - jz 2f + .p2align 4 +LABEL(tail_28): /* 29 bytes */ + mov (%rsi), %rcx + mov %rcx, (%rdi) + mov 8(%rsi), %rdx + mov %rdx, 8(%rdi) + mov 16(%rsi), %rcx + mov %rcx, 16(%rdi) + mov 21(%rsi), %rdx + mov %rdx, 21(%rdi) +#ifdef USE_AS_STRNCPY + mov $29, %cl + sub $29, %r8 + jnz LABEL(strncpy_fill_tail) +#endif + ret - .p2align 4,, 3 + .p2align 4 +LABEL(tail_29): /* 30 bytes */ + mov (%rsi), %rcx + mov %rcx, (%rdi) + mov 8(%rsi), %rdx + mov %rdx, 8(%rdi) + mov 16(%rsi), %rcx + mov %rcx, 16(%rdi) + mov 22(%rsi), %rdx + mov %rdx, 22(%rdi) +#ifdef USE_AS_STRNCPY + mov $30, %cl + sub $30, %r8 + jnz LABEL(strncpy_fill_tail) +#endif + ret -3: - dec %ecx - mov %al, (%rdi, %rcx) - jnz 3b - - .p2align 4,, 3 - -2: - mov %r8, %rax - ret + .p2align 4 +LABEL(tail_30): /* 31 bytes */ + mov (%rsi), %rcx + mov %rcx, (%rdi) + mov 8(%rsi), %rdx + mov %rdx, 8(%rdi) + mov 16(%rsi), %rcx + mov %rcx, 16(%rdi) + mov 23(%rsi), %rdx + mov %rdx, 23(%rdi) +#ifdef USE_AS_STRNCPY + mov $31, %cl + sub $31, %r8 + jnz LABEL(strncpy_fill_tail) +#endif + ret -#endif - - .p2align 4 + .pushsection .rodata + .p2align 4 +LABEL(tail_table): + .int LABEL(tail_0) - LABEL(tail_table) /* 1 byte */ + .int LABEL(tail_1) - LABEL(tail_table) + .int LABEL(tail_2) - LABEL(tail_table) + .int LABEL(tail_3) - LABEL(tail_table) + .int LABEL(tail_4) - LABEL(tail_table) + .int LABEL(tail_5) - LABEL(tail_table) + .int LABEL(tail_6) - LABEL(tail_table) + .int LABEL(tail_7) - LABEL(tail_table) + .int LABEL(tail_8) - LABEL(tail_table) + .int LABEL(tail_9) - LABEL(tail_table) + .int LABEL(tail_10) - LABEL(tail_table) + .int LABEL(tail_11) - LABEL(tail_table) + .int LABEL(tail_12) - LABEL(tail_table) + .int LABEL(tail_13) - LABEL(tail_table) + .int LABEL(tail_14) - LABEL(tail_table) + .int LABEL(tail_15) - LABEL(tail_table) + .int LABEL(tail_16) - LABEL(tail_table) + .int LABEL(tail_17) - LABEL(tail_table) + .int LABEL(tail_18) - LABEL(tail_table) + .int LABEL(tail_19) - LABEL(tail_table) + .int LABEL(tail_20) - LABEL(tail_table) + .int LABEL(tail_21) - LABEL(tail_table) + .int LABEL(tail_22) - LABEL(tail_table) + .int LABEL(tail_23) - LABEL(tail_table) + .int LABEL(tail_24) - LABEL(tail_table) + .int LABEL(tail_25) - LABEL(tail_table) + .int LABEL(tail_26) - LABEL(tail_table) + .int LABEL(tail_27) - LABEL(tail_table) + .int LABEL(tail_28) - LABEL(tail_table) + .int LABEL(tail_29) - LABEL(tail_table) + .int LABEL(tail_30) - LABEL(tail_table) + .int LABEL(tail_31) - LABEL(tail_table) /* 32 bytes */ -LABEL(exitn): -#ifdef USE_AS_STPCPY - lea (%rdi, %rdx), %rax -#else - mov %rdi, %rax -#endif - - ret + .p2align 4 +LABEL(unaligned_table): + .int LABEL(ashr_0) - LABEL(unaligned_table) + .int LABEL(ashr_1) - LABEL(unaligned_table) + .int LABEL(ashr_2) - LABEL(unaligned_table) + .int LABEL(ashr_3) - LABEL(unaligned_table) + .int LABEL(ashr_4) - LABEL(unaligned_table) + .int LABEL(ashr_5) - LABEL(unaligned_table) + .int LABEL(ashr_6) - LABEL(unaligned_table) + .int LABEL(ashr_7) - LABEL(unaligned_table) + .int LABEL(ashr_8) - LABEL(unaligned_table) + .int LABEL(ashr_9) - LABEL(unaligned_table) + .int LABEL(ashr_10) - LABEL(unaligned_table) + .int LABEL(ashr_11) - LABEL(unaligned_table) + .int LABEL(ashr_12) - LABEL(unaligned_table) + .int LABEL(ashr_13) - LABEL(unaligned_table) + .int LABEL(ashr_14) - LABEL(unaligned_table) + .int LABEL(ashr_15) - LABEL(unaligned_table) + .popsection #ifdef USE_AS_STRNCPY SET_SIZE(strncpy) #else - SET_SIZE(strcpy) /* (char *, const char *) */ + SET_SIZE(strcpy) /* (char *, const char *) */ #endif
--- a/usr/src/lib/libc/amd64/gen/strlen.s Mon Sep 21 11:25:09 2009 -0400 +++ b/usr/src/lib/libc/amd64/gen/strlen.s Mon Sep 21 11:26:40 2009 -0400 @@ -1,430 +1,199 @@ /* - * Copyright 2004 Sun Microsystems, Inc. All rights reserved. - * Use is subject to license terms. - */ - -/* - * Copyright (c) 2002 Advanced Micro Devices, Inc. - * - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the - * following conditions are met: - * - * + Redistributions of source code must retain the above - * copyright notice, this list of conditions and the - * following disclaimer. - * - * + Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the - * following disclaimer in the documentation and/or other - * materials provided with the distribution. - * - * + Neither the name of Advanced Micro Devices, Inc. nor the - * names of its contributors may be used to endorse or - * promote products derived from this software without - * specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND - * CONTRIBUTORS AS IS AND ANY EXPRESS OR IMPLIED WARRANTIES, - * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF - * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL ADVANCED MICRO DEVICES, - * INC. OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, - * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE - * GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR - * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF - * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT - * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - * - * It is licensee's responsibility to comply with any export - * regulations applicable in licensee's jurisdiction. + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END */ - .file "strlen.s" +/* + * Copyright (c) 2009, Intel Corporation + * All rights reserved. + */ + +/* + * strlen - calculate the length of string + */ #include "SYS.h" -#include "cache.h" +#include "proc64_id.h" #define LABEL(s) .strlen/**/s - ENTRY(strlen) /* (const char *s) */ - - mov %rdi, %rsi - neg %rdi - -LABEL(aligntry): - mov %rsi , %r8 - and $7, %r8d - jz LABEL(alignafter) - -LABEL(align): /* 8-byte align */ - sub $8, %r8 - - .p2align 4 - -LABEL(alignloop): - cmpb $0, (%rsi) - je LABEL(exit) + /* + * This implementation uses SSE instructions to compare up to 16 bytes + * at a time looking for the end of string (null char). + */ + ENTRY(strlen) /* (const char *s) */ + mov %rdi, %rsi /* keep original %rdi value */ + mov %rsi, %rcx + pxor %xmm0, %xmm0 /* 16 null chars */ + and $15, %rcx + jz LABEL(align16_loop) /* string is 16 byte aligned */ - inc %rsi - inc %r8 - jnz LABEL(alignloop) - - .p2align 4 - -LABEL(alignafter): - -LABEL(56try): - -LABEL(56): /* 56-byte */ - mov (%rsi), %rax - mov $0xfefefefefefefeff, %rcx - -LABEL(56loop): - mov %rcx, %r8 - add %rax, %r8 - jnc LABEL(tail) - - xor %rax, %r8 - or %rcx, %r8 - inc %r8 - jnz LABEL(tail) + /* + * Unaligned case. Round down to 16-byte boundary before comparing + * 16 bytes for a null char. The code then compensates for any extra chars + * preceding the start of the string. + */ +LABEL(unalign16): + and $0xfffffffffffffff0, %rsi - mov 8 (%rsi), %rax - lea 8 (%rsi), %rsi - - mov %rcx, %r8 - add %rax, %r8 - jnc LABEL(tail) - - xor %rax, %r8 - or %rcx, %r8 - inc %r8 - jnz LABEL(tail) - - mov 8 (%rsi), %rax - lea 8 (%rsi), %rsi - - mov %rcx, %r8 - add %rax, %r8 - jnc LABEL(tail) - - xor %rax, %r8 - or %rcx, %r8 - inc %r8 - jnz LABEL(tail) + pcmpeqb (%rsi), %xmm0 + lea 16(%rdi), %rsi + pmovmskb %xmm0, %edx - mov 8 (%rsi), %rax - lea 8 (%rsi), %rsi - - mov %rcx, %r8 - add %rax, %r8 - jnc LABEL(tail) - - xor %rax, %r8 - or %rcx, %r8 - inc %r8 - jnz LABEL(tail) + shr %cl, %edx /* Compensate for bytes preceding the string */ + test %edx, %edx + jnz LABEL(exit) + sub %rcx, %rsi /* no null, adjust to next 16-byte boundary */ + pxor %xmm0, %xmm0 /* clear xmm0, may have been changed... */ + + .p2align 4 +LABEL(align16_loop): /* 16 byte aligned */ + pcmpeqb (%rsi), %xmm0 /* look for null bytes */ + pmovmskb %xmm0, %edx /* move each byte mask of %xmm0 to edx */ - mov 8 (%rsi), %rax - lea 8 (%rsi), %rsi - - mov %rcx, %r8 - add %rax, %r8 - jnc LABEL(tail) - - xor %rax, %r8 - or %rcx, %r8 - inc %r8 - jnz LABEL(tail) + add $16, %rsi /* prepare to search next 16 bytes */ + test %edx, %edx /* if no null byte, %edx must be 0 */ + jnz LABEL(exit) /* found a null */ - mov 8 (%rsi), %rax - lea 8 (%rsi), %rsi - - mov %rcx, %r8 - add %rax, %r8 - jnc LABEL(tail) - - xor %rax, %r8 - or %rcx, %r8 - inc %r8 - jnz LABEL(tail) - - mov 8 (%rsi), %rax - lea 8 (%rsi), %rsi - - mov %rcx, %r8 - add %rax, %r8 - jnc LABEL(tail) + pcmpeqb (%rsi), %xmm0 + pmovmskb %xmm0, %edx + add $16, %rsi + test %edx, %edx + jnz LABEL(exit) - xor %rax, %r8 - or %rcx, %r8 - inc %r8 - jnz LABEL(tail) - - mov 8 (%rsi), %rax - lea 8 (%rsi), %rsi - -LABEL(56after): - -LABEL(32): /* 32-byte */ - mov _sref_(.amd64cache1), %r9 + pcmpeqb (%rsi), %xmm0 + pmovmskb %xmm0, %edx + add $16, %rsi + test %edx, %edx + jnz LABEL(exit) - .p2align 4 - -LABEL(32loop): - mov %rcx, %r8 - add %rax, %r8 - sbb %rdx, %rdx - - xor %rax, %r8 - or %rcx, %r8 - sub %rdx, %r8 - jnz LABEL(tail) - - mov 8 (%rsi), %rax - add $8, %rsi + pcmpeqb (%rsi), %xmm0 + pmovmskb %xmm0, %edx + add $16, %rsi + test %edx, %edx + jz LABEL(align16_loop) - mov %rcx, %r8 - add %rax, %r8 - sbb %rdx, %rdx - - xor %rax, %r8 - or %rcx, %r8 - sub %rdx, %r8 - jnz LABEL(tail) - - mov 8 (%rsi), %rax - add $8, %rsi - - mov %rcx, %r8 - add %rax, %r8 - sbb %rdx, %rdx - - xor %rax, %r8 - or %rcx, %r8 - sub %rdx, %r8 - jnz LABEL(tail) - - mov 8 (%rsi), %rax - add $8, %rsi + .p2align 4 +LABEL(exit): + neg %rdi + /* + * Check to see if BSF is fast on this processor. If not, use a different + * exit tail to find first bit set indicating null byte match. + */ + testl $USE_BSF, .memops_method(%rip) + jz LABEL(AMD_exit) - mov %rcx, %r8 - add %rax, %r8 - sbb %rdx, %rdx - - xor %rax, %r8 - or %rcx, %r8 - sub %rdx, %r8 - jnz LABEL(tail) - - mov 8 (%rsi), %rax - add $8, %rsi + lea -16(%rdi, %rsi), %rax /* calculate exact offset */ + bsf %edx, %ecx /* Least significant 1 bit is index of null */ + lea (%rax, %rcx),%rax + ret - mov %rcx, %r8 - add %rax, %r8 - sbb %rdx, %rdx - - xor %rax, %r8 - or %rcx, %r8 - sub %rdx, %r8 - jnz LABEL(tail) - - mov 8 (%rsi), %rax - add $8, %rsi + /* + * This exit tail does not use the bsf instruction. + */ + .p2align 4 +LABEL(AMD_exit): + lea -16(%rdi, %rsi), %rax + test %dl, %dl + jz LABEL(exit_high) + test $0x01, %dl + jnz LABEL(exit_tail0) - mov %rcx, %r8 - add %rax, %r8 - sbb %rdx, %rdx - - xor %rax, %r8 - or %rcx, %r8 - sub %rdx, %r8 - jnz LABEL(tail) + test $0x02, %dl + jnz LABEL(exit_tail1) - mov 8 (%rsi), %rax - add $8, %rsi - - mov %rcx, %r8 - add %rax, %r8 - sbb %rdx, %rdx - - xor %rax, %r8 - or %rcx, %r8 - sub %rdx, %r8 - jnz LABEL(tail) + .p2align 4 + test $0x04, %dl + jnz LABEL(exit_tail2) - mov 8 (%rsi), %rax - add $8, %rsi - - mov %rcx, %r8 - add %rax, %r8 - sbb %rdx, %rdx - - xor %rax, %r8 - or %rcx, %r8 - sub %rdx, %r8 - jnz LABEL(tail) + test $0x08, %dl + jnz LABEL(exit_tail3) - sub $32, %r9 - - mov 8 (%rsi), %rax - lea 8 (%rsi), %rsi + test $0x10, %dl + jnz LABEL(exit_tail4) - jbe LABEL(32loop) - -LABEL(32after): - -LABEL(pretry): - -LABEL(pre): /* 64-byte prefetch */ + test $0x20, %dl + jnz LABEL(exit_tail5) - .p2align 4 - -LABEL(preloop): - mov %rcx, %r8 - add %rax, %r8 - sbb %rdx, %rdx - - xor %rax, %r8 - or %rcx, %r8 - sub %rdx, %r8 - jnz LABEL(tail) + test $0x40, %dl + jnz LABEL(exit_tail6) + add $7, %rax + ret - mov 8 (%rsi), %rax - add $8, %rsi - - mov %rcx, %r8 - add %rax, %r8 - sbb %rdx, %rdx - - xor %rax, %r8 - or %rcx, %r8 - sub %rdx, %r8 - jnz LABEL(tail) + .p2align 4 +LABEL(exit_high): + add $8, %rax + test $0x01, %dh + jnz LABEL(exit_tail0) - mov 8 (%rsi), %rax - add $8, %rsi + test $0x02, %dh + jnz LABEL(exit_tail1) - mov %rcx, %r8 - add %rax, %r8 - sbb %rdx, %rdx - - xor %rax, %r8 - or %rcx, %r8 - sub %rdx, %r8 - jnz LABEL(tail) + test $0x04, %dh + jnz LABEL(exit_tail2) - mov 8 (%rsi), %rax - add $8, %rsi + test $0x08, %dh + jnz LABEL(exit_tail3) - mov %rcx, %r8 - add %rax, %r8 - sbb %rdx, %rdx - - xor %rax, %r8 - or %rcx, %r8 - sub %rdx, %r8 - jnz LABEL(tail) + test $0x10, %dh + jnz LABEL(exit_tail4) - mov 8 (%rsi), %rax - add $8, %rsi - - mov %rcx, %r8 - add %rax, %r8 - sbb %rdx, %rdx - - xor %rax, %r8 - or %rcx, %r8 - sub %rdx, %r8 - jnz LABEL(tail) + test $0x20, %dh + jnz LABEL(exit_tail5) - mov 8 (%rsi), %rax - add $8, %rsi - - mov %rcx, %r8 - add %rax, %r8 - sbb %rdx, %rdx - - xor %rax, %r8 - or %rcx, %r8 - sub %rdx, %r8 - jnz LABEL(tail) + test $0x40, %dh + jnz LABEL(exit_tail6) + add $7, %rax + ret - mov 8 (%rsi), %rax - add $8, %rsi - - mov %rcx, %r8 - add %rax, %r8 - sbb %rdx, %rdx - - xor %rax, %r8 - or %rcx, %r8 - sub %rdx, %r8 - jnz LABEL(tail) + .p2align 4 +LABEL(exit_tail0): + xor %ecx, %ecx + ret - mov 8 (%rsi), %rax - add $8, %rsi - - mov %rcx, %r8 - add %rax, %r8 - sbb %rdx, %rdx - - xor %rax, %r8 - or %rcx, %r8 - sub %rdx, %r8 - jnz LABEL(tail) + .p2align 4 +LABEL(exit_tail1): + add $1, %rax + ret - prefetchnta 512 (%rsi) /* 3DNow: use prefetch */ - - mov 8 (%rsi), %rax - add $8, %rsi - - jmp LABEL(preloop) - - .p2align 4 - -LABEL(preafter): + .p2align 4 +LABEL(exit_tail2): + add $2, %rax + ret -LABEL(tailtry): - -LABEL(tail): /* 4-byte tail */ - -LABEL(tailloop): - test %al, %al - jz LABEL(exit) - - inc %rsi - - test %ah, %ah - jz LABEL(exit) + .p2align 4 +LABEL(exit_tail3): + add $3, %rax + ret - inc %rsi - - test $0x00ff0000, %eax - jz LABEL(exit) - - inc %rsi - - test $0xff000000, %eax - jz LABEL(exit) - - inc %rsi + .p2align 4 +LABEL(exit_tail4): + add $4, %rax + ret - shr $32, %rax - jmp LABEL(tailloop) - -LABEL(tailafter): + .p2align 4 +LABEL(exit_tail5): + add $5, %rax + ret - .p2align 4 - -LABEL(exit): - lea (%rdi, %rsi), %rax - ret - + .p2align 4 +LABEL(exit_tail6): + add $6, %rax + ret SET_SIZE(strlen)
--- a/usr/src/lib/libc/i386/Makefile.com Mon Sep 21 11:25:09 2009 -0400 +++ b/usr/src/lib/libc/i386/Makefile.com Mon Sep 21 11:26:40 2009 -0400 @@ -412,6 +412,7 @@ csetlen.o \ ctime.o \ ctime_r.o \ + daemon.o \ deflt.o \ directio.o \ dirname.o \
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/usr/src/lib/libc/port/gen/daemon.c Mon Sep 21 11:26:40 2009 -0400 @@ -0,0 +1,103 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ + +/* + * Copyright 2009 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +#include "lint.h" +#include "file64.h" +#include "mtlib.h" + +#include <stdio.h> +#include <stdlib.h> +#include <unistd.h> +#include <fcntl.h> + +#include "stdiom.h" + +/* + * Use fork/setsid/fork to go into background and permanently remove + * controlling terminal. + */ +int +daemon(int nochdir, int noclose) +{ + int retv, fd; + + /* + * By the first fork+setsid, we disconnect from our current controlling + * terminal and become a session group leader. + */ + retv = fork(); + if (retv == -1) + return (-1); + if (retv != 0) + _exit(EXIT_SUCCESS); + if (setsid() == -1) + return (-1); + /* + * By forking again without calling setsid again, we make certain + * that we are not the session group leader and can never reacquire + * a controlling terminal. + */ + retv = fork(); + if (retv == -1) + return (-1); + if (retv != 0) + _exit(EXIT_SUCCESS); + + if (nochdir == 0) + (void) chdir("/"); + + if (noclose == 0) { + /* + * Missing the PRIV_FILE_READ privilege may be one of the + * reasons that prevent the opening of /dev/null to succeed. + */ + if ((fd = open("/dev/null", O_RDWR)) == -1) + return (-1); + + /* + * Also, if any of the descriptor redirects fails we should + * return with error to signal to the caller that his request + * cannot be fulfilled properly. It is up to the caller to + * do the cleanup. + */ + if ((fd != STDIN_FILENO) && (dup2(fd, STDIN_FILENO) < 0)) { + (void) close(fd); + return (-1); + } + if ((fd != STDOUT_FILENO) && (dup2(fd, STDOUT_FILENO) < 0)) { + (void) close(fd); + return (-1); + } + if ((fd != STDERR_FILENO) && (dup2(fd, STDERR_FILENO) < 0)) { + (void) close(fd); + return (-1); + } + + if (fd > STDERR_FILENO) + (void) close(fd); + } + return (0); +}
--- a/usr/src/lib/libc/port/llib-lc Mon Sep 21 11:25:09 2009 -0400 +++ b/usr/src/lib/libc/port/llib-lc Mon Sep 21 11:26:40 2009 -0400 @@ -345,6 +345,9 @@ int _tolower(int c); int toascii(int c); +/* daemon.c */ +int daemon(int nochdir, int noclose); + /* directio.c */ int directio(int filedes, int advice);
--- a/usr/src/lib/libc/port/mapfile-vers Mon Sep 21 11:25:09 2009 -0400 +++ b/usr/src/lib/libc/port/mapfile-vers Mon Sep 21 11:26:40 2009 -0400 @@ -86,6 +86,7 @@ clock_gettime; clock_nanosleep; clock_settime; + daemon; dirfd; door_bind; door_call;
--- a/usr/src/lib/libc/sparc/Makefile.com Mon Sep 21 11:25:09 2009 -0400 +++ b/usr/src/lib/libc/sparc/Makefile.com Mon Sep 21 11:26:40 2009 -0400 @@ -429,6 +429,7 @@ csetlen.o \ ctime.o \ ctime_r.o \ + daemon.o \ deflt.o \ directio.o \ dirname.o \
--- a/usr/src/lib/libc/sparcv9/Makefile.com Mon Sep 21 11:25:09 2009 -0400 +++ b/usr/src/lib/libc/sparcv9/Makefile.com Mon Sep 21 11:26:40 2009 -0400 @@ -390,6 +390,7 @@ csetlen.o \ ctime.o \ ctime_r.o \ + daemon.o \ deflt.o \ directio.o \ dirname.o \
--- a/usr/src/lib/libcryptoutil/common/config_parsing.c Mon Sep 21 11:25:09 2009 -0400 +++ b/usr/src/lib/libcryptoutil/common/config_parsing.c Mon Sep 21 11:26:40 2009 -0400 @@ -597,9 +597,10 @@ char *token1; if ((pfile = fopen(_PATH_KCF_CONF, "r")) == NULL) { - cryptoerror(LOG_ERR, + cryptoerror(LOG_DEBUG, "failed to open the kcf.conf file for read only."); - return (CKR_FUNCTION_FAILED); + *mode = CRYPTO_FIPS_MODE_DISABLED; + return (CKR_OK); } while (fgets(buffer, BUFSIZ, pfile) != NULL) {
--- a/usr/src/lib/libgrubmgmt/common/libgrub_cmd.def Mon Sep 21 11:25:09 2009 -0400 +++ b/usr/src/lib/libgrubmgmt/common/libgrub_cmd.def Mon Sep 21 11:26:40 2009 -0400 @@ -58,6 +58,9 @@ menu_cmd("args", GRBM_ARGS_CMD, GRUB_LINE_ENTRY, error_line) menu_cmd("findroot", GRBM_FINDROOT_CMD, GRUB_LINE_ENTRY, findroot) menu_cmd("bootfs", GRBM_BOOTFS_CMD, GRUB_LINE_ENTRY, bootfs) +menu_cmd("splashimage", GRBM_SPLASHIMAGE_CMD, GRUB_LINE_ENTRY, skip_line) +menu_cmd("background", GRBM_BACKGROUND_CMD, GRUB_LINE_ENTRY, skip_line) +menu_cmd("foreground", GRBM_FOREGROUND_CMD, GRUB_LINE_ENTRY, skip_line) menu_cmd_end(GRBM_CMD_NUM) /* Should be the last one */
--- a/usr/src/lib/libscf/common/highlevel.c Mon Sep 21 11:25:09 2009 -0400 +++ b/usr/src/lib/libscf/common/highlevel.c Mon Sep 21 11:26:40 2009 -0400 @@ -230,6 +230,8 @@ scf_propvec_t ua_boot_config_ovr[] = { { FASTREBOOT_DEFAULT, NULL, SCF_TYPE_BOOLEAN, NULL, UA_FASTREBOOT_DEFAULT }, + { FASTREBOOT_ONPANIC, NULL, SCF_TYPE_BOOLEAN, NULL, + UA_FASTREBOOT_ONPANIC }, { NULL } }; scf_propvec_t *prop; @@ -274,6 +276,10 @@ } } #endif /* FASTREBOOT_DEBUG */ + + if (set) + (void) smf_refresh_instance(FMRI_BOOT_CONFIG); + return (rc); } @@ -283,7 +289,7 @@ /* * Get values of properties in non-persistent "config_ovr" property group. */ -static void +void scf_get_boot_config_ovr(uint8_t *boot_config_ovr) { (void) scf_getset_boot_config_ovr(B_FALSE, boot_config_ovr); @@ -295,7 +301,10 @@ int scf_fastreboot_default_set_transient(boolean_t value) { - uint8_t boot_config_ovr = (value & UA_FASTREBOOT_DEFAULT); + uint8_t boot_config_ovr = 0; + + if (value == B_TRUE) + boot_config_ovr = UA_FASTREBOOT_DEFAULT | UA_FASTREBOOT_ONPANIC; return (scf_getset_boot_config_ovr(B_TRUE, &boot_config_ovr)); }
--- a/usr/src/lib/libscf/common/mapfile-vers Mon Sep 21 11:25:09 2009 -0400 +++ b/usr/src/lib/libscf/common/mapfile-vers Mon Sep 21 11:26:40 2009 -0400 @@ -315,6 +315,7 @@ scf_clean_propvec; scf_instance_delete_prop; scf_get_boot_config; + scf_get_boot_config_ovr; scf_is_fastboot_default; scf_fastreboot_default_set_transient; local:
--- a/usr/src/lib/libscf/inc/libscf_priv.h Mon Sep 21 11:25:09 2009 -0400 +++ b/usr/src/lib/libscf/inc/libscf_priv.h Mon Sep 21 11:26:40 2009 -0400 @@ -530,6 +530,7 @@ * Functions to extract boot config information from FMRI_BOOT_CONFIG */ void scf_get_boot_config(uint8_t *); +void scf_get_boot_config_ovr(uint8_t *); int scf_is_fastboot_default(void); /*
--- a/usr/src/lib/libstmf/common/stmf.c Mon Sep 21 11:25:09 2009 -0400 +++ b/usr/src/lib/libstmf/common/stmf.c Mon Sep 21 11:26:40 2009 -0400 @@ -4735,15 +4735,15 @@ if (iGetPersistMethod() == STMF_PERSIST_NONE) { stmfStateSet.state = STMF_STATE_OFFLINE; - stmfStateSet.config_state = STMF_CONFIG_INIT; + if ((ret = openStmf(OPEN_EXCL_STMF, &fd)) != STMF_STATUS_SUCCESS) { return (ret); } - ret = setStmfState(fd, &stmfStateSet, STMF_SERVICE_TYPE); - if (ret != STMF_STATUS_SUCCESS) { - goto done; - } + /* + * Configuration not stored persistently; nothing to + * initialize so do not set to STMF_CONFIG_INIT. + */ stmfStateSet.config_state = STMF_CONFIG_INIT_DONE; goto done; }
--- a/usr/src/lib/nsswitch/dns/common/dns_common.c Mon Sep 21 11:25:09 2009 -0400 +++ b/usr/src/lib/nsswitch/dns/common/dns_common.c Mon Sep 21 11:26:40 2009 -0400 @@ -19,12 +19,10 @@ * CDDL HEADER END */ /* - * Copyright 2008 Sun Microsystems, Inc. All rights reserved. + * Copyright 2009 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. */ -#pragma ident "%Z%%M% %I% %E% SMI" - /* * dns_common.c */ @@ -33,6 +31,7 @@ #pragma weak dn_expand #pragma weak res_ninit +#pragma weak res_ndestroy #pragma weak res_nsearch #pragma weak res_nclose #pragma weak ns_get16 @@ -274,19 +273,6 @@ } /* - * __res_ndestroy is a simplified version of the non-public function - * res_ndestroy in libresolv.so.2. Before res_ndestroy can be made - * public, __res_ndestroy will be used to make sure the memory pointed - * by statp->_u._ext.ext is freed after res_nclose() is called. - */ -static void -__res_ndestroy(res_state statp) { - res_nclose(statp); - if (statp->_u._ext.ext != NULL) - free(statp->_u._ext.ext); -} - -/* * name_is_alias(aliases_ptr, name_ptr) * Verify name matches an alias in the provided aliases list. * @@ -299,7 +285,7 @@ * INPUT: * aliases_ptr: space separated list of alias names. * name_ptr: name to look for in aliases_ptr list. - * RETURNS: NSS_SUCCESS or NSS_ERROR + * RETURNS: NSS_SUCCESS or NSS_NOTFOUND * NSS_SUCCESS indicates that the name is listed in the collected aliases. */ static nss_status_t @@ -332,7 +318,7 @@ /* Step over separator character. */ while (*aliases_ptr == ' ') aliases_ptr++; } - return (NSS_ERROR); + return (NSS_NOTFOUND); } /* @@ -397,7 +383,7 @@ /* misc variables */ int af; char *ap, *apc; - int hlen = 0, alen, iplen, len; + int hlen = 0, alen, iplen, len, isans; statp = &stat; (void) memset(statp, '\0', sizeof (struct __res_state)); @@ -414,14 +400,14 @@ blen = 0; sret = nss_packed_getkey(buffer, bufsize, &dbname, &dbop, &arg); if (sret != NSS_SUCCESS) { - __res_ndestroy(statp); + res_ndestroy(statp); return (NSS_ERROR); } if (ipnode) { /* initially only handle the simple cases */ if (arg.key.ipnode.flags != 0) { - __res_ndestroy(statp); + res_ndestroy(statp); return (NSS_ERROR); } name = arg.key.ipnode.name; @@ -439,11 +425,11 @@ pbuf->p_herrno = HOST_NOT_FOUND; pbuf->p_status = NSS_NOTFOUND; pbuf->data_len = 0; - __res_ndestroy(statp); + res_ndestroy(statp); return (NSS_NOTFOUND); } /* else lookup error - handle in general code */ - __res_ndestroy(statp); + res_ndestroy(statp); return (NSS_ERROR); } @@ -456,23 +442,23 @@ qdcount = ntohs(hp->qdcount); cp += HFIXEDSZ; if (qdcount != 1) { - __res_ndestroy(statp); + res_ndestroy(statp); return (NSS_ERROR); } n = dn_expand(bom, eom, cp, host, MAXHOSTNAMELEN); if (n < 0) { - __res_ndestroy(statp); + res_ndestroy(statp); return (NSS_ERROR); } else hlen = strlen(host); /* no host name is an error, return */ if (hlen <= 0) { - __res_ndestroy(statp); + res_ndestroy(statp); return (NSS_ERROR); } cp += n + QFIXEDSZ; if (cp > eom) { - __res_ndestroy(statp); + res_ndestroy(statp); return (NSS_ERROR); } while (ancount-- > 0 && cp < eom && blen < bsize) { @@ -482,9 +468,10 @@ * Check that the expanded name is either the * name we asked for or a learned alias. */ - if (strncasecmp(host, ans, hlen) != 0 && (alen == 0 || - name_is_alias(aliases, ans) == NSS_ERROR)) { - __res_ndestroy(statp); + if ((isans = strncasecmp(host, ans, hlen)) != 0 && + (alen == 0 || name_is_alias(aliases, ans) + == NSS_NOTFOUND)) { + res_ndestroy(statp); return (NSS_ERROR); /* spoof? */ } } @@ -494,7 +481,7 @@ cp += INT16SZ; class = ns_get16(cp); /* class */ cp += INT16SZ; - nttl = (nssuint_t)ns_get32(cp); /* ttl in sec */ + nttl = (nssuint_t)ns_get32(cp); /* ttl in sec */ if (nttl < ttl) ttl = nttl; cp += INT32SZ; @@ -507,34 +494,42 @@ eor = cp + n; if (type == T_CNAME) { /* - * The name we looked up is really an alias - * and the canonical name should be in the - * RDATA. A canonical name may have several - * aliases but an alias should only have one - * canonical name. However multiple CNAMEs and - * CNAME chains do exist! So for caching - * purposes maintain the alias as the host - * name, and the CNAME as an alias. + * The name looked up is really an alias and the + * canonical name should be in the RDATA. + * A canonical name may have several aliases but an + * alias should only have one canonical name. + * However multiple CNAMEs and CNAME chains do exist! + * + * Just error out on attempted buffer overflow exploit, + * generic code will syslog. + * */ n = dn_expand(bom, eor, cp, aname, MAXHOSTNAMELEN); - if (n > 0) { - len = strlen(aname); - if (len > 0) { + if (n > 0 && (len = strlen(aname)) > 0) { + if (isans == 0) { /* host matched ans. */ /* - * Just error out if there is an - * attempted buffer overflow exploit - * generic code will do a syslog + * Append host to alias list. */ - if (alen + len + 2 > NS_MAXMSG) { - __res_ndestroy(statp); + if (alen + hlen + 2 > NS_MAXMSG) { + res_ndestroy(statp); return (NSS_ERROR); } *apc++ = ' '; alen++; - (void) strlcpy(apc, aname, len + 1); - alen += len; - apc += len; + (void) strlcpy(apc, host, + NS_MAXMSG - alen); + alen += hlen; + apc += hlen; } + /* + * Overwrite host with canonical name. + */ + if (strlcpy(host, aname, MAXHOSTNAMELEN) >= + MAXHOSTNAMELEN) { + res_ndestroy(statp); + return (NSS_ERROR); + } + hlen = len; } cp += n; continue; @@ -552,7 +547,7 @@ af = (type == T_A ? AF_INET : AF_INET6); np = inet_ntop(af, (void *)cp, nbuf, INET6_ADDRSTRLEN); if (np == NULL) { - __res_ndestroy(statp); + res_ndestroy(statp); return (NSS_ERROR); } cp += n; @@ -563,7 +558,7 @@ if (alen > 0) len++; if (blen + len > bsize) { - __res_ndestroy(statp); + res_ndestroy(statp); return (NSS_ERROR); } (void) strlcpy(bptr, np, bsize - blen); @@ -589,7 +584,7 @@ /* still room? */ if (len + sizeof (nssuint_t) > pbuf->data_len) { /* sigh, no, what happened? */ - __res_ndestroy(statp); + res_ndestroy(statp); return (NSS_ERROR); } pbuf->ext_off = pbuf->data_off + len; @@ -597,6 +592,6 @@ pbuf->data_len = blen; pttl = (nssuint_t *)((void *)((char *)pbuf + pbuf->ext_off)); *pttl = ttl; - __res_ndestroy(statp); + res_ndestroy(statp); return (NSS_SUCCESS); }
--- a/usr/src/lib/pyzfs/common/allow.py Mon Sep 21 11:25:09 2009 -0400 +++ b/usr/src/lib/pyzfs/common/allow.py Mon Sep 21 11:26:40 2009 -0400 @@ -322,7 +322,7 @@ if sys.argv[2] == "-h": # hack to make "zfs allow -h" work usage() - ds = zfs.dataset.Dataset(sys.argv[2]) + ds = zfs.dataset.Dataset(sys.argv[2], snaps=False) p = dict() for (fs, raw) in ds.get_fsacl().items():
--- a/usr/src/pkgdefs/Makefile Mon Sep 21 11:25:09 2009 -0400 +++ b/usr/src/pkgdefs/Makefile Mon Sep 21 11:26:40 2009 -0400 @@ -118,6 +118,7 @@ SUNWatu \ SUNWaudiocmi \ SUNWaudiohd \ + SUNWaudiosolo \ SUNWaudiovia97 \ SUNWbfe \ SUNWcakr.i \
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/usr/src/pkgdefs/SUNWaudiosolo/Makefile Mon Sep 21 11:26:40 2009 -0400 @@ -0,0 +1,36 @@ +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# +# +# Copyright 2009 Sun Microsystems, Inc. All rights reserved. +# Use is subject to license terms. +# + +include ../Makefile.com + +TMPLFILES += postinstall preremove + +.KEEP_STATE: + +all: $(FILES) depend +install: all pkg + +include ../Makefile.targ +include ../Makefile.prtarg
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/usr/src/pkgdefs/SUNWaudiosolo/depend Mon Sep 21 11:26:40 2009 -0400 @@ -0,0 +1,50 @@ +# +# Copyright 2009 Sun Microsystems, Inc. All rights reserved. +# Use is subject to license terms. +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# +# This package information file defines software dependencies associated +# with the pkg. You can define three types of pkg dependencies with this file: +# P indicates a prerequisite for installation +# I indicates an incompatible package +# R indicates a reverse dependency +# <pkg.abbr> see pkginfo(4), PKG parameter +# <name> see pkginfo(4), NAME parameter +# <version> see pkginfo(4), VERSION parameter +# <arch> see pkginfo(4), ARCH parameter +# <type> <pkg.abbr> <name> +# (<arch>)<version> +# (<arch>)<version> +# ... +# <type> <pkg.abbr> <name> +# ... +# + +P SUNWcar Core Architecture, (Root) +P SUNWcakr Core Solaris Kernel Architecture (Root) +P SUNWkvm Core Architecture, (Kvm) +P SUNWcsr Core Solaris, (Root) +P SUNWckr Core Solaris Kernel (Root) +P SUNWcnetr Core Solaris Network Infrastructure (Root) +P SUNWcsu Core Solaris, (Usr) +P SUNWcsd Core Solaris Devices +P SUNWcsl Core Solaris Libraries +P SUNWaudd Audio Drivers
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/usr/src/pkgdefs/SUNWaudiosolo/pkginfo.tmpl Mon Sep 21 11:26:40 2009 -0400 @@ -0,0 +1,57 @@ +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# + +# +# Copyright 2009 Sun Microsystems, Inc. All rights reserved. +# Use is subject to license terms. +# + +# +# This required package information file describes characteristics of the +# package, such as package abbreviation, full package name, package version, +# and package architecture. +# +PKG="SUNWaudiosolo" +NAME="ESS Solo-1 Audio Driver" +ARCH="ISA" +VERSION="ONVERS,REV=0.0.0" +CATEGORY="system" +DESC="SunOS audio device driver for ESS Solo-1" +SUNW_PRODNAME="SunOS" +SUNW_PRODVERS="RELEASE/VERSION" +SUNW_PKGTYPE="root" +MAXINST="1000" +VENDOR="Sun Microsystems, Inc." +HOTLINE="Please contact your local service provider" +EMAIL="" +CLASSES="none preserve" +BASEDIR=/ +SUNW_PKGVERS="1.0" +SUNW_PKG_ALLZONES="true" +SUNW_PKG_HOLLOW="true" +SUNW_PKG_THISZONE="false" +#VSTOCK="<reserved by Release Engineering for package part #>" +#ISTATES="<developer defined>" +#RSTATES='<developer defined>' +#ULIMIT="<developer defined>" +#ORDER="<developer defined>" +#PSTAMP="<developer defined>" +#INTONLY="<developer defined>"
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/usr/src/pkgdefs/SUNWaudiosolo/postinstall.tmpl Mon Sep 21 11:26:40 2009 -0400 @@ -0,0 +1,34 @@ +#! /bin/sh +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# +# +# Copyright 2009 Sun Microsystems, Inc. All rights reserved. +# Use is subject to license terms. +# +# SUNWaudiols postinstall script + +include drv_utils + +AUDIOSOLO_ALIASES="\ + \"pci125d,1969\" \ + " + +pkg_drvadd -i "${AUDIOSOLO_ALIASES}" audiosolo || exit 1
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/usr/src/pkgdefs/SUNWaudiosolo/preremove.tmpl Mon Sep 21 11:26:40 2009 -0400 @@ -0,0 +1,30 @@ +#! /bin/sh +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# +# +# Copyright 2009 Sun Microsystems, Inc. All rights reserved. +# Use is subject to license terms. +# +# SUNWaudiosolo preremove script + +include drv_utils + +pkg_drvrem audiosolo || exit 1
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/usr/src/pkgdefs/SUNWaudiosolo/prototype_com Mon Sep 21 11:26:40 2009 -0400 @@ -0,0 +1,47 @@ +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# +# +# Copyright 2009 Sun Microsystems, Inc. All rights reserved. +# Use is subject to license terms. +# +# This required package information file contains a list of package contents. +# The 'pkgmk' command uses this file to identify the contents of a package +# and their location on the development machine when building the package. +# Can be created via a text editor or through use of the 'pkgproto' command. + +#!search <pathname pathname ...> # where to find pkg objects +#!include <filename> # include another 'prototype' file +#!default <mode> <owner> <group> # default used if not specified on entry +#!<param>=<value> # puts parameter in pkg environment + +# packaging files +i pkginfo +i copyright +i depend +i postinstall +i preremove +# +# source locations relative to the prototype file +# +# SUNWaudiosolo +# +d none kernel 755 root sys +d none kernel/drv 755 root sys
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/usr/src/pkgdefs/SUNWaudiosolo/prototype_i386 Mon Sep 21 11:26:40 2009 -0400 @@ -0,0 +1,48 @@ +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# +# +# Copyright 2009 Sun Microsystems, Inc. All rights reserved. +# Use is subject to license terms. +# +# This required package information file contains a list of package contents. +# The 'pkgmk' command uses this file to identify the contents of a package +# and their location on the development machine when building the package. +# Can be created via a text editor or through use of the 'pkgproto' command. + +#!search <pathname pathname ...> # where to find pkg objects +#!include <filename> # include another 'prototype' file +#!default <mode> <owner> <group> # default used if not specified on entry +#!<param>=<value> # puts parameter in pkg environment + +# +# Include ISA independent files (prototype_com) +# +!include prototype_com +# +# List files which are i386 specific here +# +# source locations relative to the prototype file +# +# SUNWaudiols +# +f none kernel/drv/audiosolo 755 root sys +d none kernel/drv/amd64 755 root sys +f none kernel/drv/amd64/audiosolo 755 root sys
--- a/usr/src/pkgdefs/SUNWcakr.i/prototype_com Mon Sep 21 11:25:09 2009 -0400 +++ b/usr/src/pkgdefs/SUNWcakr.i/prototype_com Mon Sep 21 11:26:40 2009 -0400 @@ -68,6 +68,7 @@ f none platform/i86pc/kernel/drv/acpippm 755 root sys f none platform/i86pc/kernel/drv/acpippm.conf 644 root sys f none platform/i86pc/kernel/drv/amd64/acpinex 755 root sys +f none platform/i86pc/kernel/drv/amd64/amd_iommu 755 root sys f none platform/i86pc/kernel/drv/amd64/ppm 755 root sys f none platform/i86pc/kernel/drv/amd64/isa 755 root sys f none platform/i86pc/kernel/drv/amd64/npe 755 root sys @@ -75,6 +76,8 @@ f none platform/i86pc/kernel/drv/amd64/pit_beep 755 root sys f none platform/i86pc/kernel/drv/amd64/rootnex 755 root sys f none platform/i86pc/kernel/drv/acpinex 755 root sys +f none platform/i86pc/kernel/drv/amd_iommu 755 root sys +f none platform/i86pc/kernel/drv/amd_iommu.conf 644 root sys f none platform/i86pc/kernel/drv/cpudrv 755 root sys f none platform/i86pc/kernel/drv/isa 755 root sys f none platform/i86pc/kernel/drv/npe 755 root sys
--- a/usr/src/pkgdefs/SUNWckr/prototype_i386 Mon Sep 21 11:25:09 2009 -0400 +++ b/usr/src/pkgdefs/SUNWckr/prototype_i386 Mon Sep 21 11:26:40 2009 -0400 @@ -68,8 +68,6 @@ f none kernel/drv/acpi_drv 755 root sys f none kernel/drv/acpi_drv.conf 644 root sys f none kernel/drv/acpi_toshiba 755 root sys -f none kernel/drv/amd_iommu 755 root sys -f none kernel/drv/amd_iommu.conf 644 root sys f none kernel/drv/bl 755 root sys f none kernel/drv/bmc 755 root sys f none kernel/drv/bmc.conf 644 root sys @@ -296,7 +294,6 @@ f none kernel/drv/amd64/arp 755 root sys f none kernel/drv/amd64/acpi_drv 755 root sys f none kernel/drv/amd64/acpi_toshiba 755 root sys -f none kernel/drv/amd64/amd_iommu 755 root sys f none kernel/drv/amd64/bl 755 root sys f none kernel/drv/amd64/bmc 755 root sys f none kernel/drv/amd64/bridge 755 root sys
--- a/usr/src/pkgdefs/SUNWhea/prototype_i386 Mon Sep 21 11:25:09 2009 -0400 +++ b/usr/src/pkgdefs/SUNWhea/prototype_i386 Mon Sep 21 11:26:40 2009 -0400 @@ -75,7 +75,6 @@ f none usr/include/ia32/sys/traptrace.h 644 root bin f none usr/include/sys/kdi_regs.h 644 root bin f none usr/include/stack_unwind.h 644 root bin -f none usr/include/sys/amd_iommu.h 644 root bin f none usr/include/sys/bootregs.h 644 root bin f none usr/include/sys/bootsvcs.h 644 root bin f none usr/include/sys/controlregs.h 644 root bin @@ -121,6 +120,7 @@ d none usr/platform/i86pc/include/sys 755 root bin f none usr/platform/i86pc/include/sys/asm_misc.h 644 root bin f none usr/platform/i86pc/include/sys/acpidev.h 644 root bin +f none usr/platform/i86pc/include/sys/amd_iommu.h 644 root bin f none usr/platform/i86pc/include/sys/clock.h 644 root bin f none usr/platform/i86pc/include/sys/cram.h 644 root bin f none usr/platform/i86pc/include/sys/debug_info.h 644 root bin
--- a/usr/src/pkgdefs/SUNWmdb/prototype_i386 Mon Sep 21 11:25:09 2009 -0400 +++ b/usr/src/pkgdefs/SUNWmdb/prototype_i386 Mon Sep 21 11:26:40 2009 -0400 @@ -81,6 +81,7 @@ f none usr/lib/mdb/kvm/amd64/nfs.so 555 root sys f none usr/lib/mdb/kvm/amd64/ptm.so 555 root sys f none usr/lib/mdb/kvm/amd64/random.so 555 root sys +f none usr/lib/mdb/kvm/amd64/rootnex.so 555 root sys f none usr/lib/mdb/kvm/amd64/s1394.so 555 root sys f none usr/lib/mdb/kvm/amd64/sata.so 555 root sys f none usr/lib/mdb/kvm/amd64/scsi_vhci.so 555 root sys @@ -117,6 +118,7 @@ f none usr/lib/mdb/kvm/nfs.so 555 root sys f none usr/lib/mdb/kvm/ptm.so 555 root sys f none usr/lib/mdb/kvm/random.so 555 root sys +f none usr/lib/mdb/kvm/rootnex.so 555 root sys f none usr/lib/mdb/kvm/s1394.so 555 root sys f none usr/lib/mdb/kvm/sata.so 555 root sys f none usr/lib/mdb/kvm/scsi_vhci.so 555 root sys
--- a/usr/src/pkgdefs/SUNWmdbr/prototype_i386 Mon Sep 21 11:25:09 2009 -0400 +++ b/usr/src/pkgdefs/SUNWmdbr/prototype_i386 Mon Sep 21 11:26:40 2009 -0400 @@ -50,6 +50,7 @@ f none kernel/kmdb/amd64/nfs 555 root sys f none kernel/kmdb/amd64/ptm 555 root sys f none kernel/kmdb/amd64/random 555 root sys +f none kernel/kmdb/amd64/rootnex 555 root sys f none kernel/kmdb/amd64/s1394 555 root sys f none kernel/kmdb/amd64/sata 555 root sys f none kernel/kmdb/amd64/scsi_vhci 555 root sys @@ -85,6 +86,7 @@ f none kernel/kmdb/nfs 555 root sys f none kernel/kmdb/ptm 555 root sys f none kernel/kmdb/random 555 root sys +f none kernel/kmdb/rootnex 555 root sys f none kernel/kmdb/s1394 555 root sys f none kernel/kmdb/sata 555 root sys f none kernel/kmdb/scsi_vhci 555 root sys
--- a/usr/src/pkgdefs/SUNWuedg/prototype_com Mon Sep 21 11:25:09 2009 -0400 +++ b/usr/src/pkgdefs/SUNWuedg/prototype_com Mon Sep 21 11:26:40 2009 -0400 @@ -2,9 +2,8 @@ # CDDL HEADER START # # The contents of this file are subject to the terms of the -# Common Development and Distribution License, Version 1.0 only -# (the "License"). You may not use this file except in compliance -# with the License. +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. # # You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE # or http://www.opensolaris.org/os/licensing. @@ -21,12 +20,10 @@ # # # -# Copyright 2002-2003 Sun Microsystems, Inc. All rights reserved. +# Copyright 2009 Sun Microsystems, Inc. All rights reserved. # Use is subject to license terms. # -#ident "%Z%%M% %I% %E% SMI" - # This required package information file contains a list of package contents. # The 'pkgmk' command uses this file to identify the contents of a package # and their location on the development machine when building the package. @@ -47,3 +44,4 @@ i preremove d none kernel 0755 root sys d none kernel/drv 0755 root sys +e none kernel/drv/usbser_edge.conf 0644 root sys
--- a/usr/src/tools/scripts/bfu.sh Mon Sep 21 11:25:09 2009 -0400 +++ b/usr/src/tools/scripts/bfu.sh Mon Sep 21 11:26:40 2009 -0400 @@ -7888,6 +7888,24 @@ rm -f $root/dev/rsr[0-9]* # + # Remove old amd_iommu driver + # + + # + # old: need to remove going forwards: + # + rm -f $root/kernel/drv/amd_iommu + rm -f $root/kernel/drv/amd_iommu.conf + rm -f $root/kernel/drv/amd64/amd_iommu + + # + # new: need to remove going backwards: + # + rm -f $root/platform/i86pc/kernel/drv/amd_iommu.conf + rm -f $root/platform/i86pc/kernel/drv/amd_iommu + rm -f $root/platform/i86pc/kernel/drv/amd64/amd_iommu + + # # The pkg* commands should not be used after this point and before # archive extraction as libcrypto/libssl may not be available. #
--- a/usr/src/uts/Makefile Mon Sep 21 11:25:09 2009 -0400 +++ b/usr/src/uts/Makefile Mon Sep 21 11:26:40 2009 -0400 @@ -177,7 +177,7 @@ # other directories that should be included and do not yet have the # necessary Makefile support. See 6414855.) # -DYNHDRDIRS = common/rpcsvc common/idmap +DYNHDRDIRS = common/rpcsvc common/idmap common/sys sparc_HDRDIRS= sun/sys i386_HDRDIRS= i86pc/vm i86xpv/vm
--- a/usr/src/uts/common/Makefile.files Mon Sep 21 11:25:09 2009 -0400 +++ b/usr/src/uts/common/Makefile.files Mon Sep 21 11:26:40 2009 -0400 @@ -452,6 +452,8 @@ AUDIOPCI_OBJS += audiopci.o +AUDIOSOLO_OBJS += audiosolo.o + AUDIOTS_OBJS += audiots.o AC97_OBJS += ac97.o ac97_ad.o ac97_alc.o ac97_cmi.o
--- a/usr/src/uts/common/Makefile.rules Mon Sep 21 11:25:09 2009 -0400 +++ b/usr/src/uts/common/Makefile.rules Mon Sep 21 11:26:40 2009 -0400 @@ -592,6 +592,10 @@ $(COMPILE.c) -o $@ $< $(CTFCONVERT_O) +$(OBJS_DIR)/%.o: $(UTSBASE)/common/io/audio/drv/audiosolo/%.c + $(COMPILE.c) -o $@ $< + $(CTFCONVERT_O) + $(OBJS_DIR)/%.o: $(UTSBASE)/common/io/audio/drv/audiots/%.c $(COMPILE.c) -o $@ $< $(CTFCONVERT_O) @@ -1862,6 +1866,9 @@ $(LINTS_DIR)/%.ln: $(UTSBASE)/common/io/audio/drv/audiop16x/%.c @($(LHEAD) $(LINT.c) $< $(LTAIL)) +$(LINTS_DIR)/%.ln: $(UTSBASE)/common/io/audio/drv/audiosolo/%.c + @($(LHEAD) $(LINT.c) $< $(LTAIL)) + $(LINTS_DIR)/%.ln: $(UTSBASE)/common/io/audio/drv/audiots/%.c @($(LHEAD) $(LINT.c) $< $(LTAIL))
--- a/usr/src/uts/common/fs/nfs/nfs4_vfsops.c Mon Sep 21 11:25:09 2009 -0400 +++ b/usr/src/uts/common/fs/nfs/nfs4_vfsops.c Mon Sep 21 11:26:40 2009 -0400 @@ -549,7 +549,7 @@ goto errout; } - nargs->addr = kmem_alloc(sizeof (struct netbuf), KM_SLEEP); + nargs->addr = kmem_zalloc(sizeof (struct netbuf), KM_SLEEP); userbufptr = STRUCT_FGETP(addr_tmp, buf); addr.len = STRUCT_FGET(addr_tmp, len); addr.buf = kmem_alloc(addr.len, KM_SLEEP);
--- a/usr/src/uts/common/fs/zfs/dsl_dataset.c Mon Sep 21 11:25:09 2009 -0400 +++ b/usr/src/uts/common/fs/zfs/dsl_dataset.c Mon Sep 21 11:26:40 2009 -0400 @@ -2017,6 +2017,10 @@ ds->ds_reserved); dsl_prop_nvlist_add_uint64(nv, ZFS_PROP_GUID, ds->ds_phys->ds_guid); + dsl_prop_nvlist_add_uint64(nv, ZFS_PROP_UNIQUE, + dsl_dataset_unique(ds)); + dsl_prop_nvlist_add_uint64(nv, ZFS_PROP_OBJSETID, + ds->ds_object); dsl_prop_nvlist_add_uint64(nv, ZFS_PROP_USERREFS, ds->ds_userrefs); dsl_prop_nvlist_add_uint64(nv, ZFS_PROP_DEFER_DESTROY, DS_IS_DEFER_DESTROY(ds) ? 1 : 0);
--- a/usr/src/uts/common/fs/zfs/spa.c Mon Sep 21 11:25:09 2009 -0400 +++ b/usr/src/uts/common/fs/zfs/spa.c Mon Sep 21 11:26:40 2009 -0400 @@ -3683,7 +3683,17 @@ if (vd->vdev_remove_wanted) { vd->vdev_remove_wanted = 0; vdev_set_state(vd, B_FALSE, VDEV_STATE_REMOVED, VDEV_AUX_NONE); - vdev_clear(spa, vd); + + /* + * We want to clear the stats, but we don't want to do a full + * vdev_clear() as that will cause us to throw away + * degraded/faulted state as well as attempt to reopen the + * device, all of which is a waste. + */ + vd->vdev_stat.vs_read_errors = 0; + vd->vdev_stat.vs_write_errors = 0; + vd->vdev_stat.vs_checksum_errors = 0; + vdev_state_dirty(vd->vdev_top); }
--- a/usr/src/uts/common/fs/zfs/vdev_disk.c Mon Sep 21 11:25:09 2009 -0400 +++ b/usr/src/uts/common/fs/zfs/vdev_disk.c Mon Sep 21 11:26:40 2009 -0400 @@ -430,12 +430,19 @@ * asynchronous removal of the device. Otherwise, probe the device and * make sure it's still accessible. */ - if (zio->io_error == EIO) { + if (zio->io_error == EIO && !vd->vdev_remove_wanted) { vdev_disk_t *dvd = vd->vdev_tsd; int state = DKIO_NONE; if (ldi_ioctl(dvd->vd_lh, DKIOCSTATE, (intptr_t)&state, FKIOCTL, kcred, NULL) == 0 && state != DKIO_INSERTED) { + /* + * We post the resource as soon as possible, instead of + * when the async removal actually happens, because the + * DE is using this information to discard previous I/O + * errors. + */ + zfs_post_remove(zio->io_spa, vd); vd->vdev_remove_wanted = B_TRUE; spa_async_request(zio->io_spa, SPA_ASYNC_REMOVE); }
--- a/usr/src/uts/common/fs/zfs/zfs_fm.c Mon Sep 21 11:25:09 2009 -0400 +++ b/usr/src/uts/common/fs/zfs/zfs_fm.c Mon Sep 21 11:26:40 2009 -0400 @@ -147,9 +147,7 @@ * not yet been asynchronously placed into the REMOVED * state. */ - if (zio->io_vd == vd && - !vdev_accessible(vd, zio) && - strcmp(subclass, FM_EREPORT_ZFS_PROBE_FAILURE) != 0) + if (zio->io_vd == vd && !vdev_accessible(vd, zio)) return; /* @@ -164,6 +162,15 @@ } } + /* + * For probe failure, we want to avoid posting ereports if we've + * already removed the device in the meantime. + */ + if (vd != NULL && + strcmp(subclass, FM_EREPORT_ZFS_PROBE_FAILURE) == 0 && + (vd->vdev_remove_wanted || vd->vdev_state == VDEV_STATE_REMOVED)) + return; + if ((ereport = fm_nvlist_create(NULL)) == NULL) return; @@ -338,6 +345,9 @@ nvlist_t *resource; char class[64]; + if (spa->spa_load_state == SPA_LOAD_TRYIMPORT) + return; + if ((resource = fm_nvlist_create(NULL)) == NULL) return;
--- a/usr/src/uts/common/inet/ip/ip.c Mon Sep 21 11:25:09 2009 -0400 +++ b/usr/src/uts/common/inet/ip/ip.c Mon Sep 21 11:26:40 2009 -0400 @@ -13947,8 +13947,8 @@ * ire_cache_lookup() can return ire of IRE_LOCAL in * transient cases. In such case, just drop the packet */ - if (ire->ire_type != IRE_CACHE) - goto drop; + if (ire != NULL && ire->ire_type != IRE_CACHE) + goto indiscard; } /*
--- a/usr/src/uts/common/inet/ipf/fil.c Mon Sep 21 11:25:09 2009 -0400 +++ b/usr/src/uts/common/inet/ipf/fil.c Mon Sep 21 11:26:40 2009 -0400 @@ -2151,6 +2151,7 @@ u_32_t *passp; { frentry_t *fr; + fr_info_t *fc; u_32_t pass; int out; ipf_stack_t *ifs = fin->fin_ifs; @@ -2164,13 +2165,51 @@ else #endif fin->fin_fr = ifs->ifs_ipfilter[out][ifs->ifs_fr_active]; - if (fin->fin_fr != NULL) + + /* + * If there are no rules loaded skip all checks and return. + */ + if (fin->fin_fr == NULL) { + + if ((pass & FR_NOMATCH)) { + IPF_BUMP(ifs->ifs_frstats[out].fr_nom); + } + + return (NULL); + } + + fc = &ifs->ifs_frcache[out][CACHE_HASH(fin)]; + READ_ENTER(&ifs->ifs_ipf_frcache); + if (!bcmp((char *)fin, (char *)fc, FI_CSIZE)) { + /* + * copy cached data so we can unlock the mutexes earlier. + */ + bcopy((char *)fc, (char *)fin, FI_COPYSIZE); + RWLOCK_EXIT(&ifs->ifs_ipf_frcache); + IPF_BUMP(ifs->ifs_frstats[out].fr_chit); + + if ((fr = fin->fin_fr) != NULL) { + IPF_BUMP(fr->fr_hits); + pass = fr->fr_flags; + } + } else { + RWLOCK_EXIT(&ifs->ifs_ipf_frcache); + pass = fr_scanlist(fin, ifs->ifs_fr_pass); + if (((pass & FR_KEEPSTATE) == 0) && + ((fin->fin_flx & FI_DONTCACHE) == 0)) { + WRITE_ENTER(&ifs->ifs_ipf_frcache); + bcopy((char *)fin, (char *)fc, FI_COPYSIZE); + RWLOCK_EXIT(&ifs->ifs_ipf_frcache); + } + + fr = fin->fin_fr; + } + if ((pass & FR_NOMATCH)) { IPF_BUMP(ifs->ifs_frstats[out].fr_nom); } - fr = fin->fin_fr; /* * Apply packets per second rate-limiting to a rule as required. @@ -3520,6 +3559,7 @@ int flushed = 0, set; WRITE_ENTER(&ifs->ifs_ipf_mutex); + bzero((char *)ifs->ifs_frcache, sizeof (ifs->ifs_frcache)); set = ifs->ifs_fr_active; if ((flags & FR_INACTIVE) == FR_INACTIVE) @@ -4504,6 +4544,7 @@ fp->fr_cksum += *p; WRITE_ENTER(&ifs->ifs_ipf_mutex); + bzero((char *)ifs->ifs_frcache, sizeof (ifs->ifs_frcache)); for (; (f = *ftail) != NULL; ftail = &f->fr_next) { if ((fp->fr_cksum != f->fr_cksum) ||
--- a/usr/src/uts/common/inet/ipf/ip_fil_solaris.c Mon Sep 21 11:25:09 2009 -0400 +++ b/usr/src/uts/common/inet/ipf/ip_fil_solaris.c Mon Sep 21 11:26:40 2009 -0400 @@ -260,6 +260,7 @@ ifs->ifs_fr_pass = (IPF_DEFAULT_PASS)|FR_NOMATCH; #endif + bzero((char *)ifs->ifs_frcache, sizeof(ifs->ifs_frcache)); MUTEX_INIT(&ifs->ifs_ipf_rw, "ipf rw mutex"); MUTEX_INIT(&ifs->ifs_ipf_timeoutlock, "ipf timeout lock mutex"); RWLOCK_INIT(&ifs->ifs_ipf_ipidfrag, "ipf IP NAT-Frag rwlock"); @@ -643,6 +644,8 @@ error = EPERM; else { WRITE_ENTER(&ifs->ifs_ipf_mutex); + bzero((char *)ifs->ifs_frcache, + sizeof (ifs->ifs_frcache)); error = COPYOUT((caddr_t)&ifs->ifs_fr_active, (caddr_t)data, sizeof(ifs->ifs_fr_active));
--- a/usr/src/uts/common/inet/ipf/netinet/ipf_stack.h Mon Sep 21 11:25:09 2009 -0400 +++ b/usr/src/uts/common/inet/ipf/netinet/ipf_stack.h Mon Sep 21 11:26:40 2009 -0400 @@ -45,6 +45,7 @@ zoneid_t ifs_zone; /* ipf module */ + fr_info_t ifs_frcache[2][8]; filterstats_t ifs_frstats[2]; frentry_t *ifs_ipfilter[2][2]; @@ -91,6 +92,7 @@ ipfmutex_t ifs_ipf_timeoutlock; ipfrwlock_t ifs_ipf_mutex; ipfrwlock_t ifs_ipf_global; + ipfrwlock_t ifs_ipf_frcache; ipfrwlock_t ifs_ip_poolrw; ipfrwlock_t ifs_ipf_frag; ipfrwlock_t ifs_ipf_state;
--- a/usr/src/uts/common/inet/ipf/solaris.c Mon Sep 21 11:25:09 2009 -0400 +++ b/usr/src/uts/common/inet/ipf/solaris.c Mon Sep 21 11:26:40 2009 -0400 @@ -3,7 +3,7 @@ * * See the IPFILTER.LICENCE file for details on licencing. * - * Copyright 2008 Sun Microsystems, Inc. All rights reserved. + * Copyright 2009 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. */ @@ -396,7 +396,7 @@ */ RWLOCK_INIT(&ifs->ifs_ipf_global, "ipf filter load/unload mutex"); RWLOCK_INIT(&ifs->ifs_ipf_mutex, "ipf filter rwlock"); - + RWLOCK_INIT(&ifs->ifs_ipf_frcache, "ipf cache rwlock"); ifs->ifs_netid = id; ifs->ifs_zone = net_getzoneidbynetid(id); ipf_kstat_init(ifs); @@ -553,6 +553,7 @@ RWLOCK_EXIT(&ifs->ifs_ipf_global); RW_DESTROY(&ifs->ifs_ipf_mutex); + RW_DESTROY(&ifs->ifs_ipf_frcache); RW_DESTROY(&ifs->ifs_ipf_global); KFREE(ifs);
--- a/usr/src/uts/common/inet/tcp/tcp.c Mon Sep 21 11:25:09 2009 -0400 +++ b/usr/src/uts/common/inet/tcp/tcp.c Mon Sep 21 11:26:40 2009 -0400 @@ -7834,12 +7834,11 @@ PRESERVE(tcp->tcp_family); if (tcp->tcp_family == AF_INET6) { - tcp->tcp_ipversion = IPV6_VERSION; tcp->tcp_mss = tcps->tcps_mss_def_ipv6; } else { - tcp->tcp_ipversion = IPV4_VERSION; tcp->tcp_mss = tcps->tcps_mss_def_ipv4; } + PRESERVE(tcp->tcp_ipversion); /* Init in tcp_init_values */ tcp->tcp_bound_if = 0; tcp->tcp_ipv6_recvancillary = 0; @@ -7983,7 +7982,7 @@ tcp->tcp_loopback_peer = NULL; /* Initialize the header template */ - if (tcp->tcp_ipversion == IPV4_VERSION) { + if (tcp->tcp_family == AF_INET) { err = tcp_header_init_ipv4(tcp); } else { err = tcp_header_init_ipv6(tcp); @@ -8048,9 +8047,19 @@ connp->conn_mlp_type = mlptSingle; connp->conn_ulp_labeled = !is_system_labeled(); ASSERT(tcp->tcp_iphc_len >= TCP_MAX_COMBINED_HEADER_LENGTH); + + /* + * tcp_do_get{sock,peer}name constructs the sockaddr from the + * ip header, and decides which header to use based on ip version. + * That operation happens outside the squeue, so we hold the lock + * here to ensure that the ip version and header remain consistent. + */ + mutex_enter(&connp->conn_lock); + tcp->tcp_ipversion = IPV4_VERSION; tcp->tcp_ipha = (ipha_t *)tcp->tcp_iphc; tcp->tcp_ip6h = NULL; - tcp->tcp_ipversion = IPV4_VERSION; + mutex_exit(&connp->conn_lock); + tcp->tcp_hdr_len = sizeof (ipha_t) + sizeof (tcph_t); tcp->tcp_tcp_hdr_len = sizeof (tcph_t); tcp->tcp_ip_hdr_len = sizeof (ipha_t); @@ -8120,12 +8129,21 @@ connp->conn_ulp_labeled = !is_system_labeled(); ASSERT(tcp->tcp_iphc_len >= TCP_MAX_COMBINED_HEADER_LENGTH); - tcp->tcp_ipversion = IPV6_VERSION; tcp->tcp_hdr_len = IPV6_HDR_LEN + sizeof (tcph_t); tcp->tcp_tcp_hdr_len = sizeof (tcph_t); tcp->tcp_ip_hdr_len = IPV6_HDR_LEN; + + /* + * tcp_do_get{sock,peer}name constructs the sockaddr from the + * ip header, and decides which header to use based on ip version. + * That operation happens outside the squeue, so we hold the lock + * here to ensure that the ip version and header remain consistent. + */ + mutex_enter(&connp->conn_lock); + tcp->tcp_ipversion = IPV6_VERSION; tcp->tcp_ip6h = (ip6_t *)tcp->tcp_iphc; tcp->tcp_ipha = NULL; + mutex_exit(&connp->conn_lock); /* Initialize the header template */ @@ -17884,12 +17902,14 @@ sin6->sin6_family = AF_INET6; if (tcp->tcp_state >= TCPS_BOUND) { sin6->sin6_port = tcp->tcp_lport; + mutex_enter(&tcp->tcp_connp->conn_lock); if (tcp->tcp_ipversion == IPV4_VERSION) { IN6_IPADDR_TO_V4MAPPED(tcp->tcp_ipha->ipha_src, &sin6->sin6_addr); } else { sin6->sin6_addr = tcp->tcp_ip6h->ip6_src; } + mutex_exit(&tcp->tcp_connp->conn_lock); } *salenp = sizeof (sin6_t); break; @@ -17930,10 +17950,12 @@ sin6->sin6_family = AF_INET6; sin6->sin6_port = tcp->tcp_fport; sin6->sin6_addr = tcp->tcp_remote_v6; + mutex_enter(&tcp->tcp_connp->conn_lock); if (tcp->tcp_ipversion == IPV6_VERSION) { sin6->sin6_flowinfo = tcp->tcp_ip6h->ip6_vcf & ~IPV6_VERS_AND_FLOW_MASK; } + mutex_exit(&tcp->tcp_connp->conn_lock); *salenp = sizeof (sin6_t); break; } @@ -26090,7 +26112,7 @@ in_port_t requested_port; ipaddr_t v4addr; in6_addr_t v6addr; - uint_t origipversion; + uint_t ipversion; int error = 0; ASSERT((uintptr_t)len <= (uintptr_t)INT_MAX); @@ -26104,7 +26126,6 @@ } return (-TOUTSTATE); } - origipversion = tcp->tcp_ipversion; ASSERT(sa != NULL && len != 0); @@ -26132,7 +26153,7 @@ return (EAFNOSUPPORT); } requested_port = ntohs(sin->sin_port); - tcp->tcp_ipversion = IPV4_VERSION; + ipversion = IPV4_VERSION; v4addr = sin->sin_addr.s_addr; IN6_IPADDR_TO_V4MAPPED(v4addr, &v6addr); break; @@ -26144,7 +26165,7 @@ return (EAFNOSUPPORT); } requested_port = ntohs(sin6->sin6_port); - tcp->tcp_ipversion = IN6_IS_ADDR_V4MAPPED(&sin6->sin6_addr) ? + ipversion = IN6_IS_ADDR_V4MAPPED(&sin6->sin6_addr) ? IPV4_VERSION : IPV6_VERSION; v6addr = sin6->sin6_addr; break; @@ -26161,9 +26182,9 @@ tcp->tcp_bound_source_v6 = v6addr; /* Check for change in ipversion */ - if (origipversion != tcp->tcp_ipversion) { + if (tcp->tcp_ipversion != ipversion) { ASSERT(tcp->tcp_family == AF_INET6); - error = tcp->tcp_ipversion == IPV6_VERSION ? + error = (ipversion == IPV6_VERSION) ? tcp_header_init_ipv6(tcp) : tcp_header_init_ipv4(tcp); if (error) { return (ENOMEM); @@ -27075,14 +27096,12 @@ sin = (sin_t *)&addr; *sin = sin_null; sin->sin_family = AF_INET; - tcp->tcp_ipversion = IPV4_VERSION; } else { ASSERT(tcp->tcp_family == AF_INET6); len = sizeof (sin6_t); sin6 = (sin6_t *)&addr; *sin6 = sin6_null; sin6->sin6_family = AF_INET6; - tcp->tcp_ipversion = IPV6_VERSION; } sa = (struct sockaddr *)&addr; }
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/usr/src/uts/common/io/audio/drv/audiosolo/audiosolo.c Mon Sep 21 11:26:40 2009 -0400 @@ -0,0 +1,1562 @@ +/* + * Copyright (c) 1999 Cameron Grant <cg@freebsd.org> + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +/* + * Copyright 2009 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +/* + * Copyright (C) 4Front Technologies 1996-2008. + */ + +#include <sys/audio/audio_driver.h> +#include <sys/note.h> +#include <sys/pci.h> +#include <sys/stdbool.h> + + +/* + * NB: The Solo-1 is a bit schizophrenic compared to most devices. + * It has two separate DMA engines for PCM data. The first can do + * either capture or playback, and supports various Sound Blaster + * compatibility features. The second is dedicated to playback. The + * two engines have very little in common when it comes to programming + * them. + * + * We configure engine 1 for record, and engine 2 for playback. Both + * are configured for 48 kHz stereo 16-bit signed PCM. + */ + +/* + * ESS Solo-1 only implements the low 24-bits on Audio1, and requires + * 64KB alignment. For Audio2, it implements the full 32-bit address + * space, but requires a 1MB address boundary. Audio1 is used for + * recording, and Audio2 is used for playback. + */ +static struct ddi_dma_attr dma_attr_audio1 = { + DMA_ATTR_VERSION, /* dma_attr_version */ + 0x0, /* dma_attr_addr_lo */ + 0x00ffffffU, /* dma_attr_addr_hi */ + 0xffff, /* dma_attr_count_max */ + 0x10000, /* dma_attr_align */ + 0x7f, /* dma_attr_burstsizes */ + 0x4, /* dma_attr_minxfer */ + 0xffff, /* dma_attr_maxxfer */ + 0xffff, /* dma_attr_seg */ + 0x1, /* dma_attr_sgllen */ + 0x1, /* dma_attr_granular */ + 0 /* dma_attr_flags */ +}; + +static struct ddi_dma_attr dma_attr_audio2 = { + DMA_ATTR_VERSION, /* dma_attr_version */ + 0x0, /* dma_attr_addr_lo */ + 0xffffffffU, /* dma_attr_addr_hi */ + 0xfff0, /* dma_attr_count_max */ + 0x100000, /* dma_attr_align */ + 0x7f, /* dma_attr_burstsizes */ + 0x4, /* dma_attr_minxfer */ + 0xfff0, /* dma_attr_maxxfer */ + 0xffff, /* dma_attr_seg */ + 0x1, /* dma_attr_sgllen */ + 0x1, /* dma_attr_granular */ + 0 /* dma_attr_flags */ +}; + +static ddi_device_acc_attr_t acc_attr = { + DDI_DEVICE_ATTR_V0, + DDI_STRUCTURE_LE_ACC, + DDI_STRICTORDER_ACC +}; + +static ddi_device_acc_attr_t buf_attr = { + DDI_DEVICE_ATTR_V0, + DDI_NEVERSWAP_ACC, + DDI_STRICTORDER_ACC +}; + + +/* + * For the sake of simplicity, this driver fixes a few parameters with + * constants. If you want these values to be tunable, upgrade to a + * nicer and newer device. This is all tuned for 100 Hz (10 + * millisecs) latency. + */ +#define SOLO_RATE 48000 +#define SOLO_INTRS 100 +#define SOLO_FRAGFR (SOLO_RATE / SOLO_INTRS) +#define SOLO_NFRAGS 8 +#define SOLO_NCHAN 2 +#define SOLO_SAMPSZ 2 +#define SOLO_FRAGSZ (SOLO_FRAGFR * (SOLO_NCHAN * SOLO_SAMPSZ)) +#define SOLO_BUFFR (SOLO_NFRAGS * SOLO_FRAGFR) +#define SOLO_BUFSZ (SOLO_NFRAGS * SOLO_FRAGSZ) + +#define INPUT_MIC 0 +#define INPUT_LINE 1 +#define INPUT_CD 2 +#define INPUT_AUX 3 +#define INPUT_MONO 4 +#define INSRCS 0x1f /* bits 0-4 */ + +#define DRVNAME "audiosolo" + +static const char *solo_insrcs[] = { + AUDIO_PORT_MIC, + AUDIO_PORT_LINEIN, + AUDIO_PORT_CD, + AUDIO_PORT_AUX1IN, + AUDIO_PORT_AUX2IN, /* this is really mono-in */ + NULL +}; + +typedef struct solo_regs { + ddi_acc_handle_t acch; + caddr_t base; +} solo_regs_t; + +typedef struct solo_engine { + struct solo_dev *dev; + audio_engine_t *engine; + ddi_dma_handle_t dmah; + ddi_acc_handle_t acch; + caddr_t kaddr; + uint32_t paddr; + + bool started; + uint64_t count; + uint16_t offset; + int syncdir; + int format; + bool swapped; + + void (*start)(struct solo_engine *); + void (*stop)(struct solo_engine *); + void (*update)(struct solo_engine *); +} solo_engine_t; + +typedef enum { + CTL_FRONT = 0, + CTL_VOLUME, + CTL_MIC, + CTL_LINE, + CTL_CD, + CTL_AUX, + CTL_MONO, + CTL_MICBOOST, + CTL_RECGAIN, + CTL_RECSRC, + CTL_MONSRC, + CTL_SPEAKER, + CTL_LOOPBACK, + CTL_NUM, /* must be last */ +} solo_ctrl_num_t; + +typedef struct solo_ctrl { + struct solo_dev *dev; + audio_ctrl_t *ctrl; + solo_ctrl_num_t num; + uint64_t val; +} solo_ctrl_t; + +typedef struct solo_dev { + dev_info_t *dip; + audio_dev_t *adev; + kmutex_t mutex; + ddi_intr_handle_t ihandle; + + bool suspended; + + /* + * Audio engines + */ + solo_engine_t rec; + solo_engine_t play; + uint32_t last_capture; + + /* + * Controls. + */ + solo_ctrl_t ctrls[CTL_NUM]; + + /* + * Mapped registers + */ + ddi_acc_handle_t pcih; + solo_regs_t io; + solo_regs_t sb; + solo_regs_t vc; + +} solo_dev_t; + +/* + * Common code for the pcm function + * + * solo_cmd write a single byte to the CMD port. + * solo_cmd1 write a CMD + 1 byte arg + * ess_get_byte returns a single byte from the DSP data port + * + * solo_write is actually solo_cmd1 + * solo_read access ext. regs via solo_cmd(0xc0, reg) followed by solo_get_byte + */ + +#define PORT_RD8(port, regno) \ + ddi_get8(port.acch, (void *)(port.base + (regno))) +#define PORT_RD16(port, regno) \ + ddi_get16(port.acch, (void *)(port.base + (regno))) +#define PORT_RD32(port, regno) \ + ddi_get32(port.acch, (void *)(port.base + (regno))) +#define PORT_WR8(port, regno, data) \ + ddi_put8(port.acch, (void *)(port.base + (regno)), data) +#define PORT_WR16(port, regno, data) \ + ddi_put16(port.acch, (void *)(port.base + (regno)), data) +#define PORT_WR32(port, regno, data) \ + ddi_put32(port.acch, (void *)(port.base + (regno)), data) + +static bool +solo_dspready(solo_dev_t *dev) +{ + return ((PORT_RD8(dev->sb, 0xc) & 0x80) == 0 ? true : false); +} + +static bool +solo_dspwr(solo_dev_t *dev, uint8_t val) +{ + int i; + + for (i = 0; i < 1000; i++) { + if (solo_dspready(dev)) { + PORT_WR8(dev->sb, 0xc, val); + return (true); + } + if (i > 10) + drv_usecwait((i > 100)? 1000 : 10); + } + audio_dev_warn(dev->adev, "solo_dspwr(0x%02x) timed out", val); + return (false); +} + +static bool +solo_cmd(solo_dev_t *dev, uint8_t val) +{ + return (solo_dspwr(dev, val)); +} + +static void +solo_cmd1(solo_dev_t *dev, uint8_t cmd, uint8_t val) +{ + if (solo_dspwr(dev, cmd)) { + (void) solo_dspwr(dev, val); + } +} + +static void +solo_setmixer(solo_dev_t *dev, uint8_t port, uint8_t value) +{ + PORT_WR8(dev->sb, 0x4, port); /* Select register */ + drv_usecwait(10); + PORT_WR8(dev->sb, 0x5, value); + drv_usecwait(10); +} + +static uint8_t +solo_getmixer(solo_dev_t *dev, uint8_t port) +{ + uint8_t val; + + PORT_WR8(dev->sb, 0x4, port); /* Select register */ + drv_usecwait(10); + val = PORT_RD8(dev->sb, 0x5); + drv_usecwait(10); + + return (val); +} + +static uint8_t +solo_get_byte(solo_dev_t *dev) +{ + for (int i = 1000; i > 0; i--) { + if (PORT_RD8(dev->sb, 0xc) & 0x40) + return (PORT_RD8(dev->sb, 0xa)); + else + drv_usecwait(20); + } + audio_dev_warn(dev->adev, "timeout waiting to read DSP port"); + return (0xff); +} + +static void +solo_write(solo_dev_t *dev, uint8_t reg, uint8_t val) +{ + solo_cmd1(dev, reg, val); +} + +static uint8_t +solo_read(solo_dev_t *dev, uint8_t reg) +{ + if (solo_cmd(dev, 0xc0) && solo_cmd(dev, reg)) { + return (solo_get_byte(dev)); + } + return (0xff); +} + +static bool +solo_reset_dsp(solo_dev_t *dev) +{ + PORT_WR8(dev->sb, 0x6, 3); + drv_usecwait(100); + PORT_WR8(dev->sb, 0x6, 0); + if (solo_get_byte(dev) != 0xAA) { + audio_dev_warn(dev->adev, "solo_reset_dsp failed"); + return (false); /* Sorry */ + } + return (true); +} + +static uint_t +solo_intr(caddr_t arg1, caddr_t arg2) +{ + solo_dev_t *dev = (void *)arg1; + audio_engine_t *prod = NULL; + audio_engine_t *cons = NULL; + uint8_t status; + uint_t rv = DDI_INTR_UNCLAIMED; + + _NOTE(ARGUNUSED(arg2)); + + mutex_enter(&dev->mutex); + + if (dev->suspended) { + mutex_exit(&dev->mutex); + return (rv); + } + + status = PORT_RD8(dev->io, 0x7); + if (status & 0x20) { + rv = DDI_INTR_CLAIMED; + cons = dev->play.engine; + /* ack the interrupt */ + solo_setmixer(dev, 0x7a, solo_getmixer(dev, 0x7a) & ~0x80); + } + + if (status & 0x10) { + rv = DDI_INTR_CLAIMED; + prod = dev->rec.engine; + /* ack the interrupt */ + (void) PORT_RD8(dev->sb, 0xe); + } + mutex_exit(&dev->mutex); + + if (cons) { + audio_engine_consume(cons); + } + + if (prod) { + audio_engine_produce(prod); + } + + return (rv); +} + +static uint8_t +solo_mixer_scale(solo_dev_t *dev, solo_ctrl_num_t num) +{ + uint32_t l, r; + uint64_t value = dev->ctrls[num].val; + + l = (value >> 8) & 0xff; + r = value & 0xff; + + l = (l * 15) / 100; + r = (r * 15) / 100; + return ((uint8_t)((l << 4) | (r))); +} + +static void +solo_configure_mixer(solo_dev_t *dev) +{ + uint32_t v; + uint32_t mon, rec; + + /* + * We disable hardware volume control (i.e. async updates to volume). + * We could in theory support this, but making it work right can be + * tricky, and we doubt it is widely used. + */ + solo_setmixer(dev, 0x64, solo_getmixer(dev, 0x64) | 0xc); + solo_setmixer(dev, 0x66, 0); + + /* master volume has 6 bits per channel, bit 6 indicates mute */ + /* left */ + v = (dev->ctrls[CTL_FRONT].val >> 8) & 0xff; + v = v ? (v * 63) / 100 : 64; + solo_setmixer(dev, 0x60, v & 0xff); + + /* right */ + v = dev->ctrls[CTL_FRONT].val & 0xff; + v = v ? (v * 63) / 100 : 64; + solo_setmixer(dev, 0x62, v & 0xff); + + v = solo_mixer_scale(dev, CTL_VOLUME); + v = v | (v << 4); + solo_setmixer(dev, 0x7c, v & 0xff); + solo_setmixer(dev, 0x14, v & 0xff); + + mon = dev->ctrls[CTL_MONSRC].val; + rec = dev->ctrls[CTL_RECSRC].val; + + /* + * The Solo-1 has dual stereo mixers (one for input and one for output), + * with separate volume controls for each. + */ + v = solo_mixer_scale(dev, CTL_MIC); + solo_setmixer(dev, 0x68, rec & (1 << INPUT_MIC) ? v : 0); + solo_setmixer(dev, 0x1a, mon & (1 << INPUT_MIC) ? v : 0); + + v = solo_mixer_scale(dev, CTL_LINE); + solo_setmixer(dev, 0x6e, rec & (1 << INPUT_LINE) ? v : 0); + solo_setmixer(dev, 0x3e, mon & (1 << INPUT_LINE) ? v : 0); + + v = solo_mixer_scale(dev, CTL_CD); + solo_setmixer(dev, 0x6a, rec & (1 << INPUT_CD) ? v : 0); + solo_setmixer(dev, 0x38, mon & (1 << INPUT_CD) ? v : 0); + + v = solo_mixer_scale(dev, CTL_AUX); + solo_setmixer(dev, 0x6c, rec & (1 << INPUT_AUX) ? v : 0); + solo_setmixer(dev, 0x3a, mon & (1 << INPUT_AUX) ? v : 0); + + v = solo_mixer_scale(dev, CTL_MONO); + v = v | (v << 4); + solo_setmixer(dev, 0x6f, rec & (1 << INPUT_MONO) ? v : 0); + solo_setmixer(dev, 0x6d, mon & (1 << INPUT_MONO) ? v : 0); + + if (dev->ctrls[CTL_MICBOOST].val) { + solo_setmixer(dev, 0x7d, solo_getmixer(dev, 0x7d) | 0x8); + } else { + solo_setmixer(dev, 0x7d, solo_getmixer(dev, 0x7d) & ~(0x8)); + } + + v = solo_mixer_scale(dev, CTL_RECGAIN); + v = v | (v << 4); + solo_write(dev, 0xb4, v & 0xff); + + v = dev->ctrls[CTL_SPEAKER].val & 0xff; + v = (v * 7) / 100; + solo_setmixer(dev, 0x3c, v & 0xff); + + if (dev->ctrls[CTL_LOOPBACK].val) { + /* record-what-you-hear mode */ + solo_setmixer(dev, 0x1c, 0x3); + } else { + /* use record mixer */ + solo_setmixer(dev, 0x1c, 0x5); + } + +} + +static int +solo_set_mixsrc(void *arg, uint64_t val) +{ + solo_ctrl_t *pc = arg; + solo_dev_t *dev = pc->dev; + + if ((val & ~INSRCS) != 0) + return (EINVAL); + + mutex_enter(&dev->mutex); + pc->val = val; + if (!dev->suspended) + solo_configure_mixer(dev); + mutex_exit(&dev->mutex); + return (0); +} + +static int +solo_set_mono(void *arg, uint64_t val) +{ + solo_ctrl_t *pc = arg; + solo_dev_t *dev = pc->dev; + + val &= 0xff; + if (val > 100) + return (EINVAL); + + val = (val & 0xff) | ((val & 0xff) << 8); + + mutex_enter(&dev->mutex); + pc->val = val; + if (!dev->suspended) + solo_configure_mixer(dev); + mutex_exit(&dev->mutex); + return (0); +} + +static int +solo_set_stereo(void *arg, uint64_t val) +{ + solo_ctrl_t *pc = arg; + solo_dev_t *dev = pc->dev; + uint8_t l; + uint8_t r; + + l = (val & 0xff00) >> 8; + r = val & 0xff; + + if ((l > 100) || (r > 100)) + return (EINVAL); + + mutex_enter(&dev->mutex); + pc->val = val; + if (!dev->suspended) + solo_configure_mixer(dev); + mutex_exit(&dev->mutex); + return (0); +} + +static int +solo_set_bool(void *arg, uint64_t val) +{ + solo_ctrl_t *pc = arg; + solo_dev_t *dev = pc->dev; + + mutex_enter(&dev->mutex); + pc->val = val; + if (!dev->suspended) + solo_configure_mixer(dev); + mutex_exit(&dev->mutex); + return (0); +} + +static int +solo_get_value(void *arg, uint64_t *val) +{ + solo_ctrl_t *pc = arg; + solo_dev_t *dev = pc->dev; + + mutex_enter(&dev->mutex); + *val = pc->val; + mutex_exit(&dev->mutex); + return (0); +} + +#define PLAYCTL (AUDIO_CTRL_FLAG_RW | AUDIO_CTRL_FLAG_PLAY) +#define RECCTL (AUDIO_CTRL_FLAG_RW | AUDIO_CTRL_FLAG_REC) +#define MONCTL (AUDIO_CTRL_FLAG_RW | AUDIO_CTRL_FLAG_MONITOR) +#define PCMVOL (PLAYCTL | AUDIO_CTRL_FLAG_PCMVOL) +#define MAINVOL (PLAYCTL | AUDIO_CTRL_FLAG_MAINVOL) +#define RECVOL (RECCTL | AUDIO_CTRL_FLAG_RECVOL) + +static void +solo_alloc_ctrl(solo_dev_t *dev, uint32_t num, uint64_t val) +{ + audio_ctrl_desc_t desc; + audio_ctrl_wr_t fn; + solo_ctrl_t *pc; + + bzero(&desc, sizeof (desc)); + + pc = &dev->ctrls[num]; + pc->num = num; + pc->dev = dev; + + switch (num) { + case CTL_VOLUME: + desc.acd_name = AUDIO_CTRL_ID_VOLUME; + desc.acd_type = AUDIO_CTRL_TYPE_MONO; + desc.acd_minvalue = 0; + desc.acd_maxvalue = 100; + desc.acd_flags = PCMVOL; + fn = solo_set_mono; + break; + + case CTL_FRONT: + desc.acd_name = AUDIO_CTRL_ID_LINEOUT; + desc.acd_type = AUDIO_CTRL_TYPE_STEREO; + desc.acd_minvalue = 0; + desc.acd_maxvalue = 100; + desc.acd_flags = MAINVOL; + fn = solo_set_stereo; + break; + + case CTL_SPEAKER: + desc.acd_name = AUDIO_CTRL_ID_SPEAKER; + desc.acd_type = AUDIO_CTRL_TYPE_MONO; + desc.acd_minvalue = 0; + desc.acd_maxvalue = 100; + desc.acd_flags = MAINVOL; + fn = solo_set_mono; + break; + + case CTL_MIC: + desc.acd_name = AUDIO_CTRL_ID_MIC; + desc.acd_type = AUDIO_CTRL_TYPE_STEREO; + desc.acd_minvalue = 0; + desc.acd_maxvalue = 100; + desc.acd_flags = RECVOL; + fn = solo_set_stereo; + break; + + case CTL_LINE: + desc.acd_name = AUDIO_CTRL_ID_LINEIN; + desc.acd_type = AUDIO_CTRL_TYPE_STEREO; + desc.acd_minvalue = 0; + desc.acd_maxvalue = 100; + desc.acd_flags = RECVOL; + fn = solo_set_stereo; + break; + + case CTL_CD: + desc.acd_name = AUDIO_CTRL_ID_CD; + desc.acd_type = AUDIO_CTRL_TYPE_STEREO; + desc.acd_minvalue = 0; + desc.acd_maxvalue = 100; + desc.acd_flags = RECVOL; + fn = solo_set_stereo; + break; + + case CTL_AUX: + desc.acd_name = AUDIO_CTRL_ID_AUX1IN; + desc.acd_type = AUDIO_CTRL_TYPE_STEREO; + desc.acd_minvalue = 0; + desc.acd_maxvalue = 100; + desc.acd_flags = RECVOL; + fn = solo_set_stereo; + break; + + case CTL_MONO: + desc.acd_name = AUDIO_CTRL_ID_AUX2IN; + desc.acd_type = AUDIO_CTRL_TYPE_MONO; + desc.acd_minvalue = 0; + desc.acd_maxvalue = 100; + desc.acd_flags = RECVOL; + fn = solo_set_mono; + break; + + case CTL_RECSRC: + desc.acd_name = AUDIO_CTRL_ID_RECSRC; + desc.acd_type = AUDIO_CTRL_TYPE_ENUM; + desc.acd_minvalue = INSRCS; + desc.acd_maxvalue = INSRCS; + desc.acd_flags = RECCTL | AUDIO_CTRL_FLAG_MULTI; + for (int i = 0; solo_insrcs[i]; i++) { + desc.acd_enum[i] = solo_insrcs[i]; + } + fn = solo_set_mixsrc; + break; + + case CTL_MONSRC: + desc.acd_name = AUDIO_CTRL_ID_MONSRC; + desc.acd_type = AUDIO_CTRL_TYPE_ENUM; + desc.acd_minvalue = INSRCS; + desc.acd_maxvalue = INSRCS; + desc.acd_flags = MONCTL | AUDIO_CTRL_FLAG_MULTI; + for (int i = 0; solo_insrcs[i]; i++) { + desc.acd_enum[i] = solo_insrcs[i]; + } + fn = solo_set_mixsrc; + break; + + case CTL_MICBOOST: + desc.acd_name = AUDIO_CTRL_ID_MICBOOST; + desc.acd_type = AUDIO_CTRL_TYPE_BOOLEAN; + desc.acd_minvalue = 0; + desc.acd_maxvalue = 1; + desc.acd_flags = RECCTL; + fn = solo_set_bool; + break; + + case CTL_LOOPBACK: + desc.acd_name = AUDIO_CTRL_ID_LOOPBACK; + desc.acd_type = AUDIO_CTRL_TYPE_BOOLEAN; + desc.acd_minvalue = 0; + desc.acd_maxvalue = 1; + desc.acd_flags = RECCTL; + fn = solo_set_bool; + break; + + case CTL_RECGAIN: + desc.acd_name = AUDIO_CTRL_ID_RECGAIN; + desc.acd_type = AUDIO_CTRL_TYPE_STEREO; + desc.acd_minvalue = 0; + desc.acd_maxvalue = 100; + desc.acd_flags = RECCTL; + fn = solo_set_stereo; + break; + } + + pc->val = val; + pc->ctrl = audio_dev_add_control(dev->adev, &desc, + solo_get_value, fn, pc); +} + +static bool +solo_add_controls(solo_dev_t *dev) +{ + solo_alloc_ctrl(dev, CTL_VOLUME, 0x4b); + solo_alloc_ctrl(dev, CTL_FRONT, 0x5a5a); + solo_alloc_ctrl(dev, CTL_SPEAKER, 0x4b); + solo_alloc_ctrl(dev, CTL_MIC, 0x3232); + solo_alloc_ctrl(dev, CTL_LINE, 0x4b4b); + solo_alloc_ctrl(dev, CTL_CD, 0x4b4b); + solo_alloc_ctrl(dev, CTL_AUX, 0); + solo_alloc_ctrl(dev, CTL_MONO, 0); + solo_alloc_ctrl(dev, CTL_RECSRC, (1U << INPUT_MIC)); + solo_alloc_ctrl(dev, CTL_MONSRC, 0); + solo_alloc_ctrl(dev, CTL_RECGAIN, 0x4b4b); + solo_alloc_ctrl(dev, CTL_MICBOOST, 1); + solo_alloc_ctrl(dev, CTL_LOOPBACK, 0); + + return (true); +} + + +/* utility functions for ESS */ +static uint8_t +solo_calcfilter(int spd) +{ + int cutoff; + + cutoff = (spd * 9 * 82) / 20; + return (256 - (7160000 / cutoff)); +} + +static void +solo_aud1_update(solo_engine_t *e) +{ + solo_dev_t *dev = e->dev; + uint16_t offset, n; + uint32_t ptr; + uint32_t count; + uint32_t diff; + + ASSERT(mutex_owned(&dev->mutex)); + + /* + * During recording, this register is known to give back + * garbage if it's not quiescent while being read. This hack + * attempts to work around it. + */ + ptr = PORT_RD32(dev->vc, 0); + count = PORT_RD16(dev->vc, 4); + diff = e->paddr + SOLO_BUFSZ - ptr - count; + if ((diff > 3) || (ptr < e->paddr) || + (ptr >= (e->paddr + SOLO_BUFSZ))) { + ptr = dev->last_capture; + } else { + dev->last_capture = ptr; + } + offset = ptr - e->paddr; + offset /= (SOLO_NCHAN * SOLO_SAMPSZ); + + n = offset >= e->offset ? + offset - e->offset : + offset + SOLO_BUFSZ - e->offset; + + e->offset = offset; + e->count += n / (SOLO_NCHAN * SOLO_SAMPSZ); +} + +static void +solo_aud1_start(solo_engine_t *e) +{ + solo_dev_t *dev = e->dev; + int len; + uint32_t v; + + ASSERT(mutex_owned(&dev->mutex)); + + len = SOLO_FRAGSZ / 2; + len = -len; + + /* sample rate - 48 kHz */ + solo_write(dev, 0xa1, 0xf0); + /* filter cutoff */ + solo_write(dev, 0xa2, solo_calcfilter(SOLO_RATE)); + + + /* mono/stereo - bit 0 set, bit 1 clear */ + solo_write(dev, 0xa8, (solo_read(dev, 0xa8) & ~0x03) | 1); + + (void) solo_cmd(dev, 0xd3); /* turn off DAC1 output */ + + /* setup fifo for signed 16-bit stereo */ + solo_write(dev, 0xb7, 0x71); + solo_write(dev, 0xb7, 0xbc); + + v = solo_mixer_scale(dev, CTL_RECGAIN); + v = v | (v << 4); + solo_write(dev, 0xb4, v & 0xff); + + PORT_WR8(dev->vc, 0x8, 0xc4); /* command */ + PORT_WR8(dev->vc, 0xd, 0xff); /* clear DMA */ + PORT_WR8(dev->vc, 0xf, 0x01); /* stop DMA */ + + PORT_WR8(dev->vc, 0xd, 0xff); /* reset */ + PORT_WR8(dev->vc, 0xf, 0x01); /* mask */ + PORT_WR8(dev->vc, 0xb, 0x14); /* mode */ + + PORT_WR32(dev->vc, 0x0, e->paddr); + PORT_WR16(dev->vc, 0x4, SOLO_BUFSZ - 1); + + /* transfer length low, high */ + solo_write(dev, 0xa4, len & 0x00ff); + solo_write(dev, 0xa5, (len & 0xff00) >> 8); + + /* autoinit, dma dir, go for it */ + solo_write(dev, 0xb8, 0x0f); + PORT_WR8(dev->vc, 0xf, 0); /* start DMA */ + + dev->last_capture = e->paddr; +} + +static void +solo_aud1_stop(solo_engine_t *e) +{ + solo_dev_t *dev = e->dev; + + /* NB: We might be in quiesce, without a lock held */ + solo_write(dev, 0xb8, solo_read(dev, 0xb8) & ~0x01); +} + +static void +solo_aud2_update(solo_engine_t *e) +{ + solo_dev_t *dev = e->dev; + uint16_t offset = 0, n; + + ASSERT(mutex_owned(&dev->mutex)); + + offset = SOLO_BUFSZ - PORT_RD16(dev->io, 0x4); + offset /= (SOLO_NCHAN * SOLO_SAMPSZ); + + n = offset >= e->offset ? + offset - e->offset : + offset + SOLO_BUFFR - e->offset; + + e->offset = offset; + e->count += n; +} + +static void +solo_aud2_start(solo_engine_t *e) +{ + solo_dev_t *dev = e->dev; + int len; + uint32_t v; + + ASSERT(mutex_owned(&dev->mutex)); + + len = SOLO_FRAGSZ / 2; + len = -len; + + /* program transfer type */ + solo_setmixer(dev, 0x78, 0x10); + /* sample rate - 48 kHz */ + solo_setmixer(dev, 0x70, 0xf0); + solo_setmixer(dev, 0x72, solo_calcfilter(SOLO_RATE)); + /* transfer length low & high */ + solo_setmixer(dev, 0x74, len & 0x00ff); + solo_setmixer(dev, 0x76, (len & 0xff00) >> 8); + /* enable irq, set signed 16-bit stereo format */ + solo_setmixer(dev, 0x7a, 0x47); + + PORT_WR8(dev->io, 0x6, 0); + PORT_WR32(dev->io, 0x0, e->paddr); + PORT_WR16(dev->io, 0x4, SOLO_BUFSZ); + + /* this crazy initialization appears to help with fifo weirdness */ + /* start the engine running */ + solo_setmixer(dev, 0x78, 0x92); + drv_usecwait(10); + solo_setmixer(dev, 0x78, 0x93); + + PORT_WR8(dev->io, 0x6, 0x0a); /* autoinit, enable */ + + v = solo_mixer_scale(dev, CTL_VOLUME); + v = v | (v << 4); + solo_setmixer(dev, 0x7c, v & 0xff); +} + +static void +solo_aud2_stop(solo_engine_t *e) +{ + solo_dev_t *dev = e->dev; + + /* NB: We might be in quiesce, without a lock held */ + PORT_WR8(dev->io, 0x6, 0); + solo_setmixer(dev, 0x78, solo_getmixer(dev, 0x78) & ~0x03); +} + +/* + * Audio entry points. + */ +static int +solo_format(void *arg) +{ + solo_engine_t *e = arg; + return (e->format); +} + +static int +solo_channels(void *arg) +{ + _NOTE(ARGUNUSED(arg)); + return (SOLO_NCHAN); +} + +static int +solo_rate(void *arg) +{ + _NOTE(ARGUNUSED(arg)); + return (SOLO_RATE); +} + +static size_t +solo_qlen(void *arg) +{ + _NOTE(ARGUNUSED(arg)); + return (0); +} + +static void +solo_chinfo(void *arg, int chan, unsigned *offset, unsigned *incr) +{ + solo_engine_t *e = arg; + + if (e->swapped) { + *offset = !chan; + } else { + *offset = chan; + } + *incr = 2; +} + +static void +solo_sync(void *arg, unsigned nframes) +{ + solo_engine_t *e = arg; + + _NOTE(ARGUNUSED(nframes)); + + (void) ddi_dma_sync(e->dmah, 0, 0, e->syncdir); +} + + +static uint64_t +solo_count(void *arg) +{ + solo_engine_t *e = arg; + solo_dev_t *dev = e->dev; + uint64_t count; + + mutex_enter(&dev->mutex); + if (!dev->suspended) + e->update(e); + count = e->count; + mutex_exit(&dev->mutex); + + return (count); +} + +static int +solo_open(void *arg, int f, unsigned *ffr, unsigned *nfr, caddr_t *buf) +{ + solo_engine_t *e = arg; + solo_dev_t *dev = e->dev; + + _NOTE(ARGUNUSED(f)); + + /* NB: For simplicity, we just fix the interrupt rate at 100 Hz */ + *ffr = SOLO_FRAGFR; + *nfr = SOLO_NFRAGS; + *buf = e->kaddr; + + mutex_enter(&dev->mutex); + e->started = false; + e->count = 0; + mutex_exit(&dev->mutex); + + return (0); +} + +void +solo_close(void *arg) +{ + solo_engine_t *e = arg; + solo_dev_t *dev = e->dev; + + mutex_enter(&dev->mutex); + if (!dev->suspended) + e->stop(e); + e->started = false; + mutex_exit(&dev->mutex); +} + + +static int +solo_start(void *arg) +{ + solo_engine_t *e = arg; + solo_dev_t *dev = e->dev; + + mutex_enter(&dev->mutex); + if (!e->started) { + if (!dev->suspended) + e->start(e); + e->started = true; + } + mutex_exit(&dev->mutex); + + return (0); +} + +static void +solo_stop(void *arg) +{ + solo_engine_t *e = arg; + solo_dev_t *dev = e->dev; + + mutex_enter(&dev->mutex); + if (e->started) { + if (!dev->suspended) + e->stop(e); + e->started = false; + } + mutex_exit(&dev->mutex); + +} + +static audio_engine_ops_t solo_engine_ops = { + AUDIO_ENGINE_VERSION, + solo_open, + solo_close, + solo_start, + solo_stop, + solo_count, + solo_format, + solo_channels, + solo_rate, + solo_sync, + solo_qlen, + solo_chinfo, +}; + +static void +solo_release_resources(solo_dev_t *dev) +{ + if (dev->ihandle != NULL) { + (void) ddi_intr_disable(dev->ihandle); + (void) ddi_intr_remove_handler(dev->ihandle); + (void) ddi_intr_free(dev->ihandle); + mutex_destroy(&dev->mutex); + } + + if (dev->io.acch != NULL) { + ddi_regs_map_free(&dev->io.acch); + } + + if (dev->sb.acch != NULL) { + ddi_regs_map_free(&dev->sb.acch); + } + + if (dev->vc.acch != NULL) { + ddi_regs_map_free(&dev->vc.acch); + } + + if (dev->pcih != NULL) { + pci_config_teardown(&dev->pcih); + } + + /* release play resources */ + if (dev->play.paddr != 0) + (void) ddi_dma_unbind_handle(dev->play.dmah); + if (dev->play.acch != NULL) + ddi_dma_mem_free(&dev->play.acch); + if (dev->play.dmah != NULL) + ddi_dma_free_handle(&dev->play.dmah); + + if (dev->play.engine != NULL) { + audio_dev_remove_engine(dev->adev, dev->play.engine); + audio_engine_free(dev->play.engine); + } + + /* release record resources */ + if (dev->rec.paddr != 0) + (void) ddi_dma_unbind_handle(dev->rec.dmah); + if (dev->rec.acch != NULL) + ddi_dma_mem_free(&dev->rec.acch); + if (dev->rec.dmah != NULL) + ddi_dma_free_handle(&dev->rec.dmah); + + if (dev->rec.engine != NULL) { + audio_dev_remove_engine(dev->adev, dev->rec.engine); + audio_engine_free(dev->rec.engine); + } + + if (dev->adev != NULL) { + audio_dev_free(dev->adev); + } + + kmem_free(dev, sizeof (*dev)); +} + +static bool +solo_setup_interrupts(solo_dev_t *dev) +{ + int actual; + uint_t ipri; + + if ((ddi_intr_alloc(dev->dip, &dev->ihandle, DDI_INTR_TYPE_FIXED, + 0, 1, &actual, DDI_INTR_ALLOC_NORMAL) != DDI_SUCCESS) || + (actual != 1)) { + audio_dev_warn(dev->adev, "can't alloc intr handle"); + return (false); + } + + if (ddi_intr_get_pri(dev->ihandle, &ipri) != DDI_SUCCESS) { + audio_dev_warn(dev->adev, "can't determine intr priority"); + (void) ddi_intr_free(dev->ihandle); + dev->ihandle = NULL; + return (false); + } + + if (ddi_intr_add_handler(dev->ihandle, solo_intr, dev, + NULL) != DDI_SUCCESS) { + audio_dev_warn(dev->adev, "can't add intr handler"); + (void) ddi_intr_free(dev->ihandle); + dev->ihandle = NULL; + return (false); + } + + mutex_init(&dev->mutex, NULL, MUTEX_DRIVER, DDI_INTR_PRI(ipri)); + + return (true); +} + +static bool +solo_map_registers(solo_dev_t *dev) +{ + dev_info_t *dip = dev->dip; + + /* map registers */ + if (ddi_regs_map_setup(dip, 1, &dev->io.base, 0, 0, &acc_attr, + &dev->io.acch) != DDI_SUCCESS) { + audio_dev_warn(dev->adev, "can't map IO registers"); + return (false); + } + if (ddi_regs_map_setup(dip, 2, &dev->sb.base, 0, 0, &acc_attr, + &dev->sb.acch) != DDI_SUCCESS) { + audio_dev_warn(dev->adev, "can't map SB registers"); + return (false); + } + if (ddi_regs_map_setup(dip, 3, &dev->vc.base, 0, 0, &acc_attr, + &dev->vc.acch) != DDI_SUCCESS) { + audio_dev_warn(dev->adev, "can't map VC registers"); + return (false); + } + + return (true); +} + +#define ESS_PCI_LEGACYCONTROL 0x40 +#define ESS_PCI_CONFIG 0x50 +#define ESS_PCI_DDMACONTROL 0x60 + +static bool +solo_init_hw(solo_dev_t *dev) +{ + uint32_t data; + + /* + * Legacy audio register -- disable legacy audio. We also + * arrange for 16-bit I/O address decoding. + */ + /* this version disables the MPU, FM synthesis (Adlib), and Game Port */ + pci_config_put16(dev->pcih, ESS_PCI_LEGACYCONTROL, 0x8041); + + /* + * Note that Solo-1 uses I/O space for all BARs, and hardwires + * the upper 32-bits to zero. + */ + data = pci_config_get32(dev->pcih, PCI_CONF_BASE2); + data |= 1; + pci_config_put16(dev->pcih, ESS_PCI_DDMACONTROL, data & 0xffff); + + /* + * Make sure that legacy IRQ and DRQ are disbled. We disable most + * other legacy features too. + */ + pci_config_put16(dev->pcih, ESS_PCI_CONFIG, 0); + + if (!solo_reset_dsp(dev)) + return (false); + + /* enable extended mode */ + (void) solo_cmd(dev, 0xc6); + + + PORT_WR8(dev->io, 0x7, 0x30); /* enable audio irqs */ + + /* demand mode, 4 bytes/xfer */ + solo_write(dev, 0xb9, 0x01); + + /* + * This sets Audio 2 (playback) to use its own independent + * rate control, and gives us 48 kHz compatible divisors. It + * also bypasses the switched capacitor filter. + */ + solo_setmixer(dev, 0x71, 0x2a); + + /* irq control */ + solo_write(dev, 0xb1, (solo_read(dev, 0xb1) & 0x0f) | 0x50); + /* drq control */ + solo_write(dev, 0xb2, (solo_read(dev, 0xb2) & 0x0f) | 0x50); + + solo_setmixer(dev, 0, 0); /* reset mixer settings */ + + solo_configure_mixer(dev); + return (true); +} + +static bool +solo_alloc_engine(solo_dev_t *dev, int engno) +{ + size_t rlen; + ddi_dma_attr_t *dattr; + ddi_dma_cookie_t c; + unsigned ccnt; + unsigned caps; + unsigned dflags; + const char *desc; + solo_engine_t *e; + + ASSERT((engno == 1) || (engno = 2)); + + switch (engno) { + case 1: /* record */ + e = &dev->rec; + desc = "record"; + dattr = &dma_attr_audio1; + caps = ENGINE_INPUT_CAP; + dflags = DDI_DMA_READ | DDI_DMA_CONSISTENT; + e->syncdir = DDI_DMA_SYNC_FORKERNEL; + e->update = solo_aud1_update; + e->start = solo_aud1_start; + e->stop = solo_aud1_stop; + e->format = AUDIO_FORMAT_S16_BE; + e->swapped = true; + break; + + case 2: /* playback */ + e = &dev->play; + desc = "playback"; + dattr = &dma_attr_audio2; + caps = ENGINE_OUTPUT_CAP; + dflags = DDI_DMA_WRITE | DDI_DMA_CONSISTENT; + e->syncdir = DDI_DMA_SYNC_FORDEV; + e->update = solo_aud2_update; + e->start = solo_aud2_start; + e->stop = solo_aud2_stop; + e->format = AUDIO_FORMAT_S16_LE; + e->swapped = false; + break; + + default: + audio_dev_warn(dev->adev, "bad engine number!"); + return (false); + } + + printf("%s %sswapped!", desc, e->swapped ? "" : "not "); + e->dev = dev; + + if (ddi_dma_alloc_handle(dev->dip, dattr, DDI_DMA_SLEEP, NULL, + &e->dmah) != DDI_SUCCESS) { + audio_dev_warn(dev->adev, "%s dma handle alloc failed", desc); + return (false); + } + if (ddi_dma_mem_alloc(e->dmah, SOLO_BUFSZ, &buf_attr, + DDI_DMA_CONSISTENT, DDI_DMA_SLEEP, NULL, &e->kaddr, + &rlen, &e->acch) != DDI_SUCCESS) { + audio_dev_warn(dev->adev, "%s dma memory alloc failed", desc); + return (false); + } + /* ensure that the buffer is zeroed out properly */ + bzero(e->kaddr, rlen); + if (ddi_dma_addr_bind_handle(e->dmah, NULL, e->kaddr, SOLO_BUFSZ, + dflags, DDI_DMA_SLEEP, NULL, &c, &ccnt) != DDI_DMA_MAPPED) { + audio_dev_warn(dev->adev, "%s dma binding failed", desc); + return (false); + } + e->paddr = c.dmac_address; + + /* + * Allocate and configure audio engine. + */ + e->engine = audio_engine_alloc(&solo_engine_ops, caps); + if (e->engine == NULL) { + audio_dev_warn(dev->adev, "record audio_engine_alloc failed"); + return (false); + } + + audio_engine_set_private(e->engine, e); + audio_dev_add_engine(dev->adev, e->engine); + + return (true); +} + + +static int +solo_suspend(solo_dev_t *dev) +{ + mutex_enter(&dev->mutex); + /* play */ + solo_aud2_stop(&dev->play); + solo_aud2_update(&dev->play); + /* record */ + solo_aud1_stop(&dev->rec); + solo_aud1_update(&dev->rec); + + dev->suspended = true; + mutex_exit(&dev->mutex); + + return (DDI_SUCCESS); +} + +static int +solo_resume(solo_dev_t *dev) +{ + solo_engine_t *e; + audio_engine_t *prod = NULL; + audio_engine_t *cons = NULL; + + audio_engine_reset(dev->rec.engine); + audio_engine_reset(dev->play.engine); + + mutex_enter(&dev->mutex); + if (!solo_init_hw(dev)) { + /* yikes! */ + audio_dev_warn(dev->adev, "unable to resume audio!"); + audio_dev_warn(dev->adev, "reboot or reload driver to reset"); + mutex_exit(&dev->mutex); + return (DDI_SUCCESS); + } + dev->suspended = false; + + /* record - audio 1 */ + e = &dev->rec; + if (e->started) { + e->start(e); + prod = e->engine; + } + + /* play - audio 2 */ + e = &dev->play; + if (e->started) { + e->start(e); + cons = e->engine; + } + + mutex_exit(&dev->mutex); + + if (cons) + audio_engine_consume(cons); + if (prod) + audio_engine_produce(prod); + + return (DDI_SUCCESS); +} + +static int +solo_attach(dev_info_t *dip) +{ + solo_dev_t *dev; + uint32_t data; + + dev = kmem_zalloc(sizeof (*dev), KM_SLEEP); + dev->dip = dip; + ddi_set_driver_private(dip, dev); + + dev->adev = audio_dev_alloc(dip, 0); + if (dev->adev == NULL) + goto no; + + audio_dev_set_description(dev->adev, "ESS Solo-1 PCI AudioDrive"); + audio_dev_set_version(dev->adev, "ES1938"); + + if (pci_config_setup(dip, &dev->pcih) != DDI_SUCCESS) { + audio_dev_warn(NULL, "pci_config_setup failed"); + goto no; + } + + data = pci_config_get16(dev->pcih, PCI_CONF_COMM); + data |= PCI_COMM_ME | PCI_COMM_IO; + pci_config_put16(dev->pcih, PCI_CONF_COMM, data); + + if ((!solo_map_registers(dev)) || + (!solo_setup_interrupts(dev)) || + (!solo_alloc_engine(dev, 1)) || + (!solo_alloc_engine(dev, 2)) || + (!solo_add_controls(dev)) || + (!solo_init_hw(dev))) { + goto no; + } + + if (audio_dev_register(dev->adev) != DDI_SUCCESS) { + audio_dev_warn(dev->adev, + "unable to register with audio framework"); + goto no; + } + + (void) ddi_intr_enable(dev->ihandle); + ddi_report_dev(dip); + + return (DDI_SUCCESS); + +no: + solo_release_resources(dev); + return (DDI_FAILURE); +} + +static int +solo_detach(solo_dev_t *dev) +{ + if (audio_dev_unregister(dev->adev) != DDI_SUCCESS) { + return (DDI_FAILURE); + } + + solo_release_resources(dev); + return (DDI_SUCCESS); +} + +static int +solo_ddi_attach(dev_info_t *dip, ddi_attach_cmd_t cmd) +{ + solo_dev_t *dev; + + switch (cmd) { + case DDI_ATTACH: + return (solo_attach(dip)); + + case DDI_RESUME: + if ((dev = ddi_get_driver_private(dip)) == NULL) { + return (DDI_FAILURE); + } + return (solo_resume(dev)); + + default: + return (DDI_FAILURE); + } +} + +static int +solo_ddi_detach(dev_info_t *dip, ddi_detach_cmd_t cmd) +{ + solo_dev_t *dev; + + if ((dev = ddi_get_driver_private(dip)) == NULL) { + return (DDI_FAILURE); + } + + switch (cmd) { + case DDI_DETACH: + return (solo_detach(dev)); + + case DDI_SUSPEND: + return (solo_suspend(dev)); + default: + return (DDI_FAILURE); + } +} + +static int +solo_quiesce(dev_info_t *dip) +{ + solo_dev_t *dev; + + dev = ddi_get_driver_private(dip); + + solo_aud1_stop(&dev->rec); + solo_aud2_stop(&dev->play); + + solo_setmixer(dev, 0, 0); + PORT_WR8(dev->io, 0x7, 0); /* disable all irqs */ + return (0); +} + +struct dev_ops solo_dev_ops = { + DEVO_REV, /* rev */ + 0, /* refcnt */ + NULL, /* getinfo */ + nulldev, /* identify */ + nulldev, /* probe */ + solo_ddi_attach, /* attach */ + solo_ddi_detach, /* detach */ + nodev, /* reset */ + NULL, /* cb_ops */ + NULL, /* bus_ops */ + NULL, /* power */ + solo_quiesce, /* quiesce */ +}; + +static struct modldrv solo_modldrv = { + &mod_driverops, /* drv_modops */ + "ESS Solo-1 Audio", /* linkinfo */ + &solo_dev_ops, /* dev_ops */ +}; + +static struct modlinkage modlinkage = { + MODREV_1, + { &solo_modldrv, NULL } +}; + +int +_init(void) +{ + int rv; + + audio_init_ops(&solo_dev_ops, DRVNAME); + if ((rv = mod_install(&modlinkage)) != 0) { + audio_fini_ops(&solo_dev_ops); + } + return (rv); +} + +int +_fini(void) +{ + int rv; + + if ((rv = mod_remove(&modlinkage)) == 0) { + audio_fini_ops(&solo_dev_ops); + } + return (rv); +} + +int +_info(struct modinfo *modinfop) +{ + return (mod_info(&modlinkage, modinfop)); +}
--- a/usr/src/uts/common/io/audio/impl/audio_client.c Mon Sep 21 11:25:09 2009 -0400 +++ b/usr/src/uts/common/io/audio/impl/audio_client.c Mon Sep 21 11:26:40 2009 -0400 @@ -58,7 +58,7 @@ * */ -const uint16_t auimpl_db_table[AUDIO_DB_SIZE + 1] = { +static const uint16_t auimpl_db_table[AUDIO_DB_SIZE + 1] = { 0, 0, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 4, 4, 5, 5, 6, 7, 8, 9, 10, 11, 12, 14, 16, 18, 20, 22, @@ -67,6 +67,10 @@ 256 }; +static list_t auimpl_clients; +static krwlock_t auimpl_client_lock; +static audio_client_ops_t *audio_client_ops[AUDIO_MN_TYPE_MASK + 1]; + void * auclnt_get_private(audio_client_t *c) { @@ -641,7 +645,8 @@ return (sp->s_user_parms->p_nchan); } -void + +static void auimpl_set_gain_master(audio_stream_t *sp, uint8_t gain) { uint32_t scaled; @@ -675,6 +680,38 @@ */ } +int +auimpl_set_pcmvol(void *arg, uint64_t val) +{ + audio_dev_t *d = arg; + list_t *l = &d->d_clients; + audio_client_t *c; + + if (val > 100) { + return (EINVAL); + } + rw_enter(&auimpl_client_lock, RW_WRITER); + d->d_pcmvol = val & 0xff; + rw_downgrade(&auimpl_client_lock); + + for (c = list_head(l); c; c = list_next(l, c)) { + /* don't need to check is_active here, its safe */ + auimpl_set_gain_master(&c->c_ostream, (uint8_t)val); + } + rw_exit(&auimpl_client_lock); + + return (0); +} + +int +auimpl_get_pcmvol(void *arg, uint64_t *val) +{ + audio_dev_t *d = arg; + + *val = d->d_pcmvol; + return (0); +} + void auclnt_set_gain(audio_stream_t *sp, uint8_t gain) { @@ -842,10 +879,6 @@ * implementations, but are for private framework use only. */ -static list_t auimpl_clients; -static krwlock_t auimpl_client_lock; -static audio_client_ops_t *audio_client_ops[AUDIO_MN_TYPE_MASK + 1]; - void auimpl_client_init(void) { @@ -1157,6 +1190,22 @@ } void +auimpl_client_activate(audio_client_t *c) +{ + rw_enter(&auimpl_client_lock, RW_WRITER); + c->c_is_active = B_TRUE; + rw_exit(&auimpl_client_lock); +} + +void +auimpl_client_deactivate(audio_client_t *c) +{ + rw_enter(&auimpl_client_lock, RW_WRITER); + c->c_is_active = B_FALSE; + rw_exit(&auimpl_client_lock); +} + +void auclnt_close(audio_client_t *c) { audio_dev_t *d = c->c_dev; @@ -1165,9 +1214,9 @@ auclnt_stop(&c->c_istream); auclnt_stop(&c->c_ostream); - rw_enter(&d->d_clnt_lock, RW_WRITER); + rw_enter(&auimpl_client_lock, RW_WRITER); list_remove(&d->d_clients, c); - rw_exit(&d->d_clnt_lock); + rw_exit(&auimpl_client_lock); mutex_enter(&c->c_lock); /* if in transition need to wait for other thread to release */ @@ -1212,7 +1261,7 @@ for (c = list_head(list); c != NULL; c = list_next(list, c)) { if ((c->c_major == mj) && (c->c_minor == mn)) { mutex_enter(&c->c_lock); - if (c->c_is_open) { + if (c->c_is_active) { c->c_refcnt++; mutex_exit(&c->c_lock); } else { @@ -1246,9 +1295,11 @@ audio_client_t *c; int rv; - rw_enter(&d->d_clnt_lock, RW_READER); + rw_enter(&auimpl_client_lock, RW_READER); restart: for (c = list_head(l); c != NULL; c = list_next(l, c)) { + if (!c->c_is_active) + continue; rv = (walker(c, arg)); if (rv == AUDIO_WALK_STOP) { break; @@ -1256,7 +1307,7 @@ goto restart; } } - rw_exit(&d->d_clnt_lock); + rw_exit(&auimpl_client_lock); } @@ -1317,11 +1368,11 @@ auimpl_engine_close(&c->c_ostream); auimpl_engine_close(&c->c_istream); } else { - rw_enter(&d->d_clnt_lock, RW_WRITER); + rw_enter(&auimpl_client_lock, RW_WRITER); list_insert_tail(&d->d_clients, c); c->c_ostream.s_gain_master = d->d_pcmvol; c->c_istream.s_gain_master = 100; - rw_exit(&d->d_clnt_lock); + rw_exit(&auimpl_client_lock); auclnt_set_gain(&c->c_ostream, 100); auclnt_set_gain(&c->c_istream, 100); } @@ -1465,14 +1516,16 @@ list_t *l = &dev->d_clients; audio_client_t *c; - rw_enter(&dev->d_clnt_lock, RW_READER); + rw_enter(&auimpl_client_lock, RW_READER); for (c = list_head(l); c != NULL; c = list_next(l, c)) { + if (!c->c_is_active) + continue; mutex_enter(&c->c_lock); c->c_do_notify = B_TRUE; cv_broadcast(&c->c_cv); mutex_exit(&c->c_lock); } - rw_exit(&dev->d_clnt_lock); + rw_exit(&auimpl_client_lock); } uint64_t
--- a/usr/src/uts/common/io/audio/impl/audio_ctrl.c Mon Sep 21 11:25:09 2009 -0400 +++ b/usr/src/uts/common/io/audio/impl/audio_ctrl.c Mon Sep 21 11:26:40 2009 -0400 @@ -239,34 +239,6 @@ kmem_free(ctrl, sizeof (*ctrl)); } -static int -auimpl_set_pcmvol(void *arg, uint64_t val) -{ - audio_dev_t *d = arg; - list_t *l = &d->d_clients; - audio_client_t *c; - - if (val > 100) { - return (EINVAL); - } - rw_enter(&d->d_clnt_lock, RW_WRITER); - d->d_pcmvol = val & 0xff; - for (c = list_head(l); c; c = list_next(l, c)) { - auimpl_set_gain_master(&c->c_ostream, (uint8_t)val); - } - rw_exit(&d->d_clnt_lock); - return (0); -} - -static int -auimpl_get_pcmvol(void *arg, uint64_t *val) -{ - audio_dev_t *d = arg; - - *val = d->d_pcmvol; - return (0); -} - int audio_dev_add_soft_volume(audio_dev_t *d) {
--- a/usr/src/uts/common/io/audio/impl/audio_ddi.c Mon Sep 21 11:25:09 2009 -0400 +++ b/usr/src/uts/common/io/audio/impl/audio_ddi.c Mon Sep 21 11:26:40 2009 -0400 @@ -180,9 +180,8 @@ /* we do device cloning! */ *devp = makedevice(c->c_major, c->c_minor); - mutex_enter(&c->c_lock); - c->c_is_open = B_TRUE; - mutex_exit(&c->c_lock); + /* now we can receive upcalls */ + auimpl_client_activate(c); auclnt_notify_dev(c->c_dev); @@ -240,14 +239,13 @@ /* we do device cloning! */ *devp = makedevice(c->c_major, c->c_minor); - mutex_enter(&c->c_lock); - c->c_is_open = B_TRUE; - mutex_exit(&c->c_lock); + qprocson(rq); + + /* now we can receive upcalls */ + auimpl_client_activate(c); auclnt_notify_dev(c->c_dev); - qprocson(rq); - return (0); } @@ -268,9 +266,8 @@ rv = auclnt_drain(c); } - mutex_enter(&c->c_lock); - c->c_is_open = B_FALSE; - mutex_exit(&c->c_lock); + /* make sure we won't get any upcalls */ + auimpl_client_deactivate(c); /* * Pick up any data sitting around in input buffers. This @@ -316,9 +313,8 @@ return (ENXIO); } - mutex_enter(&c->c_lock); - c->c_is_open = B_FALSE; - mutex_exit(&c->c_lock); + /* we don't want any upcalls anymore */ + auimpl_client_deactivate(c); /* * Pick up any data sitting around in input buffers. This
--- a/usr/src/uts/common/io/audio/impl/audio_engine.c Mon Sep 21 11:25:09 2009 -0400 +++ b/usr/src/uts/common/io/audio/impl/audio_engine.c Mon Sep 21 11:26:40 2009 -0400 @@ -75,7 +75,6 @@ mutex_init(&d->d_lock, NULL, MUTEX_DRIVER, NULL); cv_init(&d->d_cv, NULL, CV_DRIVER, NULL); rw_init(&d->d_ctrl_lock, NULL, RW_DRIVER, NULL); - rw_init(&d->d_clnt_lock, NULL, RW_DRIVER, NULL); list_create(&d->d_clients, sizeof (struct audio_client), offsetof(struct audio_client, c_dev_linkage)); list_create(&d->d_engines, sizeof (struct audio_engine), @@ -107,7 +106,6 @@ rw_destroy(&d->d_ctrl_lock); mutex_destroy(&d->d_lock); cv_destroy(&d->d_cv); - rw_destroy(&d->d_clnt_lock); kmem_free(d, sizeof (*d)); }
--- a/usr/src/uts/common/io/audio/impl/audio_impl.h Mon Sep 21 11:25:09 2009 -0400 +++ b/usr/src/uts/common/io/audio/impl/audio_impl.h Mon Sep 21 11:26:40 2009 -0400 @@ -189,7 +189,7 @@ boolean_t c_do_notify; boolean_t c_do_drain; boolean_t c_closing; - boolean_t c_is_open; + boolean_t c_is_active; /* * Client wide settings... e.g. ops vector, etc. @@ -442,9 +442,12 @@ void auimpl_client_fini(void); audio_client_t *auimpl_client_create(dev_t); void auimpl_client_destroy(audio_client_t *); +void auimpl_client_activate(audio_client_t *); +void auimpl_client_deactivate(audio_client_t *); int auimpl_create_minors(audio_dev_t *); void auimpl_remove_minors(audio_dev_t *); -void auimpl_set_gain_master(audio_stream_t *, uint8_t); +int auimpl_set_pcmvol(void *, uint64_t); +int auimpl_get_pcmvol(void *, uint64_t *); /* audio_engine.c */ void auimpl_dev_init(void);
--- a/usr/src/uts/common/io/audio/impl/audio_input.c Mon Sep 21 11:25:09 2009 -0400 +++ b/usr/src/uts/common/io/audio/impl/audio_input.c Mon Sep 21 11:26:40 2009 -0400 @@ -40,29 +40,25 @@ void \ auimpl_import_##NAME(audio_engine_t *eng, audio_stream_t *sp) \ { \ - int nch = eng->e_nchan; \ - int32_t *out; \ - TYPE *in; \ - int ch; \ - void *data; \ - int vol; \ - \ - data = sp->s_cnv_src; \ - ch = 0; \ - in = (void *)(eng->e_data + (eng->e_tidx * eng->e_framesz)); \ - out = data; \ - vol = sp->s_gain_eff; \ + int fragfr = eng->e_fragfr; \ + int nch = eng->e_nchan; \ + unsigned tidx = eng->e_tidx; \ + int32_t *out = (void *)sp->s_cnv_src; \ + TYPE *in = (void *)eng->e_data; \ + int ch = 0; \ + int vol = sp->s_gain_eff; \ \ do { /* for each channel */ \ - TYPE *ip; \ + TYPE *ip; \ int32_t *op; \ - int i; \ + int i; \ + int incr = eng->e_chincr[ch]; \ \ /* get value and adjust next channel offset */ \ op = out++; \ - ip = in++; \ + ip = in + eng->e_choffs[ch] + (tidx * incr); \ \ - i = eng->e_fragfr; \ + i = fragfr; \ \ do { /* for each frame */ \ int32_t sample = (TYPE)SWAP(*ip); \ @@ -72,7 +68,7 @@ scaled /= AUDIO_VOL_SCALE; \ \ *op = scaled; \ - ip += nch; \ + ip += incr; \ op += nch; \ \ } while (--i); \
--- a/usr/src/uts/common/io/comstar/lu/stmf_sbd/sbd.c Mon Sep 21 11:25:09 2009 -0400 +++ b/usr/src/uts/common/io/comstar/lu/stmf_sbd/sbd.c Mon Sep 21 11:26:40 2009 -0400 @@ -2962,6 +2962,8 @@ cmn_err(CE_NOTE, "ioctl failed %d", rc); } kmem_free(zc, sizeof (zfs_cmd_t)); + if (packed) + kmem_free(packed, len); out: nvlist_free(nv); (void) ldi_close(zfs_lh, FREAD|FWRITE, kcred);
--- a/usr/src/uts/common/io/comstar/stmf/stmf.c Mon Sep 21 11:25:09 2009 -0400 +++ b/usr/src/uts/common/io/comstar/stmf/stmf.c Mon Sep 21 11:26:40 2009 -0400 @@ -1387,7 +1387,7 @@ stmf_state.stmf_inventory_locked = 1; stmf_state.stmf_service_running = 0; - stmf_delete_all_ppds(); + mutex_exit(&stmf_state.stmf_lock); for (ilport = stmf_state.stmf_ilportlist; ilport != NULL; ilport = ilport->ilport_next) {
--- a/usr/src/uts/common/io/lvm/md/md_subr.c Mon Sep 21 11:25:09 2009 -0400 +++ b/usr/src/uts/common/io/lvm/md/md_subr.c Mon Sep 21 11:26:40 2009 -0400 @@ -20,7 +20,7 @@ */ /* - * Copyright 2008 Sun Microsystems, Inc. All rights reserved. + * Copyright 2009 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. */ @@ -112,6 +112,7 @@ extern void *lookup_entry(struct nm_next_hdr *, set_t, side_t, mdkey_t, md_dev64_t, int); extern struct nm_next_hdr *get_first_record(set_t, int, int); +extern dev_t getrootdev(void); struct mdq_anchor md_done_daemon; /* done request queue */ struct mdq_anchor md_mstr_daemon; /* mirror error, WOW requests */ @@ -3380,6 +3381,7 @@ mdi_unit_t *ui; md_probedev_impl_t *p; int err = 0; + set_t setno; p = (md_probedev_impl_t *)reqp->private_handle; /* @@ -3389,20 +3391,41 @@ * locks this will prevent a metaclear operation being performed * on the metadevice because metaclear takes the readerlock (via * openclose lock). + * To avoid a potential deadlock with the probe_fcn() causing i/o to + * be issued to the writerlock'd metadevice we only grab the writerlock + * if the unit is not an SVM root device. */ while (md_ioctl_lock_enter() == EINTR) ; + setno = MD_MIN2SET(reqp->mnum); ui = MDI_UNIT(reqp->mnum); if (ui != NULL) { - (void) md_unit_writerlock_common(ui, 0); + int writer_grabbed; + dev_t svm_root; + + if ((setno == MD_LOCAL_SET) && root_is_svm) { + svm_root = getrootdev(); + + if (getminor(svm_root) == reqp->mnum) { + writer_grabbed = 0; + } else { + writer_grabbed = 1; + (void) md_unit_writerlock_common(ui, 0); + } + } else { + writer_grabbed = 1; + (void) md_unit_writerlock_common(ui, 0); + } (void) md_ioctl_lock_exit(0, 0, 0, FALSE); err = (*reqp->probe_fcn)(ui, reqp->mnum); - md_unit_writerexit(ui); + if (writer_grabbed) { + md_unit_writerexit(ui); + } } else { (void) md_ioctl_lock_exit(0, 0, 0, FALSE); } - /* update the info info in the probe structure */ + /* update the info in the probe structure */ mutex_enter(PROBE_MX(p)); if (err != 0) {
--- a/usr/src/uts/common/io/nxge/nxge_espc.c Mon Sep 21 11:25:09 2009 -0400 +++ b/usr/src/uts/common/io/nxge/nxge_espc.c Mon Sep 21 11:26:40 2009 -0400 @@ -254,6 +254,8 @@ } else if (strncmp(nxgep->vpd_info.bd_model, NXGE_RFEM_BM_STR, strlen(NXGE_RFEM_BM_STR)) == 0) { nxgep->hot_swappable_phy = B_TRUE; + nxgep->platform_type = P_NEPTUNE_GENERIC; + nxgep->niu_type = NEPTUNE_2_10GF; } /* If Alonso platform, replace "mif" for the last 2 ports phy-type */
--- a/usr/src/uts/common/io/nxge/nxge_hio.c Mon Sep 21 11:25:09 2009 -0400 +++ b/usr/src/uts/common/io/nxge/nxge_hio.c Mon Sep 21 11:26:40 2009 -0400 @@ -132,9 +132,10 @@ int i; nhd = (nxge_hio_data_t *)nxge->nxge_hw_p->hio; - if (nhd == 0) { + if (nhd == NULL) { nhd = KMEM_ZALLOC(sizeof (*nhd), KM_SLEEP); MUTEX_INIT(&nhd->lock, NULL, MUTEX_DRIVER, NULL); + nhd->type = NXGE_HIO_TYPE_SERVICE; nxge->nxge_hw_p->hio = (uintptr_t)nhd; } @@ -966,8 +967,7 @@ * Any domain */ int -nxge_hio_init( - nxge_t *nxge) +nxge_hio_init(nxge_t *nxge) { nxge_hio_data_t *nhd; int i, region; @@ -976,6 +976,10 @@ if (nhd == 0) { nhd = KMEM_ZALLOC(sizeof (*nhd), KM_SLEEP); MUTEX_INIT(&nhd->lock, NULL, MUTEX_DRIVER, NULL); + if (isLDOMguest(nxge)) + nhd->type = NXGE_HIO_TYPE_GUEST; + else + nhd->type = NXGE_HIO_TYPE_SERVICE; nxge->nxge_hw_p->hio = (uintptr_t)nhd; } @@ -1917,15 +1921,12 @@ } int -nxge_hio_addres( - nxge_hio_vr_t *vr, - mac_ring_type_t type, - uint64_t *map) +nxge_hio_addres(nxge_hio_vr_t *vr, mac_ring_type_t type, uint64_t *map) { nxge_t *nxge = (nxge_t *)vr->nxge; nxge_grp_t *group; int groupid; - int i; + int i, rv = 0; int max_dcs; NXGE_DEBUG_MSG((nxge, HIO_CTL, "==> nxge_hio_addres")); @@ -1958,8 +1959,6 @@ for (i = 0; i < max_dcs; i++) { if (group->map & (1 << i)) { - int rv; - if ((rv = nxge_hio_dc_share(nxge, vr, type, i)) < 0) { if (*map == 0) /* Couldn't get even one DC. */ return (-rv); @@ -1970,8 +1969,13 @@ } } + if ((*map == 0) || (rv != 0)) { + NXGE_DEBUG_MSG((nxge, HIO_CTL, + "<== nxge_hio_addres: rv(%x)", rv)); + return (EIO); + } + NXGE_DEBUG_MSG((nxge, HIO_CTL, "<== nxge_hio_addres")); - return (0); }
--- a/usr/src/uts/common/io/nxge/nxge_hio_guest.c Mon Sep 21 11:25:09 2009 -0400 +++ b/usr/src/uts/common/io/nxge/nxge_hio_guest.c Mon Sep 21 11:26:40 2009 -0400 @@ -192,13 +192,21 @@ NXGE_DEBUG_MSG((nxge, HIO_CTL, "==> nxge_hio_vr_add")); + if (nhd->type == NXGE_HIO_TYPE_SERVICE) { + /* + * Can't add VR to the service domain from which we came. + */ + ASSERT(nhd->type == NXGE_HIO_TYPE_GUEST); + return (DDI_FAILURE); + } + /* * Get our HV cookie. */ if (ddi_prop_lookup_int_array(DDI_DEV_T_ANY, nxge->dip, 0, "reg", ®_val, ®_len) != DDI_PROP_SUCCESS) { NXGE_DEBUG_MSG((nxge, VPD_CTL, "`reg' property not found")); - return (NXGE_ERROR); + return (DDI_FAILURE); } cookie = (uint32_t)(reg_val[0]); @@ -209,7 +217,7 @@ if (hv_rv != 0) { NXGE_ERROR_MSG((nxge, NXGE_ERR_CTL, "vr->getinfo() failed")); - return (NXGE_ERROR); + return (DDI_FAILURE); } /* @@ -239,7 +247,7 @@ NXGE_ERROR_MSG((nxge, NXGE_ERR_CTL, "nxge_hio_vr_add(%d): cookie(0x%x)\n", nxge->instance, cookie)); - return (NXGE_ERROR); + return (DDI_FAILURE); } vr = &nhd->vr[vr_index]; @@ -265,7 +273,7 @@ if (nxge_hio_intr_init(nxge) != NXGE_OK) { NXGE_ERROR_MSG((nxge, NXGE_ERR_CTL, "nxge_hio_intr_init() failed")); - return (NXGE_ERROR); + return (DDI_FAILURE); } /* @@ -282,7 +290,7 @@ if (hv_rv != 0) { NXGE_ERROR_MSG((nxge, NXGE_ERR_CTL, "tx->get_map() failed")); - return (NXGE_ERROR); + return (DDI_FAILURE); } res_map_parse(nxge, NXGE_TRANSMIT_GROUP, tx_map); @@ -297,7 +305,7 @@ if (dc == 0) { NXGE_ERROR_MSG((nxge, NXGE_ERR_CTL, "DC add failed")); - return (NXGE_ERROR); + return (DDI_FAILURE); } dc->channel = (nxge_channel_t)i; } @@ -315,7 +323,7 @@ if (hv_rv != 0) { NXGE_ERROR_MSG((nxge, NXGE_ERR_CTL, "rx->get_map() failed")); - return (NXGE_ERROR); + return (DDI_FAILURE); } res_map_parse(nxge, NXGE_RECEIVE_GROUP, rx_map); @@ -330,7 +338,7 @@ if (dc == 0) { NXGE_ERROR_MSG((nxge, NXGE_ERR_CTL, "DC add failed")); - return (NXGE_ERROR); + return (DDI_FAILURE); } dc->channel = (nxge_channel_t)i; } @@ -341,14 +349,14 @@ if (status != NXGE_OK) { cmn_err(CE_WARN, "nxge(%d): nxge_mac_register failed\n", nxge->instance); - return (status); + return (DDI_FAILURE); } nxge->hio_vr = vr; /* For faster lookups. */ NXGE_DEBUG_MSG((nxge, HIO_CTL, "<== nxge_hio_vr_add")); - return (NXGE_OK); + return (DDI_SUCCESS); } /*
--- a/usr/src/uts/common/io/nxge/nxge_mac.c Mon Sep 21 11:25:09 2009 -0400 +++ b/usr/src/uts/common/io/nxge/nxge_mac.c Mon Sep 21 11:26:40 2009 -0400 @@ -312,6 +312,10 @@ */ if (nxgep->mac.portmode == PORT_HSP_MODE) { nxgep->hot_swappable_phy = B_TRUE; + if (portn > 1) { + return (NXGE_ERROR); + } + /* * If this is the 2nd NIU port, then check 2 addresses * to take care of the Goa NEM card. Port 1 can have addr 17
--- a/usr/src/uts/common/io/nxge/nxge_main.c Mon Sep 21 11:25:09 2009 -0400 +++ b/usr/src/uts/common/io/nxge/nxge_main.c Mon Sep 21 11:26:40 2009 -0400 @@ -128,10 +128,12 @@ nxge_rxbuf_threshold_t nxge_rx_threshold_lo = NXGE_RX_COPY_3; /* Use kmem_alloc() to allocate data buffers. */ -#if defined(_BIG_ENDIAN) +#if defined(__sparc) uint32_t nxge_use_kmem_alloc = 1; +#elif defined(__i386) +uint32_t nxge_use_kmem_alloc = 0; #else -uint32_t nxge_use_kmem_alloc = 0; +uint32_t nxge_use_kmem_alloc = 1; #endif rtrace_t npi_rtracebuf; @@ -822,11 +824,12 @@ if (isLDOMguest(nxgep)) { /* Find our VR & channel sets. */ status = nxge_hio_vr_add(nxgep); - if (status != NXGE_OK) { - NXGE_DEBUG_MSG((nxgep, DDI_CTL, + if (status != DDI_SUCCESS) { + NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, "nxge_hio_vr_add failed")); (void) hsvc_unregister(&nxgep->niu_hsvc); nxgep->niu_hsvc_available = B_FALSE; + goto nxge_attach_fail; } goto nxge_attach_exit; } @@ -6530,6 +6533,8 @@ p_nxge_hw_list_t hw_p; dev_info_t *p_dip; + ASSERT(nxgep != NULL); + NXGE_DEBUG_MSG((nxgep, MOD_CTL, "==> nxge_init_common_device")); p_dip = nxgep->p_dip; @@ -6644,6 +6649,8 @@ p_nxge_hw_pt_cfg_t p_cfgp; dev_info_t *p_dip; + ASSERT(nxgep != NULL); + NXGE_DEBUG_MSG((nxgep, MOD_CTL, "==> nxge_uninit_common_device")); if (nxgep->nxge_hw_p == NULL) { NXGE_DEBUG_MSG((nxgep, MOD_CTL,
--- a/usr/src/uts/common/io/nxge/nxge_rxdma.c Mon Sep 21 11:25:09 2009 -0400 +++ b/usr/src/uts/common/io/nxge/nxge_rxdma.c Mon Sep 21 11:26:40 2009 -0400 @@ -18,6 +18,7 @@ * * CDDL HEADER END */ + /* * Copyright 2009 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. @@ -4268,12 +4269,13 @@ if (isLDOMguest(nxgep)) { /* Add interrupt handler for this channel. */ - if (nxge_hio_intr_add(nxgep, VP_BOUND_RX, channel) - != NXGE_OK) { + status = nxge_hio_intr_add(nxgep, VP_BOUND_RX, channel); + if (status != NXGE_OK) { NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, " nxge_rxdma_start_channel: " " nxge_hio_intr_add failed (0x%08x channel %d)", - status, channel)); + status, channel)); + return (status); } } @@ -4556,7 +4558,6 @@ rbrp = (p_rx_rbr_ring_t)nxgep->rx_rbr_rings->rbr_rings[channel]; rcrp = (p_rx_rcr_ring_t)nxgep->rx_rcr_rings->rcr_rings[channel]; - MUTEX_ENTER(&rcrp->lock); MUTEX_ENTER(&rbrp->lock); MUTEX_ENTER(&rbrp->post_lock); @@ -4651,20 +4652,17 @@ MUTEX_EXIT(&rbrp->post_lock); MUTEX_EXIT(&rbrp->lock); - MUTEX_EXIT(&rcrp->lock); NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, "Recovery Successful, RxDMAChannel#%d Restored", channel)); NXGE_DEBUG_MSG((nxgep, RX_CTL, "==> nxge_rxdma_fatal_err_recover")); - return (NXGE_OK); + fail: MUTEX_EXIT(&rbrp->post_lock); MUTEX_EXIT(&rbrp->lock); - MUTEX_EXIT(&rcrp->lock); NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, "Recovery failed")); - return (NXGE_ERROR | rs); } @@ -4673,6 +4671,7 @@ { nxge_grp_set_t *set = &nxgep->rx_set; nxge_status_t status = NXGE_OK; + p_rx_rcr_ring_t rcrp; int rdc; NXGE_DEBUG_MSG((nxgep, RX_CTL, "<== nxge_rx_port_fatal_err_recover")); @@ -4689,10 +4688,16 @@ for (rdc = 0; rdc < NXGE_MAX_RDCS; rdc++) { if ((1 << rdc) & set->owned.map) { - if (nxge_rxdma_fatal_err_recover(nxgep, rdc) - != NXGE_OK) { - NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, - "Could not recover channel %d", rdc)); + rcrp = nxgep->rx_rcr_rings->rcr_rings[rdc]; + if (rcrp != NULL) { + MUTEX_ENTER(&rcrp->lock); + if (nxge_rxdma_fatal_err_recover(nxgep, + rdc) != NXGE_OK) { + NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, + "Could not recover " + "channel %d", rdc)); + } + MUTEX_EXIT(&rcrp->lock); } } }
--- a/usr/src/uts/common/io/nxge/nxge_txdma.c Mon Sep 21 11:25:09 2009 -0400 +++ b/usr/src/uts/common/io/nxge/nxge_txdma.c Mon Sep 21 11:26:40 2009 -0400 @@ -209,11 +209,13 @@ nxgep->statsp->tdc_ksp[channel] = 0; } - (void) nxge_txdma_stop_channel(nxgep, channel); + if (nxge_txdma_stop_channel(nxgep, channel) != NXGE_OK) + goto nxge_uninit_txdma_channel_exit; + nxge_unmap_txdma_channel(nxgep, channel); - NXGE_DEBUG_MSG((nxgep, MEM3_CTL, - "<== nxge_uninit_txdma_channel")); +nxge_uninit_txdma_channel_exit: + NXGE_DEBUG_MSG((nxgep, MEM3_CTL, "<== nxge_uninit_txdma_channel")); } void @@ -684,6 +686,8 @@ (mblk_len < stuff_len)) { stuff_len -= mblk_len; nmp = nmp->b_cont; + if (nmp) + mblk_len = MBLKL(nmp); } ASSERT(nmp); up = (uint16_t *)(nmp->b_rptr + stuff_len); @@ -2946,7 +2950,16 @@ */ (void) nxge_txdma_stop_inj_err(nxgep, channel); + if (nxgep->tx_rings == NULL) { + status = NXGE_ERROR; + goto nxge_txdma_stop_channel_exit; + } + tx_ring_p = nxgep->tx_rings->rings[channel]; + if (tx_ring_p == NULL) { + status = NXGE_ERROR; + goto nxge_txdma_stop_channel_exit; + } /* * Reset TXDMA channel
--- a/usr/src/uts/common/io/scsi/targets/st.c Mon Sep 21 11:25:09 2009 -0400 +++ b/usr/src/uts/common/io/scsi/targets/st.c Mon Sep 21 11:26:40 2009 -0400 @@ -1331,8 +1331,8 @@ (un->un_pos.blkno != 0)) || /* Or within first file */ ((un->un_pos.pmode == logical) && (un->un_pos.lgclblkno > 0))) && - ((un->un_state != ST_STATE_CLOSED) && - (un->un_laststate != ST_STATE_CLOSING)))) { + ((un->un_state == ST_STATE_CLOSED) && + (un->un_laststate == ST_STATE_CLOSING)))) { ST_DEBUG(ST_DEVINFO, st_label, SCSI_DEBUG, "cannot detach: pmode=%d fileno=0x%x, blkno=0x%x" @@ -17033,6 +17033,12 @@ case CMD_TRAN_ERR: action = QUE_COMMAND; break; + case CMD_DEV_GONE: + if (un->un_multipath) + action = PATH_FAILED; + else + action = COMMAND_DONE_ERROR; + break; default: ST_DEBUG(ST_DEVINFO, st_label, CE_PANIC, "pkt_reason not handled yet %s",
--- a/usr/src/uts/common/io/tpm/tpm.c Mon Sep 21 11:25:09 2009 -0400 +++ b/usr/src/uts/common/io/tpm/tpm.c Mon Sep 21 11:26:40 2009 -0400 @@ -421,7 +421,9 @@ ret = itpm_command(tpm, buf, sizeof (buf)); if (ret != DDI_SUCCESS) { - cmn_err(CE_WARN, "%s: itpm_command failed", myname); +#ifdef DEBUG + cmn_err(CE_WARN, "!%s: itpm_command failed", myname); +#endif return (DDI_FAILURE); } @@ -433,9 +435,11 @@ */ len = load32(buf, TPM_CAP_RESPSIZE_OFFSET); if (len != 4 * sizeof (uint32_t)) { - cmn_err(CE_WARN, "%s: capability response size should be %d" - "instead it's %d", +#ifdef DEBUG + cmn_err(CE_WARN, "!%s: capability response size should be %d" + "instead len = %d", myname, (int)(4 * sizeof (uint32_t)), (int)len); +#endif return (DDI_FAILURE); } @@ -503,8 +507,10 @@ ret = itpm_command(tpm, buf, sizeof (buf)); if (ret != DDI_SUCCESS) { - cmn_err(CE_WARN, "%s: itpm_command failed with ret code: 0x%x", +#ifdef DEBUG + cmn_err(CE_WARN, "!%s: itpm_command failed with ret code: 0x%x", myname, ret); +#endif return (DDI_FAILURE); } @@ -516,9 +522,11 @@ */ len = load32(buf, TPM_CAP_RESPSIZE_OFFSET); if (len != 3 * sizeof (uint32_t)) { - cmn_err(CE_WARN, "%s: capability response should be %d, " +#ifdef DEBUG + cmn_err(CE_WARN, "!%s: capability response should be %d, " "instead, it's %d", myname, (int)(3 * sizeof (uint32_t)), (int)len); +#endif return (DDI_FAILURE); } @@ -576,8 +584,10 @@ ret = itpm_command(tpm, buf, sizeof (buf)); if (ret != DDI_SUCCESS) { - cmn_err(CE_WARN, "%s: itpm_command failed with ret code: 0x%x", +#ifdef DEBUG + cmn_err(CE_WARN, "!%s: itpm_command failed with ret code: 0x%x", myname, ret); +#endif return (DDI_FAILURE); } @@ -586,11 +596,11 @@ */ len = load32(buf, TPM_CAP_RESPSIZE_OFFSET); if (len < TPM_CAP_VERSION_INFO_SIZE) { - cmn_err(CE_WARN, "%s: capability response should be greater" +#ifdef DEBUG + cmn_err(CE_WARN, "!%s: capability response should be greater" " than %d, instead, it's %d", - myname, - TPM_CAP_VERSION_INFO_SIZE, - len); + myname, TPM_CAP_VERSION_INFO_SIZE, len); +#endif return (DDI_FAILURE); } @@ -616,7 +626,7 @@ */ if (tpm->vers_info.version.major != 1 && tpm->vers_info.version.minor != 2) { - cmn_err(CE_WARN, "%s: Unsupported TPM version (%d.%d)", + cmn_err(CE_WARN, "!%s: Unsupported TPM version (%d.%d)", myname, tpm->vers_info.version.major, /* Version */ tpm->vers_info.version.minor); @@ -644,7 +654,9 @@ /* Need a longer timeout */ ret = itpm_command(tpm, buf, sizeof (buf)); if (ret != DDI_SUCCESS) { - cmn_err(CE_WARN, "%s: itpm_command failed", myname); +#ifdef DEBUG + cmn_err(CE_WARN, "!%s: itpm_command failed", myname); +#endif return (DDI_FAILURE); } @@ -669,25 +681,31 @@ /* Is it a TSC_ORDINAL? */ if (ordinal & TSC_ORDINAL_MASK) { if (ordinal > TSC_ORDINAL_MAX) { +#ifdef DEBUG cmn_err(CE_WARN, - "%s: tsc ordinal: %d exceeds MAX: %d", + "!%s: tsc ordinal: %d exceeds MAX: %d", myname, ordinal, TSC_ORDINAL_MAX); +#endif return (0); } index = tsc_ords_duration[ordinal]; } else { if (ordinal > TPM_ORDINAL_MAX) { +#ifdef DEBUG cmn_err(CE_WARN, - "%s: ordinal %d exceeds MAX: %d", + "!%s: ordinal %d exceeds MAX: %d", myname, ordinal, TPM_ORDINAL_MAX); +#endif return (0); } index = tpm_ords_duration[ordinal]; } if (index > TPM_DURATION_MAX_IDX) { - cmn_err(CE_WARN, "%s: FATAL:index '%d' is out of bound", +#ifdef DEBUG + cmn_err(CE_WARN, "!%s: duration index '%d' is out of bounds", myname, index); +#endif return (0); } return (tpm->duration[index]); @@ -710,22 +728,21 @@ /* The byte order is network byte order so convert it */ count = load32(buf, TPM_PARAMSIZE_OFFSET); - if (count == 0) { - cmn_err(CE_WARN, "%s: count=0, no data? %d", myname, - (int)bufsiz); - return (DDI_FAILURE); - } - if (count > bufsiz) { - cmn_err(CE_WARN, "%s: invalid count value:count:%d > bufsiz %d", - myname, (int)count, (int)bufsiz); + if (count == 0 || (count > bufsiz)) { +#ifdef DEBUG + cmn_err(CE_WARN, "!%s: invalid byte count value " + "(%d > bufsiz %d)", myname, (int)count, (int)bufsiz); +#endif return (DDI_FAILURE); } /* Send the command */ ret = tis_send_data(tpm, buf, count); if (ret != DDI_SUCCESS) { - cmn_err(CE_WARN, "%s: tis_send_data failed with error %x", +#ifdef DEBUG + cmn_err(CE_WARN, "!%s: tis_send_data failed with error %x", myname, ret); +#endif return (DDI_FAILURE); } @@ -735,7 +752,9 @@ */ ret = tis_recv_data(tpm, buf, bufsiz); if (ret < TPM_HEADER_SIZE) { - cmn_err(CE_WARN, "%s: tis_recv_data failed", myname); +#ifdef DEBUG + cmn_err(CE_WARN, "!%s: tis_recv_data failed", myname); +#endif return (DDI_FAILURE); } @@ -743,11 +762,11 @@ ret = load32(buf, TPM_RETURN_OFFSET); if (ret != TPM_SUCCESS) { if (ret == TPM_E_DEACTIVATED) - cmn_err(CE_WARN, "%s: TPM is deactivated", myname); + cmn_err(CE_WARN, "!%s: TPM is deactivated", myname); else if (ret == TPM_E_DISABLED) - cmn_err(CE_WARN, "%s: TPM is disabled", myname); + cmn_err(CE_WARN, "!%s: TPM is disabled", myname); else - cmn_err(CE_WARN, "%s: TPM error code 0x%0x", + cmn_err(CE_WARN, "!%s: TPM error code 0x%0x", myname, ret); return (DDI_FAILURE); } @@ -853,18 +872,22 @@ ASSERT(tpm != NULL && buf != NULL); if (bufsiz < TPM_HEADER_SIZE) { - /* There should be at least tag,paramsize,return code */ - cmn_err(CE_WARN, "%s: received data should contain at least " + /* There should be at least tag, paramsize, return code */ +#ifdef DEBUG + cmn_err(CE_WARN, "!%s: received data should contain at least " "the header which is %d bytes long", myname, TPM_HEADER_SIZE); +#endif goto OUT; } /* Read tag(2 bytes), paramsize(4), and result(4) */ size = receive_data(tpm, buf, TPM_HEADER_SIZE); if (size < TPM_HEADER_SIZE) { - cmn_err(CE_WARN, "%s: getting the TPM_HEADER failed: size=%d", +#ifdef DEBUG + cmn_err(CE_WARN, "!%s: recv TPM_HEADER failed, size = %d", myname, size); +#endif goto OUT; } @@ -873,9 +896,11 @@ /* Get 'paramsize'(4 bytes)--it includes tag and paramsize */ expected = load32(buf, TPM_PARAMSIZE_OFFSET); if (expected > bufsiz) { - cmn_err(CE_WARN, "%s: paramSize is bigger " +#ifdef DEBUG + cmn_err(CE_WARN, "!%s: paramSize is bigger " "than the requested size: paramSize=%d bufsiz=%d result=%d", myname, (int)expected, (int)bufsiz, cmdresult); +#endif goto OUT; } @@ -883,8 +908,10 @@ size += receive_data(tpm, (uint8_t *)&buf[TPM_HEADER_SIZE], expected - TPM_HEADER_SIZE); if (size < expected) { - cmn_err(CE_WARN, "%s: received data length=%d " - "is less than expected = %d", myname, size, expected); +#ifdef DEBUG + cmn_err(CE_WARN, "!%s: received data length (%d) " + "is less than expected (%d)", myname, size, expected); +#endif goto OUT; } @@ -893,15 +920,19 @@ status = tis_get_status(tpm); if (ret != DDI_SUCCESS) { - cmn_err(CE_WARN, "%s: TPM didn't set stsValid after its I/O: " +#ifdef DEBUG + cmn_err(CE_WARN, "!%s: TPM didn't set stsValid after its I/O: " "status = 0x%08X", myname, status); +#endif goto OUT; } /* There is still more data? */ if (status & TPM_STS_DATA_AVAIL) { - cmn_err(CE_WARN, "%s: Status TPM_STS_DATA_AVAIL set:0x%08X", +#ifdef DEBUG + cmn_err(CE_WARN, "!%s: TPM_STS_DATA_AVAIL is set:0x%08X", myname, status); +#endif goto OUT; } @@ -932,8 +963,9 @@ ASSERT(tpm != NULL && buf != NULL); if (bufsiz == 0) { - cmn_err(CE_WARN, "%s: passed in argument bufsize is zero", - myname); +#ifdef DEBUG + cmn_err(CE_WARN, "!%s: bufsiz arg is zero", myname); +#endif return (DDI_FAILURE); } @@ -944,10 +976,12 @@ tpm_set_ready(tpm); ret = tpm_wait_for_stat(tpm, TPM_STS_CMD_READY, tpm->timeout_b); if (ret != DDI_SUCCESS) { - cmn_err(CE_WARN, "%s: could not put the TPM " +#ifdef DEBUG + cmn_err(CE_WARN, "!%s: could not put the TPM " "in the command ready state:" "tpm_wait_for_stat returned error", myname); +#endif goto FAIL; } } @@ -961,8 +995,10 @@ while (count < bufsiz - 1) { burstcnt = tpm_get_burstcount(tpm); if (burstcnt == 0) { - cmn_err(CE_WARN, "%s: tpm_get_burstcnt returned error", +#ifdef DEBUG + cmn_err(CE_WARN, "!%s: tpm_get_burstcnt returned error", myname); +#endif ret = DDI_FAILURE; goto FAIL; } @@ -975,8 +1011,10 @@ ret = tpm_wait_for_stat(tpm, (TPM_STS_VALID | TPM_STS_DATA_EXPECT), tpm->timeout_c); if (ret != DDI_SUCCESS) { - cmn_err(CE_WARN, "%s: TPM didn't enter stsvalid " - "state after sending the data:", myname); +#ifdef DEBUG + cmn_err(CE_WARN, "!%s: TPM didn't enter STS_VALID " + "state", myname); +#endif goto FAIL; } } @@ -989,15 +1027,20 @@ /* Wait for the TPM to enter Valid State */ ret = tpm_wait_for_stat(tpm, TPM_STS_VALID, tpm->timeout_c); if (ret == DDI_FAILURE) { - cmn_err(CE_WARN, "%s: tpm didn't enter Valid state", myname); +#ifdef DEBUG + cmn_err(CE_WARN, "!%s: tpm didn't enter STS_VALID state", + myname); +#endif goto FAIL; } status = tis_get_status(tpm); /* The TPM should NOT be expecing more data at this point */ if ((status & TPM_STS_DATA_EXPECT) != 0) { - cmn_err(CE_WARN, "%s: DATA_EXPECT is set (shouldn't be) after " +#ifdef DEBUG + cmn_err(CE_WARN, "!%s: DATA_EXPECT should not be set after " "writing the last byte: status=0x%08X", myname, status); +#endif ret = DDI_FAILURE; goto FAIL; } @@ -1014,18 +1057,21 @@ ret = tpm_wait_for_stat(tpm, TPM_STS_DATA_AVAIL | TPM_STS_VALID, tpm_get_ordinal_duration(tpm, ordinal)); if (ret == DDI_FAILURE) { +#ifdef DEBUG status = tis_get_status(tpm); if (!(status & TPM_STS_DATA_AVAIL) || !(status & TPM_STS_VALID)) { - cmn_err(CE_WARN, "%s: TPM not ready or valid " - "(ordinal = %d timeout = %ld)", + cmn_err(CE_WARN, "!%s: TPM not ready or valid " + "(ordinal = %d timeout = %ld status = 0x%0x)", myname, ordinal, - tpm_get_ordinal_duration(tpm, ordinal)); + tpm_get_ordinal_duration(tpm, ordinal), + status); } else { - cmn_err(CE_WARN, "%s: tpm_wait_for_stat " - "(DATA_AVAIL | VALID) failed: STS = 0x%0X", + cmn_err(CE_WARN, "!%s: tpm_wait_for_stat " + "(DATA_AVAIL | VALID) failed status = 0x%0X", myname, status); } +#endif goto FAIL; } return (DDI_SUCCESS); @@ -1107,9 +1153,11 @@ while (tis_check_active_locality(tpm, locality) != DDI_SUCCESS) { if (ddi_get_lbolt() >= timeout) { - cmn_err(CE_WARN, "%s (interrupt-disabled) " +#ifdef DEBUG + cmn_err(CE_WARN, "!%s: (interrupt-disabled) " "tis_request_locality timed out (timeout_a = %ld)", myname, tpm->timeout_a); +#endif return (DDI_FAILURE); } delay(tpm->timeout_poll); @@ -1134,9 +1182,11 @@ while ((tis_get_status(tpm) & mask) != mask) { if (ddi_get_lbolt() >= absolute_timeout) { /* Timeout reached */ - cmn_err(CE_WARN, "%s: using " +#ifdef DEBUG + cmn_err(CE_WARN, "!%s: using " "polling - reached timeout (%ld usecs)", myname, drv_hztousec(timeout)); +#endif return (DDI_FAILURE); } delay(tpm->timeout_poll); @@ -1182,22 +1232,21 @@ /* Upper 3 bytes should always return 0 */ if (intf_caps & 0x7FFFFF00) { -#ifdef DEBUG - cmn_err(CE_WARN, "%s: bad intf_caps value 0x%0X", + cmn_err(CE_WARN, "!%s: bad intf_caps value 0x%0X", myname, intf_caps); -#endif return (DDI_FAILURE); } /* These two interrupts are mandatory */ if (!(intf_caps & TPM_INTF_INT_LOCALITY_CHANGE_INT)) { - cmn_err(CE_WARN, "%s: Mandatory capability Locality Change Int " + cmn_err(CE_WARN, + "!%s: Mandatory capability Locality Change Int " "not supported", myname); return (DDI_FAILURE); } if (!(intf_caps & TPM_INTF_INT_DATA_AVAIL_INT)) { - cmn_err(CE_WARN, "%s: Mandatory capability Data Available Int " - "not supported", myname); + cmn_err(CE_WARN, "!%s: Mandatory capability Data Available Int " + "not supported.", myname); return (DDI_FAILURE); } @@ -1207,7 +1256,7 @@ */ ret = tis_request_locality(tpm, DEFAULT_LOCALITY); if (ret != DDI_SUCCESS) { - cmn_err(CE_WARN, "%s: Unable to request locality %d", myname, + cmn_err(CE_WARN, "!%s: Unable to request locality %d", myname, DEFAULT_LOCALITY); return (DDI_FAILURE); } /* Now we can refer to the locality as tpm->locality */ @@ -1218,20 +1267,20 @@ /* Get the real timeouts from the TPM */ ret = tpm_get_timeouts(tpm); if (ret != DDI_SUCCESS) { - cmn_err(CE_WARN, "%s: tpm_get_timeouts error", myname); + cmn_err(CE_WARN, "!%s: tpm_get_timeouts error", myname); return (DDI_FAILURE); } ret = tpm_get_duration(tpm); if (ret != DDI_SUCCESS) { - cmn_err(CE_WARN, "%s: tpm_get_duration error", myname); + cmn_err(CE_WARN, "!%s: tpm_get_duration error", myname); return (DDI_FAILURE); } /* This gets the TPM version information */ ret = tpm_get_version(tpm); if (ret != DDI_SUCCESS) { - cmn_err(CE_WARN, "%s: tpm_get_version error", myname); + cmn_err(CE_WARN, "!%s: tpm_get_version error", myname); return (DDI_FAILURE); } @@ -1241,7 +1290,7 @@ */ ret = tpm_continue_selftest(tpm); if (ret != DDI_SUCCESS) { - cmn_err(CE_WARN, "%s: tpm_continue_selftest error", myname); + cmn_err(CE_WARN, "!%s: tpm_continue_selftest error", myname); return (DDI_FAILURE); } return (DDI_SUCCESS); @@ -1256,14 +1305,17 @@ int ret; ret = ddi_soft_state_init(&statep, sizeof (tpm_state_t), 1); - if (ret) -{ - cmn_err(CE_WARN, "ddi_soft_state_init failed: %d", ret); + if (ret) { +#ifdef DEBUG + cmn_err(CE_WARN, "!ddi_soft_state_init failed: %d", ret); +#endif return (ret); -} + } ret = mod_install(&tpm_ml); if (ret != 0) { - cmn_err(CE_WARN, "_init: mod_install returned non-zero"); +#ifdef DEBUG + cmn_err(CE_WARN, "!_init: mod_install returned non-zero"); +#endif ddi_soft_state_fini(&statep); return (ret); } @@ -1276,8 +1328,10 @@ { int ret; ret = mod_info(&tpm_ml, modinfop); +#ifdef DEBUG if (ret == 0) - cmn_err(CE_WARN, "mod_info failed: %d", ret); + cmn_err(CE_WARN, "!mod_info failed: %d", ret); +#endif return (ret); } @@ -1345,16 +1399,18 @@ if (ddi_soft_state_zalloc(statep, instance) == DDI_SUCCESS) { tpm = ddi_get_soft_state(statep, instance); if (tpm == NULL) { +#ifdef DEBUG cmn_err(CE_WARN, - "%s: cannot get state information.", + "!%s: cannot get state information.", myname); +#endif return (DDI_FAILURE); } tpm->dip = dip; } else { #ifdef DEBUG cmn_err(CE_WARN, - "%s: cannot allocate state information.", + "!%s: cannot allocate state information.", myname); #endif return (DDI_FAILURE); @@ -1363,13 +1419,17 @@ case DDI_RESUME: tpm = ddi_get_soft_state(statep, instance); if (tpm == NULL) { - cmn_err(CE_WARN, "%s: cannot get state information.", +#ifdef DEBUG + cmn_err(CE_WARN, "!%s: cannot get state information.", myname); +#endif return (DDI_FAILURE); } return (tpm_resume(tpm)); default: - cmn_err(CE_WARN, "%s: cmd %d is not implemented", myname, cmd); +#ifdef DEBUG + cmn_err(CE_WARN, "!%s: cmd %d is not implemented", myname, cmd); +#endif ret = DDI_FAILURE; goto FAIL; } @@ -1380,7 +1440,7 @@ #ifdef sun4v ret = hsvc_register(&hsvc_tpm, &hsvc_tpm_minor); if (ret != 0) { - cmn_err(CE_WARN, "%s: failed to register with " + cmn_err(CE_WARN, "!%s: failed to register with " "hypervisor: 0x%0x", myname, ret); goto FAIL; } @@ -1429,8 +1489,10 @@ /* Enable TPM device according to the TIS specification */ ret = tis_init(tpm); if (ret != DDI_SUCCESS) { - cmn_err(CE_WARN, "%s: tis_init() failed with error %d", +#ifdef DEBUG + cmn_err(CE_WARN, "!%s: tis_init() failed with error %d", myname, ret); +#endif /* We need to clean up the ddi_regs_map_setup call */ if (tpm->flags & TPM_DIDREGSMAP) { @@ -1471,7 +1533,9 @@ ret = ddi_create_minor_node(dip, "tpm", S_IFCHR, ddi_get_instance(dip), DDI_PSEUDO, 0); if (ret != DDI_SUCCESS) { - cmn_err(CE_WARN, "%s: ddi_create_minor_node failed", myname); +#ifdef DEBUG + cmn_err(CE_WARN, "!%s: ddi_create_minor_node failed", myname); +#endif goto FAIL; } tpm->flags |= TPM_DIDMINOR; @@ -1479,7 +1543,7 @@ #ifdef KCF_TPM_RNG_PROVIDER /* register RNG with kcf */ if (tpmrng_register(tpm) != DDI_SUCCESS) - cmn_err(CE_WARN, "%s: tpm RNG failed to register with kcf", + cmn_err(CE_WARN, "!%s: tpm RNG failed to register with kcf", myname); #endif @@ -1576,8 +1640,10 @@ return (DDI_FAILURE); if ((tpm = ddi_get_soft_state(statep, instance)) == NULL) { - cmn_err(CE_WARN, "%s: stored pointer to tpm state is NULL", +#ifdef DEBUG + cmn_err(CE_WARN, "!%s: stored pointer to tpm state is NULL", myname); +#endif return (ENXIO); } @@ -1588,7 +1654,9 @@ case DDI_SUSPEND: return (tpm_suspend(tpm)); default: - cmn_err(CE_WARN, "%s: case %d not implemented", myname, cmd); +#ifdef DEBUG + cmn_err(CE_WARN, "!%s: case %d not implemented", myname, cmd); +#endif return (DDI_FAILURE); } @@ -1612,8 +1680,10 @@ instance = ddi_get_instance(dip); if ((tpm = ddi_get_soft_state(statep, instance)) == NULL) { - cmn_err(CE_WARN, "%s: stored pointer to tpm state is NULL", +#ifdef DEBUG + cmn_err(CE_WARN, "!%s: stored pointer to tpm state is NULL", myname); +#endif return (DDI_FAILURE); } @@ -1625,7 +1695,9 @@ *resultp = 0; break; default: - cmn_err(CE_WARN, "%s: cmd %d is not implemented", myname, cmd); +#ifdef DEBUG + cmn_err(CE_WARN, "!%s: cmd %d is not implemented", myname, cmd); +#endif return (DDI_FAILURE); } return (DDI_SUCCESS); @@ -1647,21 +1719,27 @@ instance = getminor(*devp); if ((tpm = ddi_get_soft_state(statep, instance)) == NULL) { - cmn_err(CE_WARN, "%s: stored pointer to tpm state is NULL", +#ifdef DEBUG + cmn_err(CE_WARN, "!%s: stored pointer to tpm state is NULL", myname); +#endif return (ENXIO); } if (otyp != OTYP_CHR) { - cmn_err(CE_WARN, "%s: otyp(%d) != OTYP_CHR(%d)", +#ifdef DEBUG + cmn_err(CE_WARN, "!%s: otyp(%d) != OTYP_CHR(%d)", myname, otyp, OTYP_CHR); +#endif return (EINVAL); } TPM_EXCLUSIVE_LOCK(tpm); mutex_enter(&tpm->dev_lock); if (tpm->dev_held) { - cmn_err(CE_WARN, "%s: the device is already being used", +#ifdef DEBUG + cmn_err(CE_WARN, "!%s: the device is already being used", myname); +#endif mutex_exit(&tpm->dev_lock); return (EBUSY); } @@ -1683,13 +1761,17 @@ instance = getminor(dev); if ((tpm = ddi_get_soft_state(statep, instance)) == NULL) { - cmn_err(CE_WARN, "%s: stored pointer to tpm state is NULL", +#ifdef DEBUG + cmn_err(CE_WARN, "!%s: stored pointer to tpm state is NULL", myname); +#endif return (ENXIO); } if (otyp != OTYP_CHR) { - cmn_err(CE_WARN, "%s: otyp(%d) != OTYP_CHR(%d)", +#ifdef DEBUG + cmn_err(CE_WARN, "!%s: otyp(%d) != OTYP_CHR(%d)", myname, otyp, OTYP_CHR); +#endif return (EINVAL); } TPM_EXCLUSIVE_LOCK(tpm); @@ -1716,12 +1798,16 @@ instance = getminor(dev); if ((tpm = ddi_get_soft_state(statep, instance)) == NULL) { - cmn_err(CE_WARN, "%s: stored pointer to tpm state is NULL", +#ifdef DEBUG + cmn_err(CE_WARN, "!%s: stored pointer to tpm state is NULL", myname); +#endif return (ENXIO); } if (uiop == NULL) { - cmn_err(CE_WARN, "%s: passed in uiop is NULL", myname); +#ifdef DEBUG + cmn_err(CE_WARN, "!%s: passed in uiop is NULL", myname); +#endif return (EFAULT); } @@ -1735,25 +1821,31 @@ return (ret); if (uiop->uio_resid > tpm->bufsize) { - cmn_err(CE_WARN, "%s: read_in data is bigger " +#ifdef DEBUG + cmn_err(CE_WARN, "!%s: read_in data is bigger " "than tpm->bufsize:read in:%d, bufsiz:%d", myname, (int)uiop->uio_resid, (int)tpm->bufsize); +#endif ret = EIO; goto OUT; } ret = tis_recv_data(tpm, tpm->iobuf, tpm->bufsize); if (ret < TPM_HEADER_SIZE) { - cmn_err(CE_WARN, "%s: tis_recv_data returned error", myname); +#ifdef DEBUG + cmn_err(CE_WARN, "!%s: tis_recv_data returned error", myname); +#endif ret = EIO; goto OUT; } size = load32(tpm->iobuf, 2); if (ret != size) { - cmn_err(CE_WARN, "%s: tis_recv_data:" +#ifdef DEBUG + cmn_err(CE_WARN, "!%s: tis_recv_data:" "expected size=%d, actually read=%d", myname, size, ret); +#endif ret = EIO; goto OUT; } @@ -1761,7 +1853,9 @@ /* Send the buffer from the kernel to the userspace */ ret = uiomove(tpm->iobuf, size, UIO_READ, uiop); if (ret) { - cmn_err(CE_WARN, "%s: uiomove returned error", myname); +#ifdef DEBUG + cmn_err(CE_WARN, "!%s: uiomove returned error", myname); +#endif goto OUT; } @@ -1788,13 +1882,17 @@ instance = getminor(dev); if ((tpm = ddi_get_soft_state(statep, instance)) == NULL) { - cmn_err(CE_WARN, "%s: stored pointer to tpm state is NULL", +#ifdef DEBUG + cmn_err(CE_WARN, "!%s: stored pointer to tpm state is NULL", myname); +#endif return (ENXIO); } if (uiop == NULL) { - cmn_err(CE_WARN, "%s: passed in uiop is NULL", myname); +#ifdef DEBUG + cmn_err(CE_WARN, "!%s: passed in uiop is NULL", myname); +#endif return (EFAULT); } @@ -1802,7 +1900,9 @@ len = uiop->uio_resid; if (len == 0) { - cmn_err(CE_WARN, "%s: requested read of len 0", myname); +#ifdef DEBUG + cmn_err(CE_WARN, "!%s: requested read of len 0", myname); +#endif return (0); } @@ -1815,9 +1915,11 @@ /* Copy the header and parse the structure to find out the size... */ ret = uiomove(tpm->iobuf, TPM_HEADER_SIZE, UIO_WRITE, uiop); if (ret) { - cmn_err(CE_WARN, "%s: uiomove returned error" +#ifdef DEBUG + cmn_err(CE_WARN, "!%s: uiomove returned error" "while getting the the header", myname); +#endif goto OUT; } @@ -1826,9 +1928,11 @@ /* Copy the command to the contiguous buffer */ if (size > tpm->bufsize) { - cmn_err(CE_WARN, "%s: size %d is greater than " - "the tpm's input buffer size %d", +#ifdef DEBUG + cmn_err(CE_WARN, "!%s: size %d is greater than " + "the tpm input buffer size %d", myname, (int)size, (int)tpm->bufsize); +#endif ret = ENXIO; goto OUT; } @@ -1838,15 +1942,19 @@ UIO_WRITE, uiop); if (ret) { - cmn_err(CE_WARN, "%s: uiomove returned error" +#ifdef DEBUG + cmn_err(CE_WARN, "!%s: uiomove returned error" "while getting the rest of the command", myname); +#endif goto OUT; } /* Send the command */ ret = tis_send_data(tpm, tpm->iobuf, size); if (ret != DDI_SUCCESS) { - cmn_err(CE_WARN, "%s: tis_send_data returned error", myname); +#ifdef DEBUG + cmn_err(CE_WARN, "!%s: tis_send_data returned error", myname); +#endif ret = EFAULT; goto OUT; } @@ -1880,7 +1988,7 @@ /* Timeout reached */ mutex_exit(&tpm->iobuf_lock); #ifdef DEBUG - cmn_err(CE_WARN, "tpm_io_lock:iorequest timed out"); + cmn_err(CE_WARN, "!tpm_io_lock:iorequest timed out"); #endif return (ETIME); } @@ -1991,9 +2099,10 @@ (void) memcpy(rngmech, "random", 6); ret = crypto_load_dev_disabled("tpm", ddi_get_instance(tpm->dip), 1, rngmech); - if (ret != CRYPTO_SUCCESS) { - cmn_err(CE_WARN, "crypto_load_dev_disabled failed (%d)", ret); - } +#ifdef DEBUG + if (ret != CRYPTO_SUCCESS) + cmn_err(CE_WARN, "!crypto_load_dev_disabled failed (%d)", ret); +#endif return (DDI_SUCCESS); } @@ -2076,7 +2185,7 @@ if (ret != DDI_SUCCESS) { #ifdef DEBUG - cmn_err(CE_WARN, "tpmrng_seed_random failed"); + cmn_err(CE_WARN, "!tpmrng_seed_random failed"); #endif return (CRYPTO_FAILED); } @@ -2125,7 +2234,7 @@ ret = itpm_command(tpm, cmdbuf, buflen); if (ret != DDI_SUCCESS) { #ifdef DEBUG - cmn_err(CE_WARN, "tpmrng_generate_random failed"); + cmn_err(CE_WARN, "!tpmrng_generate_random failed"); #endif kmem_free(cmdbuf, buflen); tpm_unlock(tpm);
--- a/usr/src/uts/common/sys/Makefile Mon Sep 21 11:25:09 2009 -0400 +++ b/usr/src/uts/common/sys/Makefile Mon Sep 21 11:26:40 2009 -0400 @@ -1306,6 +1306,11 @@ LVMDERIVED_H: cd $(SRC)/uts/common/sys/lvm; pwd; $(MAKE) +clean: + $(RM) $(GENHDRS) + +clobber: clean + check: $(CHECKHDRS) FRC:
--- a/usr/src/uts/common/sys/fs/zfs.h Mon Sep 21 11:25:09 2009 -0400 +++ b/usr/src/uts/common/sys/fs/zfs.h Mon Sep 21 11:26:40 2009 -0400 @@ -117,6 +117,8 @@ ZFS_PROP_DEFER_DESTROY, ZFS_PROP_USERREFS, ZFS_PROP_LOGBIAS, + ZFS_PROP_UNIQUE, /* not exposed to the user */ + ZFS_PROP_OBJSETID, /* not exposed to the user */ ZFS_NUM_PROPS } zfs_prop_t;
--- a/usr/src/uts/common/sys/nxge/nxge_hio.h Mon Sep 21 11:25:09 2009 -0400 +++ b/usr/src/uts/common/sys/nxge/nxge_hio.h Mon Sep 21 11:26:40 2009 -0400 @@ -99,8 +99,8 @@ #define NXGE_VR_SR_MAX 8 /* There are 8 subregions (SR). */ typedef enum { - NXGE_HIO_TYPE_SERVICE, /* We are a service domain driver. */ - NXGE_HIO_TYPE_GUEST /* We are a guest domain driver. */ + NXGE_HIO_TYPE_SERVICE = 0x80, /* We are a service domain driver. */ + NXGE_HIO_TYPE_GUEST /* We are a guest domain driver. */ } nxge_hio_type_t; typedef enum {
--- a/usr/src/uts/common/syscall/uadmin.c Mon Sep 21 11:25:09 2009 -0400 +++ b/usr/src/uts/common/syscall/uadmin.c Mon Sep 21 11:26:40 2009 -0400 @@ -305,7 +305,7 @@ case A_CONFIG: switch (fcn) { case AD_UPDATE_BOOT_CONFIG: -#ifndef __sparc +#if !defined(__sparc) && !defined(__s390) { extern void fastboot_update_config(const char *); @@ -376,7 +376,7 @@ } else panic_bootstr = mdep; -#ifndef __sparc +#if !defined(__sparc) && !defined(__s390) extern void fastboot_update_and_load(int, char *); fastboot_update_and_load(fcn, mdep);
--- a/usr/src/uts/common/xen/io/blk_common.c Mon Sep 21 11:25:09 2009 -0400 +++ b/usr/src/uts/common/xen/io/blk_common.c Mon Sep 21 11:26:40 2009 -0400 @@ -558,6 +558,17 @@ goto startconnectfail_transaction_start; } + /* xentop requires the instance in xenstore */ + e = xenbus_printf(xbt, xsnode, "instance", "%d", + ddi_get_instance(ring->ri_dip)); + if (e != 0) { + cmn_err(CE_WARN, "xdb@%s: failed to write 'instance'", + ddi_get_name_addr(dip)); + xvdi_fatal_error(dip, e, "writing 'instance'"); + (void) xenbus_transaction_end(xbt, 1); + goto startconnectfail_xenbus_printf; + } + /* If feature-barrier isn't present in xenstore, add it */ e = xenbus_read(xbt, xsnode, "feature-barrier", (void **)&barrier, &len);
--- a/usr/src/uts/i86pc/Makefile.files Mon Sep 21 11:25:09 2009 -0400 +++ b/usr/src/uts/i86pc/Makefile.files Mon Sep 21 11:26:40 2009 -0400 @@ -206,6 +206,8 @@ TZMON_OBJS += tzmon.o UPPC_OBJS += uppc.o psm_common.o XSVC_OBJS += xsvc.o +AMD_IOMMU_OBJS += amd_iommu.o amd_iommu_impl.o amd_iommu_acpi.o \ + amd_iommu_cmd.o amd_iommu_log.o amd_iommu_page_tables.o # # Build up defines and paths.
--- a/usr/src/uts/i86pc/Makefile.i86pc.shared Mon Sep 21 11:25:09 2009 -0400 +++ b/usr/src/uts/i86pc/Makefile.i86pc.shared Mon Sep 21 11:26:40 2009 -0400 @@ -255,11 +255,13 @@ DRV_KMODS += tzmon DRV_KMODS += acpi_drv DRV_KMODS += acpinex +DRV_KMODS += amd_iommu DRV_KMODS += ioat DRV_KMODS += fipe DRV_KMODS += cpudrv + # # Platform Power Modules #
--- a/usr/src/uts/i86pc/Makefile.rules Mon Sep 21 11:25:09 2009 -0400 +++ b/usr/src/uts/i86pc/Makefile.rules Mon Sep 21 11:26:40 2009 -0400 @@ -79,6 +79,10 @@ $(COMPILE.c) -o $@ $< $(CTFCONVERT_O) +$(OBJS_DIR)/%.o: $(UTSBASE)/i86pc/io/amd_iommu/%.c + $(COMPILE.c) -o $@ $< + $(CTFCONVERT_O) + $(OBJS_DIR)/%.o: $(UTSBASE)/i86pc/io/ioat/%.c $(COMPILE.c) -o $@ $< $(CTFCONVERT_O) @@ -295,6 +299,9 @@ $(LINTS_DIR)/%.ln: $(UTSBASE)/i86pc/io/acpi/acpinex/%.c @($(LHEAD) $(LINT.c) $< $(LTAIL)) +$(LINTS_DIR)/%.ln: $(UTSBASE)/i86pc/io/amd_iommu/%.c + @($(LHEAD) $(LINT.c) $< $(LTAIL)) + $(LINTS_DIR)/%.ln: $(UTSBASE)/i86pc/io/ioat/%.c @($(LHEAD) $(LINT.c) $< $(LTAIL))
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/usr/src/uts/i86pc/amd_iommu/Makefile Mon Sep 21 11:26:40 2009 -0400 @@ -0,0 +1,83 @@ +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# +# Copyright 2009 Sun Microsystems, Inc. All rights reserved. +# Use is subject to license terms. +# +# This Makefile drives production of the amd_iommu driver kernel module. +# +# + +# +# Path to the base of the uts directory tree (usually /usr/src/uts). +# +UTSBASE = ../.. + +# +# Define the module and object file sets. +# +MODULE = amd_iommu +OBJECTS = $(AMD_IOMMU_OBJS:%=$(OBJS_DIR)/%) +LINTS = $(AMD_IOMMU_OBJS:%.o=$(LINTS_DIR)/%.ln) +ROOTMODULE = $(ROOT_PSM_DRV_DIR)/$(MODULE) +CONF_SRCDIR = $(UTSBASE)/i86pc/io/amd_iommu + +# +# Include common rules. +# +include $(UTSBASE)/i86pc/Makefile.i86pc + +# +# Define targets +# +ALL_TARGET = $(BINARY) $(SRC_CONFILE) +LINT_TARGET = $(MODULE).lint +INSTALL_TARGET = $(BINARY) $(ROOTMODULE) $(ROOT_CONFFILE) + +# +# depends on misc/iommulib and misc/acpica +# +LDFLAGS += -dy -Nmisc/iommulib -Nmisc/acpica + +# +# Default build targets. +# +.KEEP_STATE: + +def: $(DEF_DEPS) + +all: $(ALL_DEPS) + +clean: $(CLEAN_DEPS) + +clobber: $(CLOBBER_DEPS) + +lint: $(LINT_DEPS) + +modlintlib: $(MODLINTLIB_DEPS) + +clean.lint: $(CLEAN_LINT_DEPS) + +install: $(INSTALL_DEPS) $(CONF_INSTALL_DEPS) + +# +# Include common targets. +# +include $(UTSBASE)/i86pc/Makefile.targ
--- a/usr/src/uts/i86pc/boot/boot_console.c Mon Sep 21 11:25:09 2009 -0400 +++ b/usr/src/uts/i86pc/boot/boot_console.c Mon Sep 21 11:26:40 2009 -0400 @@ -510,6 +510,10 @@ console_value_t *consolep; size_t len, cons_len; char *cons_str; +#if !defined(_BOOT) + static char console_text[] = "text"; + extern int post_fastreboot; +#endif boot_line = bootstr; console = CONS_INVALID; @@ -522,6 +526,11 @@ if (cons_str == NULL) cons_str = find_boot_line_prop("output-device"); +#if !defined(_BOOT) + if (post_fastreboot && strcmp(cons_str, "graphics") == 0) + cons_str = console_text; +#endif + /* * Go through the console_devices array trying to match the string * we were given. The string on the command line must end with @@ -646,6 +655,10 @@ char *devnames[] = { consoledev, outputdev, inputdev, NULL }; console_value_t *consolep; int i; + extern int post_fastreboot; + + if (post_fastreboot && console == CONS_SCREEN_GRAPHICS) + console = CONS_SCREEN_TEXT; if (console != CONS_USBSER && console != CONS_SCREEN_GRAPHICS) { if (console_set) {
--- a/usr/src/uts/i86pc/cpu/genuineintel/gintel_main.c Mon Sep 21 11:25:09 2009 -0400 +++ b/usr/src/uts/i86pc/cpu/genuineintel/gintel_main.c Mon Sep 21 11:26:40 2009 -0400 @@ -131,6 +131,23 @@ case INTEL_QP_U2: case INTEL_QP_U3: case INTEL_QP_U4: + case INTEL_QP_JF: + case INTEL_QP_JF0: + case INTEL_QP_JF1: + case INTEL_QP_JF2: + case INTEL_QP_JF3: + case INTEL_QP_JF4: + case INTEL_QP_JF5: + case INTEL_QP_JF6: + case INTEL_QP_JF7: + case INTEL_QP_JF8: + case INTEL_QP_JF9: + case INTEL_QP_JFa: + case INTEL_QP_JFb: + case INTEL_QP_JFc: + case INTEL_QP_JFd: + case INTEL_QP_JFe: + case INTEL_QP_JFf: quickpath = 1; break; default:
--- a/usr/src/uts/i86pc/io/amd_iommu/amd_iommu.c Mon Sep 21 11:25:09 2009 -0400 +++ b/usr/src/uts/i86pc/io/amd_iommu/amd_iommu.c Mon Sep 21 11:26:40 2009 -0400 @@ -54,6 +54,7 @@ static int amd_iommu_close(dev_t dev, int flag, int otyp, cred_t *credp); static int amd_iommu_ioctl(dev_t dev, int cmd, intptr_t arg, int mode, cred_t *credp, int *rvalp); +static int amd_iommu_quiesce(dev_info_t *dip); static struct cb_ops amd_iommu_cb_ops = { amd_iommu_open, /* cb_open */ @@ -87,7 +88,8 @@ nodev, /* devo_reset */ &amd_iommu_cb_ops, /* devo_cb_ops */ NULL, /* devo_bus_ops */ - nulldev /* devo_power */ + nulldev, /* devo_power */ + amd_iommu_quiesce, /* devo_quiesce */ }; static struct modldrv modldrv = { @@ -442,3 +444,26 @@ return (ENOTTY); } + +static int +amd_iommu_quiesce(dev_info_t *dip) +{ + int instance = ddi_get_instance(dip); + struct amd_iommu_state *statep; + const char *f = "amd_iommu_quiesce"; + + statep = ddi_get_soft_state(amd_iommu_statep, instance); + if (statep == NULL) { + cmn_err(CE_WARN, "%s: cannot get soft state: instance %d", + f, instance); + return (DDI_FAILURE); + } + + if (amd_iommu_teardown(dip, statep, AMD_IOMMU_QUIESCE) != DDI_SUCCESS) { + cmn_err(CE_WARN, "%s: Unable to quiesce AMD IOMMU " + "%s%d", f, ddi_driver_name(dip), instance); + return (DDI_FAILURE); + } + + return (DDI_SUCCESS); +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/usr/src/uts/i86pc/io/amd_iommu/amd_iommu.conf Mon Sep 21 11:26:40 2009 -0400 @@ -0,0 +1,27 @@ +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# +# +# Copyright 2009 Sun Microsystems, Inc. All rights reserved. +# Use is subject to license terms. +# +# +# To enable IOMMU set this to "yes" and rebuild boot archive +amd-iommu="yes";
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/usr/src/uts/i86pc/io/amd_iommu/amd_iommu_acpi.c Mon Sep 21 11:26:40 2009 -0400 @@ -0,0 +1,951 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ + +/* + * Copyright 2009 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +#include "amd_iommu_acpi.h" +#include "amd_iommu_impl.h" + +static int create_acpi_hash(amd_iommu_acpi_t *acpi); +static void amd_iommu_acpi_table_fini(amd_iommu_acpi_t **acpipp); + +static void dump_acpi_aliases(void); + + +/* + * Globals + */ +static amd_iommu_acpi_global_t *amd_iommu_acpi_global; +static amd_iommu_acpi_ivhd_t **amd_iommu_acpi_ivhd_hash; +static amd_iommu_acpi_ivmd_t **amd_iommu_acpi_ivmd_hash; + +static int +type_byte_size(char *cp) +{ + uint8_t type8 = *((uint8_t *)cp); + uint8_t len_bits; + + len_bits = AMD_IOMMU_REG_GET8(&type8, AMD_IOMMU_ACPI_DEVENTRY_LEN); + + switch (len_bits) { + case 0: + return (4); + case 1: + return (8); + case 2: + return (16); + case 3: + return (32); + default: + cmn_err(CE_WARN, "%s: Invalid deventry len: %d", + amd_iommu_modname, len_bits); + return (len_bits); + } + /*NOTREACHED*/ +} + +static void +process_4byte_deventry(ivhd_container_t *c, char *cp) +{ + int entry_type = *((uint8_t *)cp); + ivhd_deventry_t deventry = {0}; + ivhd_deventry_t *devp; + uint8_t datsetting8; + align_16_t al = {0}; + int i; + + /* 4 byte entry */ + deventry.idev_len = 4; + deventry.idev_deviceid = -1; + deventry.idev_src_deviceid = -1; + + for (i = 0; i < 2; i++) { + al.ent8[i] = *((uint8_t *)&cp[i + 1]); + } + + switch (entry_type) { + case 1: + deventry.idev_type = DEVENTRY_ALL; + break; + case 2: + deventry.idev_type = DEVENTRY_SELECT; + deventry.idev_deviceid = al.ent16; + break; + case 3: + deventry.idev_type = DEVENTRY_RANGE; + deventry.idev_deviceid = al.ent16; + break; + case 4: + deventry.idev_type = DEVENTRY_RANGE_END; + deventry.idev_deviceid = al.ent16; + ASSERT(cp[3] == 0); + break; + case 0: + ASSERT(al.ent16 == 0); + ASSERT(cp[3] == 0); + default: + return; + } + + + devp = kmem_alloc(sizeof (ivhd_deventry_t), KM_SLEEP); + *devp = deventry; + + if (c->ivhdc_first_deventry == NULL) + c->ivhdc_first_deventry = devp; + else + c->ivhdc_last_deventry->idev_next = devp; + + c->ivhdc_last_deventry = devp; + + if (entry_type == 4) + return; + + datsetting8 = (*((uint8_t *)&cp[3])); + + devp->idev_Lint1Pass = AMD_IOMMU_REG_GET8(&datsetting8, + AMD_IOMMU_ACPI_LINT1PASS); + + devp->idev_Lint0Pass = AMD_IOMMU_REG_GET8(&datsetting8, + AMD_IOMMU_ACPI_LINT0PASS); + + devp->idev_SysMgt = AMD_IOMMU_REG_GET8(&datsetting8, + AMD_IOMMU_ACPI_SYSMGT); + + ASSERT(AMD_IOMMU_REG_GET8(&datsetting8, + AMD_IOMMU_ACPI_DATRSV) == 0); + + devp->idev_NMIPass = AMD_IOMMU_REG_GET8(&datsetting8, + AMD_IOMMU_ACPI_NMIPASS); + + devp->idev_ExtIntPass = AMD_IOMMU_REG_GET8(&datsetting8, + AMD_IOMMU_ACPI_EXTINTPASS); + + devp->idev_INITPass = AMD_IOMMU_REG_GET8(&datsetting8, + AMD_IOMMU_ACPI_INITPASS); +} + +static void +process_8byte_deventry(ivhd_container_t *c, char *cp) +{ + uint8_t datsetting8; + int entry_type = (uint8_t)*cp; + ivhd_deventry_t deventry = {0}; + ivhd_deventry_t *devp; + align_16_t al1 = {0}; + align_16_t al2 = {0}; + align_32_t al3 = {0}; + int i; + + /* Length is 8 bytes */ + deventry.idev_len = 8; + deventry.idev_deviceid = -1; + deventry.idev_src_deviceid = -1; + + for (i = 0; i < 2; i++) { + al1.ent8[i] = *((uint8_t *)&cp[i+1]); + al2.ent8[i] = *((uint8_t *)&cp[i+5]); + } + + datsetting8 = *((uint8_t *)&cp[3]); + + switch (entry_type) { + case 66: + deventry.idev_type = DEVENTRY_ALIAS_SELECT; + deventry.idev_deviceid = al1.ent16; + deventry.idev_src_deviceid = al2.ent16; + ASSERT(cp[4] == 0); + ASSERT(cp[7] == 0); + break; + case 67: + deventry.idev_type = DEVENTRY_ALIAS_RANGE; + deventry.idev_deviceid = al1.ent16; + deventry.idev_src_deviceid = al2.ent16; + ASSERT(cp[4] == 0); + ASSERT(cp[7] == 0); + break; + case 70: + deventry.idev_type = DEVENTRY_EXTENDED_SELECT; + deventry.idev_deviceid = al1.ent16; + break; + case 71: + deventry.idev_type = DEVENTRY_EXTENDED_RANGE; + deventry.idev_deviceid = al1.ent16; + break; + case 72: + deventry.idev_type = DEVENTRY_SPECIAL_DEVICE; + ASSERT(al1.ent16 == 0); + deventry.idev_deviceid = -1; + deventry.idev_handle = cp[4]; + deventry.idev_variety = cp[7]; + deventry.idev_src_deviceid = al2.ent16; + default: +#ifdef BROKEN_ASSERT + for (i = 0; i < 7; i++) { + ASSERT(cp[i] == 0); + } +#endif + return; + } + + + devp = kmem_alloc(sizeof (ivhd_deventry_t), KM_SLEEP); + *devp = deventry; + + if (c->ivhdc_first_deventry == NULL) + c->ivhdc_first_deventry = devp; + else + c->ivhdc_last_deventry->idev_next = devp; + + c->ivhdc_last_deventry = devp; + + devp->idev_Lint1Pass = AMD_IOMMU_REG_GET8(&datsetting8, + AMD_IOMMU_ACPI_LINT1PASS); + + devp->idev_Lint0Pass = AMD_IOMMU_REG_GET8(&datsetting8, + AMD_IOMMU_ACPI_LINT0PASS); + + devp->idev_SysMgt = AMD_IOMMU_REG_GET8(&datsetting8, + AMD_IOMMU_ACPI_SYSMGT); + + ASSERT(AMD_IOMMU_REG_GET8(&datsetting8, + AMD_IOMMU_ACPI_DATRSV) == 0); + + devp->idev_NMIPass = AMD_IOMMU_REG_GET8(&datsetting8, + AMD_IOMMU_ACPI_NMIPASS); + + devp->idev_ExtIntPass = AMD_IOMMU_REG_GET8(&datsetting8, + AMD_IOMMU_ACPI_EXTINTPASS); + + devp->idev_INITPass = AMD_IOMMU_REG_GET8(&datsetting8, + AMD_IOMMU_ACPI_INITPASS); + + if (entry_type != 70 && entry_type != 71) { + return; + } + + /* Type 70 and 71 */ + for (i = 0; i < 4; i++) { + al3.ent8[i] = *((uint8_t *)&cp[i+4]); + } + + devp->idev_AtsDisabled = AMD_IOMMU_REG_GET8(&al3.ent32, + AMD_IOMMU_ACPI_ATSDISABLED); + + ASSERT(AMD_IOMMU_REG_GET8(&al3.ent32, AMD_IOMMU_ACPI_EXTDATRSV) == 0); +} + +static void +process_ivhd(amd_iommu_acpi_t *acpi, ivhd_t *ivhdp) +{ + ivhd_container_t *c; + caddr_t ivhd_end; + caddr_t ivhd_tot_end; + caddr_t cp; + + ASSERT(ivhdp->ivhd_type == 0x10); + + c = kmem_zalloc(sizeof (ivhd_container_t), KM_SLEEP); + c->ivhdc_ivhd = kmem_alloc(sizeof (ivhd_t), KM_SLEEP); + *(c->ivhdc_ivhd) = *ivhdp; + + if (acpi->acp_first_ivhdc == NULL) + acpi->acp_first_ivhdc = c; + else + acpi->acp_last_ivhdc->ivhdc_next = c; + + acpi->acp_last_ivhdc = c; + + ivhd_end = (caddr_t)ivhdp + sizeof (ivhd_t); + ivhd_tot_end = (caddr_t)ivhdp + ivhdp->ivhd_len; + + for (cp = ivhd_end; cp < ivhd_tot_end; cp += type_byte_size(cp)) { + /* 16 byte and 32 byte size are currently reserved */ + switch (type_byte_size(cp)) { + case 4: + process_4byte_deventry(c, cp); + break; + case 8: + process_8byte_deventry(c, cp); + break; + case 16: + case 32: + /* Reserved */ + break; + default: + cmn_err(CE_WARN, "%s: unsupported length for device " + "entry in ACPI IVRS table's IVHD entry", + amd_iommu_modname); + break; + } + } +} + +static void +process_ivmd(amd_iommu_acpi_t *acpi, ivmd_t *ivmdp) +{ + ivmd_container_t *c; + + ASSERT(ivmdp->ivmd_type != 0x10); + + c = kmem_zalloc(sizeof (ivmd_container_t), KM_SLEEP); + c->ivmdc_ivmd = kmem_alloc(sizeof (ivmd_t), KM_SLEEP); + *(c->ivmdc_ivmd) = *ivmdp; + + if (acpi->acp_first_ivmdc == NULL) + acpi->acp_first_ivmdc = c; + else + acpi->acp_last_ivmdc->ivmdc_next = c; + + acpi->acp_last_ivmdc = c; +} + +int +amd_iommu_acpi_init(void) +{ + ivrs_t *ivrsp; + caddr_t ivrsp_end; + caddr_t table_end; + caddr_t cp; + uint8_t type8; + amd_iommu_acpi_t *acpi; + align_ivhd_t al_vhd = {0}; + align_ivmd_t al_vmd = {0}; + + if (AcpiGetTable(IVRS_SIG, 1, (ACPI_TABLE_HEADER **)&ivrsp) != AE_OK) { + cmn_err(CE_NOTE, "!amd_iommu: No AMD IOMMU ACPI IVRS table"); + return (DDI_FAILURE); + } + + /* + * Reserved field must be 0 + */ + ASSERT(ivrsp->ivrs_resv == 0); + + ASSERT(AMD_IOMMU_REG_GET32(&ivrsp->ivrs_ivinfo, + AMD_IOMMU_ACPI_IVINFO_RSV1) == 0); + ASSERT(AMD_IOMMU_REG_GET32(&ivrsp->ivrs_ivinfo, + AMD_IOMMU_ACPI_IVINFO_RSV2) == 0); + + ivrsp_end = (caddr_t)ivrsp + sizeof (struct ivrs); + table_end = (caddr_t)ivrsp + ivrsp->ivrs_hdr.Length; + + acpi = kmem_zalloc(sizeof (amd_iommu_acpi_t), KM_SLEEP); + acpi->acp_ivrs = kmem_alloc(sizeof (ivrs_t), KM_SLEEP); + *(acpi->acp_ivrs) = *ivrsp; + + for (cp = ivrsp_end; cp < table_end; cp += (al_vhd.ivhdp)->ivhd_len) { + al_vhd.cp = cp; + if (al_vhd.ivhdp->ivhd_type == 0x10) + process_ivhd(acpi, al_vhd.ivhdp); + } + + for (cp = ivrsp_end; cp < table_end; cp += (al_vmd.ivmdp)->ivmd_len) { + al_vmd.cp = cp; + type8 = al_vmd.ivmdp->ivmd_type; + if (type8 == 0x20 || type8 == 0x21 || type8 == 0x22) + process_ivmd(acpi, al_vmd.ivmdp); + } + + if (create_acpi_hash(acpi) != DDI_SUCCESS) { + return (DDI_FAILURE); + } + + amd_iommu_acpi_table_fini(&acpi); + + ASSERT(acpi == NULL); + + if (amd_iommu_debug & AMD_IOMMU_DEBUG_ACPI) { + dump_acpi_aliases(); + debug_enter("dump"); + } + + return (DDI_SUCCESS); +} + +static ivhd_deventry_t * +free_ivhd_deventry(ivhd_deventry_t *devp) +{ + ivhd_deventry_t *next = devp->idev_next; + + kmem_free(devp, sizeof (ivhd_deventry_t)); + + return (next); +} + +static ivhd_container_t * +free_ivhd_container(ivhd_container_t *ivhdcp) +{ + ivhd_container_t *next = ivhdcp->ivhdc_next; + ivhd_deventry_t *devp; + + for (devp = ivhdcp->ivhdc_first_deventry; devp; ) { + devp = free_ivhd_deventry(devp); + } + + kmem_free(ivhdcp->ivhdc_ivhd, sizeof (ivhd_t)); + kmem_free(ivhdcp, sizeof (ivhd_container_t)); + + return (next); +} + +static ivmd_container_t * +free_ivmd_container(ivmd_container_t *ivmdcp) +{ + ivmd_container_t *next = ivmdcp->ivmdc_next; + + kmem_free(ivmdcp->ivmdc_ivmd, sizeof (ivmd_t)); + kmem_free(ivmdcp, sizeof (ivmd_container_t)); + + return (next); +} + +void +amd_iommu_acpi_fini(void) +{ +} + +/* + * TODO: Do we need to free the ACPI table for om GetFirmwareTable() + */ +static void +amd_iommu_acpi_table_fini(amd_iommu_acpi_t **acpipp) +{ + amd_iommu_acpi_t *acpi = *acpipp; + ivhd_container_t *ivhdcp; + ivmd_container_t *ivmdcp; + + ASSERT(acpi); + + for (ivhdcp = acpi->acp_first_ivhdc; ivhdcp; ) { + ivhdcp = free_ivhd_container(ivhdcp); + } + for (ivmdcp = acpi->acp_first_ivmdc; ivmdcp; ) { + ivmdcp = free_ivmd_container(ivmdcp); + } + + kmem_free(acpi->acp_ivrs, sizeof (struct ivrs)); + kmem_free(acpi, sizeof (amd_iommu_acpi_t)); + + *acpipp = NULL; +} + +static uint16_t +deviceid_hashfn(uint16_t deviceid) +{ + return (deviceid % AMD_IOMMU_ACPI_INFO_HASH_SZ); +} + +static void +add_deventry_info(ivhd_t *ivhdp, ivhd_deventry_t *deventry, + amd_iommu_acpi_ivhd_t **hash) +{ + static amd_iommu_acpi_ivhd_t *last; + amd_iommu_acpi_ivhd_t *acpi_ivhdp; + uint8_t uint8_flags; + uint16_t uint16_info; + uint16_t idx; + + if (deventry->idev_type == DEVENTRY_RANGE_END) { + ASSERT(last); + acpi_ivhdp = last; + last = NULL; + ASSERT(acpi_ivhdp->ach_dev_type == DEVENTRY_RANGE || + acpi_ivhdp->ach_dev_type == DEVENTRY_ALIAS_RANGE || + acpi_ivhdp->ach_dev_type == DEVENTRY_EXTENDED_RANGE); + ASSERT(acpi_ivhdp->ach_deviceid_end == -1); + acpi_ivhdp->ach_deviceid_end = deventry->idev_deviceid; + /* TODO ASSERT data is 0 */ + return; + } + + ASSERT(last == NULL); + acpi_ivhdp = kmem_zalloc(sizeof (*acpi_ivhdp), KM_SLEEP); + + uint8_flags = ivhdp->ivhd_flags; + +#ifdef BROKEN_ASSERT + ASSERT(AMD_IOMMU_REG_GET8(&uint8_flags, + AMD_IOMMU_ACPI_IVHD_FLAGS_RSV) == 0); +#endif + + acpi_ivhdp->ach_IotlbSup = AMD_IOMMU_REG_GET8(&uint8_flags, + AMD_IOMMU_ACPI_IVHD_FLAGS_IOTLBSUP); + acpi_ivhdp->ach_Isoc = AMD_IOMMU_REG_GET8(&uint8_flags, + AMD_IOMMU_ACPI_IVHD_FLAGS_ISOC); + acpi_ivhdp->ach_ResPassPW = AMD_IOMMU_REG_GET8(&uint8_flags, + AMD_IOMMU_ACPI_IVHD_FLAGS_RESPASSPW); + acpi_ivhdp->ach_PassPW = AMD_IOMMU_REG_GET8(&uint8_flags, + AMD_IOMMU_ACPI_IVHD_FLAGS_PASSPW); + acpi_ivhdp->ach_HtTunEn = AMD_IOMMU_REG_GET8(&uint8_flags, + AMD_IOMMU_ACPI_IVHD_FLAGS_HTTUNEN); + + /* IVHD fields */ + acpi_ivhdp->ach_IOMMU_deviceid = ivhdp->ivhd_deviceid; + acpi_ivhdp->ach_IOMMU_cap_off = ivhdp->ivhd_cap_off; + acpi_ivhdp->ach_IOMMU_reg_base = ivhdp->ivhd_reg_base; + acpi_ivhdp->ach_IOMMU_pci_seg = ivhdp->ivhd_pci_seg; + + /* IVHD IOMMU info fields */ + uint16_info = ivhdp->ivhd_iommu_info; + +#ifdef BROKEN_ASSERT + ASSERT(AMD_IOMMU_REG_GET16(&uint16_info, + AMD_IOMMU_ACPI_IOMMU_INFO_RSV1) == 0); +#endif + + acpi_ivhdp->ach_IOMMU_UnitID = AMD_IOMMU_REG_GET16(&uint16_info, + AMD_IOMMU_ACPI_IOMMU_INFO_UNITID); + ASSERT(AMD_IOMMU_REG_GET16(&uint16_info, + AMD_IOMMU_ACPI_IOMMU_INFO_RSV2) == 0); + acpi_ivhdp->ach_IOMMU_MSInum = AMD_IOMMU_REG_GET16(&uint16_info, + AMD_IOMMU_ACPI_IOMMU_INFO_MSINUM); + + /* Initialize deviceids to -1 */ + acpi_ivhdp->ach_deviceid_start = -1; + acpi_ivhdp->ach_deviceid_end = -1; + acpi_ivhdp->ach_src_deviceid = -1; + + /* All range type entries are put on hash entry 0 */ + switch (deventry->idev_type) { + case DEVENTRY_ALL: + acpi_ivhdp->ach_deviceid_start = 0; + acpi_ivhdp->ach_deviceid_end = (uint16_t)-1; + acpi_ivhdp->ach_dev_type = DEVENTRY_ALL; + idx = AMD_IOMMU_ACPI_INFO_HASH_SZ; + break; + case DEVENTRY_SELECT: + acpi_ivhdp->ach_deviceid_start = deventry->idev_deviceid; + acpi_ivhdp->ach_deviceid_end = deventry->idev_deviceid; + acpi_ivhdp->ach_dev_type = DEVENTRY_SELECT; + idx = deviceid_hashfn(deventry->idev_deviceid); + break; + case DEVENTRY_RANGE: + acpi_ivhdp->ach_deviceid_start = deventry->idev_deviceid; + acpi_ivhdp->ach_deviceid_end = -1; + acpi_ivhdp->ach_dev_type = DEVENTRY_RANGE; + idx = AMD_IOMMU_ACPI_INFO_HASH_SZ; + last = acpi_ivhdp; + break; + case DEVENTRY_RANGE_END: + cmn_err(CE_PANIC, "%s: Unexpected Range End Deventry", + amd_iommu_modname); + /*NOTREACHED*/ + case DEVENTRY_ALIAS_SELECT: + acpi_ivhdp->ach_deviceid_start = deventry->idev_deviceid; + acpi_ivhdp->ach_deviceid_end = deventry->idev_deviceid; + acpi_ivhdp->ach_src_deviceid = deventry->idev_src_deviceid; + acpi_ivhdp->ach_dev_type = DEVENTRY_ALIAS_SELECT; + idx = deviceid_hashfn(deventry->idev_deviceid); + break; + case DEVENTRY_ALIAS_RANGE: + acpi_ivhdp->ach_deviceid_start = deventry->idev_deviceid; + acpi_ivhdp->ach_deviceid_end = -1; + acpi_ivhdp->ach_src_deviceid = deventry->idev_src_deviceid; + acpi_ivhdp->ach_dev_type = DEVENTRY_ALIAS_RANGE; + idx = AMD_IOMMU_ACPI_INFO_HASH_SZ; + last = acpi_ivhdp; + break; + case DEVENTRY_EXTENDED_SELECT: + acpi_ivhdp->ach_deviceid_start = deventry->idev_deviceid; + acpi_ivhdp->ach_deviceid_end = deventry->idev_deviceid; + acpi_ivhdp->ach_dev_type = DEVENTRY_EXTENDED_SELECT; + idx = deviceid_hashfn(deventry->idev_deviceid); + break; + case DEVENTRY_EXTENDED_RANGE: + acpi_ivhdp->ach_deviceid_start = deventry->idev_deviceid; + acpi_ivhdp->ach_deviceid_end = -1; + acpi_ivhdp->ach_dev_type = DEVENTRY_EXTENDED_RANGE; + idx = AMD_IOMMU_ACPI_INFO_HASH_SZ; + last = acpi_ivhdp; + break; + case DEVENTRY_SPECIAL_DEVICE: + acpi_ivhdp->ach_deviceid_start = -1; + acpi_ivhdp->ach_deviceid_end = -1; + acpi_ivhdp->ach_src_deviceid = deventry->idev_src_deviceid; + acpi_ivhdp->ach_special_handle = deventry->idev_handle; + acpi_ivhdp->ach_special_variety = deventry->idev_variety; + idx = AMD_IOMMU_ACPI_INFO_HASH_SZ; + default: + cmn_err(CE_PANIC, "%s: Unsupported deventry type", + amd_iommu_modname); + /*NOTREACHED*/ + } + + acpi_ivhdp->ach_Lint1Pass = deventry->idev_Lint1Pass; + acpi_ivhdp->ach_Lint0Pass = deventry->idev_Lint0Pass; + acpi_ivhdp->ach_SysMgt = deventry->idev_SysMgt; + acpi_ivhdp->ach_NMIPass = deventry->idev_NMIPass; + acpi_ivhdp->ach_ExtIntPass = deventry->idev_ExtIntPass; + acpi_ivhdp->ach_INITPass = deventry->idev_INITPass; + + + /* extended data */ + if (acpi_ivhdp->ach_dev_type == DEVENTRY_EXTENDED_SELECT || + acpi_ivhdp->ach_dev_type == DEVENTRY_EXTENDED_RANGE) { + acpi_ivhdp->ach_AtsDisabled = deventry->idev_AtsDisabled; + } + + /* + * Now add it to the hash + */ + ASSERT(hash[idx] != acpi_ivhdp); + acpi_ivhdp->ach_next = hash[idx]; + hash[idx] = acpi_ivhdp; +} + +static void +add_ivhdc_info(ivhd_container_t *ivhdcp, amd_iommu_acpi_ivhd_t **hash) +{ + ivhd_deventry_t *deventry; + ivhd_t *ivhdp = ivhdcp->ivhdc_ivhd; + + for (deventry = ivhdcp->ivhdc_first_deventry; deventry; + deventry = deventry->idev_next) { + add_deventry_info(ivhdp, deventry, hash); + } +} + +static void +add_ivhd_info(amd_iommu_acpi_t *acpi, amd_iommu_acpi_ivhd_t **hash) +{ + ivhd_container_t *ivhdcp; + + for (ivhdcp = acpi->acp_first_ivhdc; ivhdcp; + ivhdcp = ivhdcp->ivhdc_next) { + add_ivhdc_info(ivhdcp, hash); + } +} + +static void +set_ivmd_info(ivmd_t *ivmdp, amd_iommu_acpi_ivmd_t **hash) +{ + amd_iommu_acpi_ivmd_t *acpi_ivmdp; + uint8_t uint8_flags; + uint16_t idx; + + uint8_flags = ivmdp->ivmd_flags; + + acpi_ivmdp = kmem_zalloc(sizeof (*acpi_ivmdp), KM_SLEEP); + + switch (ivmdp->ivmd_type) { + case 0x20: + acpi_ivmdp->acm_deviceid_start = 0; + acpi_ivmdp->acm_deviceid_end = (uint16_t)-1; + acpi_ivmdp->acm_dev_type = IVMD_DEVICEID_ALL; + idx = AMD_IOMMU_ACPI_INFO_HASH_SZ; + break; + case 0x21: + acpi_ivmdp->acm_deviceid_start = ivmdp->ivmd_deviceid; + acpi_ivmdp->acm_deviceid_end = ivmdp->ivmd_deviceid; + acpi_ivmdp->acm_dev_type = IVMD_DEVICEID_SELECT; + idx = deviceid_hashfn(ivmdp->ivmd_deviceid); + break; + case 0x22: + acpi_ivmdp->acm_deviceid_start = ivmdp->ivmd_deviceid; + acpi_ivmdp->acm_deviceid_end = ivmdp->ivmd_auxdata; + acpi_ivmdp->acm_dev_type = IVMD_DEVICEID_RANGE; + idx = AMD_IOMMU_ACPI_INFO_HASH_SZ; + break; + default: + cmn_err(CE_PANIC, "Unknown AMD IOMMU ACPI IVMD deviceid type: " + "%x", ivmdp->ivmd_type); + /*NOTREACHED*/ + } + + ASSERT(AMD_IOMMU_REG_GET8(&uint8_flags, + AMD_IOMMU_ACPI_IVMD_RSV) == 0); + + acpi_ivmdp->acm_ExclRange = AMD_IOMMU_REG_GET8(&uint8_flags, + AMD_IOMMU_ACPI_IVMD_EXCL_RANGE); + acpi_ivmdp->acm_IW = AMD_IOMMU_REG_GET8(&uint8_flags, + AMD_IOMMU_ACPI_IVMD_IW); + acpi_ivmdp->acm_IR = AMD_IOMMU_REG_GET8(&uint8_flags, + AMD_IOMMU_ACPI_IVMD_IR); + acpi_ivmdp->acm_Unity = AMD_IOMMU_REG_GET8(&uint8_flags, + AMD_IOMMU_ACPI_IVMD_UNITY); + + acpi_ivmdp->acm_ivmd_phys_start = ivmdp->ivmd_phys_start; + acpi_ivmdp->acm_ivmd_phys_len = ivmdp->ivmd_phys_len; + + acpi_ivmdp->acm_next = hash[idx]; + hash[idx] = acpi_ivmdp; +} + +static void +add_ivmdc_info(ivmd_container_t *ivmdcp, amd_iommu_acpi_ivmd_t **hash) +{ + set_ivmd_info(ivmdcp->ivmdc_ivmd, hash); +} + +static void +add_ivmd_info(amd_iommu_acpi_t *acpi, amd_iommu_acpi_ivmd_t **hash) +{ + ivmd_container_t *ivmdcp; + + for (ivmdcp = acpi->acp_first_ivmdc; ivmdcp; + ivmdcp = ivmdcp->ivmdc_next) { + add_ivmdc_info(ivmdcp, hash); + } +} + +static void +add_global_info(amd_iommu_acpi_t *acpi, amd_iommu_acpi_global_t *global) +{ + uint32_t ivrs_ivinfo = acpi->acp_ivrs->ivrs_ivinfo; + + global->acg_HtAtsResv = + AMD_IOMMU_REG_GET32(&ivrs_ivinfo, AMD_IOMMU_ACPI_HT_ATSRSV); + global->acg_VAsize = + AMD_IOMMU_REG_GET32(&ivrs_ivinfo, AMD_IOMMU_ACPI_VA_SIZE); + global->acg_PAsize = + AMD_IOMMU_REG_GET32(&ivrs_ivinfo, AMD_IOMMU_ACPI_PA_SIZE); +} + +static int +create_acpi_hash(amd_iommu_acpi_t *acpi) +{ + /* Last hash entry is for deviceid ranges including "all" */ + + amd_iommu_acpi_global = kmem_zalloc(sizeof (amd_iommu_acpi_global_t), + KM_SLEEP); + + amd_iommu_acpi_ivhd_hash = kmem_zalloc(sizeof (amd_iommu_acpi_ivhd_t *) + * (AMD_IOMMU_ACPI_INFO_HASH_SZ + 1), KM_SLEEP); + + amd_iommu_acpi_ivmd_hash = kmem_zalloc(sizeof (amd_iommu_acpi_ivmd_t *) + * (AMD_IOMMU_ACPI_INFO_HASH_SZ + 1), KM_SLEEP); + + add_global_info(acpi, amd_iommu_acpi_global); + + add_ivhd_info(acpi, amd_iommu_acpi_ivhd_hash); + + add_ivmd_info(acpi, amd_iommu_acpi_ivmd_hash); + + return (DDI_SUCCESS); +} + +amd_iommu_acpi_global_t * +amd_iommu_lookup_acpi_global(void) +{ + ASSERT(amd_iommu_acpi_global); + + return (amd_iommu_acpi_global); +} + +amd_iommu_acpi_ivhd_t * +amd_iommu_lookup_all_ivhd(void) +{ + amd_iommu_acpi_ivhd_t *hinfop; + + hinfop = amd_iommu_acpi_ivhd_hash[AMD_IOMMU_ACPI_INFO_HASH_SZ]; + for (; hinfop; hinfop = hinfop->ach_next) { + if (hinfop->ach_deviceid_start == 0 && + hinfop->ach_deviceid_end == (uint16_t)-1) { + break; + } + } + + return (hinfop); +} + +amd_iommu_acpi_ivmd_t * +amd_iommu_lookup_all_ivmd(void) +{ + amd_iommu_acpi_ivmd_t *minfop; + + minfop = amd_iommu_acpi_ivmd_hash[AMD_IOMMU_ACPI_INFO_HASH_SZ]; + for (; minfop; minfop = minfop->acm_next) { + if (minfop->acm_deviceid_start == 0 && + minfop->acm_deviceid_end == (uint16_t)-1) { + break; + } + } + + return (minfop); +} + +amd_iommu_acpi_ivhd_t * +amd_iommu_lookup_any_ivhd(void) +{ + int i; + amd_iommu_acpi_ivhd_t *hinfop; + + for (i = AMD_IOMMU_ACPI_INFO_HASH_SZ; i >= 0; i--) { + /*LINTED*/ + if (hinfop = amd_iommu_acpi_ivhd_hash[i]) + break; + } + + return (hinfop); +} + +amd_iommu_acpi_ivmd_t * +amd_iommu_lookup_any_ivmd(void) +{ + int i; + amd_iommu_acpi_ivmd_t *minfop; + + for (i = AMD_IOMMU_ACPI_INFO_HASH_SZ; i >= 0; i--) { + /*LINTED*/ + if (minfop = amd_iommu_acpi_ivmd_hash[i]) + break; + } + + return (minfop); +} + +static void +dump_acpi_aliases(void) +{ + amd_iommu_acpi_ivhd_t *hinfop; + uint16_t idx; + + for (idx = 0; idx <= AMD_IOMMU_ACPI_INFO_HASH_SZ; idx++) { + hinfop = amd_iommu_acpi_ivhd_hash[idx]; + for (; hinfop; hinfop = hinfop->ach_next) { + cmn_err(CE_NOTE, "start=%d, end=%d, src_bdf=%d", + hinfop->ach_deviceid_start, + hinfop->ach_deviceid_end, + hinfop->ach_src_deviceid); + } + } +} + +amd_iommu_acpi_ivhd_t * +amd_iommu_lookup_ivhd(int32_t deviceid) +{ + amd_iommu_acpi_ivhd_t *hinfop; + uint16_t idx; + + if (amd_iommu_debug == AMD_IOMMU_DEBUG_ACPI) { + cmn_err(CE_NOTE, "Attempting to get ACPI IVHD info " + "for deviceid: %d", deviceid); + } + + ASSERT(amd_iommu_acpi_ivhd_hash); + + /* check if special device */ + if (deviceid == -1) { + hinfop = amd_iommu_acpi_ivhd_hash[AMD_IOMMU_ACPI_INFO_HASH_SZ]; + for (; hinfop; hinfop = hinfop->ach_next) { + if (hinfop->ach_deviceid_start == -1 && + hinfop->ach_deviceid_end == -1) { + break; + } + } + return (hinfop); + } + + /* First search for an exact match */ + + idx = deviceid_hashfn(deviceid); + + +range: + hinfop = amd_iommu_acpi_ivhd_hash[idx]; + + for (; hinfop; hinfop = hinfop->ach_next) { + if (deviceid < hinfop->ach_deviceid_start || + deviceid > hinfop->ach_deviceid_end) + continue; + + if (amd_iommu_debug == AMD_IOMMU_DEBUG_ACPI) { + cmn_err(CE_NOTE, "Found ACPI IVHD match: %p, " + "actual deviceid = %u, start = %u, end = %u", + (void *)hinfop, deviceid, + hinfop->ach_deviceid_start, + hinfop->ach_deviceid_end); + } + goto out; + } + + if (idx != AMD_IOMMU_ACPI_INFO_HASH_SZ) { + idx = AMD_IOMMU_ACPI_INFO_HASH_SZ; + goto range; + } else { + cmn_err(CE_PANIC, "IVHD not found for deviceid: %x", deviceid); + } + +out: + if (amd_iommu_debug == AMD_IOMMU_DEBUG_ACPI) { + cmn_err(CE_NOTE, "%u: %s ACPI IVHD %p", deviceid, + hinfop ? "GOT" : "Did NOT get", (void *)hinfop); + } + + return (hinfop); +} + +amd_iommu_acpi_ivmd_t * +amd_iommu_lookup_ivmd(int32_t deviceid) +{ + amd_iommu_acpi_ivmd_t *minfop; + uint16_t idx; + + if (amd_iommu_debug == AMD_IOMMU_DEBUG_ACPI) { + cmn_err(CE_NOTE, "Attempting to get ACPI IVMD info " + "for deviceid: %u", deviceid); + } + + ASSERT(amd_iommu_acpi_ivmd_hash); + + /* First search for an exact match */ + + idx = deviceid_hashfn(deviceid); + + +range: + minfop = amd_iommu_acpi_ivmd_hash[idx]; + + for (; minfop; minfop = minfop->acm_next) { + if (deviceid < minfop->acm_deviceid_start && + deviceid > minfop->acm_deviceid_end) + continue; + + if (amd_iommu_debug == AMD_IOMMU_DEBUG_ACPI) { + cmn_err(CE_NOTE, "Found ACPI IVMD match: %p, " + "actual deviceid = %u, start = %u, end = %u", + (void *)minfop, deviceid, + minfop->acm_deviceid_start, + minfop->acm_deviceid_end); + } + + goto out; + } + + if (idx != AMD_IOMMU_ACPI_INFO_HASH_SZ) { + idx = AMD_IOMMU_ACPI_INFO_HASH_SZ; + goto range; + } else { + cmn_err(CE_PANIC, "IVMD not found for deviceid: %x", deviceid); + } + +out: + if (amd_iommu_debug == AMD_IOMMU_DEBUG_ACPI) { + cmn_err(CE_NOTE, "%u: %s ACPI IVMD info %p", deviceid, + minfop ? "GOT" : "Did NOT get", (void *)minfop); + } + + return (minfop); +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/usr/src/uts/i86pc/io/amd_iommu/amd_iommu_acpi.h Mon Sep 21 11:26:40 2009 -0400 @@ -0,0 +1,306 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright 2009 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +#ifndef _AMD_IOMMU_ACPI_H +#define _AMD_IOMMU_ACPI_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include <sys/sunddi.h> +#include <sys/acpi/acpi.h> +#include <sys/acpica.h> +#include <sys/amd_iommu.h> +#include "amd_iommu_impl.h" + +#ifdef _KERNEL + +#define IVRS_SIG "IVRS" + +/* + * IVINFO settings + */ +#define AMD_IOMMU_ACPI_IVINFO_RSV1 (31 << 16 | 23) +#define AMD_IOMMU_ACPI_HT_ATSRSV (22 << 16 | 22) +#define AMD_IOMMU_ACPI_VA_SIZE (21 << 16 | 15) +#define AMD_IOMMU_ACPI_PA_SIZE (14 << 16 | 8) +#define AMD_IOMMU_ACPI_IVINFO_RSV2 (7 << 16 | 0) + +/* + * IVHD Device entry len field + */ +#define AMD_IOMMU_ACPI_DEVENTRY_LEN (7 << 16 | 6) + +/* + * IVHD flag fields definition + */ +#define AMD_IOMMU_ACPI_IVHD_FLAGS_RSV (7 << 16 | 5) +#define AMD_IOMMU_ACPI_IVHD_FLAGS_IOTLBSUP (4 << 16 | 4) +#define AMD_IOMMU_ACPI_IVHD_FLAGS_ISOC (3 << 16 | 3) +#define AMD_IOMMU_ACPI_IVHD_FLAGS_RESPASSPW (2 << 16 | 2) +#define AMD_IOMMU_ACPI_IVHD_FLAGS_PASSPW (1 << 16 | 1) +#define AMD_IOMMU_ACPI_IVHD_FLAGS_HTTUNEN (0 << 16 | 0) + +/* + * IVHD IOMMU info fields + */ +#define AMD_IOMMU_ACPI_IOMMU_INFO_RSV1 (15 << 16 | 13) +#define AMD_IOMMU_ACPI_IOMMU_INFO_UNITID (12 << 16 | 8) +#define AMD_IOMMU_ACPI_IOMMU_INFO_RSV2 (7 << 16 | 5) +#define AMD_IOMMU_ACPI_IOMMU_INFO_MSINUM (4 << 16 | 0) + +/* + * IVHD deventry data settings + */ +#define AMD_IOMMU_ACPI_LINT1PASS (7 << 16 | 7) +#define AMD_IOMMU_ACPI_LINT0PASS (6 << 16 | 6) +#define AMD_IOMMU_ACPI_SYSMGT (5 << 16 | 4) +#define AMD_IOMMU_ACPI_DATRSV (3 << 16 | 3) +#define AMD_IOMMU_ACPI_NMIPASS (2 << 16 | 2) +#define AMD_IOMMU_ACPI_EXTINTPASS (1 << 16 | 1) +#define AMD_IOMMU_ACPI_INITPASS (0 << 16 | 0) + +/* + * IVHD deventry extended data settings + */ +#define AMD_IOMMU_ACPI_ATSDISABLED (31 << 16 | 31) +#define AMD_IOMMU_ACPI_EXTDATRSV (30 << 16 | 0) + +/* + * IVMD flags fields settings + */ +#define AMD_IOMMU_ACPI_IVMD_RSV (7 << 16 | 4) +#define AMD_IOMMU_ACPI_IVMD_EXCL_RANGE (3 << 16 | 3) +#define AMD_IOMMU_ACPI_IVMD_IW (2 << 16 | 2) +#define AMD_IOMMU_ACPI_IVMD_IR (1 << 16 | 1) +#define AMD_IOMMU_ACPI_IVMD_UNITY (0 << 16 | 0) + +#define AMD_IOMMU_ACPI_INFO_HASH_SZ (256) + +/* + * Deventry special device "variety" + */ +#define AMD_IOMMU_ACPI_SPECIAL_APIC 0x1 +#define AMD_IOMMU_ACPI_SPECIAL_HPET 0x2 + +typedef enum { + DEVENTRY_INVALID = 0, + DEVENTRY_ALL = 1, + DEVENTRY_SELECT, + DEVENTRY_RANGE, + DEVENTRY_RANGE_END, + DEVENTRY_ALIAS_SELECT, + DEVENTRY_ALIAS_RANGE, + DEVENTRY_EXTENDED_SELECT, + DEVENTRY_EXTENDED_RANGE, + DEVENTRY_SPECIAL_DEVICE +} ivhd_deventry_type_t; + +typedef enum { + IVMD_DEVICE_INVALID = 0, + IVMD_DEVICEID_ALL, + IVMD_DEVICEID_SELECT, + IVMD_DEVICEID_RANGE +} ivmd_deviceid_type_t; + +typedef struct ivhd_deventry { + uint8_t idev_len; + ivhd_deventry_type_t idev_type; + int32_t idev_deviceid; + int32_t idev_src_deviceid; + uint8_t idev_handle; + uint8_t idev_variety; + uint8_t idev_Lint1Pass; + uint8_t idev_Lint0Pass; + uint8_t idev_SysMgt; + uint8_t idev_NMIPass; + uint8_t idev_ExtIntPass; + uint8_t idev_INITPass; + uint8_t idev_AtsDisabled; + struct ivhd_deventry *idev_next; +} ivhd_deventry_t; + +typedef struct ivhd { + uint8_t ivhd_type; + uint8_t ivhd_flags; + uint16_t ivhd_len; + uint16_t ivhd_deviceid; + uint16_t ivhd_cap_off; + uint64_t ivhd_reg_base; + uint16_t ivhd_pci_seg; + uint16_t ivhd_iommu_info; + uint32_t ivhd_resv; +} ivhd_t; + +typedef struct ivhd_container { + ivhd_t *ivhdc_ivhd; + ivhd_deventry_t *ivhdc_first_deventry; + ivhd_deventry_t *ivhdc_last_deventry; + struct ivhd_container *ivhdc_next; +} ivhd_container_t; + +typedef struct ivmd { + uint8_t ivmd_type; + uint8_t ivmd_flags; + uint16_t ivmd_len; + uint16_t ivmd_deviceid; + uint16_t ivmd_auxdata; + uint64_t ivmd_resv; + uint64_t ivmd_phys_start; + uint64_t ivmd_phys_len; +} ivmd_t; + +typedef struct ivmd_container { + ivmd_t *ivmdc_ivmd; + struct ivmd_container *ivmdc_next; +} ivmd_container_t; + +typedef struct ivrs { + struct acpi_table_header ivrs_hdr; + uint32_t ivrs_ivinfo; + uint64_t ivrs_resv; +} ivrs_t; + +typedef struct amd_iommu_acpi { + struct ivrs *acp_ivrs; + ivhd_container_t *acp_first_ivhdc; + ivhd_container_t *acp_last_ivhdc; + ivmd_container_t *acp_first_ivmdc; + ivmd_container_t *acp_last_ivmdc; +} amd_iommu_acpi_t; + + +/* Global IVINFo fields */ +typedef struct amd_iommu_acpi_global { + uint8_t acg_HtAtsResv; + uint8_t acg_VAsize; + uint8_t acg_PAsize; +} amd_iommu_acpi_global_t; + +typedef struct amd_iommu_acpi_ivhd { + int32_t ach_deviceid_start; + int32_t ach_deviceid_end; + + /* IVHD deventry type */ + ivhd_deventry_type_t ach_dev_type; + + /* IVHD flag fields */ + uint8_t ach_IotlbSup; + uint8_t ach_Isoc; + uint8_t ach_ResPassPW; + uint8_t ach_PassPW; + uint8_t ach_HtTunEn; + + /* IVHD fields */ + uint16_t ach_IOMMU_deviceid; + uint16_t ach_IOMMU_cap_off; + uint64_t ach_IOMMU_reg_base; + uint16_t ach_IOMMU_pci_seg; + + /* IVHD IOMMU info fields */ + uint8_t ach_IOMMU_UnitID; + uint8_t ach_IOMMU_MSInum; + + /* IVHD deventry data settings */ + uint8_t ach_Lint1Pass; + uint8_t ach_Lint0Pass; + uint8_t ach_SysMgt; + uint8_t ach_NMIPass; + uint8_t ach_ExtIntPass; + uint8_t ach_INITPass; + + /* alias */ + int32_t ach_src_deviceid; + + /* IVHD deventry extended data settings */ + uint8_t ach_AtsDisabled; + + /* IVHD deventry special device */ + uint8_t ach_special_handle; + uint8_t ach_special_variety; + + struct amd_iommu_acpi_ivhd *ach_next; +} amd_iommu_acpi_ivhd_t; + +typedef struct amd_iommu_acpi_ivmd { + int32_t acm_deviceid_start; + int32_t acm_deviceid_end; + + /* IVMD type */ + ivmd_deviceid_type_t acm_dev_type; + + /* IVMD flags */ + uint8_t acm_ExclRange; + uint8_t acm_IW; + uint8_t acm_IR; + uint8_t acm_Unity; + + /* IVMD mem block */ + uint64_t acm_ivmd_phys_start; + uint64_t acm_ivmd_phys_len; + + struct amd_iommu_acpi_ivmd *acm_next; +} amd_iommu_acpi_ivmd_t; + +typedef union { + uint16_t ent16; + uint8_t ent8[2]; +} align_16_t; + +typedef union { + uint32_t ent32; + uint8_t ent8[4]; +} align_32_t; + +typedef union { + ivhd_t *ivhdp; + char *cp; +} align_ivhd_t; + +typedef union { + ivmd_t *ivmdp; + char *cp; +} align_ivmd_t; + +#pragma pack() + +int amd_iommu_acpi_init(void); +void amd_iommu_acpi_fini(void); +amd_iommu_acpi_ivhd_t *amd_iommu_lookup_all_ivhd(void); +amd_iommu_acpi_ivmd_t *amd_iommu_lookup_all_ivmd(void); +amd_iommu_acpi_ivhd_t *amd_iommu_lookup_any_ivhd(void); +amd_iommu_acpi_ivmd_t *amd_iommu_lookup_any_ivmd(void); +amd_iommu_acpi_global_t *amd_iommu_lookup_acpi_global(void); +amd_iommu_acpi_ivhd_t *amd_iommu_lookup_ivhd(int32_t deviceid); +amd_iommu_acpi_ivmd_t *amd_iommu_lookup_ivmd(int32_t deviceid); + +#endif /* _KERNEL */ + +#ifdef __cplusplus +} +#endif + +#endif /* _AMD_IOMMU_ACPI_H */
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/usr/src/uts/i86pc/io/amd_iommu/amd_iommu_cmd.c Mon Sep 21 11:26:40 2009 -0400 @@ -0,0 +1,321 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ + +/* + * Copyright 2009 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +#include <sys/sunddi.h> +#include <sys/amd_iommu.h> +#include "amd_iommu_impl.h" + +extern int servicing_interrupt(void); + +static void +amd_iommu_wait_for_completion(amd_iommu_t *iommu) +{ + ASSERT(MUTEX_HELD(&iommu->aiomt_cmdlock)); + while (AMD_IOMMU_REG_GET64(REGADDR64( + iommu->aiomt_reg_status_va), AMD_IOMMU_COMWAIT_INT) != 1) { + AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_ctrl_va), + AMD_IOMMU_CMDBUF_ENABLE, 1); + WAIT_SEC(1); + } +} + +static int +create_compl_wait_cmd(amd_iommu_t *iommu, amd_iommu_cmdargs_t *cmdargsp, + amd_iommu_cmd_flags_t flags, uint32_t *cmdptr) +{ + const char *driver = ddi_driver_name(iommu->aiomt_dip); + int instance = ddi_get_instance(iommu->aiomt_dip); + const char *f = "create_compl_wait_cmd"; + + ASSERT(cmdargsp == NULL); + + if (flags & AMD_IOMMU_CMD_FLAGS_COMPL_WAIT_S) { + cmn_err(CE_WARN, "%s: %s%d: idx=%d: 'store' completion " + "not supported for completion wait command", + f, driver, instance, iommu->aiomt_idx); + return (DDI_FAILURE); + } + + AMD_IOMMU_REG_SET32(&cmdptr[0], AMD_IOMMU_CMD_COMPL_WAIT_S, 0); + AMD_IOMMU_REG_SET32(&cmdptr[0], AMD_IOMMU_CMD_COMPL_WAIT_I, 1); + AMD_IOMMU_REG_SET32(&cmdptr[0], AMD_IOMMU_CMD_COMPL_WAIT_F, + (flags & AMD_IOMMU_CMD_FLAGS_COMPL_WAIT_F) != 0); + AMD_IOMMU_REG_SET32(&cmdptr[0], AMD_IOMMU_CMD_COMPL_WAIT_STORE_ADDR_LO, + 0); + AMD_IOMMU_REG_SET32(&cmdptr[1], AMD_IOMMU_CMD_OPCODE, 0x01); + AMD_IOMMU_REG_SET32(&cmdptr[1], AMD_IOMMU_CMD_COMPL_WAIT_STORE_ADDR_HI, + 0); + cmdptr[2] = 0; + cmdptr[3] = 0; + + return (DDI_SUCCESS); +} + +static int +create_inval_devtab_entry_cmd(amd_iommu_t *iommu, amd_iommu_cmdargs_t *cmdargsp, + amd_iommu_cmd_flags_t flags, uint32_t *cmdptr) +{ + const char *driver = ddi_driver_name(iommu->aiomt_dip); + int instance = ddi_get_instance(iommu->aiomt_dip); + const char *f = "create_inval_devtab_entry_cmd"; + uint16_t deviceid; + + ASSERT(cmdargsp); + + if (flags != AMD_IOMMU_CMD_FLAGS_NONE) { + cmn_err(CE_WARN, "%s: %s%d: idx=%d: invalidate devtab entry " + "no flags supported", f, driver, instance, + iommu->aiomt_idx); + return (DDI_FAILURE); + } + + deviceid = cmdargsp->ca_deviceid; + + AMD_IOMMU_REG_SET32(&cmdptr[0], AMD_IOMMU_CMD_INVAL_DEVTAB_DEVICEID, + deviceid); + AMD_IOMMU_REG_SET32(&cmdptr[1], AMD_IOMMU_CMD_OPCODE, 0x02); + cmdptr[2] = 0; + cmdptr[3] = 0; + + return (DDI_SUCCESS); +} + +/*ARGSUSED*/ +static int +create_inval_iommu_pages_cmd(amd_iommu_t *iommu, amd_iommu_cmdargs_t *cmdargsp, + amd_iommu_cmd_flags_t flags, uint32_t *cmdptr) +{ + uint32_t addr_lo; + uint32_t addr_hi; + + ASSERT(cmdargsp); + + addr_lo = AMD_IOMMU_REG_GET64(REGADDR64(&cmdargsp->ca_addr), + AMD_IOMMU_CMD_INVAL_PAGES_ADDR_LO); + addr_hi = AMD_IOMMU_REG_GET64(REGADDR64(&cmdargsp->ca_addr), + AMD_IOMMU_CMD_INVAL_PAGES_ADDR_HI); + + cmdptr[0] = 0; + AMD_IOMMU_REG_SET32(&cmdptr[1], AMD_IOMMU_CMD_INVAL_PAGES_DOMAINID, + cmdargsp->ca_domainid); + AMD_IOMMU_REG_SET32(&cmdptr[1], AMD_IOMMU_CMD_OPCODE, 0x03); + AMD_IOMMU_REG_SET32(&cmdptr[2], AMD_IOMMU_CMD_INVAL_PAGES_PDE, + (flags & AMD_IOMMU_CMD_FLAGS_PAGE_PDE_INVAL) != 0); + AMD_IOMMU_REG_SET32(&cmdptr[2], AMD_IOMMU_CMD_INVAL_PAGES_S, + (flags & AMD_IOMMU_CMD_FLAGS_PAGE_INVAL_S) != 0); + AMD_IOMMU_REG_SET32(&cmdptr[2], AMD_IOMMU_CMD_INVAL_PAGES_ADDR_LO, + addr_lo); + cmdptr[3] = addr_hi; + + return (DDI_SUCCESS); + +} + +/*ARGSUSED*/ +static int +create_inval_iotlb_pages_cmd(amd_iommu_t *iommu, amd_iommu_cmdargs_t *cmdargsp, + amd_iommu_cmd_flags_t flags, uint32_t *cmdptr) +{ + uint32_t addr_lo; + uint32_t addr_hi; + + ASSERT(cmdargsp); + + addr_lo = AMD_IOMMU_REG_GET64(REGADDR64(&cmdargsp->ca_addr), + AMD_IOMMU_CMD_INVAL_IOTLB_ADDR_LO); + + addr_hi = AMD_IOMMU_REG_GET64(REGADDR64(&cmdargsp->ca_addr), + AMD_IOMMU_CMD_INVAL_IOTLB_ADDR_HI); + + AMD_IOMMU_REG_SET32(&cmdptr[0], AMD_IOMMU_CMD_INVAL_IOTLB_DEVICEID, + cmdargsp->ca_deviceid); + AMD_IOMMU_REG_SET32(&cmdptr[0], AMD_IOMMU_CMD_INVAL_IOTLB_MAXPEND, + AMD_IOMMU_DEFAULT_MAXPEND); + AMD_IOMMU_REG_SET32(&cmdptr[1], AMD_IOMMU_CMD_OPCODE, 0x04); + AMD_IOMMU_REG_SET32(&cmdptr[1], AMD_IOMMU_CMD_INVAL_IOTLB_QUEUEID, + cmdargsp->ca_deviceid); + AMD_IOMMU_REG_SET32(&cmdptr[2], AMD_IOMMU_CMD_INVAL_IOTLB_ADDR_LO, + addr_lo); + AMD_IOMMU_REG_SET32(&cmdptr[2], AMD_IOMMU_CMD_INVAL_IOTLB_S, + (flags & AMD_IOMMU_CMD_FLAGS_IOTLB_INVAL_S) != 0); + cmdptr[3] = addr_hi; + + return (DDI_SUCCESS); +} + +static int +create_inval_intr_table_cmd(amd_iommu_t *iommu, amd_iommu_cmdargs_t *cmdargsp, + amd_iommu_cmd_flags_t flags, uint32_t *cmdptr) +{ + const char *driver = ddi_driver_name(iommu->aiomt_dip); + int instance = ddi_get_instance(iommu->aiomt_dip); + const char *f = "create_inval_intr_table_cmd"; + + ASSERT(cmdargsp); + + if (flags != AMD_IOMMU_CMD_FLAGS_NONE) { + cmn_err(CE_WARN, "%s: %s%d: idx=%d: flags not supported " + "for invalidate interrupt table command", + f, driver, instance, iommu->aiomt_idx); + return (DDI_FAILURE); + } + + AMD_IOMMU_REG_SET32(&cmdptr[0], AMD_IOMMU_CMD_INVAL_INTR_DEVICEID, + cmdargsp->ca_deviceid); + AMD_IOMMU_REG_SET32(&cmdptr[1], AMD_IOMMU_CMD_OPCODE, 0x05); + cmdptr[2] = 0; + cmdptr[3] = 0; + + return (DDI_SUCCESS); +} + +int +amd_iommu_cmd(amd_iommu_t *iommu, amd_iommu_cmd_t cmd, + amd_iommu_cmdargs_t *cmdargs, amd_iommu_cmd_flags_t flags, int lock_held) +{ + int error; + int i; + uint32_t cmdptr[4] = {0}; + const char *driver = ddi_driver_name(iommu->aiomt_dip); + int instance = ddi_get_instance(iommu->aiomt_dip); + uint64_t cmdhead_off; + uint64_t cmdtail_off; + const char *f = "amd_iommu_cmd"; + + ASSERT(lock_held == 0 || lock_held == 1); + ASSERT(lock_held == 0 || MUTEX_HELD(&iommu->aiomt_cmdlock)); + + if (!lock_held) + mutex_enter(&iommu->aiomt_cmdlock); + + /* + * Prepare the command + */ + switch (cmd) { + case AMD_IOMMU_CMD_COMPL_WAIT: + if (flags & AMD_IOMMU_CMD_FLAGS_COMPL_WAIT) { + cmn_err(CE_WARN, "%s: %s%d: idx=%d: No completion wait " + " after completion wait command", + f, driver, instance, iommu->aiomt_idx); + error = DDI_FAILURE; + goto out; + } + error = create_compl_wait_cmd(iommu, cmdargs, flags, cmdptr); + break; + case AMD_IOMMU_CMD_INVAL_DEVTAB_ENTRY: + error = create_inval_devtab_entry_cmd(iommu, cmdargs, + flags & ~AMD_IOMMU_CMD_FLAGS_COMPL_WAIT, cmdptr); + break; + case AMD_IOMMU_CMD_INVAL_IOMMU_PAGES: + error = create_inval_iommu_pages_cmd(iommu, cmdargs, + flags & ~AMD_IOMMU_CMD_FLAGS_COMPL_WAIT, cmdptr); + break; + case AMD_IOMMU_CMD_INVAL_IOTLB_PAGES: + error = create_inval_iotlb_pages_cmd(iommu, cmdargs, + flags & ~AMD_IOMMU_CMD_FLAGS_COMPL_WAIT, cmdptr); + break; + case AMD_IOMMU_CMD_INVAL_INTR_TABLE: + error = create_inval_intr_table_cmd(iommu, cmdargs, + flags & ~AMD_IOMMU_CMD_FLAGS_COMPL_WAIT, cmdptr); + break; + default: + cmn_err(CE_WARN, "%s: %s%d: idx=%d: Unsupported cmd: %d", + f, driver, instance, iommu->aiomt_idx, cmd); + error = DDI_FAILURE; + goto out; + } + + if (error != DDI_SUCCESS) { + error = DDI_FAILURE; + goto out; + } + + AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_ctrl_va), + AMD_IOMMU_CMDBUF_ENABLE, 1); + + ASSERT(iommu->aiomt_cmd_tail != NULL); + + for (i = 0; i < 4; i++) { + iommu->aiomt_cmd_tail[i] = cmdptr[i]; + } + +wait_for_drain: + cmdhead_off = AMD_IOMMU_REG_GET64( + REGADDR64(iommu->aiomt_reg_cmdbuf_head_va), + AMD_IOMMU_CMDHEADPTR); + + cmdhead_off = CMD2OFF(cmdhead_off); + + ASSERT(cmdhead_off < iommu->aiomt_cmdbuf_sz); + + /* check for overflow */ + if ((caddr_t)iommu->aiomt_cmd_tail < + (cmdhead_off + iommu->aiomt_cmdbuf)) { + if ((caddr_t)iommu->aiomt_cmd_tail + 16 >= + (cmdhead_off + iommu->aiomt_cmdbuf)) +#ifdef DEBUG + cmn_err(CE_WARN, "cmdbuffer overflow: waiting for " + "drain"); +#endif + goto wait_for_drain; + } + + SYNC_FORDEV(iommu->aiomt_dmahdl); + + /* + * Update the tail pointer in soft state + * and the tail pointer register + */ + iommu->aiomt_cmd_tail += 4; + if ((caddr_t)iommu->aiomt_cmd_tail >= (iommu->aiomt_cmdbuf + + iommu->aiomt_cmdbuf_sz)) { + /* wraparound */ + /*LINTED*/ + iommu->aiomt_cmd_tail = (uint32_t *)iommu->aiomt_cmdbuf; + cmdtail_off = 0; + } else { + cmdtail_off = (caddr_t)iommu->aiomt_cmd_tail + /*LINTED*/ + - iommu->aiomt_cmdbuf; + } + + ASSERT(cmdtail_off < iommu->aiomt_cmdbuf_sz); + + AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_cmdbuf_tail_va), + AMD_IOMMU_CMDTAILPTR, OFF2CMD(cmdtail_off)); + + if (cmd == AMD_IOMMU_CMD_COMPL_WAIT) { + amd_iommu_wait_for_completion(iommu); + } else if (flags & AMD_IOMMU_CMD_FLAGS_COMPL_WAIT) { + error = amd_iommu_cmd(iommu, AMD_IOMMU_CMD_COMPL_WAIT, + NULL, 0, 1); + } + +out: + if (!lock_held) + mutex_exit(&iommu->aiomt_cmdlock); + return (error); +}
--- a/usr/src/uts/i86pc/io/amd_iommu/amd_iommu_impl.c Mon Sep 21 11:25:09 2009 -0400 +++ b/usr/src/uts/i86pc/io/amd_iommu/amd_iommu_impl.c Mon Sep 21 11:26:40 2009 -0400 @@ -35,7 +35,7 @@ #include "amd_iommu_acpi.h" #include "amd_iommu_page_tables.h" -static int amd_iommu_fini(amd_iommu_t *iommu); +static int amd_iommu_fini(amd_iommu_t *iommu, int type); static void amd_iommu_teardown_interrupts(amd_iommu_t *iommu); static void amd_iommu_stop(amd_iommu_t *iommu); @@ -481,7 +481,7 @@ } static void -amd_iommu_teardown_tables_and_buffers(amd_iommu_t *iommu) +amd_iommu_teardown_tables_and_buffers(amd_iommu_t *iommu, int type) { dev_info_t *dip = iommu->aiomt_dip; int instance = ddi_get_instance(dip); @@ -493,12 +493,22 @@ AMD_IOMMU_EVENTBASE, 0); AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_eventlog_va), AMD_IOMMU_EVENTLEN, 0); + AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_eventlog_head_va), + AMD_IOMMU_EVENTHEADPTR, 0); + AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_eventlog_head_va), + AMD_IOMMU_EVENTTAILPTR, 0); + iommu->aiomt_cmdbuf = NULL; AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_cmdbuf_va), AMD_IOMMU_COMBASE, 0); AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_cmdbuf_va), AMD_IOMMU_COMLEN, 0); + AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_cmdbuf_head_va), + AMD_IOMMU_CMDHEADPTR, 0); + AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_cmdbuf_head_va), + AMD_IOMMU_CMDTAILPTR, 0); + iommu->aiomt_devtbl = NULL; AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_devtbl_va), @@ -506,7 +516,7 @@ AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_devtbl_va), AMD_IOMMU_DEVTABSIZE, 0); - if (iommu->aiomt_dmahdl == NULL) + if (iommu->aiomt_dmahdl == NULL || type == AMD_IOMMU_QUIESCE) return; /* Unbind the handle */ @@ -1050,7 +1060,7 @@ "control regs. Skipping IOMMU idx=%d", f, driver, instance, idx); mutex_exit(&iommu->aiomt_mutex); - (void) amd_iommu_fini(iommu); + (void) amd_iommu_fini(iommu, AMD_IOMMU_TEARDOWN); return (NULL); } @@ -1094,13 +1104,13 @@ */ if (amd_iommu_setup_tables_and_buffers(iommu) != DDI_SUCCESS) { mutex_exit(&iommu->aiomt_mutex); - (void) amd_iommu_fini(iommu); + (void) amd_iommu_fini(iommu, AMD_IOMMU_TEARDOWN); return (NULL); } if (amd_iommu_setup_exclusion(iommu) != DDI_SUCCESS) { mutex_exit(&iommu->aiomt_mutex); - (void) amd_iommu_fini(iommu); + (void) amd_iommu_fini(iommu, AMD_IOMMU_TEARDOWN); return (NULL); } @@ -1108,7 +1118,7 @@ if (amd_iommu_setup_interrupts(iommu) != DDI_SUCCESS) { mutex_exit(&iommu->aiomt_mutex); - (void) amd_iommu_fini(iommu); + (void) amd_iommu_fini(iommu, AMD_IOMMU_TEARDOWN); return (NULL); } @@ -1125,20 +1135,20 @@ */ if (amd_iommu_setup_passthru(iommu) != DDI_SUCCESS) { mutex_exit(&iommu->aiomt_mutex); - (void) amd_iommu_fini(iommu); + (void) amd_iommu_fini(iommu, AMD_IOMMU_TEARDOWN); return (NULL); } if (amd_iommu_start(iommu) != DDI_SUCCESS) { mutex_exit(&iommu->aiomt_mutex); - (void) amd_iommu_fini(iommu); + (void) amd_iommu_fini(iommu, AMD_IOMMU_TEARDOWN); return (NULL); } /* xxx register/start race */ if (amd_iommu_register(iommu) != DDI_SUCCESS) { mutex_exit(&iommu->aiomt_mutex); - (void) amd_iommu_fini(iommu); + (void) amd_iommu_fini(iommu, AMD_IOMMU_TEARDOWN); return (NULL); } @@ -1151,7 +1161,7 @@ } static int -amd_iommu_fini(amd_iommu_t *iommu) +amd_iommu_fini(amd_iommu_t *iommu, int type) { int idx = iommu->aiomt_idx; dev_info_t *dip = iommu->aiomt_dip; @@ -1159,17 +1169,28 @@ const char *driver = ddi_driver_name(dip); const char *f = "amd_iommu_fini"; - mutex_enter(&iommu->aiomt_mutex); - if (amd_iommu_unregister(iommu) != DDI_SUCCESS) { - cmn_err(CE_NOTE, "%s: %s%d: Fini of IOMMU unit failed. " - "idx = %d", f, driver, instance, idx); - return (DDI_FAILURE); + if (type == AMD_IOMMU_TEARDOWN) { + mutex_enter(&iommu->aiomt_mutex); + if (amd_iommu_unregister(iommu) != DDI_SUCCESS) { + cmn_err(CE_NOTE, "%s: %s%d: Fini of IOMMU unit failed. " + "idx = %d", f, driver, instance, idx); + return (DDI_FAILURE); + } } + amd_iommu_stop(iommu); - amd_iommu_fini_page_tables(iommu); - amd_iommu_teardown_interrupts(iommu); - amd_iommu_teardown_exclusion(iommu); - amd_iommu_teardown_tables_and_buffers(iommu); + + if (type == AMD_IOMMU_TEARDOWN) { + amd_iommu_fini_page_tables(iommu); + amd_iommu_teardown_interrupts(iommu); + amd_iommu_teardown_exclusion(iommu); + } + + amd_iommu_teardown_tables_and_buffers(iommu, type); + + if (type == AMD_IOMMU_QUIESCE) + return (DDI_SUCCESS); + if (iommu->aiomt_va != NULL) { hat_unload(kas.a_hat, (void *)(uintptr_t)iommu->aiomt_va, iommu->aiomt_reg_size, HAT_UNLOAD_UNLOCK); @@ -1246,7 +1267,7 @@ /* check if cap ID is secure device cap id */ if (id != PCI_CAP_ID_SECURE_DEV) { if (amd_iommu_debug) { - cmn_err(CE_WARN, + cmn_err(CE_NOTE, "%s: %s%d: skipping IOMMU: idx(0x%x) " "cap ID (0x%x) != secure dev capid (0x%x)", f, driver, instance, idx, id, @@ -1299,20 +1320,21 @@ } int -amd_iommu_teardown(dev_info_t *dip, amd_iommu_state_t *statep) +amd_iommu_teardown(dev_info_t *dip, amd_iommu_state_t *statep, int type) { int instance = ddi_get_instance(dip); const char *driver = ddi_driver_name(dip); - amd_iommu_t *iommu; + amd_iommu_t *iommu, *next_iommu; int teardown; int error = DDI_SUCCESS; const char *f = "amd_iommu_teardown"; teardown = 0; for (iommu = statep->aioms_iommu_start; iommu; - iommu = iommu->aiomt_next) { + iommu = next_iommu) { ASSERT(statep->aioms_nunits > 0); - if (amd_iommu_fini(iommu) != DDI_SUCCESS) { + next_iommu = iommu->aiomt_next; + if (amd_iommu_fini(iommu, type) != DDI_SUCCESS) { error = DDI_FAILURE; continue; } @@ -1394,7 +1416,7 @@ mutex_enter(&amd_iommu_pgtable_lock); if (amd_iommu_debug == AMD_IOMMU_DEBUG_PAGE_TABLES) { - cmn_err(CE_WARN, "%s: %s%d: idx=%d Attempting to get cookies " + cmn_err(CE_NOTE, "%s: %s%d: idx=%d Attempting to get cookies " "from handle for device %s", f, driver, instance, idx, path); }
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/usr/src/uts/i86pc/io/amd_iommu/amd_iommu_impl.h Mon Sep 21 11:26:40 2009 -0400 @@ -0,0 +1,494 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright 2009 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +#ifndef _AMD_IOMMU_IMPL_H +#define _AMD_IOMMU_IMPL_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include <sys/pci.h> + +#ifdef _KERNEL + +#define AMD_IOMMU_PCI_PROG_IF (0x0) + +#define AMD_IOMMU_CAP (0x3) + +#define AMD_IOMMU_REG_SIZE (0x2028) +#define AMD_IOMMU_DEVTBL_SZ (16) +#define AMD_IOMMU_CMDBUF_SZ (15) +#define AMD_IOMMU_EVENTLOG_SZ (15) +#define AMD_IOMMU_DEVENT_SZ (32) +#define AMD_IOMMU_CMD_SZ (16) +#define AMD_IOMMU_EVENT_SZ (16) + +/* Capability Register offsets */ +#define AMD_IOMMU_CAP_HDR_OFF (0x00) +#define AMD_IOMMU_CAP_ADDR_LOW_OFF (0x04) +#define AMD_IOMMU_CAP_ADDR_HI_OFF (0x08) +#define AMD_IOMMU_CAP_RANGE_OFF (0x0C) +#define AMD_IOMMU_CAP_MISC_OFF (0x10) + +/* ControL Registers offsets */ +#define AMD_IOMMU_DEVTBL_REG_OFF (0x00) +#define AMD_IOMMU_CMDBUF_REG_OFF (0x08) +#define AMD_IOMMU_EVENTLOG_REG_OFF (0x10) +#define AMD_IOMMU_CTRL_REG_OFF (0x18) +#define AMD_IOMMU_EXCL_BASE_REG_OFF (0x20) +#define AMD_IOMMU_EXCL_LIM_REG_OFF (0x28) +#define AMD_IOMMU_CMDBUF_HEAD_REG_OFF (0x2000) +#define AMD_IOMMU_CMDBUF_TAIL_REG_OFF (0x2008) +#define AMD_IOMMU_EVENTLOG_HEAD_REG_OFF (0x2010) +#define AMD_IOMMU_EVENTLOG_TAIL_REG_OFF (0x2018) +#define AMD_IOMMU_STATUS_REG_OFF (0x2020) + +/* Capability Header Register Bits */ +#define AMD_IOMMU_CAP_NPCACHE (26 << 16 | 26) +#define AMD_IOMMU_CAP_HTTUN (25 << 16 | 25) +#define AMD_IOMMU_CAP_IOTLB (24 << 16 | 24) +#define AMD_IOMMU_CAP_TYPE (18 << 16 | 16) +#define AMD_IOMMU_CAP_ID (7 << 16 | 0) + +/* Capability Range Register bits */ +#define AMD_IOMMU_LAST_DEVFN (31 << 16 | 24) +#define AMD_IOMMU_FIRST_DEVFN (23 << 16 | 16) +#define AMD_IOMMU_RNG_BUS (15 << 16 | 8) +#define AMD_IOMMU_RNG_VALID (7 << 16 | 7) +#define AMD_IOMMU_HT_UNITID (4 << 16 | 0) + + +/* Capability Misc Register bits */ +#define AMD_IOMMU_HT_ATSRSV (22 << 16 | 22) +#define AMD_IOMMU_VA_SIZE (21 << 16 | 15) +#define AMD_IOMMU_PA_SIZE (14 << 16 | 8) +#define AMD_IOMMU_MSINUM (4 << 16 | 0) + +/* Device Table Base Address register bits */ +#define AMD_IOMMU_DEVTABBASE (51 << 16 | 12) +#define AMD_IOMMU_DEVTABSIZE (8 << 16 | 0) + +/* Command Buffer Base Address register bits */ +#define AMD_IOMMU_COMLEN (59 << 16 | 56) +#define AMD_IOMMU_COMBASE (51 << 16 | 12) + +#define AMD_IOMMU_CMDBUF_MINSZ (8) +#define AMD_IOMMU_CMDBUF_MAXSZ (15) + +/* Event Log Base Address register bits */ +#define AMD_IOMMU_EVENTLEN (59 << 16 | 56) +#define AMD_IOMMU_EVENTBASE (51 << 16 | 12) + +#define AMD_IOMMU_EVENTLOG_MINSZ (8) +#define AMD_IOMMU_EVENTLOG_MAXSZ (15) + +/* Control register bits */ +#define AMD_IOMMU_CMDBUF_ENABLE (12 << 16 | 12) +#define AMD_IOMMU_ISOC (11 << 16 | 11) +#define AMD_IOMMU_COHERENT (10 << 16 | 10) +#define AMD_IOMMU_RESPASSPW (9 << 16 | 9) +#define AMD_IOMMU_PASSPW (8 << 16 | 8) +#define AMD_IOMMU_INVTO (7 << 16 | 5) +#define AMD_IOMMU_COMWAITINT_ENABLE (4 << 16 | 4) +#define AMD_IOMMU_EVENTINT_ENABLE (3 << 16 | 3) +#define AMD_IOMMU_EVENTLOG_ENABLE (2 << 16 | 2) +#define AMD_IOMMU_HT_TUN_ENABLE (1 << 16 | 1) +#define AMD_IOMMU_ENABLE (0 << 16 | 0) + +/* Exclusion Base Register bits */ +#define AMD_IOMMU_EXCL_BASE_ADDR (51 << 16 | 12) +#define AMD_IOMMU_EXCL_BASE_ALLOW (1 << 16 | 1) +#define AMD_IOMMU_EXCL_BASE_EXEN (0 << 16 | 0) + +/* Exclusion Limit Register bits */ +#define AMD_IOMMU_EXCL_LIM (51 << 16 | 12) + +/* Command Buffer Head Pointer Register bits */ +#define AMD_IOMMU_CMDHEADPTR (18 << 16 | 4) + +/* Command Buffer Tail Pointer Register bits */ +#define AMD_IOMMU_CMDTAILPTR (18 << 16 | 4) + +/* Event Log Head Pointer Register bits */ +#define AMD_IOMMU_EVENTHEADPTR (18 << 16 | 4) + +/* Event Log Tail Pointer Register bits */ +#define AMD_IOMMU_EVENTTAILPTR (18 << 16 | 4) + +/* Status Register bits */ +#define AMD_IOMMU_CMDBUF_RUN (4 << 16 | 4) +#define AMD_IOMMU_EVENT_LOG_RUN (3 << 16 | 3) +#define AMD_IOMMU_COMWAIT_INT (2 << 16 | 2) +#define AMD_IOMMU_EVENT_LOG_INT (1 << 16 | 1) +#define AMD_IOMMU_EVENT_OVERFLOW_INT (0 << 16 | 0) + +/* Device Table Bits */ + +/* size in bytes of each device table entry */ +#define AMD_IOMMU_DEVTBL_ENTRY_SZ (32) + +/* Interrupt Remapping related Device Table bits */ +#define AMD_IOMMU_DEVTBL_LINT1PASS ((191-128) << 16 | (191-128)) +#define AMD_IOMMU_DEVTBL_LINT0PASS ((190-128) << 16 | (190-128)) +#define AMD_IOMMU_DEVTBL_INTCTL ((189-128) << 16 | (188-128)) +#define AMD_IOMMU_DEVTBL_NMIPASS ((186-128) << 16 | (186-128)) +#define AMD_IOMMU_DEVTBL_EXTINTPAS ((185-128) << 16 | (185-128)) +#define AMD_IOMMU_DEVTBL_INITPASS ((184-128) << 16 | (184-128)) +#define AMD_IOMMU_DEVTBL_INTR_ROOT ((179-128) << 16 | (134-128)) +#define AMD_IOMMU_DEVTBL_IG ((133-128) << 16 | (133-128)) +#define AMD_IOMMU_DEVTBL_INTTABLEN ((132-128) << 16 | (129-128)) +#define AMD_IOMMU_DEVTBL_IV ((128-128) << 16 | (128-128)) + +/* DMA Remapping related Device Table Bits */ +#define AMD_IOMMU_DEVTBL_SYSMGT ((105-64) << 16 | (104-64)) +#define AMD_IOMMU_DEVTBL_EX ((103-64) << 16 | (103-64)) +#define AMD_IOMMU_DEVTBL_SD ((102-64) << 16 | (102-64)) +#define AMD_IOMMU_DEVTBL_CACHE ((101-64) << 16 | (101-64)) +#define AMD_IOMMU_DEVTBL_IOCTL ((100-64) << 16 | (99-64)) +#define AMD_IOMMU_DEVTBL_SA ((98-64) << 16 | (98-64)) +#define AMD_IOMMU_DEVTBL_SE ((97-64) << 16 | (97-64)) +#define AMD_IOMMU_DEVTBL_IOTLB ((96-64) << 16 | (96-64)) +#define AMD_IOMMU_DEVTBL_DOMAINID ((79-64) << 16 | (64-64)) +#define AMD_IOMMU_DEVTBL_IW (62 << 16 | 62) +#define AMD_IOMMU_DEVTBL_IR (61 << 16 | 61) +#define AMD_IOMMU_DEVTBL_ROOT_PGTBL (51 << 16 | 12) +#define AMD_IOMMU_DEVTBL_PG_MODE (11 << 16 | 9) +#define AMD_IOMMU_DEVTBL_TV (1 << 16 | 1) +#define AMD_IOMMU_DEVTBL_V (0 << 16 | 0) + +#define BUS_DEVFN_TO_BDF(b, devfn) (devfn) +#define AMD_IOMMU_ALIAS_HASH_SZ (256) + +#define AMD_IOMMU_REG_ADDR_LOCKED (0x1) + +/* + * IOMMU Command bits + */ + +typedef enum { + AMD_IOMMU_CMD_INVAL = 0, + AMD_IOMMU_CMD_COMPL_WAIT, + AMD_IOMMU_CMD_INVAL_DEVTAB_ENTRY, + AMD_IOMMU_CMD_INVAL_IOMMU_PAGES, + AMD_IOMMU_CMD_INVAL_IOTLB_PAGES, + AMD_IOMMU_CMD_INVAL_INTR_TABLE, +} amd_iommu_cmd_t; + +typedef enum { + AMD_IOMMU_CMD_FLAGS_NONE = 0, + AMD_IOMMU_CMD_FLAGS_COMPL_WAIT = 1, + AMD_IOMMU_CMD_FLAGS_COMPL_WAIT_F = 2, + AMD_IOMMU_CMD_FLAGS_COMPL_WAIT_S = 4, + AMD_IOMMU_CMD_FLAGS_PAGE_PDE_INVAL = 8, + AMD_IOMMU_CMD_FLAGS_PAGE_INVAL_S = 16, + AMD_IOMMU_CMD_FLAGS_IOTLB_INVAL_S = 32 +} amd_iommu_cmd_flags_t; + +/* Common command bits */ +#define AMD_IOMMU_CMD_OPCODE (31 << 16 | 28) + +/* Completion Wait command bits */ +#define AMD_IOMMU_CMD_COMPL_WAIT_S (0 << 16 | 0) +#define AMD_IOMMU_CMD_COMPL_WAIT_I (1 << 16 | 1) +#define AMD_IOMMU_CMD_COMPL_WAIT_F (2 << 16 | 2) +#define AMD_IOMMU_CMD_COMPL_WAIT_STORE_ADDR_LO (31 << 16 | 3) +#define AMD_IOMMU_CMD_COMPL_WAIT_STORE_ADDR_HI (19 << 16 | 0) + +/* Invalidate Device Table entry command bits */ +#define AMD_IOMMU_CMD_INVAL_DEVTAB_DEVICEID (15 << 16 | 0) + +/* Invalidate IOMMU Pages command bits */ +#define AMD_IOMMU_CMD_INVAL_PAGES_DOMAINID (15 << 16 | 0) +#define AMD_IOMMU_CMD_INVAL_PAGES_S (0 << 16 | 0) +#define AMD_IOMMU_CMD_INVAL_PAGES_PDE (1 << 16 | 1) +#define AMD_IOMMU_CMD_INVAL_PAGES_ADDR_LO (31 << 16 | 12) +#define AMD_IOMMU_CMD_INVAL_PAGES_ADDR_HI (63 << 16 | 32) + + +/* Invalidate IOTLB command bits */ +#define AMD_IOMMU_CMD_INVAL_IOTLB_DEVICEID (15 << 16 | 0) +#define AMD_IOMMU_CMD_INVAL_IOTLB_MAXPEND (31 << 16 | 24) +#define AMD_IOMMU_CMD_INVAL_IOTLB_QUEUEID (15 << 16 | 0) +#define AMD_IOMMU_CMD_INVAL_IOTLB_S (0 << 16 | 0) +#define AMD_IOMMU_CMD_INVAL_IOTLB_ADDR_LO (31 << 16 | 12) +#define AMD_IOMMU_CMD_INVAL_IOTLB_ADDR_HI (31 << 16 | 0) + +#define AMD_IOMMU_DEFAULT_MAXPEND (10) + +/* Invalidate Interrupt Table bits */ +#define AMD_IOMMU_CMD_INVAL_INTR_DEVICEID (15 << 16 | 0) + +#if defined(__amd64) +#define dmac_cookie_addr dmac_laddress +#else +#define dmac_cookie_addr dmac_address +#endif + +#define AMD_IOMMU_TABLE_ALIGN ((1ULL << 12) - 1) + +#define AMD_IOMMU_MAX_DEVICEID (0xFFFF) + +/* + * DMA sync macros + * TODO: optimize sync only small ranges + */ +#define SYNC_FORDEV(h) (void) ddi_dma_sync(h, 0, 0, DDI_DMA_SYNC_FORDEV) +#define SYNC_FORKERN(h) (void) ddi_dma_sync(h, 0, 0, DDI_DMA_SYNC_FORKERNEL) + +#define WAIT_SEC(s) drv_usecwait(1000000*(s)) + +#define CMD2OFF(c) ((c) << 4) +#define OFF2CMD(o) ((o) >> 4) + +typedef union split { + uint64_t u64; + uint32_t u32[2]; +} split_t; + +#define BITPOS_START(b) ((b) >> 16) +#define BITPOS_END(b) ((b) & 0xFFFF) + +#define START_MASK64(s) (((s) == 63) ? ~((uint64_t)0) : \ + (uint64_t)((1ULL << ((s)+1)) - 1)) +#define START_MASK32(s) (((s) == 31) ? ~((uint32_t)0) : \ + (uint32_t)((1ULL << ((s)+1)) - 1)) +#define START_MASK16(s) (((s) == 15) ? ~((uint16_t)0) : \ + (uint16_t)((1ULL << ((s)+1)) - 1)) +#define START_MASK8(s) (((s) == 7) ? ~((uint8_t)0) : \ + (uint8_t)((1ULL << ((s)+1)) - 1)) + +#define END_MASK(e) ((1ULL << (e)) - 1) + +#define BIT_MASK64(s, e) (uint64_t)(START_MASK64(s) & ~END_MASK(e)) +#define BIT_MASK32(s, e) (uint32_t)(START_MASK32(s) & ~END_MASK(e)) +#define BIT_MASK16(s, e) (uint16_t)(START_MASK16(s) & ~END_MASK(e)) +#define BIT_MASK8(s, e) (uint8_t)(START_MASK8(s) & ~END_MASK(e)) + +#define AMD_IOMMU_REG_GET64_IMPL(rp, b) \ + (((*(rp)) & (START_MASK64(BITPOS_START(b)))) >> BITPOS_END(b)) +#define AMD_IOMMU_REG_GET64(rp, b) \ + ((amd_iommu_64bit_bug) ? amd_iommu_reg_get64_workaround(rp, b) : \ + AMD_IOMMU_REG_GET64_IMPL(rp, b)) +#define AMD_IOMMU_REG_GET32(rp, b) \ + (((*(rp)) & (START_MASK32(BITPOS_START(b)))) >> BITPOS_END(b)) +#define AMD_IOMMU_REG_GET16(rp, b) \ + (((*(rp)) & (START_MASK16(BITPOS_START(b)))) >> BITPOS_END(b)) +#define AMD_IOMMU_REG_GET8(rp, b) \ + (((*(rp)) & (START_MASK8(BITPOS_START(b)))) >> BITPOS_END(b)) + +#define AMD_IOMMU_REG_SET64_IMPL(rp, b, v) \ + ((*(rp)) = \ + (((uint64_t)(*(rp)) & ~(BIT_MASK64(BITPOS_START(b), BITPOS_END(b)))) \ + | ((uint64_t)(v) << BITPOS_END(b)))) + +#define AMD_IOMMU_REG_SET64(rp, b, v) \ + (void) ((amd_iommu_64bit_bug) ? \ + amd_iommu_reg_set64_workaround(rp, b, v) : \ + AMD_IOMMU_REG_SET64_IMPL(rp, b, v)) + +#define AMD_IOMMU_REG_SET32(rp, b, v) \ + ((*(rp)) = \ + (((uint32_t)(*(rp)) & ~(BIT_MASK32(BITPOS_START(b), BITPOS_END(b)))) \ + | ((uint32_t)(v) << BITPOS_END(b)))) + +#define AMD_IOMMU_REG_SET16(rp, b, v) \ + ((*(rp)) = \ + (((uint16_t)(*(rp)) & ~(BIT_MASK16(BITPOS_START(b), BITPOS_END(b)))) \ + | ((uint16_t)(v) << BITPOS_END(b)))) + +#define AMD_IOMMU_REG_SET8(rp, b, v) \ + ((*(rp)) = \ + (((uint8_t)(*(rp)) & ~(BIT_MASK8(BITPOS_START(b), BITPOS_END(b)))) \ + | ((uint8_t)(v) << BITPOS_END(b)))) + +/* + * Cast a 64 bit pointer to a uint64_t * + */ +#define REGADDR64(a) ((uint64_t *)(uintptr_t)(a)) + +typedef enum { + AMD_IOMMU_INTR_INVALID = 0, + AMD_IOMMU_INTR_TABLE, + AMD_IOMMU_INTR_ALLOCED, + AMD_IOMMU_INTR_HANDLER, + AMD_IOMMU_INTR_ENABLED +} amd_iommu_intr_state_t; + + +typedef struct amd_iommu { + kmutex_t aiomt_mutex; + kmutex_t aiomt_eventlock; + kmutex_t aiomt_cmdlock; + dev_info_t *aiomt_dip; + int aiomt_idx; + iommulib_handle_t aiomt_iommulib_handle; + iommulib_ops_t *aiomt_iommulib_ops; + uint32_t aiomt_cap_hdr; + uint8_t aiomt_npcache; + uint8_t aiomt_httun; + uint8_t aiomt_iotlb; + uint8_t aiomt_captype; + uint8_t aiomt_capid; + uint32_t aiomt_low_addr32; + uint32_t aiomt_hi_addr32; + uint64_t aiomt_reg_pa; + uint64_t aiomt_va; + uint64_t aiomt_reg_va; + uint32_t aiomt_range; + uint8_t aiomt_rng_bus; + uint8_t aiomt_first_devfn; + uint8_t aiomt_last_devfn; + uint8_t aiomt_rng_valid; + uint8_t aiomt_ht_unitid; + uint32_t aiomt_misc; + uint8_t aiomt_htatsresv; + uint8_t aiomt_vasize; + uint8_t aiomt_pasize; + uint8_t aiomt_msinum; + uint8_t aiomt_reg_pages; + uint32_t aiomt_reg_size; + uint32_t aiomt_devtbl_sz; + uint32_t aiomt_cmdbuf_sz; + uint32_t aiomt_eventlog_sz; + caddr_t aiomt_devtbl; + caddr_t aiomt_cmdbuf; + caddr_t aiomt_eventlog; + uint32_t *aiomt_cmd_tail; + uint32_t *aiomt_event_head; + ddi_dma_handle_t aiomt_dmahdl; + void *aiomt_dma_bufva; + uint64_t aiomt_dma_mem_realsz; + ddi_acc_handle_t aiomt_dma_mem_hdl; + ddi_dma_cookie_t aiomt_buf_dma_cookie; + uint_t aiomt_buf_dma_ncookie; + amd_iommu_intr_state_t aiomt_intr_state; + ddi_intr_handle_t *aiomt_intr_htable; + uint32_t aiomt_intr_htable_sz; + uint32_t aiomt_actual_intrs; + uint32_t aiomt_intr_cap; + uint64_t aiomt_reg_devtbl_va; + uint64_t aiomt_reg_cmdbuf_va; + uint64_t aiomt_reg_eventlog_va; + uint64_t aiomt_reg_ctrl_va; + uint64_t aiomt_reg_excl_base_va; + uint64_t aiomt_reg_excl_lim_va; + uint64_t aiomt_reg_cmdbuf_head_va; + uint64_t aiomt_reg_cmdbuf_tail_va; + uint64_t aiomt_reg_eventlog_head_va; + uint64_t aiomt_reg_eventlog_tail_va; + uint64_t aiomt_reg_status_va; + struct amd_iommu *aiomt_next; +} amd_iommu_t; + +typedef struct amd_iommu_dma_devtbl_ent { + uint16_t de_domainid; + uint8_t de_R; + uint8_t de_W; + caddr_t de_root_pgtbl; + uint8_t de_pgmode; +} amd_iommu_dma_devtbl_entry_t; + +typedef struct amd_iommu_alias { + uint16_t al_bdf; + uint16_t al_src_bdf; + struct amd_iommu_alias *al_next; +} amd_iommu_alias_t; + +typedef struct amd_iommu_cmdargs { + uint64_t ca_addr; + uint16_t ca_domainid; + uint16_t ca_deviceid; +} amd_iommu_cmdargs_t; + +struct amd_iommu_page_table; + +typedef struct amd_iommu_page_table_hash { + kmutex_t ampt_lock; + struct amd_iommu_page_table **ampt_hash; +} amd_iommu_page_table_hash_t; + +typedef enum { + AMD_IOMMU_LOG_INVALID_OP = 0, + AMD_IOMMU_LOG_DISPLAY, + AMD_IOMMU_LOG_DISCARD +} amd_iommu_log_op_t; + +typedef enum { + AMD_IOMMU_DEBUG_NONE = 0, + AMD_IOMMU_DEBUG_ALLOCHDL = 0x1, + AMD_IOMMU_DEBUG_FREEHDL = 0x2, + AMD_IOMMU_DEBUG_BIND = 0x4, + AMD_IOMMU_DEBUG_UNBIND = 0x8, + AMD_IOMMU_DEBUG_WIN = 0x10, + AMD_IOMMU_DEBUG_PAGE_TABLES = 0x20, + AMD_IOMMU_DEBUG_DEVTBL = 0x40, + AMD_IOMMU_DEBUG_CMDBUF = 0x80, + AMD_IOMMU_DEBUG_EVENTLOG = 0x100, + AMD_IOMMU_DEBUG_ACPI = 0x200, + AMD_IOMMU_DEBUG_PA2VA = 0x400, + AMD_IOMMU_DEBUG_TABLES = 0x800, + AMD_IOMMU_DEBUG_EXCL = 0x1000, + AMD_IOMMU_DEBUG_INTR = 0x2000 +} amd_iommu_debug_t; + +extern const char *amd_iommu_modname; +extern kmutex_t amd_iommu_global_lock; +extern amd_iommu_alias_t **amd_iommu_alias; +extern amd_iommu_page_table_hash_t amd_iommu_page_table_hash; +extern ddi_device_acc_attr_t amd_iommu_devacc; +extern amd_iommu_debug_t amd_iommu_debug; + +extern uint8_t amd_iommu_htatsresv; +extern uint8_t amd_iommu_vasize; +extern uint8_t amd_iommu_pasize; +extern int amd_iommu_64bit_bug; +extern int amd_iommu_unity_map; +extern int amd_iommu_no_RW_perms; +extern int amd_iommu_no_unmap; +extern int amd_iommu_pageva_inval_all; +extern int amd_iommu_disable; +extern char *amd_iommu_disable_list; + +extern uint64_t amd_iommu_reg_get64_workaround(uint64_t *regp, uint32_t bits); +extern uint64_t amd_iommu_reg_set64_workaround(uint64_t *regp, uint32_t bits, + uint64_t value); + +int amd_iommu_cmd(amd_iommu_t *iommu, amd_iommu_cmd_t cmd, + amd_iommu_cmdargs_t *cmdargs, amd_iommu_cmd_flags_t flags, int lock_held); +int amd_iommu_page_table_hash_init(amd_iommu_page_table_hash_t *ampt); +void amd_iommu_page_table_hash_fini(amd_iommu_page_table_hash_t *ampt); + +int amd_iommu_read_log(amd_iommu_t *iommu, amd_iommu_log_op_t op); +void amd_iommu_read_boot_props(void); +void amd_iommu_lookup_conf_props(dev_info_t *dip); + +#endif /* _KERNEL */ + +#ifdef __cplusplus +} +#endif + +#endif /* _AMD_IOMMU_IMPL_H */
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/usr/src/uts/i86pc/io/amd_iommu/amd_iommu_log.c Mon Sep 21 11:26:40 2009 -0400 @@ -0,0 +1,582 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ + +/* + * Copyright 2009 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +#include <sys/sunddi.h> +#include <sys/amd_iommu.h> +#include "amd_iommu_impl.h" +#include "amd_iommu_log.h" + + +static const char * +get_hw_error(uint8_t type) +{ + const char *hwerr; + + switch (type) { + case 0: + hwerr = "Reserved"; + break; + case 1: + hwerr = "Master Abort"; + break; + case 2: + hwerr = "Target Abort"; + break; + case 3: + hwerr = "Data Error"; + break; + default: + hwerr = "Unknown"; + break; + } + + return (hwerr); +} + +const char * +get_illegal_req(uint8_t type, uint8_t TR) +{ + const char *illreq; + + switch (type) { + case 0: + illreq = (TR == 1) ? "Translation I=0/V=0/V=1&&TV=0" : + "Read or Non-posted Write in INTR Range"; + break; + case 1: + illreq = (TR == 1) ? "Translation INTR/Port-IO/SysMgt; OR" + "Translation when SysMgt=11b/Port-IO when IOCTL=10b " + "while V=1 && TV=0" : + "Pre-translated transaction from device with I=0 or V=0"; + break; + case 2: + illreq = (TR == 1) ? "Reserved": + "Port-IO transaction for device with IoCtl = 00b"; + break; + case 3: + illreq = (TR == 1) ? "Reserved": + "Posted write to SysMgt with device SysMgt=00b " + "OR SysMgt=10b && message not INTx " + "OR Posted write to addr transaltion range with " + "HtAtsResv=1"; + break; + case 4: + illreq = (TR == 1) ? "Reserved": + "Read request or non-posted write in SysMgt with " + "device SysMgt=10b or 0xb" + "OR Read request or non-posted write in " + "addr translation range with HtAtsResv=1"; + break; + case 5: + illreq = (TR == 1) ? "Reserved": + "Posted write to Interrupt/EOI Range " + "for device that has IntCtl=00b"; + break; + case 6: + illreq = (TR == 1) ? "Reserved": + "Posted write to reserved Interrupt Address Range"; + break; + case 7: + illreq = (TR == 1) ? "Reserved": + "transaction to SysMgt when SysMgt=11b OR " + "transaction to Port-IO when IoCtl=10b while " + "while V=1 TV=0"; + break; + default: + illreq = "Unknown error"; + break; + } + return (illreq); +} + +static void +devtab_illegal_entry(amd_iommu_t *iommu, uint32_t *event) +{ + uint16_t deviceid; + uint8_t TR; + uint8_t RZ; + uint8_t RW; + uint8_t I; + uint32_t vaddr_lo; + uint32_t vaddr_hi; + const char *driver = ddi_driver_name(iommu->aiomt_dip); + int instance = ddi_get_instance(iommu->aiomt_dip); + const char *f = "devtab_illegal_entry"; + + ASSERT(AMD_IOMMU_REG_GET32(&event[1], AMD_IOMMU_EVENT_TYPE) == + AMD_IOMMU_EVENT_DEVTAB_ILLEGAL_ENTRY); + + deviceid = AMD_IOMMU_REG_GET32(&event[0], + AMD_IOMMU_EVENT_DEVTAB_ILL_DEVICEID); + + TR = AMD_IOMMU_REG_GET32(&event[1], + AMD_IOMMU_EVENT_DEVTAB_ILL_TR); + + RZ = AMD_IOMMU_REG_GET32(&event[1], + AMD_IOMMU_EVENT_DEVTAB_ILL_RZ); + + RW = AMD_IOMMU_REG_GET32(&event[1], + AMD_IOMMU_EVENT_DEVTAB_ILL_RW); + + I = AMD_IOMMU_REG_GET32(&event[1], + AMD_IOMMU_EVENT_DEVTAB_ILL_INTR); + + vaddr_lo = AMD_IOMMU_REG_GET32(&event[2], + AMD_IOMMU_EVENT_DEVTAB_ILL_VADDR_LO); + + vaddr_hi = event[3]; + + cmn_err(CE_WARN, "%s: %s%d: idx = %d. Illegal device table entry " + "deviceid=%u, %s request, %s %s transaction, %s request, " + "virtual address = %p", + f, driver, instance, iommu->aiomt_idx, + deviceid, + TR == 1 ? "Translation" : "Transaction", + RZ == 1 ? "Non-zero reserved bit" : "Illegal Level encoding", + RW == 1 ? "Write" : "Read", + I == 1 ? "Interrupt" : "Memory", + (void *)(uintptr_t)(((uint64_t)vaddr_hi) << 32 | vaddr_lo)); +} + +static void +io_page_fault(amd_iommu_t *iommu, uint32_t *event) +{ + uint16_t deviceid; + uint16_t domainid; + uint8_t TR; + uint8_t RZ; + uint8_t RW; + uint8_t PE; + uint8_t PR; + uint8_t I; + uint32_t vaddr_lo; + uint32_t vaddr_hi; + const char *driver = ddi_driver_name(iommu->aiomt_dip); + int instance = ddi_get_instance(iommu->aiomt_dip); + const char *f = "io_page_fault"; + + ASSERT(AMD_IOMMU_REG_GET32(&event[1], AMD_IOMMU_EVENT_TYPE) == + AMD_IOMMU_EVENT_IO_PAGE_FAULT); + + deviceid = AMD_IOMMU_REG_GET32(&event[0], + AMD_IOMMU_EVENT_IO_PGFAULT_DEVICEID); + + TR = AMD_IOMMU_REG_GET32(&event[1], AMD_IOMMU_EVENT_IO_PGFAULT_TR); + + RZ = AMD_IOMMU_REG_GET32(&event[1], AMD_IOMMU_EVENT_IO_PGFAULT_RZ); + + PE = AMD_IOMMU_REG_GET32(&event[1], AMD_IOMMU_EVENT_IO_PGFAULT_PE); + + RW = AMD_IOMMU_REG_GET32(&event[1], AMD_IOMMU_EVENT_IO_PGFAULT_RW); + + PR = AMD_IOMMU_REG_GET32(&event[1], AMD_IOMMU_EVENT_IO_PGFAULT_PR); + + I = AMD_IOMMU_REG_GET32(&event[1], AMD_IOMMU_EVENT_IO_PGFAULT_INTR); + + domainid = AMD_IOMMU_REG_GET32(&event[1], + AMD_IOMMU_EVENT_IO_PGFAULT_DOMAINID); + + vaddr_lo = event[2]; + + vaddr_hi = event[3]; + + cmn_err(CE_WARN, "%s: %s%d: idx = %d. IO Page Fault. " + "deviceid=%u, %s request, %s, %s permissions, %s transaction, " + "%s, %s request, domainid=%u, virtual address = %p", + f, driver, instance, iommu->aiomt_idx, + deviceid, + TR == 1 ? "Translation" : "Transaction", + RZ == 1 ? "Non-zero reserved bit" : "Illegal Level encoding", + PE == 1 ? "did not have" : "had", + RW == 1 ? "Write" : "Read", + PR == 1 ? "Page present or Interrupt Remapped" : + "Page not present or Interrupt Blocked", + I == 1 ? "Interrupt" : "Memory", + domainid, + (void *)(uintptr_t)(((uint64_t)vaddr_hi) << 32 | vaddr_lo)); +} + +static void +devtab_hw_error(amd_iommu_t *iommu, uint32_t *event) +{ + uint16_t deviceid; + uint8_t type; + uint8_t TR; + uint8_t RW; + uint8_t I; + uint32_t physaddr_lo; + uint32_t physaddr_hi; + const char *hwerr; + const char *driver = ddi_driver_name(iommu->aiomt_dip); + int instance = ddi_get_instance(iommu->aiomt_dip); + const char *f = "devtab_hw_error"; + + ASSERT(AMD_IOMMU_REG_GET32(&event[1], AMD_IOMMU_EVENT_TYPE) == + AMD_IOMMU_EVENT_DEVTAB_HW_ERROR); + + deviceid = AMD_IOMMU_REG_GET32(&event[0], + AMD_IOMMU_EVENT_DEVTAB_HWERR_DEVICEID); + + type = AMD_IOMMU_REG_GET32(&event[1], + AMD_IOMMU_EVENT_DEVTAB_HWERR_TYPE); + + hwerr = get_hw_error(type); + + TR = AMD_IOMMU_REG_GET32(&event[1], AMD_IOMMU_EVENT_DEVTAB_HWERR_TR); + + RW = AMD_IOMMU_REG_GET32(&event[1], AMD_IOMMU_EVENT_DEVTAB_HWERR_RW); + + I = AMD_IOMMU_REG_GET32(&event[1], AMD_IOMMU_EVENT_DEVTAB_HWERR_INTR); + + physaddr_lo = AMD_IOMMU_REG_GET32(&event[2], + AMD_IOMMU_EVENT_DEVTAB_HWERR_PHYSADDR_LO); + + physaddr_hi = event[3]; + + cmn_err(CE_WARN, "%s: %s%d: idx = %d. Device Table HW Error. " + "deviceid=%u, HW error type: %s, %s request, %s transaction, " + "%s request, physical address = %p", + f, driver, instance, iommu->aiomt_idx, + deviceid, hwerr, + TR == 1 ? "Translation" : "Transaction", + RW == 1 ? "Write" : "Read", + I == 1 ? "Interrupt" : "Memory", + (void *)(uintptr_t)(((uint64_t)physaddr_hi) << 32 | physaddr_lo)); +} + + +static void +pgtable_hw_error(amd_iommu_t *iommu, uint32_t *event) +{ + uint16_t deviceid; + uint16_t domainid; + uint8_t type; + uint8_t TR; + uint8_t RW; + uint8_t I; + uint32_t physaddr_lo; + uint32_t physaddr_hi; + const char *hwerr; + const char *driver = ddi_driver_name(iommu->aiomt_dip); + int instance = ddi_get_instance(iommu->aiomt_dip); + const char *f = "pgtable_hw_error"; + + ASSERT(AMD_IOMMU_REG_GET32(&event[1], AMD_IOMMU_EVENT_TYPE) == + AMD_IOMMU_EVENT_PGTABLE_HW_ERROR); + + deviceid = AMD_IOMMU_REG_GET32(&event[0], + AMD_IOMMU_EVENT_PGTABLE_HWERR_DEVICEID); + + type = AMD_IOMMU_REG_GET32(&event[1], + AMD_IOMMU_EVENT_DEVTAB_HWERR_TYPE); + + hwerr = get_hw_error(type); + + TR = AMD_IOMMU_REG_GET32(&event[1], AMD_IOMMU_EVENT_PGTABLE_HWERR_TR); + + RW = AMD_IOMMU_REG_GET32(&event[1], AMD_IOMMU_EVENT_PGTABLE_HWERR_RW); + + I = AMD_IOMMU_REG_GET32(&event[1], AMD_IOMMU_EVENT_PGTABLE_HWERR_INTR); + + domainid = AMD_IOMMU_REG_GET32(&event[1], + AMD_IOMMU_EVENT_PGTABLE_HWERR_DOMAINID); + + physaddr_lo = AMD_IOMMU_REG_GET32(&event[2], + AMD_IOMMU_EVENT_PGTABLE_HWERR_PHYSADDR_LO); + + physaddr_hi = event[3]; + + cmn_err(CE_WARN, "%s: %s%d: idx = %d. Page Table HW Error. " + "deviceid=%u, HW error type: %s, %s request, %s transaction, " + "%s request, domainid=%u, physical address = %p", + f, driver, instance, iommu->aiomt_idx, + deviceid, hwerr, + TR == 1 ? "Translation" : "Transaction", + RW == 1 ? "Write" : "Read", + I == 1 ? "Interrupt" : "Memory", + domainid, + (void *)(uintptr_t)(((uint64_t)physaddr_hi) << 32 | physaddr_lo)); +} + +static void +cmdbuf_illegal_cmd(amd_iommu_t *iommu, uint32_t *event) +{ + uint32_t physaddr_lo; + uint32_t physaddr_hi; + const char *driver = ddi_driver_name(iommu->aiomt_dip); + int instance = ddi_get_instance(iommu->aiomt_dip); + const char *f = "cmdbuf_illegal_cmd"; + + ASSERT(AMD_IOMMU_REG_GET32(&event[1], AMD_IOMMU_EVENT_TYPE) == + AMD_IOMMU_EVENT_CMDBUF_ILLEGAL_CMD); + + physaddr_lo = AMD_IOMMU_REG_GET32(&event[2], + AMD_IOMMU_EVENT_CMDBUF_ILLEGAL_CMD_PHYS_LO); + + physaddr_hi = event[3]; + + cmn_err(CE_WARN, "%s: %s%d: idx = %d. Illegal IOMMU command. " + "command physical address = %p", + f, driver, instance, iommu->aiomt_idx, + (void *)(uintptr_t)(((uint64_t)physaddr_hi) << 32 | physaddr_lo)); +} + +static void +cmdbuf_hw_error(amd_iommu_t *iommu, uint32_t *event) +{ + uint32_t physaddr_lo; + uint32_t physaddr_hi; + uint8_t type; + const char *hwerr; + const char *driver = ddi_driver_name(iommu->aiomt_dip); + int instance = ddi_get_instance(iommu->aiomt_dip); + const char *f = "cmdbuf_hw_error"; + + ASSERT(AMD_IOMMU_REG_GET32(&event[1], AMD_IOMMU_EVENT_TYPE) == + AMD_IOMMU_EVENT_CMDBUF_HW_ERROR); + + type = AMD_IOMMU_REG_GET32(&event[1], + AMD_IOMMU_EVENT_CMDBUF_HWERR_TYPE); + + hwerr = get_hw_error(type); + + physaddr_lo = AMD_IOMMU_REG_GET32(&event[2], + AMD_IOMMU_EVENT_CMDBUF_HWERR_PHYS_LO); + + physaddr_hi = event[3]; + + cmn_err(CE_WARN, "%s: %s%d: idx = %d. Command Buffer HW error. " + "HW error type = %s, command buffer physical address = %p", + f, driver, instance, iommu->aiomt_idx, + hwerr, + (void *)(uintptr_t)(((uint64_t)physaddr_hi) << 32 | physaddr_lo)); +} + +static void +iotlb_inval_to(amd_iommu_t *iommu, uint32_t *event) +{ + uint16_t deviceid; + uint32_t physaddr_lo; + uint32_t physaddr_hi; + uint8_t type; + const char *hwerr; + const char *driver = ddi_driver_name(iommu->aiomt_dip); + int instance = ddi_get_instance(iommu->aiomt_dip); + const char *f = "iotlb_inval_to"; + + ASSERT(AMD_IOMMU_REG_GET32(&event[1], AMD_IOMMU_EVENT_TYPE) == + AMD_IOMMU_EVENT_IOTLB_INVAL_TO); + + deviceid = AMD_IOMMU_REG_GET32(&event[0], + AMD_IOMMU_EVENT_IOTLB_INVAL_TO_DEVICEID); + + /* + * XXX bug in spec. Is the type field available +04 26:25 or is + * it reserved + */ + type = AMD_IOMMU_REG_GET32(&event[1], + AMD_IOMMU_EVENT_IOTLB_INVAL_TO_TYPE); + hwerr = get_hw_error(type); + + physaddr_lo = AMD_IOMMU_REG_GET32(&event[2], + AMD_IOMMU_EVENT_IOTLB_INVAL_TO_PHYS_LO); + + physaddr_hi = event[3]; + + cmn_err(CE_WARN, "%s: %s%d: idx = %d. deviceid = %u " + "IOTLB invalidation Timeout. " + "HW error type = %s, invalidation command physical address = %p", + f, driver, instance, iommu->aiomt_idx, deviceid, + hwerr, + (void *)(uintptr_t)(((uint64_t)physaddr_hi) << 32 | physaddr_lo)); +} + +static void +device_illegal_req(amd_iommu_t *iommu, uint32_t *event) +{ + uint16_t deviceid; + uint8_t TR; + uint32_t addr_lo; + uint32_t addr_hi; + uint8_t type; + const char *reqerr; + const char *driver = ddi_driver_name(iommu->aiomt_dip); + int instance = ddi_get_instance(iommu->aiomt_dip); + const char *f = "device_illegal_req"; + + ASSERT(AMD_IOMMU_REG_GET32(&event[1], AMD_IOMMU_EVENT_TYPE) == + AMD_IOMMU_EVENT_DEVICE_ILLEGAL_REQ); + + deviceid = AMD_IOMMU_REG_GET32(&event[0], + AMD_IOMMU_EVENT_DEVICE_ILLEGAL_REQ_DEVICEID); + + TR = AMD_IOMMU_REG_GET32(&event[1], + AMD_IOMMU_EVENT_DEVICE_ILLEGAL_REQ_TR); + + type = AMD_IOMMU_REG_GET32(&event[1], + AMD_IOMMU_EVENT_DEVICE_ILLEGAL_REQ_TYPE); + + reqerr = get_illegal_req(type, TR); + + + addr_lo = event[2]; + addr_hi = event[3]; + + cmn_err(CE_WARN, "%s: %s%d: idx = %d. deviceid = %d " + "Illegal Device Request. " + "Illegal Request type = %s, %s request, address accessed = %p", + f, driver, instance, iommu->aiomt_idx, deviceid, + reqerr, + TR == 1 ? "Translation" : "Transaction", + (void *)(uintptr_t)(((uint64_t)addr_hi) << 32 | addr_lo)); +} + +static void +amd_iommu_process_one_event(amd_iommu_t *iommu) +{ + uint32_t event[4]; + amd_iommu_event_t event_type; + int i; + const char *driver = ddi_driver_name(iommu->aiomt_dip); + int instance = ddi_get_instance(iommu->aiomt_dip); + const char *f = "amd_iommu_process_one_event"; + + ASSERT(MUTEX_HELD(&iommu->aiomt_eventlock)); + + SYNC_FORKERN(iommu->aiomt_dmahdl); + for (i = 0; i < 4; i++) { + event[i] = iommu->aiomt_event_head[i]; + } + + event_type = AMD_IOMMU_REG_GET32(&event[1], AMD_IOMMU_EVENT_TYPE); + + switch (event_type) { + case AMD_IOMMU_EVENT_DEVTAB_ILLEGAL_ENTRY: + devtab_illegal_entry(iommu, event); + break; + case AMD_IOMMU_EVENT_IO_PAGE_FAULT: + io_page_fault(iommu, event); + break; + case AMD_IOMMU_EVENT_DEVTAB_HW_ERROR: + devtab_hw_error(iommu, event); + break; + case AMD_IOMMU_EVENT_PGTABLE_HW_ERROR: + pgtable_hw_error(iommu, event); + break; + case AMD_IOMMU_EVENT_CMDBUF_HW_ERROR: + cmdbuf_hw_error(iommu, event); + break; + case AMD_IOMMU_EVENT_CMDBUF_ILLEGAL_CMD: + cmdbuf_illegal_cmd(iommu, event); + break; + case AMD_IOMMU_EVENT_IOTLB_INVAL_TO: + iotlb_inval_to(iommu, event); + break; + case AMD_IOMMU_EVENT_DEVICE_ILLEGAL_REQ: + device_illegal_req(iommu, event); + break; + default: + cmn_err(CE_WARN, "%s: %s%d: idx = %d. Unknown event: %u", + f, driver, instance, iommu->aiomt_idx, event_type); + break; + } +} + +int +amd_iommu_read_log(amd_iommu_t *iommu, amd_iommu_log_op_t op) +{ + caddr_t evtail; + uint64_t evtail_off; + uint64_t evhead_off; + + ASSERT(op != AMD_IOMMU_LOG_INVALID_OP); + + mutex_enter(&iommu->aiomt_eventlock); + + ASSERT(iommu->aiomt_event_head != NULL); + + /* XXX verify */ + evtail_off = AMD_IOMMU_REG_GET64( + REGADDR64(iommu->aiomt_reg_eventlog_tail_va), + AMD_IOMMU_EVENTTAILPTR); + + evtail_off = EV2OFF(evtail_off); + + ASSERT(evtail_off < iommu->aiomt_eventlog_sz); + + evtail = iommu->aiomt_eventlog + evtail_off; + + if (op == AMD_IOMMU_LOG_DISCARD) { + /*LINTED*/ + iommu->aiomt_event_head = (uint32_t *)evtail; + AMD_IOMMU_REG_SET64(REGADDR64( + iommu->aiomt_reg_eventlog_head_va), + AMD_IOMMU_EVENTHEADPTR, OFF2EV(evtail_off)); + cmn_err(CE_NOTE, "Discarded IOMMU event log"); + mutex_exit(&iommu->aiomt_eventlock); + return (DDI_SUCCESS); + } + + /*LINTED*/ + while (1) { + if ((caddr_t)iommu->aiomt_event_head == evtail) + break; + + cmn_err(CE_WARN, "evtail_off = %p, head = %p, tail = %p", + (void *)(uintptr_t)evtail_off, + (void *)iommu->aiomt_event_head, + (void *)evtail); + + amd_iommu_process_one_event(iommu); + + /* + * Update the head pointer in soft state + * and the head pointer register + */ + iommu->aiomt_event_head += 4; + if ((caddr_t)iommu->aiomt_event_head >= + iommu->aiomt_eventlog + iommu->aiomt_eventlog_sz) { + /* wraparound */ + iommu->aiomt_event_head = + /*LINTED*/ + (uint32_t *)iommu->aiomt_eventlog; + evhead_off = 0; + } else { + evhead_off = (caddr_t)iommu->aiomt_event_head + /*LINTED*/ + - iommu->aiomt_eventlog; + } + + ASSERT(evhead_off < iommu->aiomt_eventlog_sz); + + AMD_IOMMU_REG_SET64(REGADDR64( + iommu->aiomt_reg_eventlog_head_va), + AMD_IOMMU_EVENTHEADPTR, OFF2EV(evhead_off)); + } + mutex_exit(&iommu->aiomt_eventlock); + + return (DDI_SUCCESS); +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/usr/src/uts/i86pc/io/amd_iommu/amd_iommu_log.h Mon Sep 21 11:26:40 2009 -0400 @@ -0,0 +1,116 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright 2009 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +#ifndef _AMD_IOMMU_LOG_H +#define _AMD_IOMMU_LOG_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include <sys/amd_iommu.h> + +#ifdef _KERNEL + +#define EV2OFF(e) ((e) << 4) +#define OFF2EV(o) ((o) >> 4) + +typedef enum { + AMD_IOMMU_EVENT_INVALID = 0, + AMD_IOMMU_EVENT_DEVTAB_ILLEGAL_ENTRY = 1, + AMD_IOMMU_EVENT_IO_PAGE_FAULT = 2, + AMD_IOMMU_EVENT_DEVTAB_HW_ERROR = 3, + AMD_IOMMU_EVENT_PGTABLE_HW_ERROR = 4, + AMD_IOMMU_EVENT_CMDBUF_ILLEGAL_CMD = 5, + AMD_IOMMU_EVENT_CMDBUF_HW_ERROR = 6, + AMD_IOMMU_EVENT_IOTLB_INVAL_TO = 7, + AMD_IOMMU_EVENT_DEVICE_ILLEGAL_REQ = 8 +} amd_iommu_event_t; + +/* Common to all events */ +#define AMD_IOMMU_EVENT_TYPE (31 << 16 | 28) + +/* Illegal device Table Entry Event bits */ +#define AMD_IOMMU_EVENT_DEVTAB_ILL_DEVICEID (15 << 16 | 0) +#define AMD_IOMMU_EVENT_DEVTAB_ILL_TR (24 << 16 | 24) +#define AMD_IOMMU_EVENT_DEVTAB_ILL_RZ (23 << 16 | 23) +#define AMD_IOMMU_EVENT_DEVTAB_ILL_RW (21 << 16 | 21) +#define AMD_IOMMU_EVENT_DEVTAB_ILL_INTR (19 << 16 | 19) +#define AMD_IOMMU_EVENT_DEVTAB_ILL_VADDR_LO (31 << 16 | 2) + +/* IO Page Fault event bits */ +#define AMD_IOMMU_EVENT_IO_PGFAULT_DEVICEID (15 << 16 | 0) +#define AMD_IOMMU_EVENT_IO_PGFAULT_TR (24 << 16 | 24) +#define AMD_IOMMU_EVENT_IO_PGFAULT_RZ (23 << 16 | 23) +#define AMD_IOMMU_EVENT_IO_PGFAULT_PE (22 << 16 | 22) +#define AMD_IOMMU_EVENT_IO_PGFAULT_RW (21 << 16 | 21) +#define AMD_IOMMU_EVENT_IO_PGFAULT_PR (20 << 16 | 20) +#define AMD_IOMMU_EVENT_IO_PGFAULT_INTR (19 << 16 | 19) +#define AMD_IOMMU_EVENT_IO_PGFAULT_DOMAINID (15 << 16 | 0) + + +/* Device Table HW Error event bits */ +#define AMD_IOMMU_EVENT_DEVTAB_HWERR_DEVICEID (15 << 16 | 0) +#define AMD_IOMMU_EVENT_DEVTAB_HWERR_TYPE (26 << 16 | 25) +#define AMD_IOMMU_EVENT_DEVTAB_HWERR_TR (24 << 16 | 24) +#define AMD_IOMMU_EVENT_DEVTAB_HWERR_RW (21 << 16 | 21) +#define AMD_IOMMU_EVENT_DEVTAB_HWERR_INTR (19 << 16 | 19) +#define AMD_IOMMU_EVENT_DEVTAB_HWERR_PHYSADDR_LO (31 << 16 | 4) + + +/* Page Table HW Error event bits */ +#define AMD_IOMMU_EVENT_PGTABLE_HWERR_DEVICEID (15 << 16 | 0) +#define AMD_IOMMU_EVENT_DEVTAB_HWERR_TYPE (26 << 16 | 25) +#define AMD_IOMMU_EVENT_PGTABLE_HWERR_TR (24 << 16 | 24) +#define AMD_IOMMU_EVENT_PGTABLE_HWERR_RW (21 << 16 | 21) +#define AMD_IOMMU_EVENT_PGTABLE_HWERR_INTR (19 << 16 | 19) +#define AMD_IOMMU_EVENT_PGTABLE_HWERR_DOMAINID (15 << 16 | 0) +#define AMD_IOMMU_EVENT_PGTABLE_HWERR_PHYSADDR_LO (31 << 16 | 3) + +/* Illegal Command Error event bits */ +#define AMD_IOMMU_EVENT_CMDBUF_ILLEGAL_CMD_PHYS_LO (31 << 16 | 4) + +/* Command Buffer HW Error event bits */ +#define AMD_IOMMU_EVENT_CMDBUF_HWERR_TYPE (26 << 16 | 25) +#define AMD_IOMMU_EVENT_CMDBUF_HWERR_PHYS_LO (31 << 16 | 4) + + +/* IOTLB Invalidation TO event bits */ +#define AMD_IOMMU_EVENT_IOTLB_INVAL_TO_DEVICEID (15 << 16 | 0) +#define AMD_IOMMU_EVENT_IOTLB_INVAL_TO_TYPE (26 << 16 | 25) +#define AMD_IOMMU_EVENT_IOTLB_INVAL_TO_PHYS_LO (31 << 16 | 4) + +/* Illegal Device request event bits */ +#define AMD_IOMMU_EVENT_DEVICE_ILLEGAL_REQ_DEVICEID (15 << 16 | 0) +#define AMD_IOMMU_EVENT_DEVICE_ILLEGAL_REQ_TYPE (27 << 16 | 25) +#define AMD_IOMMU_EVENT_DEVICE_ILLEGAL_REQ_TR (24 << 16 | 24) + +#endif /* _KERNEL */ + +#ifdef __cplusplus +} +#endif + +#endif /* _AMD_IOMMU_LOG_H */
--- a/usr/src/uts/i86pc/io/amd_iommu/amd_iommu_page_tables.c Mon Sep 21 11:25:09 2009 -0400 +++ b/usr/src/uts/i86pc/io/amd_iommu/amd_iommu_page_tables.c Mon Sep 21 11:26:40 2009 -0400 @@ -510,14 +510,14 @@ { uint64_t *devtbl_entry; amd_iommu_cmdargs_t cmdargs = {0}; - int error; + int error, flags; dev_info_t *idip = iommu->aiomt_dip; const char *driver = ddi_driver_name(idip); int instance = ddi_get_instance(idip); const char *f = "amd_iommu_set_devtbl_entry"; if (amd_iommu_debug & AMD_IOMMU_DEBUG_DEVTBL) { - cmn_err(CE_WARN, "%s: attempting to set devtbl entry for %s", + cmn_err(CE_NOTE, "%s: attempting to set devtbl entry for %s", f, path); } @@ -536,10 +536,39 @@ [deviceid * AMD_IOMMU_DEVTBL_ENTRY_SZ]; if (amd_iommu_debug & AMD_IOMMU_DEBUG_DEVTBL) { - cmn_err(CE_WARN, "%s: deviceid=%u devtbl entry (%p) for %s", + cmn_err(CE_NOTE, "%s: deviceid=%u devtbl entry (%p) for %s", f, deviceid, (void *)(uintptr_t)(*devtbl_entry), path); } + /* + * Flush internal caches, need to do this if we came up from + * fast boot + */ + cmdargs.ca_deviceid = deviceid; + error = amd_iommu_cmd(iommu, AMD_IOMMU_CMD_INVAL_DEVTAB_ENTRY, + &cmdargs, 0, 0); + if (error != DDI_SUCCESS) { + cmn_err(CE_WARN, "%s: idx=%d: deviceid=%d" + "Failed to invalidate domain in IOMMU HW cache", + f, iommu->aiomt_idx, deviceid); + return (error); + } + + cmdargs.ca_domainid = (uint16_t)domainid; + cmdargs.ca_addr = (uintptr_t)0x7FFFFFFFFFFFF000; + flags = AMD_IOMMU_CMD_FLAGS_PAGE_PDE_INVAL | + AMD_IOMMU_CMD_FLAGS_PAGE_INVAL_S; + + error = amd_iommu_cmd(iommu, AMD_IOMMU_CMD_INVAL_IOMMU_PAGES, + &cmdargs, flags, 0); + if (error != DDI_SUCCESS) { + cmn_err(CE_WARN, "%s: idx=%d: domainid=%d" + "Failed to invalidate translations in IOMMU HW cache", + f, iommu->aiomt_idx, cmdargs.ca_domainid); + return (error); + } + + /* Initialize device table entry */ if (init_devtbl(iommu, devtbl_entry, domainid, dp)) { cmdargs.ca_deviceid = deviceid; error = amd_iommu_cmd(iommu, AMD_IOMMU_CMD_INVAL_DEVTAB_ENTRY, @@ -582,7 +611,7 @@ [deviceid * AMD_IOMMU_DEVTBL_ENTRY_SZ]; if (amd_iommu_debug & AMD_IOMMU_DEBUG_DEVTBL) { - cmn_err(CE_WARN, "%s: deviceid=%u devtbl entry (%p) for %s", + cmn_err(CE_NOTE, "%s: deviceid=%u devtbl entry (%p) for %s", f, deviceid, (void *)(uintptr_t)(*devtbl_entry), path); } @@ -1548,7 +1577,7 @@ for (pfn = pfn_start; pfn <= pfn_end; pfn++, pg++) { if (amd_iommu_debug & AMD_IOMMU_DEBUG_PAGE_TABLES) { - cmn_err(CE_WARN, "%s: attempting to create page tables " + cmn_err(CE_NOTE, "%s: attempting to create page tables " "for pfn = %p, va = %p, path = %s", f, (void *)(uintptr_t)(pfn << MMU_PAGESHIFT), (void *)(uintptr_t)(pg << MMU_PAGESHIFT), path); @@ -1568,7 +1597,7 @@ } if (amd_iommu_debug & AMD_IOMMU_DEBUG_PAGE_TABLES) { - cmn_err(CE_WARN, "%s: successfuly created page tables " + cmn_err(CE_NOTE, "%s: successfuly created page tables " "for pfn = %p, vapg = %p, path = %s", f, (void *)(uintptr_t)pfn, (void *)(uintptr_t)pg, path);
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/usr/src/uts/i86pc/io/amd_iommu/amd_iommu_page_tables.h Mon Sep 21 11:26:40 2009 -0400 @@ -0,0 +1,134 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright 2009 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +#ifndef _AMD_IOMMU_PAGE_TABLES_H +#define _AMD_IOMMU_PAGE_TABLES_H + +#ifdef __cplusplus +extern "C" { +#endif + +#ifdef _KERNEL + +/* Common to PTEs and PDEs */ +#define AMD_IOMMU_PTDE_IW (62 << 16 | 62) +#define AMD_IOMMU_PTDE_IR (61 << 16 | 61) +#define AMD_IOMMU_PTDE_ADDR (51 << 16 | 12) +#define AMD_IOMMU_PTDE_NXT_LVL (11 << 16 | 9) +#define AMD_IOMMU_PTDE_PR (0 << 16 | 0) + +#define AMD_IOMMU_PTE_FC (60 << 16 | 60) +#define AMD_IOMMU_PTE_U (59 << 16 | 59) + +#define AMD_IOMMU_VA_NBITS(l) ((l) == 6 ? 7 : 9) +#define AMD_IOMMU_VA_BITMASK(l) ((1 << AMD_IOMMU_VA_NBITS(l)) - 1) +#define AMD_IOMMU_VA_SHIFT(v, l) \ + ((v) >> (MMU_PAGESHIFT + (AMD_IOMMU_VA_NBITS(l - 1) * (l - 1)))) +#define AMD_IOMMU_VA_BITS(v, l) \ + (AMD_IOMMU_VA_SHIFT(v, l) & AMD_IOMMU_VA_BITMASK(l)) +#define AMD_IOMMU_VA_TOTBITS(l) \ + (((l) == 6 ? 7 + (l - 1) * 9: l*9) + MMU_PAGESHIFT) +#define AMD_IOMMU_VA_TOTMASK(l) ((1 << AMD_IOMMU_VA_TOTBITS(l)) - 1) +#define AMD_IOMMU_VA_INVAL_SETMASK(l) \ + (((1 << AMD_IOMMU_VA_TOTBITS(l)) - 1) >> 1) +#define AMD_IOMMU_VA_INVAL_CLRMASK(l) \ + (~(1 << (AMD_IOMMU_VA_TOTBITS(l) - 1))) +#define AMD_IOMMU_VA_INVAL(v, l) \ + (((v) & AMD_IOMMU_VA_INVAL_CLRMASK(l)) | AMD_IOMMU_VA_INVAL_SETMASK(l)) + +#define AMD_IOMMU_PGTABLE_SZ (4096) +#define AMD_IOMMU_PGTABLE_MAXLEVEL (6) +#define AMD_IOMMU_PGTABLE_HASH_SZ (256) + +#define AMD_IOMMU_PGTABLE_ALIGN ((1ULL << 12) - 1) +#define AMD_IOMMU_PGTABLE_SIZE (1ULL << 12) + +#define AMD_IOMMU_MAX_PDTE (1ULL << AMD_IOMMU_VA_NBITS(1)) +#define PT_REF_VALID(p) ((p)->pt_ref >= 0 && \ + (p)->pt_ref <= AMD_IOMMU_MAX_PDTE) + +#define AMD_IOMMU_DOMAIN_HASH_SZ (256) +#define AMD_IOMMU_PGTABLE_FREELIST_MAX (256) +#define AMD_IOMMU_PA2VA_HASH_SZ (256) + +#define AMD_IOMMU_SIZE_4G ((uint64_t)1 << 32) +#define AMD_IOMMU_VMEM_NAMELEN (30) + +typedef enum { + AMD_IOMMU_INVALID_DOMAIN = 0, + AMD_IOMMU_IDENTITY_DOMAIN = 0xFFFD, + AMD_IOMMU_PASSTHRU_DOMAIN = 0xFFFE, + AMD_IOMMU_SYS_DOMAIN = 0xFFFF +} domain_id_t; + +typedef enum { + AMD_IOMMU_INVALID_MAP = 0, + AMD_IOMMU_UNITY_MAP, + AMD_IOMMU_VMEM_MAP +} map_type_t; + +typedef struct amd_iommu_page_table { + domain_id_t pt_domainid; + int pt_level; + ddi_dma_handle_t pt_dma_hdl; + ddi_acc_handle_t pt_mem_hdl; + uint64_t pt_mem_reqsz; + uint64_t pt_mem_realsz; + uint64_t *pt_pgtblva; + uint64_t pt_pte_ref[AMD_IOMMU_MAX_PDTE]; + uint16_t pt_index; + int pt_ref; + ddi_dma_cookie_t pt_cookie; + struct amd_iommu_page_table *pt_next; + struct amd_iommu_page_table *pt_prev; + struct amd_iommu_page_table *pt_parent; +} amd_iommu_page_table_t; + +typedef struct amd_iommu_domain { + domain_id_t d_domainid; + uint64_t d_pgtable_root_4K; + int64_t d_ref; + vmem_t *d_vmem; + struct amd_iommu_domain *d_prev; + struct amd_iommu_domain *d_next; +} amd_iommu_domain_t; + +int amd_iommu_map_pa2va(amd_iommu_t *iommu, dev_info_t *rdip, + ddi_dma_attr_t *attrp, struct ddi_dma_req *dmareq, + uint64_t pa, uint64_t pa_sz, map_type_t type, + uint64_t *start_vap, int km_flags); +int amd_iommu_unmap_va(amd_iommu_t *iommu, dev_info_t *rdip, + uint64_t va, uint64_t va_sz, map_type_t type); +void amd_iommu_init_page_tables(amd_iommu_t *iommu); +void amd_iommu_fini_page_tables(amd_iommu_t *iommu); +void amd_iommu_set_passthru(amd_iommu_t *iommu, dev_info_t *rdip); + +#endif /* _KERNEL */ + +#ifdef __cplusplus +} +#endif + +#endif /* _AMD_IOMMU_PAGE_TABLES_H */
--- a/usr/src/uts/i86pc/io/intel_iommu.c Mon Sep 21 11:25:09 2009 -0400 +++ b/usr/src/uts/i86pc/io/intel_iommu.c Mon Sep 21 11:26:40 2009 -0400 @@ -92,10 +92,14 @@ /* * internal variables - * iommu_state - the list of iommu structures + * iommu_states - the list of iommu + * domain_states - the list of domain + * rmrr_states - the list of rmrr * page_num - the count of pages for iommu page tables */ static list_t iommu_states; +static list_t domain_states; +static list_t rmrr_states; static uint_t page_num; /* @@ -2042,6 +2046,8 @@ offsetof(dvma_cache_node_t, node)); } + list_insert_tail(&domain_states, domain); + return (DDI_SUCCESS); } @@ -2759,6 +2765,7 @@ if (list_is_empty(&(dmar_info->dmari_rmrr[i]))) break; for_each_in_list(&(dmar_info->dmari_rmrr[i]), rmrr) { + list_insert_tail(&rmrr_states, rmrr); build_single_rmrr_identity_map(rmrr); } } @@ -3030,6 +3037,10 @@ */ list_create(&iommu_states, sizeof (intel_iommu_state_t), offsetof(intel_iommu_state_t, node)); + list_create(&domain_states, sizeof (dmar_domain_state_t), + offsetof(dmar_domain_state_t, node)); + list_create(&rmrr_states, sizeof (rmrr_info_t), + offsetof(rmrr_info_t, node4states)); root_devinfo = ddi_root_node(); ASSERT(root_devinfo);
--- a/usr/src/uts/i86pc/os/cpuid_subr.c Mon Sep 21 11:25:09 2009 -0400 +++ b/usr/src/uts/i86pc/os/cpuid_subr.c Mon Sep 21 11:26:40 2009 -0400 @@ -67,36 +67,48 @@ * Second index by (model & 0x3) for family 0fh * or CPUID bits for later families */ -static uint32_t amd_skts[4][4] = { +static uint32_t amd_skts[4][8] = { /* * Family 0xf revisions B through E */ #define A_SKTS_0 0 { - X86_SOCKET_754, /* 0b00 */ - X86_SOCKET_940, /* 0b01 */ - X86_SOCKET_754, /* 0b10 */ - X86_SOCKET_939 /* 0b11 */ + X86_SOCKET_754, /* 0b000 */ + X86_SOCKET_940, /* 0b001 */ + X86_SOCKET_754, /* 0b010 */ + X86_SOCKET_939, /* 0b011 */ + X86_SOCKET_UNKNOWN, /* 0b100 */ + X86_SOCKET_UNKNOWN, /* 0b101 */ + X86_SOCKET_UNKNOWN, /* 0b110 */ + X86_SOCKET_UNKNOWN /* 0b111 */ }, /* * Family 0xf revisions F and G */ #define A_SKTS_1 1 { - X86_SOCKET_S1g1, /* 0b00 */ - X86_SOCKET_F1207, /* 0b01 */ - X86_SOCKET_UNKNOWN, /* 0b10 */ - X86_SOCKET_AM2 /* 0b11 */ + X86_SOCKET_S1g1, /* 0b000 */ + X86_SOCKET_F1207, /* 0b001 */ + X86_SOCKET_UNKNOWN, /* 0b010 */ + X86_SOCKET_AM2, /* 0b011 */ + X86_SOCKET_UNKNOWN, /* 0b100 */ + X86_SOCKET_UNKNOWN, /* 0b101 */ + X86_SOCKET_UNKNOWN, /* 0b110 */ + X86_SOCKET_UNKNOWN /* 0b111 */ }, /* * Family 0x10 */ #define A_SKTS_2 2 { - X86_SOCKET_F1207, /* 0b00 */ - X86_SOCKET_AM, /* 0b01 */ - X86_SOCKET_S1g3, /* 0b10 */ - X86_SOCKET_G34, /* 0b11 */ + X86_SOCKET_F1207, /* 0b000 */ + X86_SOCKET_AM, /* 0b001 */ + X86_SOCKET_S1g3, /* 0b010 */ + X86_SOCKET_G34, /* 0b011 */ + X86_SOCKET_ASB2, /* 0b100 */ + X86_SOCKET_C32, /* 0b101 */ + X86_SOCKET_UNKNOWN, /* 0b110 */ + X86_SOCKET_UNKNOWN /* 0b111 */ }, /* @@ -104,10 +116,14 @@ */ #define A_SKTS_3 3 { - X86_SOCKET_UNKNOWN, /* 0b00 */ - X86_SOCKET_UNKNOWN, /* 0b01 */ - X86_SOCKET_S1g2, /* 0b10 */ - X86_SOCKET_UNKNOWN, /* 0b11 */ + X86_SOCKET_UNKNOWN, /* 0b000 */ + X86_SOCKET_UNKNOWN, /* 0b001 */ + X86_SOCKET_S1g2, /* 0b010 */ + X86_SOCKET_UNKNOWN, /* 0b011 */ + X86_SOCKET_UNKNOWN, /* 0b100 */ + X86_SOCKET_UNKNOWN, /* 0b101 */ + X86_SOCKET_UNKNOWN, /* 0b110 */ + X86_SOCKET_UNKNOWN /* 0b111 */ } }; @@ -115,7 +131,7 @@ uint32_t skt_code; char sktstr[16]; }; -static struct amd_sktmap_s amd_sktmap[13] = { +static struct amd_sktmap_s amd_sktmap[15] = { { X86_SOCKET_754, "754" }, { X86_SOCKET_939, "939" }, { X86_SOCKET_940, "940" }, @@ -128,6 +144,8 @@ { X86_SOCKET_AM2R2, "AM2r2" }, { X86_SOCKET_AM3, "AM3" }, { X86_SOCKET_G34, "G34" }, + { X86_SOCKET_ASB2, "ASB2" }, + { X86_SOCKET_C32, "C32" }, { X86_SOCKET_UNKNOWN, "Unknown" } }; @@ -277,7 +295,7 @@ /* PkgType bits */ idx = BITX(cp.cp_ebx, 31, 28); - if (idx > 3) { + if (idx > 7) { /* Reserved bits */ *skt_p = X86_SOCKET_UNKNOWN; } else if (family == 0x10 &&
--- a/usr/src/uts/i86pc/os/fakebop.c Mon Sep 21 11:25:09 2009 -0400 +++ b/usr/src/uts/i86pc/os/fakebop.c Mon Sep 21 11:26:40 2009 -0400 @@ -124,7 +124,7 @@ static int early_allocation = 1; int force_fastreboot = 0; -int fastreboot_onpanic = 0; +volatile int fastreboot_onpanic = 0; int post_fastreboot = 0; #ifdef __xpv int fastreboot_capable = 0; @@ -696,10 +696,17 @@ consoledev = outputdev + v_len + 1; v_len = do_bsys_getproplen(NULL, "console"); - if (v_len > 0) + if (v_len > 0) { (void) do_bsys_getprop(NULL, "console", consoledev); - else + if (post_fastreboot && + strcmp(consoledev, "graphics") == 0) { + bsetprops("console", "text"); + v_len = strlen("text"); + bcopy("text", consoledev, v_len); + } + } else { v_len = 0; + } consoledev[v_len] = 0; bcons_init2(inputdev, outputdev, consoledev); } else { @@ -1696,8 +1703,6 @@ HYPERVISOR_shared_info = (void *)xbootp->bi_shared_info; xen_info = xbootp->bi_xen_start_info; #endif - bcons_init((void *)xbootp->bi_cmdline); - have_console = 1; #ifndef __xpv if (*((uint32_t *)(FASTBOOT_SWTCH_PA + FASTBOOT_STACK_OFFSET)) == @@ -1707,6 +1712,9 @@ } #endif + bcons_init((void *)xbootp->bi_cmdline); + have_console = 1; + /* * enable debugging */
--- a/usr/src/uts/i86pc/os/fastboot.c Mon Sep 21 11:25:09 2009 -0400 +++ b/usr/src/uts/i86pc/os/fastboot.c Mon Sep 21 11:26:40 2009 -0400 @@ -136,6 +136,11 @@ int reserve_mem_enabled = 1; /* + * Mutex to protect fastreboot_onpanic. + */ +kmutex_t fastreboot_config_mutex; + +/* * Amount of memory below PA 1G to reserve for constructing the multiboot * data structure and the page tables as we tend to run out of those * when more drivers are loaded. @@ -868,7 +873,8 @@ int is_retry = 0; uint64_t end_addr; - ASSERT(fastreboot_capable); + if (!fastreboot_capable) + return; if (newkernel.fi_valid) fastboot_free_newkernel(&newkernel); @@ -1408,12 +1414,16 @@ if (!fastreboot_capable) return; + mutex_enter(&fastreboot_config_mutex); + fastboot_get_bootprop(); if (fastreboot_onpanic) fastboot_load_kernel(fastreboot_onpanic_cmdline); else if (reserve_mem_enabled) fastboot_reserve_mem(&newkernel); + + mutex_exit(&fastreboot_config_mutex); } /* @@ -1427,15 +1437,69 @@ fastboot_update_config(const char *mdep) { uint8_t boot_config = (uint8_t)*mdep; - int cur_fastreboot_onpanic = fastreboot_onpanic; + int cur_fastreboot_onpanic; if (!fastreboot_capable) return; + mutex_enter(&fastreboot_config_mutex); + + cur_fastreboot_onpanic = fastreboot_onpanic; fastreboot_onpanic = boot_config & UA_FASTREBOOT_ONPANIC; + if (fastreboot_onpanic && (!cur_fastreboot_onpanic || !newkernel.fi_valid)) fastboot_load_kernel(fastreboot_onpanic_cmdline); if (cur_fastreboot_onpanic && !fastreboot_onpanic) fastboot_free_newkernel(&newkernel); + + mutex_exit(&fastreboot_config_mutex); } + +/* + * This is the interface to be called by other kernel components to + * disable fastreboot_onpanic. + */ +void +fastreboot_disable() +{ + uint8_t boot_config = (uint8_t)(~UA_FASTREBOOT_ONPANIC); + fastboot_update_config((const char *)&boot_config); +} + +/* + * This is the interface to be called by fm_panic() in case FMA has diagnosed + * a terminal machine check exception. It does not free up memory allocated + * for the backup kernel. General disabling fastreboot_onpanic in a + * non-panicking situation must go through fastboot_update_config(). + */ +void +fastreboot_disable_highpil() +{ + fastreboot_onpanic = 0; +} + + +/* + * A simplified interface for uadmin to call to update the configuration + * setting and load a new kernel if necessary. + */ +void +fastboot_update_and_load(int fcn, char *mdep) +{ + if (fcn != AD_FASTREBOOT) { + /* + * If user has explicitly requested reboot to prom, + * or uadmin(1M) was invoked with other functions, + * don't try to fast reboot after dumping. + */ + fastreboot_disable(); + } + + mutex_enter(&fastreboot_config_mutex); + + if (fastreboot_onpanic) + fastboot_load_kernel(mdep); + + mutex_exit(&fastreboot_config_mutex); +}
--- a/usr/src/uts/i86pc/os/mlsetup.c Mon Sep 21 11:25:09 2009 -0400 +++ b/usr/src/uts/i86pc/os/mlsetup.c Mon Sep 21 11:26:40 2009 -0400 @@ -49,10 +49,10 @@ #include <sys/machsystm.h> #include <sys/ontrap.h> #include <sys/bootconf.h> +#include <sys/boot_console.h> #include <sys/kdi_machimpl.h> #include <sys/archsystm.h> #include <sys/promif.h> -#include <sys/bootconf.h> #include <sys/pci_cfgspace.h> #ifdef __xpv #include <sys/hypervisor.h> @@ -78,6 +78,21 @@ 0xf, 0xf, 0xf, 0xf, 0xf, 0xf, 0xf, 0xf, 0xf }; +/* + * Set console mode + */ +static void +set_console_mode(uint8_t val) +{ + struct bop_regs rp = {0}; + + rp.eax.byte.ah = 0x0; + rp.eax.byte.al = val; + rp.ebx.word.bx = 0x0; + + BOP_DOINT(bootops, 0x10, &rp); +} + /* * Setup routine called right before main(). Interposing this function @@ -90,6 +105,8 @@ extern struct classfuncs sys_classfuncs; extern disp_t cpu0_disp; extern char t0stack[]; + extern int post_fastreboot; + extern int console; ASSERT_STACK_ALIGNED(); @@ -304,6 +321,13 @@ kdi_idt_sync(); /* + * Explicitly set console to text mode (0x3) if this is a boot + * post Fast Reboot, and the console is set to CONS_SCREEN_TEXT. + */ + if (post_fastreboot && console == CONS_SCREEN_TEXT) + set_console_mode(0x3); + + /* * If requested (boot -d) drop into kmdb. * * This must be done after cpu_list_init() on the 64-bit kernel
--- a/usr/src/uts/i86pc/sys/Makefile Mon Sep 21 11:25:09 2009 -0400 +++ b/usr/src/uts/i86pc/sys/Makefile Mon Sep 21 11:26:40 2009 -0400 @@ -38,6 +38,7 @@ HDRS= \ acpidev.h \ + amd_iommu.h \ asm_misc.h \ clock.h \ cram.h \
--- a/usr/src/uts/i86pc/sys/amd_iommu.h Mon Sep 21 11:25:09 2009 -0400 +++ b/usr/src/uts/i86pc/sys/amd_iommu.h Mon Sep 21 11:26:40 2009 -0400 @@ -43,8 +43,11 @@ int aioms_nunits; /* # of IOMMUs in function */ } amd_iommu_state_t; +#define AMD_IOMMU_QUIESCE (0) +#define AMD_IOMMU_TEARDOWN (1) + int amd_iommu_setup(dev_info_t *dip, amd_iommu_state_t *statep); -int amd_iommu_teardown(dev_info_t *dip, amd_iommu_state_t *statep); +int amd_iommu_teardown(dev_info_t *dip, amd_iommu_state_t *statep, int type); int amd_iommu_lookup_src_bdf(uint16_t bdf, uint16_t *src_bdfp); #endif /* _KERNEL */
--- a/usr/src/uts/i86pc/sys/dmar_acpi.h Mon Sep 21 11:25:09 2009 -0400 +++ b/usr/src/uts/i86pc/sys/dmar_acpi.h Mon Sep 21 11:26:40 2009 -0400 @@ -160,6 +160,7 @@ */ typedef struct rmrr_info { list_node_t node; + list_node_t node4states; uint16_t ri_segment; uint64_t ri_baseaddr; uint64_t ri_limiaddr;
--- a/usr/src/uts/i86pc/sys/fastboot.h Mon Sep 21 11:25:09 2009 -0400 +++ b/usr/src/uts/i86pc/sys/fastboot.h Mon Sep 21 11:26:40 2009 -0400 @@ -173,7 +173,7 @@ extern int force_fastreboot; /* If set, fast reboot after panic. */ -extern int fastreboot_onpanic; +extern volatile int fastreboot_onpanic; extern char fastreboot_onpanic_cmdline[FASTBOOT_SAVED_CMDLINE_LEN]; #endif /* _ASM */
--- a/usr/src/uts/i86pc/sys/intel_iommu.h Mon Sep 21 11:25:09 2009 -0400 +++ b/usr/src/uts/i86pc/sys/intel_iommu.h Mon Sep 21 11:26:40 2009 -0400 @@ -145,20 +145,29 @@ #define IOMMU_CAP_GET_DWD(x) (((x) >> 54) & 1) #define IOMMU_CAP_GET_DRD(x) (((x) >> 55) & 1) #define IOMMU_CAP_GET_PSI(x) (((x) >> 39) & 1) +#define IOMMU_CAP_GET_SPS(x) (((x) >> 34) & 0xf) +#define IOMMU_CAP_GET_ISOCH(x) (((x) >> 23) & 1) +#define IOMMU_CAP_GET_ZLR(x) (((x) >> 22) & 1) #define IOMMU_CAP_GET_MAMV(x) (((x) >> 48) & 0x3f) #define IOMMU_CAP_GET_CM(x) (((x) >> 7) & 1) +#define IOMMU_CAP_GET_PHMR(x) (((x) >> 6) & 1) +#define IOMMU_CAP_GET_PLMR(x) (((x) >> 5) & 1) #define IOMMU_CAP_GET_RWBF(x) (((x) >> 4) & 1) +#define IOMMU_CAP_GET_AFL(x) (((x) >> 3) & 1) #define IOMMU_CAP_GET_FRO(x) ((((x) >> 24) & 0x3ff) * 16) #define IOMMU_CAP_MGAW(x) (((((uint64_t)x) >> 16) & 0x3f) + 1) #define IOMMU_CAP_SAGAW(x) (((x) >> 8) & 0x1f) #define IOMMU_CAP_ND(x) (1 << (((x) & 0x7) *2 + 4)) -1 #define IOMMU_ECAP_GET_IRO(x) ((((x) >> 8) & 0x3ff) << 4) -#define IOMMU_ECAP_GET_C(x) ((x) & 0x1) #define IOMMU_ECAP_GET_MHMV(x) ((x >> 20) & 0xf) +#define IOMMU_ECAP_GET_SC(x) ((x) & 0x80) +#define IOMMU_ECAP_GET_PT(x) ((x) & 0x40) +#define IOMMU_ECAP_GET_CH(x) ((x) & 0x20) #define IOMMU_ECAP_GET_EIM(x) ((x) & 0x10) #define IOMMU_ECAP_GET_IR(x) ((x) & 0x8) #define IOMMU_ECAP_GET_DI(x) ((x) & 0x4) #define IOMMU_ECAP_GET_QI(x) ((x) & 0x2) +#define IOMMU_ECAP_GET_C(x) ((x) & 0x1) /* iotlb invalidation */ @@ -343,7 +352,7 @@ typedef enum { IEC_INV_GLOBAL = 0, - IEC_INV_INDEX, + IEC_INV_INDEX } iec_inv_g_t; /* @@ -523,6 +532,7 @@ * dm_identity - does this domain identity mapped */ typedef struct dmar_domain_state { + list_node_t node; uint_t dm_domain_id; intel_iommu_state_t *dm_iommu; vmem_t *dm_dvma_map;
--- a/usr/src/uts/intel/Makefile.files Mon Sep 21 11:25:09 2009 -0400 +++ b/usr/src/uts/intel/Makefile.files Mon Sep 21 11:26:40 2009 -0400 @@ -252,17 +252,6 @@ AMR_OBJS = amr.o # -# AMD_IOMMU module -# -AMD_IOMMU_OBJS = \ - amd_iommu.o \ - amd_iommu_impl.o \ - amd_iommu_acpi.o \ - amd_iommu_cmd.o \ - amd_iommu_log.o \ - amd_iommu_page_tables.o - -# # IOMMULIB module # IOMMULIB_OBJS = iommulib.o
--- a/usr/src/uts/intel/Makefile.intel.shared Mon Sep 21 11:25:09 2009 -0400 +++ b/usr/src/uts/intel/Makefile.intel.shared Mon Sep 21 11:26:40 2009 -0400 @@ -206,6 +206,7 @@ DRV_KMODS += audiols DRV_KMODS += audiop16x DRV_KMODS += audiopci +DRV_KMODS += audiosolo DRV_KMODS += audiots DRV_KMODS += audiovia823x DRV_KMODS_32 += audiovia97 @@ -400,7 +401,6 @@ DRV_KMODS += rtls DRV_KMODS += sfe DRV_KMODS += amd8111s -DRV_KMODS += amd_iommu DRV_KMODS += igb DRV_KMODS += ixgbe DRV_KMODS += vr
--- a/usr/src/uts/intel/Makefile.rules Mon Sep 21 11:25:09 2009 -0400 +++ b/usr/src/uts/intel/Makefile.rules Mon Sep 21 11:26:40 2009 -0400 @@ -153,10 +153,6 @@ $(COMPILE.c) -o $@ $< $(CTFCONVERT_O) -$(OBJS_DIR)/%.o: $(UTSBASE)/intel/io/amd_iommu/%.c - $(COMPILE.c) -o $@ $< - $(CTFCONVERT_O) - $(OBJS_DIR)/%.o: $(UTSBASE)/intel/io/amr/%.c $(COMPILE.c) -o $@ $< $(CTFCONVERT_O) @@ -394,9 +390,6 @@ $(LINTS_DIR)/%.ln: $(UTSBASE)/intel/io/amd8111s/%.c @($(LHEAD) $(LINT.c) $< $(LTAIL)) -$(LINTS_DIR)/%.ln: $(UTSBASE)/intel/io/amd_iommu/%.c - @($(LHEAD) $(LINT.c) $< $(LTAIL)) - $(LINTS_DIR)/%.ln: $(UTSBASE)/intel/io/amr/%.c @($(LHEAD) $(LINT.c) $< $(LTAIL))
--- a/usr/src/uts/intel/amd_iommu/Makefile Mon Sep 21 11:25:09 2009 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,84 +0,0 @@ -# -# CDDL HEADER START -# -# The contents of this file are subject to the terms of the -# Common Development and Distribution License (the "License"). -# You may not use this file except in compliance with the License. -# -# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE -# or http://www.opensolaris.org/os/licensing. -# See the License for the specific language governing permissions -# and limitations under the License. -# -# When distributing Covered Code, include this CDDL HEADER in each -# file and include the License file at usr/src/OPENSOLARIS.LICENSE. -# If applicable, add the following below this CDDL HEADER, with the -# fields enclosed by brackets "[]" replaced with your own identifying -# information: Portions Copyright [yyyy] [name of copyright owner] -# -# CDDL HEADER END -# -# Copyright 2009 Sun Microsystems, Inc. All rights reserved. -# Use is subject to license terms. -# -# This Makefile drives production of the amd_iommu driver kernel module. -# -# intel implementation architecture dependent -# - -# -# Path to the base of the uts directory tree (usually /usr/src/uts). -# -UTSBASE = ../.. - -# -# Define the module and object file sets. -# -MODULE = amd_iommu -OBJECTS = $(AMD_IOMMU_OBJS:%=$(OBJS_DIR)/%) -LINTS = $(AMD_IOMMU_OBJS:%.o=$(LINTS_DIR)/%.ln) -ROOTMODULE = $(ROOT_DRV_DIR)/$(MODULE) -CONF_SRCDIR = $(UTSBASE)/intel/io/amd_iommu - -# -# Include common rules. -# -include $(UTSBASE)/intel/Makefile.intel - -# -# Define targets -# -ALL_TARGET = $(BINARY) $(SRC_CONFILE) -LINT_TARGET = $(MODULE).lint -INSTALL_TARGET = $(BINARY) $(ROOTMODULE) $(ROOT_CONFFILE) - -# -# depends on misc/iommulib and misc/acpica -# -LDFLAGS += -dy -Nmisc/iommulib -Nmisc/acpica - -# -# Default build targets. -# -.KEEP_STATE: - -def: $(DEF_DEPS) - -all: $(ALL_DEPS) - -clean: $(CLEAN_DEPS) - -clobber: $(CLOBBER_DEPS) - -lint: $(LINT_DEPS) - -modlintlib: $(MODLINTLIB_DEPS) - -clean.lint: $(CLEAN_LINT_DEPS) - -install: $(INSTALL_DEPS) $(CONF_INSTALL_DEPS) - -# -# Include common targets. -# -include $(UTSBASE)/intel/Makefile.targ
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/usr/src/uts/intel/audiosolo/Makefile Mon Sep 21 11:26:40 2009 -0400 @@ -0,0 +1,81 @@ +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# +# +# uts/intel/audiosolo/Makefile +# +# Copyright 2009 Sun Microsystems, Inc. All rights reserved. +# Use is subject to license terms. +# +# This makefile drives the production of the audiosolo driver. +# + +# +# Path to the base of the uts directory tree (usually /usr/src/uts). +# +UTSBASE = ../.. + +# +# Define the module and object file sets. +# +MODULE = audiosolo +OBJECTS = $(AUDIOSOLO_OBJS:%=$(OBJS_DIR)/%) +LINTS = $(AUDIOSOLO_OBJS:%.o=$(LINTS_DIR)/%.ln) +ROOTMODULE = $(ROOT_DRV_DIR)/$(MODULE) + +# +# Include common rules. +# +include $(UTSBASE)/intel/Makefile.intel + +# +# Define targets +# +ALL_TARGET = $(BINARY) +LINT_TARGET = $(MODULE).lint +INSTALL_TARGET = $(BINARY) $(ROOTMODULE) + +LDFLAGS += -dy -Ndrv/audio + +# +# Default build targets. +# +.KEEP_STATE: + +def: $(DEF_DEPS) + +all: $(ALL_DEPS) + +clean: $(CLEAN_DEPS) + +clobber: $(CLOBBER_DEPS) + +lint: $(LINT_DEPS) + +modlintlib: $(MODLINTLIB_DEPS) + +clean.lint: $(CLEAN_LINT_DEPS) + +install: $(INSTALL_DEPS) + +# +# Include common targets. +# +include $(UTSBASE)/intel/Makefile.targ
--- a/usr/src/uts/intel/io/amd_iommu/amd_iommu.c Mon Sep 21 11:25:09 2009 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,440 +0,0 @@ -/* - * CDDL HEADER START - * - * The contents of this file are subject to the terms of the - * Common Development and Distribution License (the "License"). - * You may not use this file except in compliance with the License. - * - * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE - * or http://www.opensolaris.org/os/licensing. - * See the License for the specific language governing permissions - * and limitations under the License. - * - * When distributing Covered Code, include this CDDL HEADER in each - * file and include the License file at usr/src/OPENSOLARIS.LICENSE. - * If applicable, add the following below this CDDL HEADER, with the - * fields enclosed by brackets "[]" replaced with your own identifying - * information: Portions Copyright [yyyy] [name of copyright owner] - * - * CDDL HEADER END - */ - -/* - * Copyright 2009 Sun Microsystems, Inc. All rights reserved. - * Use is subject to license terms. - */ - -#include <sys/types.h> -#include <sys/file.h> -#include <sys/errno.h> -#include <sys/open.h> -#include <sys/stat.h> -#include <sys/cred.h> -#include <sys/modctl.h> -#include <sys/conf.h> -#include <sys/devops.h> -#include <sys/ddi.h> - -#include <sys/amd_iommu.h> -#include "amd_iommu_impl.h" -#include "amd_iommu_acpi.h" - - -#define AMD_IOMMU_MINOR2INST(x) (x) -#define AMD_IOMMU_INST2MINOR(x) (x) -#define AMD_IOMMU_NODETYPE "ddi_iommu" -#define AMD_IOMMU_MINOR_NAME "amd-iommu" - -static int amd_iommu_getinfo(dev_info_t *dip, ddi_info_cmd_t cmd, void *arg, - void **result); -static int amd_iommu_attach(dev_info_t *dip, ddi_attach_cmd_t cmd); -static int amd_iommu_detach(dev_info_t *dip, ddi_detach_cmd_t cmd); -static int amd_iommu_open(dev_t *devp, int flag, int otyp, cred_t *credp); -static int amd_iommu_close(dev_t dev, int flag, int otyp, cred_t *credp); -static int amd_iommu_ioctl(dev_t dev, int cmd, intptr_t arg, int mode, - cred_t *credp, int *rvalp); - -static struct cb_ops amd_iommu_cb_ops = { - amd_iommu_open, /* cb_open */ - amd_iommu_close, /* cb_close */ - nodev, /* cb_strategy */ - nodev, /* cb_print */ - nodev, /* cb_dump */ - nodev, /* cb_read */ - nodev, /* cb_write */ - amd_iommu_ioctl, /* cb_ioctl */ - nodev, /* cb_devmap */ - nodev, /* cb_mmap */ - nodev, /* cb_segmap */ - nochpoll, /* cb_chpoll */ - ddi_prop_op, /* cb_prop_op */ - NULL, /* cb_str */ - D_NEW | D_MP, /* cb_flag */ - CB_REV, /* cb_rev */ - nodev, /* cb_aread */ - nodev /* cb_awrite */ -}; - -static struct dev_ops amd_iommu_dev_ops = { - DEVO_REV, /* devo_rev */ - 0, /* devo_refcnt */ - amd_iommu_getinfo, /* devo_getinfo */ - nulldev, /* devo_identify */ - nulldev, /* devo_probe */ - amd_iommu_attach, /* devo_attach */ - amd_iommu_detach, /* devo_detach */ - nodev, /* devo_reset */ - &amd_iommu_cb_ops, /* devo_cb_ops */ - NULL, /* devo_bus_ops */ - nulldev /* devo_power */ -}; - -static struct modldrv modldrv = { - &mod_driverops, - "AMD IOMMU 0.1", - &amd_iommu_dev_ops -}; - -static struct modlinkage modlinkage = { - MODREV_1, - (void *)&modldrv, - NULL -}; - -amd_iommu_debug_t amd_iommu_debug; -kmutex_t amd_iommu_global_lock; -const char *amd_iommu_modname = "amd_iommu"; -amd_iommu_alias_t **amd_iommu_alias; -amd_iommu_page_table_hash_t amd_iommu_page_table_hash; -static void *amd_iommu_statep; -int amd_iommu_64bit_bug; -int amd_iommu_unity_map; -int amd_iommu_no_RW_perms; -int amd_iommu_no_unmap; -int amd_iommu_pageva_inval_all; -int amd_iommu_disable; /* disable IOMMU */ -char *amd_iommu_disable_list; /* list of drivers bypassing IOMMU */ - -int -_init(void) -{ - int error = ENOTSUP; - -#if defined(__amd64) && !defined(__xpv) - - error = ddi_soft_state_init(&amd_iommu_statep, - sizeof (struct amd_iommu_state), 1); - if (error) { - cmn_err(CE_WARN, "%s: _init: failed to init soft state.", - amd_iommu_modname); - return (error); - } - - if (amd_iommu_acpi_init() != DDI_SUCCESS) { - if (amd_iommu_debug) { - cmn_err(CE_WARN, "%s: _init: ACPI init failed.", - amd_iommu_modname); - } - ddi_soft_state_fini(&amd_iommu_statep); - return (ENOTSUP); - } - - amd_iommu_read_boot_props(); - - if (amd_iommu_page_table_hash_init(&amd_iommu_page_table_hash) - != DDI_SUCCESS) { - cmn_err(CE_WARN, "%s: _init: Page table hash init failed.", - amd_iommu_modname); - if (amd_iommu_disable_list) { - kmem_free(amd_iommu_disable_list, - strlen(amd_iommu_disable_list) + 1); - amd_iommu_disable_list = NULL; - } - amd_iommu_acpi_fini(); - ddi_soft_state_fini(&amd_iommu_statep); - amd_iommu_statep = NULL; - return (EFAULT); - } - - error = mod_install(&modlinkage); - if (error) { - cmn_err(CE_WARN, "%s: _init: mod_install failed.", - amd_iommu_modname); - amd_iommu_page_table_hash_fini(&amd_iommu_page_table_hash); - if (amd_iommu_disable_list) { - kmem_free(amd_iommu_disable_list, - strlen(amd_iommu_disable_list) + 1); - amd_iommu_disable_list = NULL; - } - amd_iommu_acpi_fini(); - ddi_soft_state_fini(&amd_iommu_statep); - amd_iommu_statep = NULL; - return (error); - } - error = 0; -#endif - - return (error); -} - -int -_info(struct modinfo *modinfop) -{ - return (mod_info(&modlinkage, modinfop)); -} - -int -_fini(void) -{ - int error; - - error = mod_remove(&modlinkage); - if (error) - return (error); - - amd_iommu_page_table_hash_fini(&amd_iommu_page_table_hash); - if (amd_iommu_disable_list) { - kmem_free(amd_iommu_disable_list, - strlen(amd_iommu_disable_list) + 1); - amd_iommu_disable_list = NULL; - } - amd_iommu_acpi_fini(); - ddi_soft_state_fini(&amd_iommu_statep); - amd_iommu_statep = NULL; - - return (0); -} - -/*ARGSUSED*/ -static int -amd_iommu_getinfo(dev_info_t *dip, ddi_info_cmd_t cmd, void *arg, void **result) -{ - struct amd_iommu_state *statep; - - ASSERT(result); - - *result = NULL; - - switch (cmd) { - case DDI_INFO_DEVT2DEVINFO: - statep = ddi_get_soft_state(amd_iommu_statep, - AMD_IOMMU_MINOR2INST(getminor((dev_t)arg))); - if (statep) { - *result = statep->aioms_devi; - return (DDI_SUCCESS); - } - break; - case DDI_INFO_DEVT2INSTANCE: - *result = (void *)(uintptr_t) - AMD_IOMMU_MINOR2INST(getminor((dev_t)arg)); - return (DDI_SUCCESS); - } - - return (DDI_FAILURE); -} - -static int -amd_iommu_attach(dev_info_t *dip, ddi_attach_cmd_t cmd) -{ - int instance = ddi_get_instance(dip); - const char *driver = ddi_driver_name(dip); - struct amd_iommu_state *statep; - - ASSERT(instance >= 0); - ASSERT(driver); - - switch (cmd) { - case DDI_ATTACH: - if (ddi_soft_state_zalloc(amd_iommu_statep, instance) - != DDI_SUCCESS) { - cmn_err(CE_WARN, "Unable to allocate soft state for " - "%s%d", driver, instance); - return (DDI_FAILURE); - } - - statep = ddi_get_soft_state(amd_iommu_statep, instance); - if (statep == NULL) { - cmn_err(CE_WARN, "Unable to get soft state for " - "%s%d", driver, instance); - ddi_soft_state_free(amd_iommu_statep, instance); - return (DDI_FAILURE); - } - - if (ddi_create_minor_node(dip, AMD_IOMMU_MINOR_NAME, S_IFCHR, - AMD_IOMMU_INST2MINOR(instance), AMD_IOMMU_NODETYPE, - 0) != DDI_SUCCESS) { - cmn_err(CE_WARN, "Unable to create minor node for " - "%s%d", driver, instance); - ddi_remove_minor_node(dip, NULL); - ddi_soft_state_free(amd_iommu_statep, instance); - return (DDI_FAILURE); - } - - statep->aioms_devi = dip; - statep->aioms_instance = instance; - statep->aioms_iommu_start = NULL; - statep->aioms_iommu_end = NULL; - - amd_iommu_lookup_conf_props(dip); - - if (amd_iommu_disable_list) { - cmn_err(CE_NOTE, "AMD IOMMU disabled for the following" - " drivers:\n%s", amd_iommu_disable_list); - } - - if (amd_iommu_disable) { - cmn_err(CE_NOTE, "AMD IOMMU disabled by user"); - } else if (amd_iommu_setup(dip, statep) != DDI_SUCCESS) { - cmn_err(CE_WARN, "Unable to initialize AMD IOMMU " - "%s%d", driver, instance); - ddi_remove_minor_node(dip, NULL); - ddi_soft_state_free(amd_iommu_statep, instance); - return (DDI_FAILURE); - } - - ddi_report_dev(dip); - - return (DDI_SUCCESS); - - case DDI_RESUME: - return (DDI_SUCCESS); - default: - return (DDI_FAILURE); - } -} - -static int -amd_iommu_detach(dev_info_t *dip, ddi_detach_cmd_t cmd) -{ - int instance = ddi_get_instance(dip); - const char *driver = ddi_driver_name(dip); - struct amd_iommu_state *statep; - - ASSERT(instance >= 0); - ASSERT(driver); - - switch (cmd) { - case DDI_DETACH: - statep = ddi_get_soft_state(amd_iommu_statep, instance); - if (statep == NULL) { - cmn_err(CE_WARN, "%s%d: Cannot get soft state", - driver, instance); - return (DDI_FAILURE); - } - return (DDI_FAILURE); - case DDI_SUSPEND: - return (DDI_SUCCESS); - default: - return (DDI_FAILURE); - } -} - -/*ARGSUSED*/ -static int -amd_iommu_open(dev_t *devp, int flag, int otyp, cred_t *credp) -{ - int instance = AMD_IOMMU_MINOR2INST(getminor(*devp)); - struct amd_iommu_state *statep; - const char *f = "amd_iommu_open"; - - if (instance < 0) { - cmn_err(CE_WARN, "%s: invalid instance %d", - f, instance); - return (ENXIO); - } - - if (!(flag & (FREAD|FWRITE))) { - cmn_err(CE_WARN, "%s: invalid flags %d", f, flag); - return (EINVAL); - } - - if (otyp != OTYP_CHR) { - cmn_err(CE_WARN, "%s: invalid otyp %d", f, otyp); - return (EINVAL); - } - - statep = ddi_get_soft_state(amd_iommu_statep, instance); - if (statep == NULL) { - cmn_err(CE_WARN, "%s: cannot get soft state: instance %d", - f, instance); - return (ENXIO); - } - - ASSERT(statep->aioms_instance == instance); - - return (0); -} - -/*ARGSUSED*/ -static int -amd_iommu_close(dev_t dev, int flag, int otyp, cred_t *credp) -{ - int instance = AMD_IOMMU_MINOR2INST(getminor(dev)); - struct amd_iommu_state *statep; - const char *f = "amd_iommu_close"; - - if (instance < 0) { - cmn_err(CE_WARN, "%s: invalid instance %d", f, instance); - return (ENXIO); - } - - if (!(flag & (FREAD|FWRITE))) { - cmn_err(CE_WARN, "%s: invalid flags %d", f, flag); - return (EINVAL); - } - - if (otyp != OTYP_CHR) { - cmn_err(CE_WARN, "%s: invalid otyp %d", f, otyp); - return (EINVAL); - } - - statep = ddi_get_soft_state(amd_iommu_statep, instance); - if (statep == NULL) { - cmn_err(CE_WARN, "%s: cannot get soft state: instance %d", - f, instance); - return (ENXIO); - } - - ASSERT(statep->aioms_instance == instance); - return (0); - -} - -/*ARGSUSED*/ -static int -amd_iommu_ioctl(dev_t dev, int cmd, intptr_t arg, int mode, cred_t *credp, - int *rvalp) -{ - int instance = AMD_IOMMU_MINOR2INST(getminor(dev)); - struct amd_iommu_state *statep; - const char *f = "amd_iommu_ioctl"; - - ASSERT(*rvalp); - - if (instance < 0) { - cmn_err(CE_WARN, "%s: invalid instance %d", f, instance); - return (ENXIO); - } - - - if (!(mode & (FREAD|FWRITE))) { - cmn_err(CE_WARN, "%s: invalid mode %d", f, mode); - return (EINVAL); - } - - if (mode & FKIOCTL) { - cmn_err(CE_WARN, "%s: FKIOCTL unsupported mode %d", f, mode); - return (EINVAL); - } - - statep = ddi_get_soft_state(amd_iommu_statep, instance); - if (statep == NULL) { - cmn_err(CE_WARN, "%s: cannot get soft state: instance %d", - f, instance); - return (ENXIO); - } - - ASSERT(statep->aioms_instance == instance); - - return (ENOTTY); -}
--- a/usr/src/uts/intel/io/amd_iommu/amd_iommu.conf Mon Sep 21 11:25:09 2009 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,27 +0,0 @@ -# -# CDDL HEADER START -# -# The contents of this file are subject to the terms of the -# Common Development and Distribution License (the "License"). -# You may not use this file except in compliance with the License. -# -# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE -# or http://www.opensolaris.org/os/licensing. -# See the License for the specific language governing permissions -# and limitations under the License. -# -# When distributing Covered Code, include this CDDL HEADER in each -# file and include the License file at usr/src/OPENSOLARIS.LICENSE. -# If applicable, add the following below this CDDL HEADER, with the -# fields enclosed by brackets "[]" replaced with your own identifying -# information: Portions Copyright [yyyy] [name of copyright owner] -# -# CDDL HEADER END -# -# -# Copyright 2009 Sun Microsystems, Inc. All rights reserved. -# Use is subject to license terms. -# -# -# To enable IOMMU set this to "yes" and rebuild boot archive -amd-iommu="yes";
--- a/usr/src/uts/intel/io/amd_iommu/amd_iommu_acpi.c Mon Sep 21 11:25:09 2009 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,951 +0,0 @@ -/* - * CDDL HEADER START - * - * The contents of this file are subject to the terms of the - * Common Development and Distribution License (the "License"). - * You may not use this file except in compliance with the License. - * - * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE - * or http://www.opensolaris.org/os/licensing. - * See the License for the specific language governing permissions - * and limitations under the License. - * - * When distributing Covered Code, include this CDDL HEADER in each - * file and include the License file at usr/src/OPENSOLARIS.LICENSE. - * If applicable, add the following below this CDDL HEADER, with the - * fields enclosed by brackets "[]" replaced with your own identifying - * information: Portions Copyright [yyyy] [name of copyright owner] - * - * CDDL HEADER END - */ - -/* - * Copyright 2008 Sun Microsystems, Inc. All rights reserved. - * Use is subject to license terms. - */ - -#include "amd_iommu_acpi.h" -#include "amd_iommu_impl.h" - -static int create_acpi_hash(amd_iommu_acpi_t *acpi); -static void amd_iommu_acpi_table_fini(amd_iommu_acpi_t **acpipp); - -static void dump_acpi_aliases(void); - - -/* - * Globals - */ -static amd_iommu_acpi_global_t *amd_iommu_acpi_global; -static amd_iommu_acpi_ivhd_t **amd_iommu_acpi_ivhd_hash; -static amd_iommu_acpi_ivmd_t **amd_iommu_acpi_ivmd_hash; - -static int -type_byte_size(char *cp) -{ - uint8_t type8 = *((uint8_t *)cp); - uint8_t len_bits; - - len_bits = AMD_IOMMU_REG_GET8(&type8, AMD_IOMMU_ACPI_DEVENTRY_LEN); - - switch (len_bits) { - case 0: - return (4); - case 1: - return (8); - case 2: - return (16); - case 3: - return (32); - default: - cmn_err(CE_WARN, "%s: Invalid deventry len: %d", - amd_iommu_modname, len_bits); - return (len_bits); - } - /*NOTREACHED*/ -} - -static void -process_4byte_deventry(ivhd_container_t *c, char *cp) -{ - int entry_type = *((uint8_t *)cp); - ivhd_deventry_t deventry = {0}; - ivhd_deventry_t *devp; - uint8_t datsetting8; - align_16_t al = {0}; - int i; - - /* 4 byte entry */ - deventry.idev_len = 4; - deventry.idev_deviceid = -1; - deventry.idev_src_deviceid = -1; - - for (i = 0; i < 2; i++) { - al.ent8[i] = *((uint8_t *)&cp[i + 1]); - } - - switch (entry_type) { - case 1: - deventry.idev_type = DEVENTRY_ALL; - break; - case 2: - deventry.idev_type = DEVENTRY_SELECT; - deventry.idev_deviceid = al.ent16; - break; - case 3: - deventry.idev_type = DEVENTRY_RANGE; - deventry.idev_deviceid = al.ent16; - break; - case 4: - deventry.idev_type = DEVENTRY_RANGE_END; - deventry.idev_deviceid = al.ent16; - ASSERT(cp[3] == 0); - break; - case 0: - ASSERT(al.ent16 == 0); - ASSERT(cp[3] == 0); - default: - return; - } - - - devp = kmem_alloc(sizeof (ivhd_deventry_t), KM_SLEEP); - *devp = deventry; - - if (c->ivhdc_first_deventry == NULL) - c->ivhdc_first_deventry = devp; - else - c->ivhdc_last_deventry->idev_next = devp; - - c->ivhdc_last_deventry = devp; - - if (entry_type == 4) - return; - - datsetting8 = (*((uint8_t *)&cp[3])); - - devp->idev_Lint1Pass = AMD_IOMMU_REG_GET8(&datsetting8, - AMD_IOMMU_ACPI_LINT1PASS); - - devp->idev_Lint0Pass = AMD_IOMMU_REG_GET8(&datsetting8, - AMD_IOMMU_ACPI_LINT0PASS); - - devp->idev_SysMgt = AMD_IOMMU_REG_GET8(&datsetting8, - AMD_IOMMU_ACPI_SYSMGT); - - ASSERT(AMD_IOMMU_REG_GET8(&datsetting8, - AMD_IOMMU_ACPI_DATRSV) == 0); - - devp->idev_NMIPass = AMD_IOMMU_REG_GET8(&datsetting8, - AMD_IOMMU_ACPI_NMIPASS); - - devp->idev_ExtIntPass = AMD_IOMMU_REG_GET8(&datsetting8, - AMD_IOMMU_ACPI_EXTINTPASS); - - devp->idev_INITPass = AMD_IOMMU_REG_GET8(&datsetting8, - AMD_IOMMU_ACPI_INITPASS); -} - -static void -process_8byte_deventry(ivhd_container_t *c, char *cp) -{ - uint8_t datsetting8; - int entry_type = (uint8_t)*cp; - ivhd_deventry_t deventry = {0}; - ivhd_deventry_t *devp; - align_16_t al1 = {0}; - align_16_t al2 = {0}; - align_32_t al3 = {0}; - int i; - - /* Length is 8 bytes */ - deventry.idev_len = 8; - deventry.idev_deviceid = -1; - deventry.idev_src_deviceid = -1; - - for (i = 0; i < 2; i++) { - al1.ent8[i] = *((uint8_t *)&cp[i+1]); - al2.ent8[i] = *((uint8_t *)&cp[i+5]); - } - - datsetting8 = *((uint8_t *)&cp[3]); - - switch (entry_type) { - case 66: - deventry.idev_type = DEVENTRY_ALIAS_SELECT; - deventry.idev_deviceid = al1.ent16; - deventry.idev_src_deviceid = al2.ent16; - ASSERT(cp[4] == 0); - ASSERT(cp[7] == 0); - break; - case 67: - deventry.idev_type = DEVENTRY_ALIAS_RANGE; - deventry.idev_deviceid = al1.ent16; - deventry.idev_src_deviceid = al2.ent16; - ASSERT(cp[4] == 0); - ASSERT(cp[7] == 0); - break; - case 70: - deventry.idev_type = DEVENTRY_EXTENDED_SELECT; - deventry.idev_deviceid = al1.ent16; - break; - case 71: - deventry.idev_type = DEVENTRY_EXTENDED_RANGE; - deventry.idev_deviceid = al1.ent16; - break; - case 72: - deventry.idev_type = DEVENTRY_SPECIAL_DEVICE; - ASSERT(al1.ent16 == 0); - deventry.idev_deviceid = -1; - deventry.idev_handle = cp[4]; - deventry.idev_variety = cp[7]; - deventry.idev_src_deviceid = al2.ent16; - default: -#ifdef BROKEN_ASSERT - for (i = 0; i < 7; i++) { - ASSERT(cp[i] == 0); - } -#endif - return; - } - - - devp = kmem_alloc(sizeof (ivhd_deventry_t), KM_SLEEP); - *devp = deventry; - - if (c->ivhdc_first_deventry == NULL) - c->ivhdc_first_deventry = devp; - else - c->ivhdc_last_deventry->idev_next = devp; - - c->ivhdc_last_deventry = devp; - - devp->idev_Lint1Pass = AMD_IOMMU_REG_GET8(&datsetting8, - AMD_IOMMU_ACPI_LINT1PASS); - - devp->idev_Lint0Pass = AMD_IOMMU_REG_GET8(&datsetting8, - AMD_IOMMU_ACPI_LINT0PASS); - - devp->idev_SysMgt = AMD_IOMMU_REG_GET8(&datsetting8, - AMD_IOMMU_ACPI_SYSMGT); - - ASSERT(AMD_IOMMU_REG_GET8(&datsetting8, - AMD_IOMMU_ACPI_DATRSV) == 0); - - devp->idev_NMIPass = AMD_IOMMU_REG_GET8(&datsetting8, - AMD_IOMMU_ACPI_NMIPASS); - - devp->idev_ExtIntPass = AMD_IOMMU_REG_GET8(&datsetting8, - AMD_IOMMU_ACPI_EXTINTPASS); - - devp->idev_INITPass = AMD_IOMMU_REG_GET8(&datsetting8, - AMD_IOMMU_ACPI_INITPASS); - - if (entry_type != 70 && entry_type != 71) { - return; - } - - /* Type 70 and 71 */ - for (i = 0; i < 4; i++) { - al3.ent8[i] = *((uint8_t *)&cp[i+4]); - } - - devp->idev_AtsDisabled = AMD_IOMMU_REG_GET8(&al3.ent32, - AMD_IOMMU_ACPI_ATSDISABLED); - - ASSERT(AMD_IOMMU_REG_GET8(&al3.ent32, AMD_IOMMU_ACPI_EXTDATRSV) == 0); -} - -static void -process_ivhd(amd_iommu_acpi_t *acpi, ivhd_t *ivhdp) -{ - ivhd_container_t *c; - caddr_t ivhd_end; - caddr_t ivhd_tot_end; - caddr_t cp; - - ASSERT(ivhdp->ivhd_type == 0x10); - - c = kmem_zalloc(sizeof (ivhd_container_t), KM_SLEEP); - c->ivhdc_ivhd = kmem_alloc(sizeof (ivhd_t), KM_SLEEP); - *(c->ivhdc_ivhd) = *ivhdp; - - if (acpi->acp_first_ivhdc == NULL) - acpi->acp_first_ivhdc = c; - else - acpi->acp_last_ivhdc->ivhdc_next = c; - - acpi->acp_last_ivhdc = c; - - ivhd_end = (caddr_t)ivhdp + sizeof (ivhd_t); - ivhd_tot_end = (caddr_t)ivhdp + ivhdp->ivhd_len; - - for (cp = ivhd_end; cp < ivhd_tot_end; cp += type_byte_size(cp)) { - /* 16 byte and 32 byte size are currently reserved */ - switch (type_byte_size(cp)) { - case 4: - process_4byte_deventry(c, cp); - break; - case 8: - process_8byte_deventry(c, cp); - break; - case 16: - case 32: - /* Reserved */ - break; - default: - cmn_err(CE_WARN, "%s: unsupported length for device " - "entry in ACPI IVRS table's IVHD entry", - amd_iommu_modname); - break; - } - } -} - -static void -process_ivmd(amd_iommu_acpi_t *acpi, ivmd_t *ivmdp) -{ - ivmd_container_t *c; - - ASSERT(ivmdp->ivmd_type != 0x10); - - c = kmem_zalloc(sizeof (ivmd_container_t), KM_SLEEP); - c->ivmdc_ivmd = kmem_alloc(sizeof (ivmd_t), KM_SLEEP); - *(c->ivmdc_ivmd) = *ivmdp; - - if (acpi->acp_first_ivmdc == NULL) - acpi->acp_first_ivmdc = c; - else - acpi->acp_last_ivmdc->ivmdc_next = c; - - acpi->acp_last_ivmdc = c; -} - -int -amd_iommu_acpi_init(void) -{ - ivrs_t *ivrsp; - caddr_t ivrsp_end; - caddr_t table_end; - caddr_t cp; - uint8_t type8; - amd_iommu_acpi_t *acpi; - align_ivhd_t al_vhd = {0}; - align_ivmd_t al_vmd = {0}; - - if (AcpiGetTable(IVRS_SIG, 1, (ACPI_TABLE_HEADER **)&ivrsp) != AE_OK) { - cmn_err(CE_NOTE, "!amd_iommu: No AMD IOMMU ACPI IVRS table"); - return (DDI_FAILURE); - } - - /* - * Reserved field must be 0 - */ - ASSERT(ivrsp->ivrs_resv == 0); - - ASSERT(AMD_IOMMU_REG_GET32(&ivrsp->ivrs_ivinfo, - AMD_IOMMU_ACPI_IVINFO_RSV1) == 0); - ASSERT(AMD_IOMMU_REG_GET32(&ivrsp->ivrs_ivinfo, - AMD_IOMMU_ACPI_IVINFO_RSV2) == 0); - - ivrsp_end = (caddr_t)ivrsp + sizeof (struct ivrs); - table_end = (caddr_t)ivrsp + ivrsp->ivrs_hdr.Length; - - acpi = kmem_zalloc(sizeof (amd_iommu_acpi_t), KM_SLEEP); - acpi->acp_ivrs = kmem_alloc(sizeof (ivrs_t), KM_SLEEP); - *(acpi->acp_ivrs) = *ivrsp; - - for (cp = ivrsp_end; cp < table_end; cp += (al_vhd.ivhdp)->ivhd_len) { - al_vhd.cp = cp; - if (al_vhd.ivhdp->ivhd_type == 0x10) - process_ivhd(acpi, al_vhd.ivhdp); - } - - for (cp = ivrsp_end; cp < table_end; cp += (al_vmd.ivmdp)->ivmd_len) { - al_vmd.cp = cp; - type8 = al_vmd.ivmdp->ivmd_type; - if (type8 == 0x20 || type8 == 0x21 || type8 == 0x22) - process_ivmd(acpi, al_vmd.ivmdp); - } - - if (create_acpi_hash(acpi) != DDI_SUCCESS) { - return (DDI_FAILURE); - } - - amd_iommu_acpi_table_fini(&acpi); - - ASSERT(acpi == NULL); - - if (amd_iommu_debug & AMD_IOMMU_DEBUG_ACPI) { - dump_acpi_aliases(); - debug_enter("dump"); - } - - return (DDI_SUCCESS); -} - -static ivhd_deventry_t * -free_ivhd_deventry(ivhd_deventry_t *devp) -{ - ivhd_deventry_t *next = devp->idev_next; - - kmem_free(devp, sizeof (ivhd_deventry_t)); - - return (next); -} - -static ivhd_container_t * -free_ivhd_container(ivhd_container_t *ivhdcp) -{ - ivhd_container_t *next = ivhdcp->ivhdc_next; - ivhd_deventry_t *devp; - - for (devp = ivhdcp->ivhdc_first_deventry; devp; ) { - devp = free_ivhd_deventry(devp); - } - - kmem_free(ivhdcp->ivhdc_ivhd, sizeof (ivhd_t)); - kmem_free(ivhdcp, sizeof (ivhd_container_t)); - - return (next); -} - -static ivmd_container_t * -free_ivmd_container(ivmd_container_t *ivmdcp) -{ - ivmd_container_t *next = ivmdcp->ivmdc_next; - - kmem_free(ivmdcp->ivmdc_ivmd, sizeof (ivmd_t)); - kmem_free(ivmdcp, sizeof (ivmd_container_t)); - - return (next); -} - -void -amd_iommu_acpi_fini(void) -{ -} - -/* - * TODO: Do we need to free the ACPI table for om GetFirmwareTable() - */ -static void -amd_iommu_acpi_table_fini(amd_iommu_acpi_t **acpipp) -{ - amd_iommu_acpi_t *acpi = *acpipp; - ivhd_container_t *ivhdcp; - ivmd_container_t *ivmdcp; - - ASSERT(acpi); - - for (ivhdcp = acpi->acp_first_ivhdc; ivhdcp; ) { - ivhdcp = free_ivhd_container(ivhdcp); - } - for (ivmdcp = acpi->acp_first_ivmdc; ivmdcp; ) { - ivmdcp = free_ivmd_container(ivmdcp); - } - - kmem_free(acpi->acp_ivrs, sizeof (struct ivrs)); - kmem_free(acpi, sizeof (amd_iommu_acpi_t)); - - *acpipp = NULL; -} - -static uint16_t -deviceid_hashfn(uint16_t deviceid) -{ - return (deviceid % AMD_IOMMU_ACPI_INFO_HASH_SZ); -} - -static void -add_deventry_info(ivhd_t *ivhdp, ivhd_deventry_t *deventry, - amd_iommu_acpi_ivhd_t **hash) -{ - static amd_iommu_acpi_ivhd_t *last; - amd_iommu_acpi_ivhd_t *acpi_ivhdp; - uint8_t uint8_flags; - uint16_t uint16_info; - uint16_t idx; - - if (deventry->idev_type == DEVENTRY_RANGE_END) { - ASSERT(last); - acpi_ivhdp = last; - last = NULL; - ASSERT(acpi_ivhdp->ach_dev_type == DEVENTRY_RANGE || - acpi_ivhdp->ach_dev_type == DEVENTRY_ALIAS_RANGE || - acpi_ivhdp->ach_dev_type == DEVENTRY_EXTENDED_RANGE); - ASSERT(acpi_ivhdp->ach_deviceid_end == -1); - acpi_ivhdp->ach_deviceid_end = deventry->idev_deviceid; - /* TODO ASSERT data is 0 */ - return; - } - - ASSERT(last == NULL); - acpi_ivhdp = kmem_zalloc(sizeof (*acpi_ivhdp), KM_SLEEP); - - uint8_flags = ivhdp->ivhd_flags; - -#ifdef BROKEN_ASSERT - ASSERT(AMD_IOMMU_REG_GET8(&uint8_flags, - AMD_IOMMU_ACPI_IVHD_FLAGS_RSV) == 0); -#endif - - acpi_ivhdp->ach_IotlbSup = AMD_IOMMU_REG_GET8(&uint8_flags, - AMD_IOMMU_ACPI_IVHD_FLAGS_IOTLBSUP); - acpi_ivhdp->ach_Isoc = AMD_IOMMU_REG_GET8(&uint8_flags, - AMD_IOMMU_ACPI_IVHD_FLAGS_ISOC); - acpi_ivhdp->ach_ResPassPW = AMD_IOMMU_REG_GET8(&uint8_flags, - AMD_IOMMU_ACPI_IVHD_FLAGS_RESPASSPW); - acpi_ivhdp->ach_PassPW = AMD_IOMMU_REG_GET8(&uint8_flags, - AMD_IOMMU_ACPI_IVHD_FLAGS_PASSPW); - acpi_ivhdp->ach_HtTunEn = AMD_IOMMU_REG_GET8(&uint8_flags, - AMD_IOMMU_ACPI_IVHD_FLAGS_HTTUNEN); - - /* IVHD fields */ - acpi_ivhdp->ach_IOMMU_deviceid = ivhdp->ivhd_deviceid; - acpi_ivhdp->ach_IOMMU_cap_off = ivhdp->ivhd_cap_off; - acpi_ivhdp->ach_IOMMU_reg_base = ivhdp->ivhd_reg_base; - acpi_ivhdp->ach_IOMMU_pci_seg = ivhdp->ivhd_pci_seg; - - /* IVHD IOMMU info fields */ - uint16_info = ivhdp->ivhd_iommu_info; - -#ifdef BROKEN_ASSERT - ASSERT(AMD_IOMMU_REG_GET16(&uint16_info, - AMD_IOMMU_ACPI_IOMMU_INFO_RSV1) == 0); -#endif - - acpi_ivhdp->ach_IOMMU_UnitID = AMD_IOMMU_REG_GET16(&uint16_info, - AMD_IOMMU_ACPI_IOMMU_INFO_UNITID); - ASSERT(AMD_IOMMU_REG_GET16(&uint16_info, - AMD_IOMMU_ACPI_IOMMU_INFO_RSV2) == 0); - acpi_ivhdp->ach_IOMMU_MSInum = AMD_IOMMU_REG_GET16(&uint16_info, - AMD_IOMMU_ACPI_IOMMU_INFO_MSINUM); - - /* Initialize deviceids to -1 */ - acpi_ivhdp->ach_deviceid_start = -1; - acpi_ivhdp->ach_deviceid_end = -1; - acpi_ivhdp->ach_src_deviceid = -1; - - /* All range type entries are put on hash entry 0 */ - switch (deventry->idev_type) { - case DEVENTRY_ALL: - acpi_ivhdp->ach_deviceid_start = 0; - acpi_ivhdp->ach_deviceid_end = (uint16_t)-1; - acpi_ivhdp->ach_dev_type = DEVENTRY_ALL; - idx = AMD_IOMMU_ACPI_INFO_HASH_SZ; - break; - case DEVENTRY_SELECT: - acpi_ivhdp->ach_deviceid_start = deventry->idev_deviceid; - acpi_ivhdp->ach_deviceid_end = deventry->idev_deviceid; - acpi_ivhdp->ach_dev_type = DEVENTRY_SELECT; - idx = deviceid_hashfn(deventry->idev_deviceid); - break; - case DEVENTRY_RANGE: - acpi_ivhdp->ach_deviceid_start = deventry->idev_deviceid; - acpi_ivhdp->ach_deviceid_end = -1; - acpi_ivhdp->ach_dev_type = DEVENTRY_RANGE; - idx = AMD_IOMMU_ACPI_INFO_HASH_SZ; - last = acpi_ivhdp; - break; - case DEVENTRY_RANGE_END: - cmn_err(CE_PANIC, "%s: Unexpected Range End Deventry", - amd_iommu_modname); - /*NOTREACHED*/ - case DEVENTRY_ALIAS_SELECT: - acpi_ivhdp->ach_deviceid_start = deventry->idev_deviceid; - acpi_ivhdp->ach_deviceid_end = deventry->idev_deviceid; - acpi_ivhdp->ach_src_deviceid = deventry->idev_src_deviceid; - acpi_ivhdp->ach_dev_type = DEVENTRY_ALIAS_SELECT; - idx = deviceid_hashfn(deventry->idev_deviceid); - break; - case DEVENTRY_ALIAS_RANGE: - acpi_ivhdp->ach_deviceid_start = deventry->idev_deviceid; - acpi_ivhdp->ach_deviceid_end = -1; - acpi_ivhdp->ach_src_deviceid = deventry->idev_src_deviceid; - acpi_ivhdp->ach_dev_type = DEVENTRY_ALIAS_RANGE; - idx = AMD_IOMMU_ACPI_INFO_HASH_SZ; - last = acpi_ivhdp; - break; - case DEVENTRY_EXTENDED_SELECT: - acpi_ivhdp->ach_deviceid_start = deventry->idev_deviceid; - acpi_ivhdp->ach_deviceid_end = deventry->idev_deviceid; - acpi_ivhdp->ach_dev_type = DEVENTRY_EXTENDED_SELECT; - idx = deviceid_hashfn(deventry->idev_deviceid); - break; - case DEVENTRY_EXTENDED_RANGE: - acpi_ivhdp->ach_deviceid_start = deventry->idev_deviceid; - acpi_ivhdp->ach_deviceid_end = -1; - acpi_ivhdp->ach_dev_type = DEVENTRY_EXTENDED_RANGE; - idx = AMD_IOMMU_ACPI_INFO_HASH_SZ; - last = acpi_ivhdp; - break; - case DEVENTRY_SPECIAL_DEVICE: - acpi_ivhdp->ach_deviceid_start = -1; - acpi_ivhdp->ach_deviceid_end = -1; - acpi_ivhdp->ach_src_deviceid = deventry->idev_src_deviceid; - acpi_ivhdp->ach_special_handle = deventry->idev_handle; - acpi_ivhdp->ach_special_variety = deventry->idev_variety; - idx = AMD_IOMMU_ACPI_INFO_HASH_SZ; - default: - cmn_err(CE_PANIC, "%s: Unsupported deventry type", - amd_iommu_modname); - /*NOTREACHED*/ - } - - acpi_ivhdp->ach_Lint1Pass = deventry->idev_Lint1Pass; - acpi_ivhdp->ach_Lint0Pass = deventry->idev_Lint0Pass; - acpi_ivhdp->ach_SysMgt = deventry->idev_SysMgt; - acpi_ivhdp->ach_NMIPass = deventry->idev_NMIPass; - acpi_ivhdp->ach_ExtIntPass = deventry->idev_ExtIntPass; - acpi_ivhdp->ach_INITPass = deventry->idev_INITPass; - - - /* extended data */ - if (acpi_ivhdp->ach_dev_type == DEVENTRY_EXTENDED_SELECT || - acpi_ivhdp->ach_dev_type == DEVENTRY_EXTENDED_RANGE) { - acpi_ivhdp->ach_AtsDisabled = deventry->idev_AtsDisabled; - } - - /* - * Now add it to the hash - */ - ASSERT(hash[idx] != acpi_ivhdp); - acpi_ivhdp->ach_next = hash[idx]; - hash[idx] = acpi_ivhdp; -} - -static void -add_ivhdc_info(ivhd_container_t *ivhdcp, amd_iommu_acpi_ivhd_t **hash) -{ - ivhd_deventry_t *deventry; - ivhd_t *ivhdp = ivhdcp->ivhdc_ivhd; - - for (deventry = ivhdcp->ivhdc_first_deventry; deventry; - deventry = deventry->idev_next) { - add_deventry_info(ivhdp, deventry, hash); - } -} - -static void -add_ivhd_info(amd_iommu_acpi_t *acpi, amd_iommu_acpi_ivhd_t **hash) -{ - ivhd_container_t *ivhdcp; - - for (ivhdcp = acpi->acp_first_ivhdc; ivhdcp; - ivhdcp = ivhdcp->ivhdc_next) { - add_ivhdc_info(ivhdcp, hash); - } -} - -static void -set_ivmd_info(ivmd_t *ivmdp, amd_iommu_acpi_ivmd_t **hash) -{ - amd_iommu_acpi_ivmd_t *acpi_ivmdp; - uint8_t uint8_flags; - uint16_t idx; - - uint8_flags = ivmdp->ivmd_flags; - - acpi_ivmdp = kmem_zalloc(sizeof (*acpi_ivmdp), KM_SLEEP); - - switch (ivmdp->ivmd_type) { - case 0x20: - acpi_ivmdp->acm_deviceid_start = 0; - acpi_ivmdp->acm_deviceid_end = (uint16_t)-1; - acpi_ivmdp->acm_dev_type = IVMD_DEVICEID_ALL; - idx = AMD_IOMMU_ACPI_INFO_HASH_SZ; - break; - case 0x21: - acpi_ivmdp->acm_deviceid_start = ivmdp->ivmd_deviceid; - acpi_ivmdp->acm_deviceid_end = ivmdp->ivmd_deviceid; - acpi_ivmdp->acm_dev_type = IVMD_DEVICEID_SELECT; - idx = deviceid_hashfn(ivmdp->ivmd_deviceid); - break; - case 0x22: - acpi_ivmdp->acm_deviceid_start = ivmdp->ivmd_deviceid; - acpi_ivmdp->acm_deviceid_end = ivmdp->ivmd_auxdata; - acpi_ivmdp->acm_dev_type = IVMD_DEVICEID_RANGE; - idx = AMD_IOMMU_ACPI_INFO_HASH_SZ; - break; - default: - cmn_err(CE_PANIC, "Unknown AMD IOMMU ACPI IVMD deviceid type: " - "%x", ivmdp->ivmd_type); - /*NOTREACHED*/ - } - - ASSERT(AMD_IOMMU_REG_GET8(&uint8_flags, - AMD_IOMMU_ACPI_IVMD_RSV) == 0); - - acpi_ivmdp->acm_ExclRange = AMD_IOMMU_REG_GET8(&uint8_flags, - AMD_IOMMU_ACPI_IVMD_EXCL_RANGE); - acpi_ivmdp->acm_IW = AMD_IOMMU_REG_GET8(&uint8_flags, - AMD_IOMMU_ACPI_IVMD_IW); - acpi_ivmdp->acm_IR = AMD_IOMMU_REG_GET8(&uint8_flags, - AMD_IOMMU_ACPI_IVMD_IR); - acpi_ivmdp->acm_Unity = AMD_IOMMU_REG_GET8(&uint8_flags, - AMD_IOMMU_ACPI_IVMD_UNITY); - - acpi_ivmdp->acm_ivmd_phys_start = ivmdp->ivmd_phys_start; - acpi_ivmdp->acm_ivmd_phys_len = ivmdp->ivmd_phys_len; - - acpi_ivmdp->acm_next = hash[idx]; - hash[idx] = acpi_ivmdp; -} - -static void -add_ivmdc_info(ivmd_container_t *ivmdcp, amd_iommu_acpi_ivmd_t **hash) -{ - set_ivmd_info(ivmdcp->ivmdc_ivmd, hash); -} - -static void -add_ivmd_info(amd_iommu_acpi_t *acpi, amd_iommu_acpi_ivmd_t **hash) -{ - ivmd_container_t *ivmdcp; - - for (ivmdcp = acpi->acp_first_ivmdc; ivmdcp; - ivmdcp = ivmdcp->ivmdc_next) { - add_ivmdc_info(ivmdcp, hash); - } -} - -static void -add_global_info(amd_iommu_acpi_t *acpi, amd_iommu_acpi_global_t *global) -{ - uint32_t ivrs_ivinfo = acpi->acp_ivrs->ivrs_ivinfo; - - global->acg_HtAtsResv = - AMD_IOMMU_REG_GET32(&ivrs_ivinfo, AMD_IOMMU_ACPI_HT_ATSRSV); - global->acg_VAsize = - AMD_IOMMU_REG_GET32(&ivrs_ivinfo, AMD_IOMMU_ACPI_VA_SIZE); - global->acg_PAsize = - AMD_IOMMU_REG_GET32(&ivrs_ivinfo, AMD_IOMMU_ACPI_PA_SIZE); -} - -static int -create_acpi_hash(amd_iommu_acpi_t *acpi) -{ - /* Last hash entry is for deviceid ranges including "all" */ - - amd_iommu_acpi_global = kmem_zalloc(sizeof (amd_iommu_acpi_global_t), - KM_SLEEP); - - amd_iommu_acpi_ivhd_hash = kmem_zalloc(sizeof (amd_iommu_acpi_ivhd_t *) - * (AMD_IOMMU_ACPI_INFO_HASH_SZ + 1), KM_SLEEP); - - amd_iommu_acpi_ivmd_hash = kmem_zalloc(sizeof (amd_iommu_acpi_ivmd_t *) - * (AMD_IOMMU_ACPI_INFO_HASH_SZ + 1), KM_SLEEP); - - add_global_info(acpi, amd_iommu_acpi_global); - - add_ivhd_info(acpi, amd_iommu_acpi_ivhd_hash); - - add_ivmd_info(acpi, amd_iommu_acpi_ivmd_hash); - - return (DDI_SUCCESS); -} - -amd_iommu_acpi_global_t * -amd_iommu_lookup_acpi_global(void) -{ - ASSERT(amd_iommu_acpi_global); - - return (amd_iommu_acpi_global); -} - -amd_iommu_acpi_ivhd_t * -amd_iommu_lookup_all_ivhd(void) -{ - amd_iommu_acpi_ivhd_t *hinfop; - - hinfop = amd_iommu_acpi_ivhd_hash[AMD_IOMMU_ACPI_INFO_HASH_SZ]; - for (; hinfop; hinfop = hinfop->ach_next) { - if (hinfop->ach_deviceid_start == 0 && - hinfop->ach_deviceid_end == (uint16_t)-1) { - break; - } - } - - return (hinfop); -} - -amd_iommu_acpi_ivmd_t * -amd_iommu_lookup_all_ivmd(void) -{ - amd_iommu_acpi_ivmd_t *minfop; - - minfop = amd_iommu_acpi_ivmd_hash[AMD_IOMMU_ACPI_INFO_HASH_SZ]; - for (; minfop; minfop = minfop->acm_next) { - if (minfop->acm_deviceid_start == 0 && - minfop->acm_deviceid_end == (uint16_t)-1) { - break; - } - } - - return (minfop); -} - -amd_iommu_acpi_ivhd_t * -amd_iommu_lookup_any_ivhd(void) -{ - int i; - amd_iommu_acpi_ivhd_t *hinfop; - - for (i = AMD_IOMMU_ACPI_INFO_HASH_SZ; i >= 0; i--) { - /*LINTED*/ - if (hinfop = amd_iommu_acpi_ivhd_hash[i]) - break; - } - - return (hinfop); -} - -amd_iommu_acpi_ivmd_t * -amd_iommu_lookup_any_ivmd(void) -{ - int i; - amd_iommu_acpi_ivmd_t *minfop; - - for (i = AMD_IOMMU_ACPI_INFO_HASH_SZ; i >= 0; i--) { - /*LINTED*/ - if (minfop = amd_iommu_acpi_ivmd_hash[i]) - break; - } - - return (minfop); -} - -static void -dump_acpi_aliases(void) -{ - amd_iommu_acpi_ivhd_t *hinfop; - uint16_t idx; - - for (idx = 0; idx <= AMD_IOMMU_ACPI_INFO_HASH_SZ; idx++) { - hinfop = amd_iommu_acpi_ivhd_hash[idx]; - for (; hinfop; hinfop = hinfop->ach_next) { - cmn_err(CE_NOTE, "start=%d, end=%d, src_bdf=%d", - hinfop->ach_deviceid_start, - hinfop->ach_deviceid_end, - hinfop->ach_src_deviceid); - } - } -} - -amd_iommu_acpi_ivhd_t * -amd_iommu_lookup_ivhd(int32_t deviceid) -{ - amd_iommu_acpi_ivhd_t *hinfop; - uint16_t idx; - - if (amd_iommu_debug == AMD_IOMMU_DEBUG_ACPI) { - cmn_err(CE_NOTE, "Attempting to get ACPI IVHD info " - "for deviceid: %d", deviceid); - } - - ASSERT(amd_iommu_acpi_ivhd_hash); - - /* check if special device */ - if (deviceid == -1) { - hinfop = amd_iommu_acpi_ivhd_hash[AMD_IOMMU_ACPI_INFO_HASH_SZ]; - for (; hinfop; hinfop = hinfop->ach_next) { - if (hinfop->ach_deviceid_start == -1 && - hinfop->ach_deviceid_end == -1) { - break; - } - } - return (hinfop); - } - - /* First search for an exact match */ - - idx = deviceid_hashfn(deviceid); - - -range: - hinfop = amd_iommu_acpi_ivhd_hash[idx]; - - for (; hinfop; hinfop = hinfop->ach_next) { - if (deviceid < hinfop->ach_deviceid_start || - deviceid > hinfop->ach_deviceid_end) - continue; - - if (amd_iommu_debug == AMD_IOMMU_DEBUG_ACPI) { - cmn_err(CE_NOTE, "Found ACPI IVHD match: %p, " - "actual deviceid = %u, start = %u, end = %u", - (void *)hinfop, deviceid, - hinfop->ach_deviceid_start, - hinfop->ach_deviceid_end); - } - goto out; - } - - if (idx != AMD_IOMMU_ACPI_INFO_HASH_SZ) { - idx = AMD_IOMMU_ACPI_INFO_HASH_SZ; - goto range; - } else { - cmn_err(CE_PANIC, "IVHD not found for deviceid: %x", deviceid); - } - -out: - if (amd_iommu_debug == AMD_IOMMU_DEBUG_ACPI) { - cmn_err(CE_NOTE, "%u: %s ACPI IVHD %p", deviceid, - hinfop ? "GOT" : "Did NOT get", (void *)hinfop); - } - - return (hinfop); -} - -amd_iommu_acpi_ivmd_t * -amd_iommu_lookup_ivmd(int32_t deviceid) -{ - amd_iommu_acpi_ivmd_t *minfop; - uint16_t idx; - - if (amd_iommu_debug == AMD_IOMMU_DEBUG_ACPI) { - cmn_err(CE_NOTE, "Attempting to get ACPI IVMD info " - "for deviceid: %u", deviceid); - } - - ASSERT(amd_iommu_acpi_ivmd_hash); - - /* First search for an exact match */ - - idx = deviceid_hashfn(deviceid); - - -range: - minfop = amd_iommu_acpi_ivmd_hash[idx]; - - for (; minfop; minfop = minfop->acm_next) { - if (deviceid < minfop->acm_deviceid_start && - deviceid > minfop->acm_deviceid_end) - continue; - - if (amd_iommu_debug == AMD_IOMMU_DEBUG_ACPI) { - cmn_err(CE_NOTE, "Found ACPI IVMD match: %p, " - "actual deviceid = %u, start = %u, end = %u", - (void *)minfop, deviceid, - minfop->acm_deviceid_start, - minfop->acm_deviceid_end); - } - - goto out; - } - - if (idx != AMD_IOMMU_ACPI_INFO_HASH_SZ) { - idx = AMD_IOMMU_ACPI_INFO_HASH_SZ; - goto range; - } else { - cmn_err(CE_PANIC, "IVMD not found for deviceid: %x", deviceid); - } - -out: - if (amd_iommu_debug == AMD_IOMMU_DEBUG_ACPI) { - cmn_err(CE_NOTE, "%u: %s ACPI IVMD info %p", deviceid, - minfop ? "GOT" : "Did NOT get", (void *)minfop); - } - - return (minfop); -}
--- a/usr/src/uts/intel/io/amd_iommu/amd_iommu_acpi.h Mon Sep 21 11:25:09 2009 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,306 +0,0 @@ -/* - * CDDL HEADER START - * - * The contents of this file are subject to the terms of the - * Common Development and Distribution License (the "License"). - * You may not use this file except in compliance with the License. - * - * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE - * or http://www.opensolaris.org/os/licensing. - * See the License for the specific language governing permissions - * and limitations under the License. - * - * When distributing Covered Code, include this CDDL HEADER in each - * file and include the License file at usr/src/OPENSOLARIS.LICENSE. - * If applicable, add the following below this CDDL HEADER, with the - * fields enclosed by brackets "[]" replaced with your own identifying - * information: Portions Copyright [yyyy] [name of copyright owner] - * - * CDDL HEADER END - */ -/* - * Copyright 2008 Sun Microsystems, Inc. All rights reserved. - * Use is subject to license terms. - */ - -#ifndef _AMD_IOMMU_ACPI_H -#define _AMD_IOMMU_ACPI_H - -#ifdef __cplusplus -extern "C" { -#endif - -#include <sys/sunddi.h> -#include <sys/acpi/acpi.h> -#include <sys/acpica.h> -#include <sys/amd_iommu.h> -#include "amd_iommu_impl.h" - -#ifdef _KERNEL - -#define IVRS_SIG "IVRS" - -/* - * IVINFO settings - */ -#define AMD_IOMMU_ACPI_IVINFO_RSV1 (31 << 16 | 23) -#define AMD_IOMMU_ACPI_HT_ATSRSV (22 << 16 | 22) -#define AMD_IOMMU_ACPI_VA_SIZE (21 << 16 | 15) -#define AMD_IOMMU_ACPI_PA_SIZE (14 << 16 | 8) -#define AMD_IOMMU_ACPI_IVINFO_RSV2 (7 << 16 | 0) - -/* - * IVHD Device entry len field - */ -#define AMD_IOMMU_ACPI_DEVENTRY_LEN (7 << 16 | 6) - -/* - * IVHD flag fields definition - */ -#define AMD_IOMMU_ACPI_IVHD_FLAGS_RSV (7 << 16 | 5) -#define AMD_IOMMU_ACPI_IVHD_FLAGS_IOTLBSUP (4 << 16 | 4) -#define AMD_IOMMU_ACPI_IVHD_FLAGS_ISOC (3 << 16 | 3) -#define AMD_IOMMU_ACPI_IVHD_FLAGS_RESPASSPW (2 << 16 | 2) -#define AMD_IOMMU_ACPI_IVHD_FLAGS_PASSPW (1 << 16 | 1) -#define AMD_IOMMU_ACPI_IVHD_FLAGS_HTTUNEN (0 << 16 | 0) - -/* - * IVHD IOMMU info fields - */ -#define AMD_IOMMU_ACPI_IOMMU_INFO_RSV1 (15 << 16 | 13) -#define AMD_IOMMU_ACPI_IOMMU_INFO_UNITID (12 << 16 | 8) -#define AMD_IOMMU_ACPI_IOMMU_INFO_RSV2 (7 << 16 | 5) -#define AMD_IOMMU_ACPI_IOMMU_INFO_MSINUM (4 << 16 | 0) - -/* - * IVHD deventry data settings - */ -#define AMD_IOMMU_ACPI_LINT1PASS (7 << 16 | 7) -#define AMD_IOMMU_ACPI_LINT0PASS (6 << 16 | 6) -#define AMD_IOMMU_ACPI_SYSMGT (5 << 16 | 4) -#define AMD_IOMMU_ACPI_DATRSV (3 << 16 | 3) -#define AMD_IOMMU_ACPI_NMIPASS (2 << 16 | 2) -#define AMD_IOMMU_ACPI_EXTINTPASS (1 << 16 | 1) -#define AMD_IOMMU_ACPI_INITPASS (0 << 16 | 0) - -/* - * IVHD deventry extended data settings - */ -#define AMD_IOMMU_ACPI_ATSDISABLED (31 << 16 | 31) -#define AMD_IOMMU_ACPI_EXTDATRSV (30 << 16 | 0) - -/* - * IVMD flags fields settings - */ -#define AMD_IOMMU_ACPI_IVMD_RSV (7 << 16 | 4) -#define AMD_IOMMU_ACPI_IVMD_EXCL_RANGE (3 << 16 | 3) -#define AMD_IOMMU_ACPI_IVMD_IW (2 << 16 | 2) -#define AMD_IOMMU_ACPI_IVMD_IR (1 << 16 | 1) -#define AMD_IOMMU_ACPI_IVMD_UNITY (0 << 16 | 0) - -#define AMD_IOMMU_ACPI_INFO_HASH_SZ (256) - -/* - * Deventry special device "variety" - */ -#define AMD_IOMMU_ACPI_SPECIAL_APIC 0x1 -#define AMD_IOMMU_ACPI_SPECIAL_HPET 0x2 - -typedef enum { - DEVENTRY_INVALID = 0, - DEVENTRY_ALL = 1, - DEVENTRY_SELECT, - DEVENTRY_RANGE, - DEVENTRY_RANGE_END, - DEVENTRY_ALIAS_SELECT, - DEVENTRY_ALIAS_RANGE, - DEVENTRY_EXTENDED_SELECT, - DEVENTRY_EXTENDED_RANGE, - DEVENTRY_SPECIAL_DEVICE -} ivhd_deventry_type_t; - -typedef enum { - IVMD_DEVICE_INVALID = 0, - IVMD_DEVICEID_ALL, - IVMD_DEVICEID_SELECT, - IVMD_DEVICEID_RANGE -} ivmd_deviceid_type_t; - -typedef struct ivhd_deventry { - uint8_t idev_len; - ivhd_deventry_type_t idev_type; - int32_t idev_deviceid; - int32_t idev_src_deviceid; - uint8_t idev_handle; - uint8_t idev_variety; - uint8_t idev_Lint1Pass; - uint8_t idev_Lint0Pass; - uint8_t idev_SysMgt; - uint8_t idev_NMIPass; - uint8_t idev_ExtIntPass; - uint8_t idev_INITPass; - uint8_t idev_AtsDisabled; - struct ivhd_deventry *idev_next; -} ivhd_deventry_t; - -typedef struct ivhd { - uint8_t ivhd_type; - uint8_t ivhd_flags; - uint16_t ivhd_len; - uint16_t ivhd_deviceid; - uint16_t ivhd_cap_off; - uint64_t ivhd_reg_base; - uint16_t ivhd_pci_seg; - uint16_t ivhd_iommu_info; - uint32_t ivhd_resv; -} ivhd_t; - -typedef struct ivhd_container { - ivhd_t *ivhdc_ivhd; - ivhd_deventry_t *ivhdc_first_deventry; - ivhd_deventry_t *ivhdc_last_deventry; - struct ivhd_container *ivhdc_next; -} ivhd_container_t; - -typedef struct ivmd { - uint8_t ivmd_type; - uint8_t ivmd_flags; - uint16_t ivmd_len; - uint16_t ivmd_deviceid; - uint16_t ivmd_auxdata; - uint64_t ivmd_resv; - uint64_t ivmd_phys_start; - uint64_t ivmd_phys_len; -} ivmd_t; - -typedef struct ivmd_container { - ivmd_t *ivmdc_ivmd; - struct ivmd_container *ivmdc_next; -} ivmd_container_t; - -typedef struct ivrs { - struct acpi_table_header ivrs_hdr; - uint32_t ivrs_ivinfo; - uint64_t ivrs_resv; -} ivrs_t; - -typedef struct amd_iommu_acpi { - struct ivrs *acp_ivrs; - ivhd_container_t *acp_first_ivhdc; - ivhd_container_t *acp_last_ivhdc; - ivmd_container_t *acp_first_ivmdc; - ivmd_container_t *acp_last_ivmdc; -} amd_iommu_acpi_t; - - -/* Global IVINFo fields */ -typedef struct amd_iommu_acpi_global { - uint8_t acg_HtAtsResv; - uint8_t acg_VAsize; - uint8_t acg_PAsize; -} amd_iommu_acpi_global_t; - -typedef struct amd_iommu_acpi_ivhd { - int32_t ach_deviceid_start; - int32_t ach_deviceid_end; - - /* IVHD deventry type */ - ivhd_deventry_type_t ach_dev_type; - - /* IVHD flag fields */ - uint8_t ach_IotlbSup; - uint8_t ach_Isoc; - uint8_t ach_ResPassPW; - uint8_t ach_PassPW; - uint8_t ach_HtTunEn; - - /* IVHD fields */ - uint16_t ach_IOMMU_deviceid; - uint16_t ach_IOMMU_cap_off; - uint64_t ach_IOMMU_reg_base; - uint16_t ach_IOMMU_pci_seg; - - /* IVHD IOMMU info fields */ - uint8_t ach_IOMMU_UnitID; - uint8_t ach_IOMMU_MSInum; - - /* IVHD deventry data settings */ - uint8_t ach_Lint1Pass; - uint8_t ach_Lint0Pass; - uint8_t ach_SysMgt; - uint8_t ach_NMIPass; - uint8_t ach_ExtIntPass; - uint8_t ach_INITPass; - - /* alias */ - int32_t ach_src_deviceid; - - /* IVHD deventry extended data settings */ - uint8_t ach_AtsDisabled; - - /* IVHD deventry special device */ - uint8_t ach_special_handle; - uint8_t ach_special_variety; - - struct amd_iommu_acpi_ivhd *ach_next; -} amd_iommu_acpi_ivhd_t; - -typedef struct amd_iommu_acpi_ivmd { - int32_t acm_deviceid_start; - int32_t acm_deviceid_end; - - /* IVMD type */ - ivmd_deviceid_type_t acm_dev_type; - - /* IVMD flags */ - uint8_t acm_ExclRange; - uint8_t acm_IW; - uint8_t acm_IR; - uint8_t acm_Unity; - - /* IVMD mem block */ - uint64_t acm_ivmd_phys_start; - uint64_t acm_ivmd_phys_len; - - struct amd_iommu_acpi_ivmd *acm_next; -} amd_iommu_acpi_ivmd_t; - -typedef union { - uint16_t ent16; - uint8_t ent8[2]; -} align_16_t; - -typedef union { - uint32_t ent32; - uint8_t ent8[4]; -} align_32_t; - -typedef union { - ivhd_t *ivhdp; - char *cp; -} align_ivhd_t; - -typedef union { - ivmd_t *ivmdp; - char *cp; -} align_ivmd_t; - -#pragma pack() - -int amd_iommu_acpi_init(void); -void amd_iommu_acpi_fini(void); -amd_iommu_acpi_ivhd_t *amd_iommu_lookup_all_ivhd(void); -amd_iommu_acpi_ivmd_t *amd_iommu_lookup_all_ivmd(void); -amd_iommu_acpi_ivhd_t *amd_iommu_lookup_any_ivhd(void); -amd_iommu_acpi_ivmd_t *amd_iommu_lookup_any_ivmd(void); -amd_iommu_acpi_global_t *amd_iommu_lookup_acpi_global(void); -amd_iommu_acpi_ivhd_t *amd_iommu_lookup_ivhd(int32_t deviceid); -amd_iommu_acpi_ivmd_t *amd_iommu_lookup_ivmd(int32_t deviceid); - -#endif /* _KERNEL */ - -#ifdef __cplusplus -} -#endif - -#endif /* _AMD_IOMMU_ACPI_H */
--- a/usr/src/uts/intel/io/amd_iommu/amd_iommu_cmd.c Mon Sep 21 11:25:09 2009 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,321 +0,0 @@ -/* - * CDDL HEADER START - * - * The contents of this file are subject to the terms of the - * Common Development and Distribution License (the "License"). - * You may not use this file except in compliance with the License. - * - * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE - * or http://www.opensolaris.org/os/licensing. - * See the License for the specific language governing permissions - * and limitations under the License. - * - * When distributing Covered Code, include this CDDL HEADER in each - * file and include the License file at usr/src/OPENSOLARIS.LICENSE. - * If applicable, add the following below this CDDL HEADER, with the - * fields enclosed by brackets "[]" replaced with your own identifying - * information: Portions Copyright [yyyy] [name of copyright owner] - * - * CDDL HEADER END - */ - -/* - * Copyright 2008 Sun Microsystems, Inc. All rights reserved. - * Use is subject to license terms. - */ - -#include <sys/sunddi.h> -#include <sys/amd_iommu.h> -#include "amd_iommu_impl.h" - -extern int servicing_interrupt(void); - -static void -amd_iommu_wait_for_completion(amd_iommu_t *iommu) -{ - ASSERT(MUTEX_HELD(&iommu->aiomt_cmdlock)); - while (AMD_IOMMU_REG_GET64(REGADDR64( - iommu->aiomt_reg_status_va), AMD_IOMMU_COMWAIT_INT) != 1) { - AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_ctrl_va), - AMD_IOMMU_CMDBUF_ENABLE, 1); - WAIT_SEC(1); - } -} - -static int -create_compl_wait_cmd(amd_iommu_t *iommu, amd_iommu_cmdargs_t *cmdargsp, - amd_iommu_cmd_flags_t flags, uint32_t *cmdptr) -{ - const char *driver = ddi_driver_name(iommu->aiomt_dip); - int instance = ddi_get_instance(iommu->aiomt_dip); - const char *f = "create_compl_wait_cmd"; - - ASSERT(cmdargsp == NULL); - - if (flags & AMD_IOMMU_CMD_FLAGS_COMPL_WAIT_S) { - cmn_err(CE_WARN, "%s: %s%d: idx=%d: 'store' completion " - "not supported for completion wait command", - f, driver, instance, iommu->aiomt_idx); - return (DDI_FAILURE); - } - - AMD_IOMMU_REG_SET32(&cmdptr[0], AMD_IOMMU_CMD_COMPL_WAIT_S, 0); - AMD_IOMMU_REG_SET32(&cmdptr[0], AMD_IOMMU_CMD_COMPL_WAIT_I, 1); - AMD_IOMMU_REG_SET32(&cmdptr[0], AMD_IOMMU_CMD_COMPL_WAIT_F, - (flags & AMD_IOMMU_CMD_FLAGS_COMPL_WAIT_F) != 0); - AMD_IOMMU_REG_SET32(&cmdptr[0], AMD_IOMMU_CMD_COMPL_WAIT_STORE_ADDR_LO, - 0); - AMD_IOMMU_REG_SET32(&cmdptr[1], AMD_IOMMU_CMD_OPCODE, 0x01); - AMD_IOMMU_REG_SET32(&cmdptr[1], AMD_IOMMU_CMD_COMPL_WAIT_STORE_ADDR_HI, - 0); - cmdptr[2] = 0; - cmdptr[3] = 0; - - return (DDI_SUCCESS); -} - -static int -create_inval_devtab_entry_cmd(amd_iommu_t *iommu, amd_iommu_cmdargs_t *cmdargsp, - amd_iommu_cmd_flags_t flags, uint32_t *cmdptr) -{ - const char *driver = ddi_driver_name(iommu->aiomt_dip); - int instance = ddi_get_instance(iommu->aiomt_dip); - const char *f = "create_inval_devtab_entry_cmd"; - uint16_t deviceid; - - ASSERT(cmdargsp); - - if (flags != AMD_IOMMU_CMD_FLAGS_NONE) { - cmn_err(CE_WARN, "%s: %s%d: idx=%d: invalidate devtab entry " - "no flags supported", f, driver, instance, - iommu->aiomt_idx); - return (DDI_FAILURE); - } - - deviceid = cmdargsp->ca_deviceid; - - AMD_IOMMU_REG_SET32(&cmdptr[0], AMD_IOMMU_CMD_INVAL_DEVTAB_DEVICEID, - deviceid); - AMD_IOMMU_REG_SET32(&cmdptr[1], AMD_IOMMU_CMD_OPCODE, 0x02); - cmdptr[2] = 0; - cmdptr[3] = 0; - - return (DDI_SUCCESS); -} - -/*ARGSUSED*/ -static int -create_inval_iommu_pages_cmd(amd_iommu_t *iommu, amd_iommu_cmdargs_t *cmdargsp, - amd_iommu_cmd_flags_t flags, uint32_t *cmdptr) -{ - uint32_t addr_lo; - uint32_t addr_hi; - - ASSERT(cmdargsp); - - addr_lo = AMD_IOMMU_REG_GET64(REGADDR64(&cmdargsp->ca_addr), - AMD_IOMMU_CMD_INVAL_PAGES_ADDR_LO); - addr_hi = AMD_IOMMU_REG_GET64(REGADDR64(&cmdargsp->ca_addr), - AMD_IOMMU_CMD_INVAL_PAGES_ADDR_HI); - - cmdptr[0] = 0; - AMD_IOMMU_REG_SET32(&cmdptr[1], AMD_IOMMU_CMD_INVAL_PAGES_DOMAINID, - cmdargsp->ca_domainid); - AMD_IOMMU_REG_SET32(&cmdptr[1], AMD_IOMMU_CMD_OPCODE, 0x03); - AMD_IOMMU_REG_SET32(&cmdptr[2], AMD_IOMMU_CMD_INVAL_PAGES_PDE, - (flags & AMD_IOMMU_CMD_FLAGS_PAGE_PDE_INVAL) != 0); - AMD_IOMMU_REG_SET32(&cmdptr[2], AMD_IOMMU_CMD_INVAL_PAGES_S, - (flags & AMD_IOMMU_CMD_FLAGS_PAGE_INVAL_S) != 0); - AMD_IOMMU_REG_SET32(&cmdptr[2], AMD_IOMMU_CMD_INVAL_PAGES_ADDR_LO, - addr_lo); - cmdptr[3] = addr_hi; - - return (DDI_SUCCESS); - -} - -/*ARGSUSED*/ -static int -create_inval_iotlb_pages_cmd(amd_iommu_t *iommu, amd_iommu_cmdargs_t *cmdargsp, - amd_iommu_cmd_flags_t flags, uint32_t *cmdptr) -{ - uint32_t addr_lo; - uint32_t addr_hi; - - ASSERT(cmdargsp); - - addr_lo = AMD_IOMMU_REG_GET64(REGADDR64(&cmdargsp->ca_addr), - AMD_IOMMU_CMD_INVAL_IOTLB_ADDR_LO); - - addr_hi = AMD_IOMMU_REG_GET64(REGADDR64(&cmdargsp->ca_addr), - AMD_IOMMU_CMD_INVAL_IOTLB_ADDR_HI); - - AMD_IOMMU_REG_SET32(&cmdptr[0], AMD_IOMMU_CMD_INVAL_IOTLB_DEVICEID, - cmdargsp->ca_deviceid); - AMD_IOMMU_REG_SET32(&cmdptr[0], AMD_IOMMU_CMD_INVAL_IOTLB_MAXPEND, - AMD_IOMMU_DEFAULT_MAXPEND); - AMD_IOMMU_REG_SET32(&cmdptr[1], AMD_IOMMU_CMD_OPCODE, 0x04); - AMD_IOMMU_REG_SET32(&cmdptr[1], AMD_IOMMU_CMD_INVAL_IOTLB_QUEUEID, - cmdargsp->ca_deviceid); - AMD_IOMMU_REG_SET32(&cmdptr[2], AMD_IOMMU_CMD_INVAL_IOTLB_ADDR_LO, - addr_lo); - AMD_IOMMU_REG_SET32(&cmdptr[2], AMD_IOMMU_CMD_INVAL_IOTLB_S, - (flags & AMD_IOMMU_CMD_FLAGS_IOTLB_INVAL_S) != 0); - cmdptr[3] = addr_hi; - - return (DDI_SUCCESS); -} - -static int -create_inval_intr_table_cmd(amd_iommu_t *iommu, amd_iommu_cmdargs_t *cmdargsp, - amd_iommu_cmd_flags_t flags, uint32_t *cmdptr) -{ - const char *driver = ddi_driver_name(iommu->aiomt_dip); - int instance = ddi_get_instance(iommu->aiomt_dip); - const char *f = "create_inval_intr_table_cmd"; - - ASSERT(cmdargsp); - - if (flags != AMD_IOMMU_CMD_FLAGS_NONE) { - cmn_err(CE_WARN, "%s: %s%d: idx=%d: flags not supported " - "for invalidate interrupt table command", - f, driver, instance, iommu->aiomt_idx); - return (DDI_FAILURE); - } - - AMD_IOMMU_REG_SET32(&cmdptr[0], AMD_IOMMU_CMD_INVAL_INTR_DEVICEID, - cmdargsp->ca_deviceid); - AMD_IOMMU_REG_SET32(&cmdptr[1], AMD_IOMMU_CMD_OPCODE, 0x05); - cmdptr[2] = 0; - cmdptr[3] = 0; - - return (DDI_SUCCESS); -} - -int -amd_iommu_cmd(amd_iommu_t *iommu, amd_iommu_cmd_t cmd, - amd_iommu_cmdargs_t *cmdargs, amd_iommu_cmd_flags_t flags, int lock_held) -{ - int error; - int i; - uint32_t cmdptr[4] = {0}; - const char *driver = ddi_driver_name(iommu->aiomt_dip); - int instance = ddi_get_instance(iommu->aiomt_dip); - uint64_t cmdhead_off; - uint64_t cmdtail_off; - const char *f = "amd_iommu_cmd"; - - ASSERT(lock_held == 0 || lock_held == 1); - ASSERT(lock_held == 0 || MUTEX_HELD(&iommu->aiomt_cmdlock)); - - if (!lock_held) - mutex_enter(&iommu->aiomt_cmdlock); - - /* - * Prepare the command - */ - switch (cmd) { - case AMD_IOMMU_CMD_COMPL_WAIT: - if (flags & AMD_IOMMU_CMD_FLAGS_COMPL_WAIT) { - cmn_err(CE_WARN, "%s: %s%d: idx=%d: No completion wait " - " after completion wait command", - f, driver, instance, iommu->aiomt_idx); - error = DDI_FAILURE; - goto out; - } - error = create_compl_wait_cmd(iommu, cmdargs, flags, cmdptr); - break; - case AMD_IOMMU_CMD_INVAL_DEVTAB_ENTRY: - error = create_inval_devtab_entry_cmd(iommu, cmdargs, - flags & ~AMD_IOMMU_CMD_FLAGS_COMPL_WAIT, cmdptr); - break; - case AMD_IOMMU_CMD_INVAL_IOMMU_PAGES: - error = create_inval_iommu_pages_cmd(iommu, cmdargs, - flags & ~AMD_IOMMU_CMD_FLAGS_COMPL_WAIT, cmdptr); - break; - case AMD_IOMMU_CMD_INVAL_IOTLB_PAGES: - error = create_inval_iotlb_pages_cmd(iommu, cmdargs, - flags & ~AMD_IOMMU_CMD_FLAGS_COMPL_WAIT, cmdptr); - break; - case AMD_IOMMU_CMD_INVAL_INTR_TABLE: - error = create_inval_intr_table_cmd(iommu, cmdargs, - flags & ~AMD_IOMMU_CMD_FLAGS_COMPL_WAIT, cmdptr); - break; - default: - cmn_err(CE_WARN, "%s: %s%d: idx=%d: Unsupported cmd: %d", - f, driver, instance, iommu->aiomt_idx, cmd); - error = DDI_FAILURE; - goto out; - } - - if (error != DDI_SUCCESS) { - error = DDI_FAILURE; - goto out; - } - - AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_ctrl_va), - AMD_IOMMU_CMDBUF_ENABLE, 1); - - ASSERT(iommu->aiomt_cmd_tail != NULL); - - for (i = 0; i < 4; i++) { - iommu->aiomt_cmd_tail[i] = cmdptr[i]; - } - -wait_for_drain: - cmdhead_off = AMD_IOMMU_REG_GET64( - REGADDR64(iommu->aiomt_reg_cmdbuf_head_va), - AMD_IOMMU_CMDHEADPTR); - - cmdhead_off = CMD2OFF(cmdhead_off); - - ASSERT(cmdhead_off < iommu->aiomt_cmdbuf_sz); - - /* check for overflow */ - if ((caddr_t)iommu->aiomt_cmd_tail < - (cmdhead_off + iommu->aiomt_cmdbuf)) { - if ((caddr_t)iommu->aiomt_cmd_tail + 16 >= - (cmdhead_off + iommu->aiomt_cmdbuf)) -#ifdef DEBUG - cmn_err(CE_WARN, "cmdbuffer overflow: waiting for " - "drain"); -#endif - goto wait_for_drain; - } - - SYNC_FORDEV(iommu->aiomt_dmahdl); - - /* - * Update the tail pointer in soft state - * and the tail pointer register - */ - iommu->aiomt_cmd_tail += 4; - if ((caddr_t)iommu->aiomt_cmd_tail >= (iommu->aiomt_cmdbuf - + iommu->aiomt_cmdbuf_sz)) { - /* wraparound */ - /*LINTED*/ - iommu->aiomt_cmd_tail = (uint32_t *)iommu->aiomt_cmdbuf; - cmdtail_off = 0; - } else { - cmdtail_off = (caddr_t)iommu->aiomt_cmd_tail - /*LINTED*/ - - iommu->aiomt_cmdbuf; - } - - ASSERT(cmdtail_off < iommu->aiomt_cmdbuf_sz); - - AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_cmdbuf_tail_va), - AMD_IOMMU_CMDTAILPTR, OFF2CMD(cmdtail_off)); - - if (cmd == AMD_IOMMU_CMD_COMPL_WAIT) { - amd_iommu_wait_for_completion(iommu); - } else if (flags & AMD_IOMMU_CMD_FLAGS_COMPL_WAIT) { - error = amd_iommu_cmd(iommu, AMD_IOMMU_CMD_COMPL_WAIT, - NULL, 0, 1); - } - -out: - if (!lock_held) - mutex_exit(&iommu->aiomt_cmdlock); - return (error); -}
--- a/usr/src/uts/intel/io/amd_iommu/amd_iommu_impl.c Mon Sep 21 11:25:09 2009 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,1880 +0,0 @@ -/* - * CDDL HEADER START - * - * The contents of this file are subject to the terms of the - * Common Development and Distribution License (the "License"). - * You may not use this file except in compliance with the License. - * - * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE - * or http://www.opensolaris.org/os/licensing. - * See the License for the specific language governing permissions - * and limitations under the License. - * - * When distributing Covered Code, include this CDDL HEADER in each - * file and include the License file at usr/src/OPENSOLARIS.LICENSE. - * If applicable, add the following below this CDDL HEADER, with the - * fields enclosed by brackets "[]" replaced with your own identifying - * information: Portions Copyright [yyyy] [name of copyright owner] - * - * CDDL HEADER END - */ - -/* - * Copyright 2009 Sun Microsystems, Inc. All rights reserved. - * Use is subject to license terms. - */ - -#include <sys/sunddi.h> -#include <sys/iommulib.h> -#include <sys/amd_iommu.h> -#include <sys/pci_cap.h> -#include <sys/bootconf.h> -#include <sys/ddidmareq.h> - -#include "amd_iommu_impl.h" -#include "amd_iommu_acpi.h" -#include "amd_iommu_page_tables.h" - -static int amd_iommu_fini(amd_iommu_t *iommu); -static void amd_iommu_teardown_interrupts(amd_iommu_t *iommu); -static void amd_iommu_stop(amd_iommu_t *iommu); - -static int amd_iommu_probe(iommulib_handle_t handle, dev_info_t *rdip); -static int amd_iommu_allochdl(iommulib_handle_t handle, - dev_info_t *dip, dev_info_t *rdip, ddi_dma_attr_t *attr, - int (*waitfp)(caddr_t), caddr_t arg, ddi_dma_handle_t *dma_handlep); -static int amd_iommu_freehdl(iommulib_handle_t handle, - dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t dma_handle); -static int amd_iommu_bindhdl(iommulib_handle_t handle, dev_info_t *dip, - dev_info_t *rdip, ddi_dma_handle_t dma_handle, - struct ddi_dma_req *dmareq, ddi_dma_cookie_t *cookiep, - uint_t *ccountp); -static int amd_iommu_unbindhdl(iommulib_handle_t handle, - dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t dma_handle); -static int amd_iommu_sync(iommulib_handle_t handle, dev_info_t *dip, - dev_info_t *rdip, ddi_dma_handle_t dma_handle, off_t off, - size_t len, uint_t cache_flags); -static int amd_iommu_win(iommulib_handle_t handle, dev_info_t *dip, - dev_info_t *rdip, ddi_dma_handle_t dma_handle, uint_t win, - off_t *offp, size_t *lenp, ddi_dma_cookie_t *cookiep, - uint_t *ccountp); -static int amd_iommu_map(iommulib_handle_t handle, dev_info_t *dip, - dev_info_t *rdip, struct ddi_dma_req *dmareq, - ddi_dma_handle_t *dma_handle); -static int amd_iommu_mctl(iommulib_handle_t handle, dev_info_t *dip, - dev_info_t *rdip, ddi_dma_handle_t dma_handle, - enum ddi_dma_ctlops request, off_t *offp, size_t *lenp, - caddr_t *objpp, uint_t cache_flags); - -static int unmap_current_window(amd_iommu_t *iommu, dev_info_t *rdip, - ddi_dma_cookie_t *cookie_array, uint_t ccount, int ncookies, int locked); - -extern void *device_arena_alloc(size_t size, int vm_flag); -extern void device_arena_free(void * vaddr, size_t size); - -ddi_dma_attr_t amd_iommu_dma_attr = { - DMA_ATTR_V0, - 0U, /* dma_attr_addr_lo */ - 0xffffffffffffffffULL, /* dma_attr_addr_hi */ - 0xffffffffU, /* dma_attr_count_max */ - (uint64_t)4096, /* dma_attr_align */ - 1, /* dma_attr_burstsizes */ - 64, /* dma_attr_minxfer */ - 0xffffffffU, /* dma_attr_maxxfer */ - 0xffffffffU, /* dma_attr_seg */ - 1, /* dma_attr_sgllen, variable */ - 64, /* dma_attr_granular */ - 0 /* dma_attr_flags */ -}; - -ddi_device_acc_attr_t amd_iommu_devacc = { - DDI_DEVICE_ATTR_V0, - DDI_NEVERSWAP_ACC, - DDI_STRICTORDER_ACC -}; - -struct iommulib_ops amd_iommulib_ops = { - IOMMU_OPS_VERSION, - AMD_IOMMU, - "AMD IOMMU Vers. 1", - NULL, - amd_iommu_probe, - amd_iommu_allochdl, - amd_iommu_freehdl, - amd_iommu_bindhdl, - amd_iommu_unbindhdl, - amd_iommu_sync, - amd_iommu_win, - amd_iommu_map, - amd_iommu_mctl -}; - -static kmutex_t amd_iommu_pgtable_lock; - -static int -amd_iommu_register(amd_iommu_t *iommu) -{ - dev_info_t *dip = iommu->aiomt_dip; - const char *driver = ddi_driver_name(dip); - int instance = ddi_get_instance(dip); - iommulib_ops_t *iommulib_ops; - iommulib_handle_t handle; - const char *f = "amd_iommu_register"; - - iommulib_ops = kmem_zalloc(sizeof (iommulib_ops_t), KM_SLEEP); - - *iommulib_ops = amd_iommulib_ops; - - iommulib_ops->ilops_data = (void *)iommu; - iommu->aiomt_iommulib_ops = iommulib_ops; - - if (iommulib_iommu_register(dip, iommulib_ops, &handle) - != DDI_SUCCESS) { - cmn_err(CE_WARN, "%s: %s%d: Register with iommulib " - "failed idx=%d", f, driver, instance, iommu->aiomt_idx); - kmem_free(iommulib_ops, sizeof (iommulib_ops_t)); - return (DDI_FAILURE); - } - - iommu->aiomt_iommulib_handle = handle; - - return (DDI_SUCCESS); -} - -static int -amd_iommu_unregister(amd_iommu_t *iommu) -{ - if (iommu->aiomt_iommulib_handle == NULL) { - /* we never registered */ - return (DDI_SUCCESS); - } - - if (iommulib_iommu_unregister(iommu->aiomt_iommulib_handle) - != DDI_SUCCESS) { - return (DDI_FAILURE); - } - - kmem_free(iommu->aiomt_iommulib_ops, sizeof (iommulib_ops_t)); - iommu->aiomt_iommulib_ops = NULL; - iommu->aiomt_iommulib_handle = NULL; - - return (DDI_SUCCESS); -} - -static int -amd_iommu_setup_passthru(amd_iommu_t *iommu) -{ - gfx_entry_t *gfxp; - dev_info_t *dip; - - /* - * Setup passthru mapping for "special" devices - */ - amd_iommu_set_passthru(iommu, NULL); - - for (gfxp = gfx_devinfo_list; gfxp; gfxp = gfxp->g_next) { - gfxp->g_ref++; - dip = gfxp->g_dip; - if (dip) { - amd_iommu_set_passthru(iommu, dip); - } - gfxp->g_ref--; - } - - return (DDI_SUCCESS); -} - -static int -amd_iommu_start(amd_iommu_t *iommu) -{ - dev_info_t *dip = iommu->aiomt_dip; - int instance = ddi_get_instance(dip); - const char *driver = ddi_driver_name(dip); - amd_iommu_acpi_ivhd_t *hinfop; - const char *f = "amd_iommu_start"; - - hinfop = amd_iommu_lookup_all_ivhd(); - - /* - * Disable HT tunnel translation. - * XXX use ACPI - */ - AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_ctrl_va), - AMD_IOMMU_HT_TUN_ENABLE, 0); - - if (hinfop) { - if (amd_iommu_debug) { - cmn_err(CE_NOTE, - "amd_iommu: using ACPI for CTRL registers"); - } - AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_ctrl_va), - AMD_IOMMU_ISOC, hinfop->ach_Isoc); - AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_ctrl_va), - AMD_IOMMU_RESPASSPW, hinfop->ach_ResPassPW); - AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_ctrl_va), - AMD_IOMMU_PASSPW, hinfop->ach_PassPW); - } - - AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_ctrl_va), - AMD_IOMMU_INVTO, 5); - - - /* - * The Device table entry bit 0 (V) controls whether the device - * table entry is valid for address translation and Device table - * entry bit 128 (IV) controls whether interrupt remapping is valid. - * By setting both to zero we are essentially doing pass-thru. Since - * this table is zeroed on allocation, essentially we will have - * pass-thru when IOMMU is enabled. - */ - - /* Finally enable the IOMMU ... */ - AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_ctrl_va), - AMD_IOMMU_ENABLE, 1); - - if (amd_iommu_debug) { - cmn_err(CE_NOTE, "%s: %s%d: AMD IOMMU idx=%d. " - "Successfully started AMD IOMMU", f, driver, instance, - iommu->aiomt_idx); - } - cmn_err(CE_NOTE, "AMD IOMMU (%d,%d) enabled", - instance, iommu->aiomt_idx); - - return (DDI_SUCCESS); -} - -static void -amd_iommu_stop(amd_iommu_t *iommu) -{ - dev_info_t *dip = iommu->aiomt_dip; - int instance = ddi_get_instance(dip); - const char *driver = ddi_driver_name(dip); - const char *f = "amd_iommu_stop"; - - AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_ctrl_va), - AMD_IOMMU_ENABLE, 0); - AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_ctrl_va), - AMD_IOMMU_EVENTINT_ENABLE, 0); - AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_ctrl_va), - AMD_IOMMU_COMWAITINT_ENABLE, 0); - AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_ctrl_va), - AMD_IOMMU_EVENTLOG_ENABLE, 0); - - /* - * Disable translation on HT tunnel traffic - */ - AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_ctrl_va), - AMD_IOMMU_HT_TUN_ENABLE, 0); - AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_ctrl_va), - AMD_IOMMU_CMDBUF_ENABLE, 0); - - cmn_err(CE_NOTE, "%s: %s%d: AMD IOMMYU idx=%d. " - "Successfully stopped AMD IOMMU", f, driver, instance, - iommu->aiomt_idx); -} - -static int -amd_iommu_setup_tables_and_buffers(amd_iommu_t *iommu) -{ - dev_info_t *dip = iommu->aiomt_dip; - int instance = ddi_get_instance(dip); - const char *driver = ddi_driver_name(dip); - uint32_t dma_bufsz; - caddr_t addr; - uint32_t sz; - uint32_t p2sz; - int i; - uint64_t *dentry; - int err; - const char *f = "amd_iommu_setup_tables_and_buffers"; - - /* - * We will put the Device Table, Command Buffer and - * Event Log in contiguous memory. Allocate the maximum - * size allowed for such structures - * Device Table: 256b * 64K = 32B * 64K - * Command Buffer: 128b * 32K = 16B * 32K - * Event Log: 128b * 32K = 16B * 32K - */ - iommu->aiomt_devtbl_sz = (1<<AMD_IOMMU_DEVTBL_SZ) * AMD_IOMMU_DEVENT_SZ; - iommu->aiomt_cmdbuf_sz = (1<<AMD_IOMMU_CMDBUF_SZ) * AMD_IOMMU_CMD_SZ; - iommu->aiomt_eventlog_sz = - (1<<AMD_IOMMU_EVENTLOG_SZ) * AMD_IOMMU_EVENT_SZ; - - dma_bufsz = iommu->aiomt_devtbl_sz + iommu->aiomt_cmdbuf_sz - + iommu->aiomt_eventlog_sz; - - /* - * Alloc a DMA handle. - */ - err = ddi_dma_alloc_handle(dip, &amd_iommu_dma_attr, - DDI_DMA_SLEEP, NULL, &iommu->aiomt_dmahdl); - if (err != DDI_SUCCESS) { - cmn_err(CE_WARN, "%s: %s%d: Cannot alloc DMA handle for " - "AMD IOMMU tables and buffers", f, driver, instance); - return (DDI_FAILURE); - } - - /* - * Alloc memory for tables and buffers - * XXX remove cast to size_t - */ - err = ddi_dma_mem_alloc(iommu->aiomt_dmahdl, dma_bufsz, - &amd_iommu_devacc, DDI_DMA_CONSISTENT|IOMEM_DATA_UNCACHED, - DDI_DMA_SLEEP, NULL, (caddr_t *)&iommu->aiomt_dma_bufva, - (size_t *)&iommu->aiomt_dma_mem_realsz, &iommu->aiomt_dma_mem_hdl); - if (err != DDI_SUCCESS) { - cmn_err(CE_WARN, "%s: %s%d: Cannot alloc memory for DMA " - "to AMD IOMMU tables and buffers", f, driver, instance); - iommu->aiomt_dma_bufva = NULL; - iommu->aiomt_dma_mem_realsz = 0; - ddi_dma_free_handle(&iommu->aiomt_dmahdl); - iommu->aiomt_dmahdl = NULL; - return (DDI_FAILURE); - } - - /* - * The VA must be 4K aligned and >= table size - */ - ASSERT(((uintptr_t)iommu->aiomt_dma_bufva & - AMD_IOMMU_TABLE_ALIGN) == 0); - ASSERT(iommu->aiomt_dma_mem_realsz >= dma_bufsz); - - /* - * Now bind the handle - */ - err = ddi_dma_addr_bind_handle(iommu->aiomt_dmahdl, NULL, - iommu->aiomt_dma_bufva, iommu->aiomt_dma_mem_realsz, - DDI_DMA_RDWR | DDI_DMA_CONSISTENT, DDI_DMA_SLEEP, - NULL, &iommu->aiomt_buf_dma_cookie, &iommu->aiomt_buf_dma_ncookie); - if (err != DDI_DMA_MAPPED) { - cmn_err(CE_WARN, "%s: %s%d: Cannot bind memory for DMA " - "to AMD IOMMU tables and buffers. bufrealsz=%p", - f, driver, instance, - (void *)(uintptr_t)iommu->aiomt_dma_mem_realsz); - iommu->aiomt_buf_dma_cookie.dmac_laddress = 0; - iommu->aiomt_buf_dma_cookie.dmac_size = 0; - iommu->aiomt_buf_dma_cookie.dmac_type = 0; - iommu->aiomt_buf_dma_ncookie = 0; - ddi_dma_mem_free(&iommu->aiomt_dma_mem_hdl); - iommu->aiomt_dma_mem_hdl = NULL; - iommu->aiomt_dma_bufva = NULL; - iommu->aiomt_dma_mem_realsz = 0; - ddi_dma_free_handle(&iommu->aiomt_dmahdl); - iommu->aiomt_dmahdl = NULL; - return (DDI_FAILURE); - } - - /* - * We assume the DMA engine on the IOMMU is capable of handling the - * whole table buffer in a single cookie. If not and multiple cookies - * are needed we fail. - */ - if (iommu->aiomt_buf_dma_ncookie != 1) { - cmn_err(CE_WARN, "%s: %s%d: Cannot handle multiple " - "cookies for DMA to AMD IOMMU tables and buffers. " - "#cookies=%u", f, driver, instance, - iommu->aiomt_buf_dma_ncookie); - (void) ddi_dma_unbind_handle(iommu->aiomt_dmahdl); - iommu->aiomt_buf_dma_cookie.dmac_laddress = 0; - iommu->aiomt_buf_dma_cookie.dmac_size = 0; - iommu->aiomt_buf_dma_cookie.dmac_type = 0; - iommu->aiomt_buf_dma_ncookie = 0; - ddi_dma_mem_free(&iommu->aiomt_dma_mem_hdl); - iommu->aiomt_dma_mem_hdl = NULL; - iommu->aiomt_dma_bufva = NULL; - iommu->aiomt_dma_mem_realsz = 0; - ddi_dma_free_handle(&iommu->aiomt_dmahdl); - iommu->aiomt_dmahdl = NULL; - return (DDI_FAILURE); - } - - /* - * The address in the cookie must be 4K aligned and >= table size - */ - ASSERT((iommu->aiomt_buf_dma_cookie.dmac_cookie_addr - & AMD_IOMMU_TABLE_ALIGN) == 0); - ASSERT(iommu->aiomt_buf_dma_cookie.dmac_size - <= iommu->aiomt_dma_mem_realsz); - ASSERT(iommu->aiomt_buf_dma_cookie.dmac_size >= dma_bufsz); - - /* - * Setup the device table pointers in the iommu struct as - * well as the IOMMU device table register - */ - iommu->aiomt_devtbl = iommu->aiomt_dma_bufva; - bzero(iommu->aiomt_devtbl, iommu->aiomt_devtbl_sz); - - /* - * Set V=1 and TV = 0, so any inadvertant pass-thrus cause - * page faults. Also set SE bit so we aren't swamped with - * page fault messages - */ - for (i = 0; i <= AMD_IOMMU_MAX_DEVICEID; i++) { - /*LINTED*/ - dentry = (uint64_t *)&iommu->aiomt_devtbl - [i * AMD_IOMMU_DEVTBL_ENTRY_SZ]; - AMD_IOMMU_REG_SET64(dentry, AMD_IOMMU_DEVTBL_V, 1); - AMD_IOMMU_REG_SET64(&(dentry[1]), AMD_IOMMU_DEVTBL_SE, 1); - } - - addr = (caddr_t)(uintptr_t)iommu->aiomt_buf_dma_cookie.dmac_cookie_addr; - AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_devtbl_va), - AMD_IOMMU_DEVTABBASE, ((uint64_t)(uintptr_t)addr) >> 12); - sz = (iommu->aiomt_devtbl_sz >> 12) - 1; - ASSERT(sz <= ((1 << 9) - 1)); - AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_devtbl_va), - AMD_IOMMU_DEVTABSIZE, sz); - - /* - * Setup the command buffer pointers - */ - iommu->aiomt_cmdbuf = iommu->aiomt_devtbl + - iommu->aiomt_devtbl_sz; - bzero(iommu->aiomt_cmdbuf, iommu->aiomt_cmdbuf_sz); - addr += iommu->aiomt_devtbl_sz; - AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_cmdbuf_va), - AMD_IOMMU_COMBASE, ((uint64_t)(uintptr_t)addr) >> 12); - - p2sz = AMD_IOMMU_CMDBUF_SZ; - ASSERT(p2sz >= AMD_IOMMU_CMDBUF_MINSZ && - p2sz <= AMD_IOMMU_CMDBUF_MAXSZ); - AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_cmdbuf_va), - AMD_IOMMU_COMLEN, p2sz); - /*LINTED*/ - iommu->aiomt_cmd_tail = (uint32_t *)iommu->aiomt_cmdbuf; - AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_cmdbuf_head_va), - AMD_IOMMU_CMDHEADPTR, 0); - AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_cmdbuf_tail_va), - AMD_IOMMU_CMDTAILPTR, 0); - - /* - * Setup the event log pointers - */ - iommu->aiomt_eventlog = iommu->aiomt_cmdbuf + - iommu->aiomt_eventlog_sz; - bzero(iommu->aiomt_eventlog, iommu->aiomt_eventlog_sz); - addr += iommu->aiomt_cmdbuf_sz; - AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_eventlog_va), - AMD_IOMMU_EVENTBASE, ((uint64_t)(uintptr_t)addr) >> 12); - p2sz = AMD_IOMMU_EVENTLOG_SZ; - ASSERT(p2sz >= AMD_IOMMU_EVENTLOG_MINSZ && - p2sz <= AMD_IOMMU_EVENTLOG_MAXSZ); - AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_eventlog_va), - AMD_IOMMU_EVENTLEN, sz); - /*LINTED*/ - iommu->aiomt_event_head = (uint32_t *)iommu->aiomt_eventlog; - AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_eventlog_head_va), - AMD_IOMMU_EVENTHEADPTR, 0); - AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_eventlog_tail_va), - AMD_IOMMU_EVENTTAILPTR, 0); - - /* dma sync so device sees this init */ - SYNC_FORDEV(iommu->aiomt_dmahdl); - - if (amd_iommu_debug & AMD_IOMMU_DEBUG_TABLES) { - cmn_err(CE_NOTE, "%s: %s%d: successfully setup AMD IOMMU " - "tables, idx=%d", f, driver, instance, iommu->aiomt_idx); - } - - return (DDI_SUCCESS); -} - -static void -amd_iommu_teardown_tables_and_buffers(amd_iommu_t *iommu) -{ - dev_info_t *dip = iommu->aiomt_dip; - int instance = ddi_get_instance(dip); - const char *driver = ddi_driver_name(dip); - const char *f = "amd_iommu_teardown_tables_and_buffers"; - - iommu->aiomt_eventlog = NULL; - AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_eventlog_va), - AMD_IOMMU_EVENTBASE, 0); - AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_eventlog_va), - AMD_IOMMU_EVENTLEN, 0); - - iommu->aiomt_cmdbuf = NULL; - AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_cmdbuf_va), - AMD_IOMMU_COMBASE, 0); - AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_cmdbuf_va), - AMD_IOMMU_COMLEN, 0); - - iommu->aiomt_devtbl = NULL; - AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_devtbl_va), - AMD_IOMMU_DEVTABBASE, 0); - AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_devtbl_va), - AMD_IOMMU_DEVTABSIZE, 0); - - if (iommu->aiomt_dmahdl == NULL) - return; - - /* Unbind the handle */ - if (ddi_dma_unbind_handle(iommu->aiomt_dmahdl) != DDI_SUCCESS) { - cmn_err(CE_WARN, "%s: %s%d: failed to unbind handle: " - "%p for IOMMU idx=%d", f, driver, instance, - (void *)iommu->aiomt_dmahdl, iommu->aiomt_idx); - } - iommu->aiomt_buf_dma_cookie.dmac_laddress = 0; - iommu->aiomt_buf_dma_cookie.dmac_size = 0; - iommu->aiomt_buf_dma_cookie.dmac_type = 0; - iommu->aiomt_buf_dma_ncookie = 0; - - /* Free the table memory allocated for DMA */ - ddi_dma_mem_free(&iommu->aiomt_dma_mem_hdl); - iommu->aiomt_dma_mem_hdl = NULL; - iommu->aiomt_dma_bufva = NULL; - iommu->aiomt_dma_mem_realsz = 0; - - /* Free the DMA handle */ - ddi_dma_free_handle(&iommu->aiomt_dmahdl); - iommu->aiomt_dmahdl = NULL; -} - -static void -amd_iommu_enable_interrupts(amd_iommu_t *iommu) -{ - ASSERT(AMD_IOMMU_REG_GET64(REGADDR64(iommu->aiomt_reg_status_va), - AMD_IOMMU_CMDBUF_RUN) == 0); - ASSERT(AMD_IOMMU_REG_GET64(REGADDR64(iommu->aiomt_reg_status_va), - AMD_IOMMU_EVENT_LOG_RUN) == 0); - - /* Must be set prior to enabling command buffer */ - /* Must be set prior to enabling event logging */ - AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_ctrl_va), - AMD_IOMMU_CMDBUF_ENABLE, 1); - /* No interrupts for completion wait - too heavy weight. use polling */ - AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_ctrl_va), - AMD_IOMMU_COMWAITINT_ENABLE, 0); - AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_ctrl_va), - AMD_IOMMU_EVENTLOG_ENABLE, 1); - AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_ctrl_va), - AMD_IOMMU_EVENTINT_ENABLE, 1); -} - -static int -amd_iommu_setup_exclusion(amd_iommu_t *iommu) -{ - amd_iommu_acpi_ivmd_t *minfop; - - minfop = amd_iommu_lookup_all_ivmd(); - - if (minfop && minfop->acm_ExclRange == 1) { - cmn_err(CE_NOTE, "Programming exclusion range"); - AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_excl_base_va), - AMD_IOMMU_EXCL_BASE_ADDR, - minfop->acm_ivmd_phys_start >> 12); - AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_excl_base_va), - AMD_IOMMU_EXCL_BASE_ALLOW, 1); - AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_excl_base_va), - AMD_IOMMU_EXCL_BASE_EXEN, 1); - AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_excl_lim_va), - AMD_IOMMU_EXCL_LIM, (minfop->acm_ivmd_phys_start + - minfop->acm_ivmd_phys_len) >> 12); - } else { - if (amd_iommu_debug) { - cmn_err(CE_NOTE, "Skipping exclusion range"); - } - AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_excl_base_va), - AMD_IOMMU_EXCL_BASE_ADDR, 0); - AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_excl_base_va), - AMD_IOMMU_EXCL_BASE_ALLOW, 1); - AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_excl_base_va), - AMD_IOMMU_EXCL_BASE_EXEN, 0); - AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_excl_lim_va), - AMD_IOMMU_EXCL_LIM, 0); - } - - return (DDI_SUCCESS); -} - -static void -amd_iommu_teardown_exclusion(amd_iommu_t *iommu) -{ - (void) amd_iommu_setup_exclusion(iommu); -} - -static uint_t -amd_iommu_intr_handler(caddr_t arg1, caddr_t arg2) -{ - /*LINTED*/ - amd_iommu_t *iommu = (amd_iommu_t *)arg1; - dev_info_t *dip = iommu->aiomt_dip; - int instance = ddi_get_instance(dip); - const char *driver = ddi_driver_name(dip); - const char *f = "amd_iommu_intr_handler"; - - ASSERT(arg1); - ASSERT(arg2 == NULL); - - if (amd_iommu_debug & AMD_IOMMU_DEBUG_INTR) { - cmn_err(CE_NOTE, "%s: %s%d: IOMMU unit idx=%d. In INTR handler", - f, driver, instance, iommu->aiomt_idx); - } - - if (AMD_IOMMU_REG_GET64(REGADDR64(iommu->aiomt_reg_status_va), - AMD_IOMMU_EVENT_LOG_INT) == 1) { - if (amd_iommu_debug & AMD_IOMMU_DEBUG_INTR) { - cmn_err(CE_NOTE, "%s: %s%d: IOMMU unit idx=%d " - "Event Log Interrupt", f, driver, instance, - iommu->aiomt_idx); - } - (void) amd_iommu_read_log(iommu, AMD_IOMMU_LOG_DISPLAY); - WAIT_SEC(1); - AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_status_va), - AMD_IOMMU_EVENT_LOG_INT, 1); - return (DDI_INTR_CLAIMED); - } - - if (AMD_IOMMU_REG_GET64(REGADDR64(iommu->aiomt_reg_status_va), - AMD_IOMMU_EVENT_OVERFLOW_INT) == 1) { - cmn_err(CE_NOTE, "!%s: %s%d: IOMMU unit idx=%d " - "Event Overflow Interrupt", f, driver, instance, - iommu->aiomt_idx); - (void) amd_iommu_read_log(iommu, AMD_IOMMU_LOG_DISCARD); - AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_status_va), - AMD_IOMMU_EVENT_LOG_INT, 1); - AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_status_va), - AMD_IOMMU_EVENT_OVERFLOW_INT, 1); - return (DDI_INTR_CLAIMED); - } - - return (DDI_INTR_UNCLAIMED); -} - - -static int -amd_iommu_setup_interrupts(amd_iommu_t *iommu) -{ - dev_info_t *dip = iommu->aiomt_dip; - int instance = ddi_get_instance(dip); - const char *driver = ddi_driver_name(dip); - int intrcap0; - int intrcapN; - int type; - int err; - int req; - int avail; - int p2req; - int actual; - int i; - int j; - const char *f = "amd_iommu_setup_interrupts"; - - if (ddi_intr_get_supported_types(dip, &type) != DDI_SUCCESS) { - cmn_err(CE_WARN, "%s: %s%d: ddi_intr_get_supported_types " - "failed: idx=%d", f, driver, instance, iommu->aiomt_idx); - return (DDI_FAILURE); - } - - if (amd_iommu_debug & AMD_IOMMU_DEBUG_INTR) { - cmn_err(CE_NOTE, "%s: %s%d: AMD IOMMU idx=%d. " - "Interrupt types supported = 0x%x", f, driver, instance, - iommu->aiomt_idx, type); - } - - /* - * for now we only support MSI - */ - if ((type & DDI_INTR_TYPE_MSI) == 0) { - cmn_err(CE_WARN, "%s: %s%d: AMD IOMMU idx=%d. " - "MSI interrupts not supported. Failing init.", - f, driver, instance, iommu->aiomt_idx); - return (DDI_FAILURE); - } - - if (amd_iommu_debug & AMD_IOMMU_DEBUG_INTR) { - cmn_err(CE_NOTE, "%s: %s%d: AMD IOMMU idx=%d. MSI supported", - f, driver, instance, iommu->aiomt_idx); - } - - err = ddi_intr_get_nintrs(dip, DDI_INTR_TYPE_MSI, &req); - if (err != DDI_SUCCESS) { - cmn_err(CE_WARN, "%s: %s%d: AMD IOMMU idx=%d. " - "ddi_intr_get_nintrs failed err = %d", - f, driver, instance, iommu->aiomt_idx, err); - return (DDI_FAILURE); - } - - if (amd_iommu_debug & AMD_IOMMU_DEBUG_INTR) { - cmn_err(CE_NOTE, "%s: %s%d: AMD IOMMU idx=%d. " - "MSI number of interrupts requested: %d", - f, driver, instance, iommu->aiomt_idx, req); - } - - if (req == 0) { - cmn_err(CE_WARN, "%s: %s%d: AMD IOMMU idx=%d: 0 MSI " - "interrupts requested. Failing init", f, - driver, instance, iommu->aiomt_idx); - return (DDI_FAILURE); - } - - err = ddi_intr_get_navail(dip, DDI_INTR_TYPE_MSI, &avail); - if (err != DDI_SUCCESS) { - cmn_err(CE_WARN, "%s: %s%d: AMD IOMMU idx=%d " - "ddi_intr_get_navail failed err = %d", f, - driver, instance, iommu->aiomt_idx, err); - return (DDI_FAILURE); - } - - if (amd_iommu_debug & AMD_IOMMU_DEBUG_INTR) { - cmn_err(CE_NOTE, "%s: %s%d: AMD IOMMU idx=%d. " - "MSI number of interrupts available: %d", - f, driver, instance, iommu->aiomt_idx, avail); - } - - if (avail == 0) { - cmn_err(CE_WARN, "%s: %s%d: AMD IOMMU idx=%d: 0 MSI " - "interrupts available. Failing init", f, - driver, instance, iommu->aiomt_idx); - return (DDI_FAILURE); - } - - if (avail < req) { - cmn_err(CE_WARN, "%s: %s%d: AMD IOMMU idx=%d: MSI " - "interrupts: requested (%d) > available (%d). " - "Failing init", f, driver, instance, iommu->aiomt_idx, - req, avail); - return (DDI_FAILURE); - } - - /* Allocate memory for DDI interrupt handles */ - iommu->aiomt_intr_htable_sz = req * sizeof (ddi_intr_handle_t); - iommu->aiomt_intr_htable = kmem_zalloc(iommu->aiomt_intr_htable_sz, - KM_SLEEP); - - iommu->aiomt_intr_state = AMD_IOMMU_INTR_TABLE; - - /* Convert req to a power of two as required by ddi_intr_alloc */ - p2req = 0; - while (1<<p2req <= req) - p2req++; - p2req--; - req = 1<<p2req; - - if (amd_iommu_debug & AMD_IOMMU_DEBUG_INTR) { - cmn_err(CE_NOTE, "%s: %s%d: AMD IOMMU idx=%d. " - "MSI power of 2 number of interrupts: %d,%d", - f, driver, instance, iommu->aiomt_idx, p2req, req); - } - - err = ddi_intr_alloc(iommu->aiomt_dip, iommu->aiomt_intr_htable, - DDI_INTR_TYPE_MSI, 0, req, &actual, DDI_INTR_ALLOC_STRICT); - if (err != DDI_SUCCESS) { - cmn_err(CE_WARN, "%s: %s%d: AMD IOMMU idx=%d: " - "ddi_intr_alloc failed: err = %d", - f, driver, instance, iommu->aiomt_idx, err); - amd_iommu_teardown_interrupts(iommu); - return (DDI_FAILURE); - } - - iommu->aiomt_actual_intrs = actual; - iommu->aiomt_intr_state = AMD_IOMMU_INTR_ALLOCED; - - if (amd_iommu_debug & AMD_IOMMU_DEBUG_INTR) { - cmn_err(CE_NOTE, "%s: %s%d: AMD IOMMU idx=%d. " - "number of interrupts actually allocated %d", - f, driver, instance, iommu->aiomt_idx, actual); - } - - if (iommu->aiomt_actual_intrs < req) { - cmn_err(CE_WARN, "%s: %s%d: AMD IOMMU idx=%d: " - "ddi_intr_alloc failed: actual (%d) < req (%d)", - f, driver, instance, iommu->aiomt_idx, - iommu->aiomt_actual_intrs, req); - amd_iommu_teardown_interrupts(iommu); - return (DDI_FAILURE); - } - - for (i = 0; i < iommu->aiomt_actual_intrs; i++) { - if (ddi_intr_add_handler(iommu->aiomt_intr_htable[i], - amd_iommu_intr_handler, (void *)iommu, NULL) - != DDI_SUCCESS) { - cmn_err(CE_WARN, "%s: %s%d: AMD IOMMU idx=%d: " - "ddi_intr_add_handler failed: intr = %d, err = %d", - f, driver, instance, iommu->aiomt_idx, i, err); - for (j = 0; j < i; j++) { - (void) ddi_intr_remove_handler( - iommu->aiomt_intr_htable[j]); - } - amd_iommu_teardown_interrupts(iommu); - return (DDI_FAILURE); - } - } - iommu->aiomt_intr_state = AMD_IOMMU_INTR_HANDLER; - - intrcap0 = intrcapN = -1; - if (ddi_intr_get_cap(iommu->aiomt_intr_htable[0], &intrcap0) - != DDI_SUCCESS || - ddi_intr_get_cap( - iommu->aiomt_intr_htable[iommu->aiomt_actual_intrs - 1], &intrcapN) - != DDI_SUCCESS || intrcap0 != intrcapN) { - cmn_err(CE_WARN, "%s: %s%d: AMD IOMMU idx=%d: " - "ddi_intr_get_cap failed or inconsistent cap among " - "interrupts: intrcap0 (%d) < intrcapN (%d)", - f, driver, instance, iommu->aiomt_idx, intrcap0, intrcapN); - amd_iommu_teardown_interrupts(iommu); - return (DDI_FAILURE); - } - iommu->aiomt_intr_cap = intrcap0; - - if (intrcap0 & DDI_INTR_FLAG_BLOCK) { - /* Need to call block enable */ - if (amd_iommu_debug & AMD_IOMMU_DEBUG_INTR) { - cmn_err(CE_NOTE, "%s: %s%d: AMD IOMMU idx=%d: " - "Need to call block enable", - f, driver, instance, iommu->aiomt_idx); - } - if (ddi_intr_block_enable(iommu->aiomt_intr_htable, - iommu->aiomt_actual_intrs) != DDI_SUCCESS) { - cmn_err(CE_WARN, "%s: %s%d: AMD IOMMU idx=%d: " - "ddi_intr_block enable failed ", f, driver, - instance, iommu->aiomt_idx); - (void) ddi_intr_block_disable(iommu->aiomt_intr_htable, - iommu->aiomt_actual_intrs); - amd_iommu_teardown_interrupts(iommu); - return (DDI_FAILURE); - } - } else { - if (amd_iommu_debug & AMD_IOMMU_DEBUG_INTR) { - cmn_err(CE_NOTE, "%s: %s%d: AMD IOMMU idx=%d: " - "Need to call individual enable", - f, driver, instance, iommu->aiomt_idx); - } - for (i = 0; i < iommu->aiomt_actual_intrs; i++) { - if (ddi_intr_enable(iommu->aiomt_intr_htable[i]) - != DDI_SUCCESS) { - cmn_err(CE_WARN, "%s: %s%d: AMD IOMMU idx=%d: " - "ddi_intr_enable failed: intr = %d", f, - driver, instance, iommu->aiomt_idx, i); - for (j = 0; j < i; j++) { - (void) ddi_intr_disable( - iommu->aiomt_intr_htable[j]); - } - amd_iommu_teardown_interrupts(iommu); - return (DDI_FAILURE); - } - } - } - iommu->aiomt_intr_state = AMD_IOMMU_INTR_ENABLED; - - if (amd_iommu_debug & AMD_IOMMU_DEBUG_INTR) { - cmn_err(CE_NOTE, "%s: %s%d: AMD IOMMU idx=%d: " - "Interrupts successfully %s enabled. # of interrupts = %d", - f, driver, instance, iommu->aiomt_idx, - (intrcap0 & DDI_INTR_FLAG_BLOCK) ? "(block)" : - "(individually)", iommu->aiomt_actual_intrs); - } - - return (DDI_SUCCESS); -} - -static void -amd_iommu_teardown_interrupts(amd_iommu_t *iommu) -{ - int i; - - if (iommu->aiomt_intr_state & AMD_IOMMU_INTR_ENABLED) { - if (iommu->aiomt_intr_cap & DDI_INTR_FLAG_BLOCK) { - (void) ddi_intr_block_disable(iommu->aiomt_intr_htable, - iommu->aiomt_actual_intrs); - } else { - for (i = 0; i < iommu->aiomt_actual_intrs; i++) { - (void) ddi_intr_disable( - iommu->aiomt_intr_htable[i]); - } - } - } - - if (iommu->aiomt_intr_state & AMD_IOMMU_INTR_HANDLER) { - for (i = 0; i < iommu->aiomt_actual_intrs; i++) { - (void) ddi_intr_remove_handler( - iommu->aiomt_intr_htable[i]); - } - } - - if (iommu->aiomt_intr_state & AMD_IOMMU_INTR_ALLOCED) { - for (i = 0; i < iommu->aiomt_actual_intrs; i++) { - (void) ddi_intr_free(iommu->aiomt_intr_htable[i]); - } - } - if (iommu->aiomt_intr_state & AMD_IOMMU_INTR_TABLE) { - kmem_free(iommu->aiomt_intr_htable, - iommu->aiomt_intr_htable_sz); - } - iommu->aiomt_intr_htable = NULL; - iommu->aiomt_intr_htable_sz = 0; - iommu->aiomt_intr_state = AMD_IOMMU_INTR_INVALID; -} - -static amd_iommu_t * -amd_iommu_init(dev_info_t *dip, ddi_acc_handle_t handle, int idx, - uint16_t cap_base) -{ - amd_iommu_t *iommu; - int instance = ddi_get_instance(dip); - const char *driver = ddi_driver_name(dip); - uint32_t caphdr; - uint32_t low_addr32; - uint32_t hi_addr32; - uint32_t range; - uint32_t misc; - uint64_t pgoffset; - amd_iommu_acpi_global_t *global; - amd_iommu_acpi_ivhd_t *hinfop; - const char *f = "amd_iommu_init"; - - global = amd_iommu_lookup_acpi_global(); - hinfop = amd_iommu_lookup_any_ivhd(); - - low_addr32 = PCI_CAP_GET32(handle, 0, cap_base, - AMD_IOMMU_CAP_ADDR_LOW_OFF); - if (!(low_addr32 & AMD_IOMMU_REG_ADDR_LOCKED)) { - cmn_err(CE_WARN, "%s: %s%d: capability registers not locked. " - "Unable to use IOMMU unit idx=%d - skipping ...", f, driver, - instance, idx); - return (NULL); - } - - iommu = kmem_zalloc(sizeof (amd_iommu_t), KM_SLEEP); - mutex_init(&iommu->aiomt_mutex, NULL, MUTEX_DRIVER, NULL); - mutex_enter(&iommu->aiomt_mutex); - - mutex_init(&iommu->aiomt_cmdlock, NULL, MUTEX_DRIVER, NULL); - mutex_init(&iommu->aiomt_eventlock, NULL, MUTEX_DRIVER, NULL); - - iommu->aiomt_dip = dip; - iommu->aiomt_idx = idx; - - /* - * Since everything in the capability block is locked and RO at this - * point, copy everything into the IOMMU struct - */ - - /* Get cap header */ - caphdr = PCI_CAP_GET32(handle, 0, cap_base, AMD_IOMMU_CAP_HDR_OFF); - iommu->aiomt_cap_hdr = caphdr; - iommu->aiomt_npcache = AMD_IOMMU_REG_GET32(&caphdr, - AMD_IOMMU_CAP_NPCACHE); - iommu->aiomt_httun = AMD_IOMMU_REG_GET32(&caphdr, AMD_IOMMU_CAP_HTTUN); - - if (hinfop) - iommu->aiomt_iotlb = hinfop->ach_IotlbSup; - else - iommu->aiomt_iotlb = - AMD_IOMMU_REG_GET32(&caphdr, AMD_IOMMU_CAP_IOTLB); - - iommu->aiomt_captype = AMD_IOMMU_REG_GET32(&caphdr, AMD_IOMMU_CAP_TYPE); - iommu->aiomt_capid = AMD_IOMMU_REG_GET32(&caphdr, AMD_IOMMU_CAP_ID); - - /* - * Get address of IOMMU control registers - */ - hi_addr32 = PCI_CAP_GET32(handle, 0, cap_base, - AMD_IOMMU_CAP_ADDR_HI_OFF); - iommu->aiomt_low_addr32 = low_addr32; - iommu->aiomt_hi_addr32 = hi_addr32; - low_addr32 &= ~AMD_IOMMU_REG_ADDR_LOCKED; - - if (hinfop) { - iommu->aiomt_reg_pa = hinfop->ach_IOMMU_reg_base; - ASSERT(hinfop->ach_IOMMU_pci_seg == 0); - } else { - iommu->aiomt_reg_pa = ((uint64_t)hi_addr32 << 32 | low_addr32); - } - - /* - * Get cap range reg - */ - range = PCI_CAP_GET32(handle, 0, cap_base, AMD_IOMMU_CAP_RANGE_OFF); - iommu->aiomt_range = range; - iommu->aiomt_rng_valid = AMD_IOMMU_REG_GET32(&range, - AMD_IOMMU_RNG_VALID); - if (iommu->aiomt_rng_valid) { - iommu->aiomt_rng_bus = AMD_IOMMU_REG_GET32(&range, - AMD_IOMMU_RNG_BUS); - iommu->aiomt_first_devfn = AMD_IOMMU_REG_GET32(&range, - AMD_IOMMU_FIRST_DEVFN); - iommu->aiomt_last_devfn = AMD_IOMMU_REG_GET32(&range, - AMD_IOMMU_LAST_DEVFN); - } else { - iommu->aiomt_rng_bus = 0; - iommu->aiomt_first_devfn = 0; - iommu->aiomt_last_devfn = 0; - } - - if (hinfop) - iommu->aiomt_ht_unitid = hinfop->ach_IOMMU_UnitID; - else - iommu->aiomt_ht_unitid = AMD_IOMMU_REG_GET32(&range, - AMD_IOMMU_HT_UNITID); - - /* - * Get cap misc reg - */ - misc = PCI_CAP_GET32(handle, 0, cap_base, AMD_IOMMU_CAP_MISC_OFF); - iommu->aiomt_misc = misc; - - if (global) { - iommu->aiomt_htatsresv = global->acg_HtAtsResv; - iommu->aiomt_vasize = global->acg_VAsize; - iommu->aiomt_pasize = global->acg_PAsize; - } else { - iommu->aiomt_htatsresv = AMD_IOMMU_REG_GET32(&misc, - AMD_IOMMU_HT_ATSRSV); - iommu->aiomt_vasize = AMD_IOMMU_REG_GET32(&misc, - AMD_IOMMU_VA_SIZE); - iommu->aiomt_pasize = AMD_IOMMU_REG_GET32(&misc, - AMD_IOMMU_PA_SIZE); - } - - if (hinfop) { - iommu->aiomt_msinum = hinfop->ach_IOMMU_MSInum; - } else { - iommu->aiomt_msinum = - AMD_IOMMU_REG_GET32(&misc, AMD_IOMMU_MSINUM); - } - - /* - * Set up mapping between control registers PA and VA - */ - pgoffset = iommu->aiomt_reg_pa & MMU_PAGEOFFSET; - ASSERT(pgoffset == 0); - iommu->aiomt_reg_pages = mmu_btopr(AMD_IOMMU_REG_SIZE + pgoffset); - iommu->aiomt_reg_size = mmu_ptob(iommu->aiomt_reg_pages); - - iommu->aiomt_va = (uintptr_t)device_arena_alloc( - ptob(iommu->aiomt_reg_pages), VM_SLEEP); - if (iommu->aiomt_va == 0) { - cmn_err(CE_WARN, "%s: %s%d: Failed to alloc VA for IOMMU " - "control regs. Skipping IOMMU idx=%d", f, driver, - instance, idx); - mutex_exit(&iommu->aiomt_mutex); - (void) amd_iommu_fini(iommu); - return (NULL); - } - - hat_devload(kas.a_hat, (void *)(uintptr_t)iommu->aiomt_va, - iommu->aiomt_reg_size, - mmu_btop(iommu->aiomt_reg_pa), PROT_READ | PROT_WRITE - | HAT_STRICTORDER, HAT_LOAD_LOCK); - - iommu->aiomt_reg_va = iommu->aiomt_va + pgoffset; - - /* - * Setup the various control register's VA - */ - iommu->aiomt_reg_devtbl_va = iommu->aiomt_reg_va + - AMD_IOMMU_DEVTBL_REG_OFF; - iommu->aiomt_reg_cmdbuf_va = iommu->aiomt_reg_va + - AMD_IOMMU_CMDBUF_REG_OFF; - iommu->aiomt_reg_eventlog_va = iommu->aiomt_reg_va + - AMD_IOMMU_EVENTLOG_REG_OFF; - iommu->aiomt_reg_ctrl_va = iommu->aiomt_reg_va + - AMD_IOMMU_CTRL_REG_OFF; - iommu->aiomt_reg_excl_base_va = iommu->aiomt_reg_va + - AMD_IOMMU_EXCL_BASE_REG_OFF; - iommu->aiomt_reg_excl_lim_va = iommu->aiomt_reg_va + - AMD_IOMMU_EXCL_LIM_REG_OFF; - iommu->aiomt_reg_cmdbuf_head_va = iommu->aiomt_reg_va + - AMD_IOMMU_CMDBUF_HEAD_REG_OFF; - iommu->aiomt_reg_cmdbuf_tail_va = iommu->aiomt_reg_va + - AMD_IOMMU_CMDBUF_TAIL_REG_OFF; - iommu->aiomt_reg_eventlog_head_va = iommu->aiomt_reg_va + - AMD_IOMMU_EVENTLOG_HEAD_REG_OFF; - iommu->aiomt_reg_eventlog_tail_va = iommu->aiomt_reg_va + - AMD_IOMMU_EVENTLOG_TAIL_REG_OFF; - iommu->aiomt_reg_status_va = iommu->aiomt_reg_va + - AMD_IOMMU_STATUS_REG_OFF; - - - /* - * Setup the DEVICE table, CMD buffer, and LOG buffer in - * memory and setup DMA access to this memory location - */ - if (amd_iommu_setup_tables_and_buffers(iommu) != DDI_SUCCESS) { - mutex_exit(&iommu->aiomt_mutex); - (void) amd_iommu_fini(iommu); - return (NULL); - } - - if (amd_iommu_setup_exclusion(iommu) != DDI_SUCCESS) { - mutex_exit(&iommu->aiomt_mutex); - (void) amd_iommu_fini(iommu); - return (NULL); - } - - amd_iommu_enable_interrupts(iommu); - - if (amd_iommu_setup_interrupts(iommu) != DDI_SUCCESS) { - mutex_exit(&iommu->aiomt_mutex); - (void) amd_iommu_fini(iommu); - return (NULL); - } - - /* - * need to setup domain table before gfx bypass - */ - amd_iommu_init_page_tables(iommu); - - /* - * Set pass-thru for special devices like IOAPIC and HPET - * - * Also, gfx devices don't use DDI for DMA. No need to register - * before setting up gfx passthru - */ - if (amd_iommu_setup_passthru(iommu) != DDI_SUCCESS) { - mutex_exit(&iommu->aiomt_mutex); - (void) amd_iommu_fini(iommu); - return (NULL); - } - - if (amd_iommu_start(iommu) != DDI_SUCCESS) { - mutex_exit(&iommu->aiomt_mutex); - (void) amd_iommu_fini(iommu); - return (NULL); - } - - /* xxx register/start race */ - if (amd_iommu_register(iommu) != DDI_SUCCESS) { - mutex_exit(&iommu->aiomt_mutex); - (void) amd_iommu_fini(iommu); - return (NULL); - } - - if (amd_iommu_debug) { - cmn_err(CE_NOTE, "%s: %s%d: IOMMU idx=%d inited.", f, driver, - instance, idx); - } - - return (iommu); -} - -static int -amd_iommu_fini(amd_iommu_t *iommu) -{ - int idx = iommu->aiomt_idx; - dev_info_t *dip = iommu->aiomt_dip; - int instance = ddi_get_instance(dip); - const char *driver = ddi_driver_name(dip); - const char *f = "amd_iommu_fini"; - - mutex_enter(&iommu->aiomt_mutex); - if (amd_iommu_unregister(iommu) != DDI_SUCCESS) { - cmn_err(CE_NOTE, "%s: %s%d: Fini of IOMMU unit failed. " - "idx = %d", f, driver, instance, idx); - return (DDI_FAILURE); - } - amd_iommu_stop(iommu); - amd_iommu_fini_page_tables(iommu); - amd_iommu_teardown_interrupts(iommu); - amd_iommu_teardown_exclusion(iommu); - amd_iommu_teardown_tables_and_buffers(iommu); - if (iommu->aiomt_va != NULL) { - hat_unload(kas.a_hat, (void *)(uintptr_t)iommu->aiomt_va, - iommu->aiomt_reg_size, HAT_UNLOAD_UNLOCK); - device_arena_free((void *)(uintptr_t)iommu->aiomt_va, - ptob(iommu->aiomt_reg_pages)); - iommu->aiomt_va = NULL; - iommu->aiomt_reg_va = NULL; - } - mutex_destroy(&iommu->aiomt_eventlock); - mutex_destroy(&iommu->aiomt_cmdlock); - mutex_exit(&iommu->aiomt_mutex); - mutex_destroy(&iommu->aiomt_mutex); - kmem_free(iommu, sizeof (amd_iommu_t)); - - cmn_err(CE_NOTE, "%s: %s%d: Fini of IOMMU unit complete. idx = %d", - f, driver, instance, idx); - - return (DDI_SUCCESS); -} - -int -amd_iommu_setup(dev_info_t *dip, amd_iommu_state_t *statep) -{ - int instance = ddi_get_instance(dip); - const char *driver = ddi_driver_name(dip); - ddi_acc_handle_t handle; - uint8_t base_class; - uint8_t sub_class; - uint8_t prog_class; - int idx; - uint32_t id; - uint16_t cap_base; - uint32_t caphdr; - uint8_t cap_type; - uint8_t cap_id; - amd_iommu_t *iommu; - const char *f = "amd_iommu_setup"; - - ASSERT(instance >= 0); - ASSERT(driver); - - /* First setup PCI access to config space */ - - if (pci_config_setup(dip, &handle) != DDI_SUCCESS) { - cmn_err(CE_WARN, "%s: PCI config setup failed: %s%d", - f, driver, instance); - return (DDI_FAILURE); - } - - /* - * The AMD IOMMU is part of an independent PCI function. There may be - * more than one IOMMU in that PCI function - */ - base_class = pci_config_get8(handle, PCI_CONF_BASCLASS); - sub_class = pci_config_get8(handle, PCI_CONF_SUBCLASS); - prog_class = pci_config_get8(handle, PCI_CONF_PROGCLASS); - - if (base_class != PCI_CLASS_PERIPH || sub_class != PCI_PERIPH_IOMMU || - prog_class != AMD_IOMMU_PCI_PROG_IF) { - cmn_err(CE_WARN, "%s: %s%d: invalid PCI class(0x%x)/" - "subclass(0x%x)/programming interface(0x%x)", f, driver, - instance, base_class, sub_class, prog_class); - pci_config_teardown(&handle); - return (DDI_FAILURE); - } - - /* - * Find and initialize all IOMMU units in this function - */ - for (idx = 0; ; idx++) { - if (pci_cap_probe(handle, idx, &id, &cap_base) != DDI_SUCCESS) - break; - - /* check if cap ID is secure device cap id */ - if (id != PCI_CAP_ID_SECURE_DEV) { - if (amd_iommu_debug) { - cmn_err(CE_WARN, - "%s: %s%d: skipping IOMMU: idx(0x%x) " - "cap ID (0x%x) != secure dev capid (0x%x)", - f, driver, instance, idx, id, - PCI_CAP_ID_SECURE_DEV); - } - continue; - } - - /* check if cap type is IOMMU cap type */ - caphdr = PCI_CAP_GET32(handle, 0, cap_base, - AMD_IOMMU_CAP_HDR_OFF); - cap_type = AMD_IOMMU_REG_GET32(&caphdr, AMD_IOMMU_CAP_TYPE); - cap_id = AMD_IOMMU_REG_GET32(&caphdr, AMD_IOMMU_CAP_ID); - - if (cap_type != AMD_IOMMU_CAP) { - cmn_err(CE_WARN, "%s: %s%d: skipping IOMMU: idx(0x%x) " - "cap type (0x%x) != AMD IOMMU CAP (0x%x)", f, - driver, instance, idx, cap_type, AMD_IOMMU_CAP); - continue; - } - ASSERT(cap_id == PCI_CAP_ID_SECURE_DEV); - ASSERT(cap_id == id); - - iommu = amd_iommu_init(dip, handle, idx, cap_base); - if (iommu == NULL) { - cmn_err(CE_WARN, "%s: %s%d: skipping IOMMU: idx(0x%x) " - "failed to init IOMMU", f, - driver, instance, idx); - continue; - } - - if (statep->aioms_iommu_start == NULL) { - statep->aioms_iommu_start = iommu; - } else { - statep->aioms_iommu_end->aiomt_next = iommu; - } - statep->aioms_iommu_end = iommu; - - statep->aioms_nunits++; - } - - pci_config_teardown(&handle); - - if (amd_iommu_debug) { - cmn_err(CE_NOTE, "%s: %s%d: state=%p: setup %d IOMMU units", - f, driver, instance, (void *)statep, statep->aioms_nunits); - } - - return (DDI_SUCCESS); -} - -int -amd_iommu_teardown(dev_info_t *dip, amd_iommu_state_t *statep) -{ - int instance = ddi_get_instance(dip); - const char *driver = ddi_driver_name(dip); - amd_iommu_t *iommu; - int teardown; - int error = DDI_SUCCESS; - const char *f = "amd_iommu_teardown"; - - teardown = 0; - for (iommu = statep->aioms_iommu_start; iommu; - iommu = iommu->aiomt_next) { - ASSERT(statep->aioms_nunits > 0); - if (amd_iommu_fini(iommu) != DDI_SUCCESS) { - error = DDI_FAILURE; - continue; - } - statep->aioms_nunits--; - teardown++; - } - - cmn_err(CE_NOTE, "%s: %s%d: state=%p: toredown %d units. " - "%d units left", f, driver, instance, (void *)statep, - teardown, statep->aioms_nunits); - - return (error); -} - -/* Interface with IOMMULIB */ -/*ARGSUSED*/ -static int -amd_iommu_probe(iommulib_handle_t handle, dev_info_t *rdip) -{ - const char *driver = ddi_driver_name(rdip); - char *s; - amd_iommu_t *iommu = iommulib_iommu_getdata(handle); - - if (amd_iommu_disable_list) { - s = strstr(amd_iommu_disable_list, driver); - if (s == NULL) - return (DDI_SUCCESS); - if (s == amd_iommu_disable_list || *(s - 1) == ':') { - s += strlen(driver); - if (*s == '\0' || *s == ':') { - amd_iommu_set_passthru(iommu, rdip); - return (DDI_FAILURE); - } - } - } - - return (DDI_SUCCESS); -} - -/*ARGSUSED*/ -static int -amd_iommu_allochdl(iommulib_handle_t handle, - dev_info_t *dip, dev_info_t *rdip, ddi_dma_attr_t *attr, - int (*waitfp)(caddr_t), caddr_t arg, ddi_dma_handle_t *dma_handlep) -{ - return (iommulib_iommu_dma_allochdl(dip, rdip, attr, waitfp, - arg, dma_handlep)); -} - -/*ARGSUSED*/ -static int -amd_iommu_freehdl(iommulib_handle_t handle, - dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t dma_handle) -{ - return (iommulib_iommu_dma_freehdl(dip, rdip, dma_handle)); -} - -/*ARGSUSED*/ -static int -map_current_window(amd_iommu_t *iommu, dev_info_t *rdip, ddi_dma_attr_t *attrp, - struct ddi_dma_req *dmareq, ddi_dma_cookie_t *cookie_array, uint_t ccount, - int km_flags) -{ - const char *driver = ddi_driver_name(iommu->aiomt_dip); - int instance = ddi_get_instance(iommu->aiomt_dip); - int idx = iommu->aiomt_idx; - int i; - uint64_t start_va; - char *path; - int error = DDI_FAILURE; - const char *f = "map_current_window"; - - path = kmem_alloc(MAXPATHLEN, km_flags); - if (path == NULL) { - return (DDI_DMA_NORESOURCES); - } - - (void) ddi_pathname(rdip, path); - mutex_enter(&amd_iommu_pgtable_lock); - - if (amd_iommu_debug == AMD_IOMMU_DEBUG_PAGE_TABLES) { - cmn_err(CE_WARN, "%s: %s%d: idx=%d Attempting to get cookies " - "from handle for device %s", - f, driver, instance, idx, path); - } - - start_va = 0; - for (i = 0; i < ccount; i++) { - if ((error = amd_iommu_map_pa2va(iommu, rdip, attrp, dmareq, - cookie_array[i].dmac_cookie_addr, - cookie_array[i].dmac_size, - AMD_IOMMU_VMEM_MAP, &start_va, km_flags)) != DDI_SUCCESS) { - break; - } - cookie_array[i].dmac_cookie_addr = (uintptr_t)start_va; - cookie_array[i].dmac_type = 0; - } - - if (i != ccount) { - cmn_err(CE_WARN, "%s: %s%d: idx=%d Cannot map cookie# %d " - "for device %s", f, driver, instance, idx, i, path); - (void) unmap_current_window(iommu, rdip, cookie_array, - ccount, i, 1); - goto out; - } - - if (amd_iommu_debug & AMD_IOMMU_DEBUG_PAGE_TABLES) { - cmn_err(CE_NOTE, "%s: return SUCCESS", f); - } - - error = DDI_DMA_MAPPED; -out: - mutex_exit(&amd_iommu_pgtable_lock); - kmem_free(path, MAXPATHLEN); - return (error); -} - -/*ARGSUSED*/ -static int -unmap_current_window(amd_iommu_t *iommu, dev_info_t *rdip, - ddi_dma_cookie_t *cookie_array, uint_t ccount, int ncookies, int locked) -{ - const char *driver = ddi_driver_name(iommu->aiomt_dip); - int instance = ddi_get_instance(iommu->aiomt_dip); - int idx = iommu->aiomt_idx; - int i; - int error = DDI_FAILURE; - char *path; - int pathfree; - const char *f = "unmap_current_window"; - - if (!locked) - mutex_enter(&amd_iommu_pgtable_lock); - - path = kmem_alloc(MAXPATHLEN, KM_NOSLEEP); - if (path) { - (void) ddi_pathname(rdip, path); - pathfree = 1; - } else { - path = "<path-mem-alloc-failed>"; - pathfree = 0; - } - - if (ncookies == -1) - ncookies = ccount; - - for (i = 0; i < ncookies; i++) { - if (amd_iommu_unmap_va(iommu, rdip, - cookie_array[i].dmac_cookie_addr, - cookie_array[i].dmac_size, - AMD_IOMMU_VMEM_MAP) != DDI_SUCCESS) { - break; - } - } - - if (amd_iommu_cmd(iommu, AMD_IOMMU_CMD_COMPL_WAIT, NULL, 0, 0) - != DDI_SUCCESS) { - cmn_err(CE_WARN, "%s: AMD IOMMU completion wait failed for: %s", - f, path); - } - - if (i != ncookies) { - cmn_err(CE_WARN, "%s: %s%d: idx=%d Cannot unmap cookie# %d " - "for device %s", f, driver, instance, idx, i, path); - error = DDI_FAILURE; - goto out; - } - - error = DDI_SUCCESS; - -out: - if (pathfree) - kmem_free(path, MAXPATHLEN); - if (!locked) - mutex_exit(&amd_iommu_pgtable_lock); - return (error); -} - -/*ARGSUSED*/ -static int -amd_iommu_bindhdl(iommulib_handle_t handle, dev_info_t *dip, - dev_info_t *rdip, ddi_dma_handle_t dma_handle, - struct ddi_dma_req *dmareq, ddi_dma_cookie_t *cookiep, - uint_t *ccountp) -{ - int dma_error = DDI_DMA_NOMAPPING; - int error; - char *path; - ddi_dma_cookie_t *cookie_array = NULL; - uint_t ccount = 0; - ddi_dma_impl_t *hp; - ddi_dma_attr_t *attrp; - int km_flags; - amd_iommu_t *iommu = iommulib_iommu_getdata(handle); - int instance = ddi_get_instance(rdip); - const char *driver = ddi_driver_name(rdip); - const char *f = "amd_iommu_bindhdl"; - - dma_error = iommulib_iommu_dma_bindhdl(dip, rdip, dma_handle, - dmareq, cookiep, ccountp); - - if (dma_error != DDI_DMA_MAPPED && dma_error != DDI_DMA_PARTIAL_MAP) - return (dma_error); - - km_flags = iommulib_iommu_dma_get_sleep_flags(dip, dma_handle); - - path = kmem_alloc(MAXPATHLEN, km_flags); - if (path) { - (void) ddi_pathname(rdip, path); - } else { - dma_error = DDI_DMA_NORESOURCES; - goto unbind; - } - - if (amd_iommu_debug & AMD_IOMMU_DEBUG_BIND) { - cmn_err(CE_NOTE, "%s: %s got cookie (%p), #cookies: %d", - f, path, - (void *)cookiep->dmac_cookie_addr, - *ccountp); - } - - cookie_array = NULL; - ccount = 0; - if ((error = iommulib_iommu_dma_get_cookies(dip, dma_handle, - &cookie_array, &ccount)) != DDI_SUCCESS) { - cmn_err(CE_WARN, "%s: %s%d: Cannot get cookies " - "for device %s", f, driver, instance, path); - dma_error = error; - goto unbind; - } - - hp = (ddi_dma_impl_t *)dma_handle; - attrp = &hp->dmai_attr; - - error = map_current_window(iommu, rdip, attrp, dmareq, - cookie_array, ccount, km_flags); - if (error != DDI_SUCCESS) { - dma_error = error; - goto unbind; - } - - if ((error = iommulib_iommu_dma_set_cookies(dip, dma_handle, - cookie_array, ccount)) != DDI_SUCCESS) { - cmn_err(CE_WARN, "%s: %s%d: Cannot set cookies " - "for device %s", f, driver, instance, path); - dma_error = error; - goto unbind; - } - - *cookiep = cookie_array[0]; - - if (amd_iommu_debug & AMD_IOMMU_DEBUG_BIND) { - cmn_err(CE_NOTE, "%s: %s remapped cookie (%p), #cookies: %d", - f, path, - (void *)(uintptr_t)cookiep->dmac_cookie_addr, - *ccountp); - } - - kmem_free(path, MAXPATHLEN); - ASSERT(dma_error == DDI_DMA_MAPPED || dma_error == DDI_DMA_PARTIAL_MAP); - return (dma_error); -unbind: - kmem_free(path, MAXPATHLEN); - (void) iommulib_iommu_dma_unbindhdl(dip, rdip, dma_handle); - return (dma_error); -} - -/*ARGSUSED*/ -static int -amd_iommu_unbindhdl(iommulib_handle_t handle, - dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t dma_handle) -{ - amd_iommu_t *iommu = iommulib_iommu_getdata(handle); - ddi_dma_cookie_t *cookie_array = NULL; - uint_t ccount = 0; - int error = DDI_FAILURE; - int instance = ddi_get_instance(rdip); - const char *driver = ddi_driver_name(rdip); - const char *f = "amd_iommu_unbindhdl"; - - cookie_array = NULL; - ccount = 0; - if (iommulib_iommu_dma_get_cookies(dip, dma_handle, &cookie_array, - &ccount) != DDI_SUCCESS) { - cmn_err(CE_WARN, "%s: %s%d: Cannot get cookies " - "for device %p", f, driver, instance, (void *)rdip); - error = DDI_FAILURE; - goto out; - } - - if (iommulib_iommu_dma_clear_cookies(dip, dma_handle) != DDI_SUCCESS) { - cmn_err(CE_WARN, "%s: %s%d: Cannot clear cookies " - "for device %p", f, driver, instance, (void *)rdip); - error = DDI_FAILURE; - goto out; - } - - if (iommulib_iommu_dma_unbindhdl(dip, rdip, dma_handle) - != DDI_SUCCESS) { - cmn_err(CE_WARN, "%s: %s%d: failed to unbindhdl for dip=%p", - f, driver, instance, (void *)rdip); - error = DDI_FAILURE; - goto out; - } - - if (unmap_current_window(iommu, rdip, cookie_array, ccount, -1, 0) - != DDI_SUCCESS) { - cmn_err(CE_WARN, "%s: %s%d: failed to unmap current window " - "for dip=%p", f, driver, instance, (void *)rdip); - error = DDI_FAILURE; - } else { - error = DDI_SUCCESS; - } -out: - if (cookie_array) - kmem_free(cookie_array, sizeof (ddi_dma_cookie_t) * ccount); - return (error); -} - -/*ARGSUSED*/ -static int -amd_iommu_sync(iommulib_handle_t handle, dev_info_t *dip, - dev_info_t *rdip, ddi_dma_handle_t dma_handle, off_t off, - size_t len, uint_t cache_flags) -{ - ddi_dma_cookie_t *cookie_array = NULL; - uint_t ccount = 0; - int error; - const char *f = "amd_iommu_sync"; - - cookie_array = NULL; - ccount = 0; - if (iommulib_iommu_dma_get_cookies(dip, dma_handle, &cookie_array, - &ccount) != DDI_SUCCESS) { - ASSERT(cookie_array == NULL); - cmn_err(CE_WARN, "%s: Cannot get cookies " - "for device %p", f, (void *)rdip); - error = DDI_FAILURE; - goto out; - } - - if (iommulib_iommu_dma_clear_cookies(dip, dma_handle) != DDI_SUCCESS) { - cmn_err(CE_WARN, "%s: Cannot clear cookies " - "for device %p", f, (void *)rdip); - error = DDI_FAILURE; - goto out; - } - - error = iommulib_iommu_dma_sync(dip, rdip, dma_handle, off, - len, cache_flags); - - if (iommulib_iommu_dma_set_cookies(dip, dma_handle, cookie_array, - ccount) != DDI_SUCCESS) { - cmn_err(CE_WARN, "%s: Cannot set cookies " - "for device %p", f, (void *)rdip); - error = DDI_FAILURE; - } else { - cookie_array = NULL; - ccount = 0; - } - -out: - if (cookie_array) - kmem_free(cookie_array, sizeof (ddi_dma_cookie_t) * ccount); - return (error); -} - -/*ARGSUSED*/ -static int -amd_iommu_win(iommulib_handle_t handle, dev_info_t *dip, - dev_info_t *rdip, ddi_dma_handle_t dma_handle, uint_t win, - off_t *offp, size_t *lenp, ddi_dma_cookie_t *cookiep, - uint_t *ccountp) -{ - int error = DDI_FAILURE; - amd_iommu_t *iommu = iommulib_iommu_getdata(handle); - ddi_dma_cookie_t *cookie_array = NULL; - uint_t ccount = 0; - int km_flags; - ddi_dma_impl_t *hp; - ddi_dma_attr_t *attrp; - struct ddi_dma_req sdmareq = {0}; - int instance = ddi_get_instance(rdip); - const char *driver = ddi_driver_name(rdip); - const char *f = "amd_iommu_win"; - - km_flags = iommulib_iommu_dma_get_sleep_flags(dip, dma_handle); - - cookie_array = NULL; - ccount = 0; - if (iommulib_iommu_dma_get_cookies(dip, dma_handle, &cookie_array, - &ccount) != DDI_SUCCESS) { - cmn_err(CE_WARN, "%s: %s%d: Cannot get cookies " - "for device %p", f, driver, instance, (void *)rdip); - error = DDI_FAILURE; - goto out; - } - - if (iommulib_iommu_dma_clear_cookies(dip, dma_handle) != DDI_SUCCESS) { - cmn_err(CE_WARN, "%s: %s%d: Cannot clear cookies " - "for device %p", f, driver, instance, (void *)rdip); - error = DDI_FAILURE; - goto out; - } - - if (iommulib_iommu_dma_win(dip, rdip, dma_handle, win, - offp, lenp, cookiep, ccountp) != DDI_SUCCESS) { - cmn_err(CE_WARN, "%s: %s%d: failed switch windows for dip=%p", - f, driver, instance, (void *)rdip); - error = DDI_FAILURE; - goto out; - } - - (void) unmap_current_window(iommu, rdip, cookie_array, ccount, -1, 0); - - if (cookie_array) { - kmem_free(cookie_array, sizeof (ddi_dma_cookie_t) * ccount); - cookie_array = NULL; - ccount = 0; - } - - cookie_array = NULL; - ccount = 0; - if (iommulib_iommu_dma_get_cookies(dip, dma_handle, &cookie_array, - &ccount) != DDI_SUCCESS) { - cmn_err(CE_WARN, "%s: %s%d: Cannot get cookies " - "for device %p", f, driver, instance, (void *)rdip); - error = DDI_FAILURE; - goto out; - } - - hp = (ddi_dma_impl_t *)dma_handle; - attrp = &hp->dmai_attr; - - sdmareq.dmar_flags = DDI_DMA_RDWR; - error = map_current_window(iommu, rdip, attrp, &sdmareq, - cookie_array, ccount, km_flags); - - if (iommulib_iommu_dma_set_cookies(dip, dma_handle, cookie_array, - ccount) != DDI_SUCCESS) { - cmn_err(CE_WARN, "%s: %s%d: Cannot set cookies " - "for device %p", f, driver, instance, (void *)rdip); - error = DDI_FAILURE; - goto out; - } - - *cookiep = cookie_array[0]; - - return (error == DDI_SUCCESS ? DDI_SUCCESS : DDI_FAILURE); -out: - if (cookie_array) - kmem_free(cookie_array, sizeof (ddi_dma_cookie_t) * ccount); - - return (error); -} - -/* Obsoleted DMA routines */ - -/*ARGSUSED*/ -static int -amd_iommu_map(iommulib_handle_t handle, dev_info_t *dip, - dev_info_t *rdip, struct ddi_dma_req *dmareq, - ddi_dma_handle_t *dma_handle) -{ - ASSERT(0); - return (iommulib_iommu_dma_map(dip, rdip, dmareq, dma_handle)); -} - -/*ARGSUSED*/ -static int -amd_iommu_mctl(iommulib_handle_t handle, dev_info_t *dip, - dev_info_t *rdip, ddi_dma_handle_t dma_handle, - enum ddi_dma_ctlops request, off_t *offp, size_t *lenp, - caddr_t *objpp, uint_t cache_flags) -{ - ASSERT(0); - return (iommulib_iommu_dma_mctl(dip, rdip, dma_handle, - request, offp, lenp, objpp, cache_flags)); -} - -uint64_t -amd_iommu_reg_get64_workaround(uint64_t *regp, uint32_t bits) -{ - split_t s; - uint32_t *ptr32 = (uint32_t *)regp; - uint64_t *s64p = &(s.u64); - - s.u32[0] = ptr32[0]; - s.u32[1] = ptr32[1]; - - return (AMD_IOMMU_REG_GET64_IMPL(s64p, bits)); -} - -uint64_t -amd_iommu_reg_set64_workaround(uint64_t *regp, uint32_t bits, uint64_t value) -{ - split_t s; - uint32_t *ptr32 = (uint32_t *)regp; - uint64_t *s64p = &(s.u64); - - s.u32[0] = ptr32[0]; - s.u32[1] = ptr32[1]; - - AMD_IOMMU_REG_SET64_IMPL(s64p, bits, value); - - *regp = s.u64; - - return (s.u64); -} - -void -amd_iommu_read_boot_props(void) -{ - char *propval; - - /* - * if "amd-iommu = no/false" boot property is set, - * ignore AMD iommu - */ - if (ddi_prop_lookup_string(DDI_DEV_T_ANY, ddi_root_node(), - DDI_PROP_DONTPASS, "amd-iommu", &propval) == DDI_SUCCESS) { - if (strcmp(propval, "no") == 0 || - strcmp(propval, "false") == 0) { - amd_iommu_disable = 1; - } - ddi_prop_free(propval); - } - - /* - * Copy the list of drivers for which IOMMU is disabled by user. - */ - if (ddi_prop_lookup_string(DDI_DEV_T_ANY, ddi_root_node(), - DDI_PROP_DONTPASS, "amd-iommu-disable-list", &propval) - == DDI_SUCCESS) { - amd_iommu_disable_list = kmem_alloc(strlen(propval) + 1, - KM_SLEEP); - (void) strcpy(amd_iommu_disable_list, propval); - ddi_prop_free(propval); - } - -} - -void -amd_iommu_lookup_conf_props(dev_info_t *dip) -{ - char *disable; - - if (ddi_prop_lookup_string(DDI_DEV_T_ANY, dip, - DDI_PROP_DONTPASS|DDI_PROP_NOTPROM, "amd-iommu", &disable) - == DDI_PROP_SUCCESS) { - if (strcmp(disable, "no") == 0) { - amd_iommu_disable = 1; - } - ddi_prop_free(disable); - } - - if (ddi_prop_lookup_string(DDI_DEV_T_ANY, dip, - DDI_PROP_DONTPASS|DDI_PROP_NOTPROM, "amd-iommu-disable-list", - &disable) == DDI_PROP_SUCCESS) { - amd_iommu_disable_list = kmem_alloc(strlen(disable) + 1, - KM_SLEEP); - (void) strcpy(amd_iommu_disable_list, disable); - ddi_prop_free(disable); - } -}
--- a/usr/src/uts/intel/io/amd_iommu/amd_iommu_impl.h Mon Sep 21 11:25:09 2009 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,494 +0,0 @@ -/* - * CDDL HEADER START - * - * The contents of this file are subject to the terms of the - * Common Development and Distribution License (the "License"). - * You may not use this file except in compliance with the License. - * - * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE - * or http://www.opensolaris.org/os/licensing. - * See the License for the specific language governing permissions - * and limitations under the License. - * - * When distributing Covered Code, include this CDDL HEADER in each - * file and include the License file at usr/src/OPENSOLARIS.LICENSE. - * If applicable, add the following below this CDDL HEADER, with the - * fields enclosed by brackets "[]" replaced with your own identifying - * information: Portions Copyright [yyyy] [name of copyright owner] - * - * CDDL HEADER END - */ -/* - * Copyright 2009 Sun Microsystems, Inc. All rights reserved. - * Use is subject to license terms. - */ - -#ifndef _AMD_IOMMU_IMPL_H -#define _AMD_IOMMU_IMPL_H - -#ifdef __cplusplus -extern "C" { -#endif - -#include <sys/pci.h> - -#ifdef _KERNEL - -#define AMD_IOMMU_PCI_PROG_IF (0x0) - -#define AMD_IOMMU_CAP (0x3) - -#define AMD_IOMMU_REG_SIZE (0x2028) -#define AMD_IOMMU_DEVTBL_SZ (16) -#define AMD_IOMMU_CMDBUF_SZ (15) -#define AMD_IOMMU_EVENTLOG_SZ (15) -#define AMD_IOMMU_DEVENT_SZ (32) -#define AMD_IOMMU_CMD_SZ (16) -#define AMD_IOMMU_EVENT_SZ (16) - -/* Capability Register offsets */ -#define AMD_IOMMU_CAP_HDR_OFF (0x00) -#define AMD_IOMMU_CAP_ADDR_LOW_OFF (0x04) -#define AMD_IOMMU_CAP_ADDR_HI_OFF (0x08) -#define AMD_IOMMU_CAP_RANGE_OFF (0x0C) -#define AMD_IOMMU_CAP_MISC_OFF (0x10) - -/* ControL Registers offsets */ -#define AMD_IOMMU_DEVTBL_REG_OFF (0x00) -#define AMD_IOMMU_CMDBUF_REG_OFF (0x08) -#define AMD_IOMMU_EVENTLOG_REG_OFF (0x10) -#define AMD_IOMMU_CTRL_REG_OFF (0x18) -#define AMD_IOMMU_EXCL_BASE_REG_OFF (0x20) -#define AMD_IOMMU_EXCL_LIM_REG_OFF (0x28) -#define AMD_IOMMU_CMDBUF_HEAD_REG_OFF (0x2000) -#define AMD_IOMMU_CMDBUF_TAIL_REG_OFF (0x2008) -#define AMD_IOMMU_EVENTLOG_HEAD_REG_OFF (0x2010) -#define AMD_IOMMU_EVENTLOG_TAIL_REG_OFF (0x2018) -#define AMD_IOMMU_STATUS_REG_OFF (0x2020) - -/* Capability Header Register Bits */ -#define AMD_IOMMU_CAP_NPCACHE (26 << 16 | 26) -#define AMD_IOMMU_CAP_HTTUN (25 << 16 | 25) -#define AMD_IOMMU_CAP_IOTLB (24 << 16 | 24) -#define AMD_IOMMU_CAP_TYPE (18 << 16 | 16) -#define AMD_IOMMU_CAP_ID (7 << 16 | 0) - -/* Capability Range Register bits */ -#define AMD_IOMMU_LAST_DEVFN (31 << 16 | 24) -#define AMD_IOMMU_FIRST_DEVFN (23 << 16 | 16) -#define AMD_IOMMU_RNG_BUS (15 << 16 | 8) -#define AMD_IOMMU_RNG_VALID (7 << 16 | 7) -#define AMD_IOMMU_HT_UNITID (4 << 16 | 0) - - -/* Capability Misc Register bits */ -#define AMD_IOMMU_HT_ATSRSV (22 << 16 | 22) -#define AMD_IOMMU_VA_SIZE (21 << 16 | 15) -#define AMD_IOMMU_PA_SIZE (14 << 16 | 8) -#define AMD_IOMMU_MSINUM (4 << 16 | 0) - -/* Device Table Base Address register bits */ -#define AMD_IOMMU_DEVTABBASE (51 << 16 | 12) -#define AMD_IOMMU_DEVTABSIZE (8 << 16 | 0) - -/* Command Buffer Base Address register bits */ -#define AMD_IOMMU_COMLEN (59 << 16 | 56) -#define AMD_IOMMU_COMBASE (51 << 16 | 12) - -#define AMD_IOMMU_CMDBUF_MINSZ (8) -#define AMD_IOMMU_CMDBUF_MAXSZ (15) - -/* Event Log Base Address register bits */ -#define AMD_IOMMU_EVENTLEN (59 << 16 | 56) -#define AMD_IOMMU_EVENTBASE (51 << 16 | 12) - -#define AMD_IOMMU_EVENTLOG_MINSZ (8) -#define AMD_IOMMU_EVENTLOG_MAXSZ (15) - -/* Control register bits */ -#define AMD_IOMMU_CMDBUF_ENABLE (12 << 16 | 12) -#define AMD_IOMMU_ISOC (11 << 16 | 11) -#define AMD_IOMMU_COHERENT (10 << 16 | 10) -#define AMD_IOMMU_RESPASSPW (9 << 16 | 9) -#define AMD_IOMMU_PASSPW (8 << 16 | 8) -#define AMD_IOMMU_INVTO (7 << 16 | 5) -#define AMD_IOMMU_COMWAITINT_ENABLE (4 << 16 | 4) -#define AMD_IOMMU_EVENTINT_ENABLE (3 << 16 | 3) -#define AMD_IOMMU_EVENTLOG_ENABLE (2 << 16 | 2) -#define AMD_IOMMU_HT_TUN_ENABLE (1 << 16 | 1) -#define AMD_IOMMU_ENABLE (0 << 16 | 0) - -/* Exclusion Base Register bits */ -#define AMD_IOMMU_EXCL_BASE_ADDR (51 << 16 | 12) -#define AMD_IOMMU_EXCL_BASE_ALLOW (1 << 16 | 1) -#define AMD_IOMMU_EXCL_BASE_EXEN (0 << 16 | 0) - -/* Exclusion Limit Register bits */ -#define AMD_IOMMU_EXCL_LIM (51 << 16 | 12) - -/* Command Buffer Head Pointer Register bits */ -#define AMD_IOMMU_CMDHEADPTR (18 << 16 | 4) - -/* Command Buffer Tail Pointer Register bits */ -#define AMD_IOMMU_CMDTAILPTR (18 << 16 | 4) - -/* Event Log Head Pointer Register bits */ -#define AMD_IOMMU_EVENTHEADPTR (18 << 16 | 4) - -/* Event Log Tail Pointer Register bits */ -#define AMD_IOMMU_EVENTTAILPTR (18 << 16 | 4) - -/* Status Register bits */ -#define AMD_IOMMU_CMDBUF_RUN (4 << 16 | 4) -#define AMD_IOMMU_EVENT_LOG_RUN (3 << 16 | 3) -#define AMD_IOMMU_COMWAIT_INT (2 << 16 | 2) -#define AMD_IOMMU_EVENT_LOG_INT (1 << 16 | 1) -#define AMD_IOMMU_EVENT_OVERFLOW_INT (0 << 16 | 0) - -/* Device Table Bits */ - -/* size in bytes of each device table entry */ -#define AMD_IOMMU_DEVTBL_ENTRY_SZ (32) - -/* Interrupt Remapping related Device Table bits */ -#define AMD_IOMMU_DEVTBL_LINT1PASS ((191-128) << 16 | (191-128)) -#define AMD_IOMMU_DEVTBL_LINT0PASS ((190-128) << 16 | (190-128)) -#define AMD_IOMMU_DEVTBL_INTCTL ((189-128) << 16 | (188-128)) -#define AMD_IOMMU_DEVTBL_NMIPASS ((186-128) << 16 | (186-128)) -#define AMD_IOMMU_DEVTBL_EXTINTPAS ((185-128) << 16 | (185-128)) -#define AMD_IOMMU_DEVTBL_INITPASS ((184-128) << 16 | (184-128)) -#define AMD_IOMMU_DEVTBL_INTR_ROOT ((179-128) << 16 | (134-128)) -#define AMD_IOMMU_DEVTBL_IG ((133-128) << 16 | (133-128)) -#define AMD_IOMMU_DEVTBL_INTTABLEN ((132-128) << 16 | (129-128)) -#define AMD_IOMMU_DEVTBL_IV ((128-128) << 16 | (128-128)) - -/* DMA Remapping related Device Table Bits */ -#define AMD_IOMMU_DEVTBL_SYSMGT ((105-64) << 16 | (104-64)) -#define AMD_IOMMU_DEVTBL_EX ((103-64) << 16 | (103-64)) -#define AMD_IOMMU_DEVTBL_SD ((102-64) << 16 | (102-64)) -#define AMD_IOMMU_DEVTBL_CACHE ((101-64) << 16 | (101-64)) -#define AMD_IOMMU_DEVTBL_IOCTL ((100-64) << 16 | (99-64)) -#define AMD_IOMMU_DEVTBL_SA ((98-64) << 16 | (98-64)) -#define AMD_IOMMU_DEVTBL_SE ((97-64) << 16 | (97-64)) -#define AMD_IOMMU_DEVTBL_IOTLB ((96-64) << 16 | (96-64)) -#define AMD_IOMMU_DEVTBL_DOMAINID ((79-64) << 16 | (64-64)) -#define AMD_IOMMU_DEVTBL_IW (62 << 16 | 62) -#define AMD_IOMMU_DEVTBL_IR (61 << 16 | 61) -#define AMD_IOMMU_DEVTBL_ROOT_PGTBL (51 << 16 | 12) -#define AMD_IOMMU_DEVTBL_PG_MODE (11 << 16 | 9) -#define AMD_IOMMU_DEVTBL_TV (1 << 16 | 1) -#define AMD_IOMMU_DEVTBL_V (0 << 16 | 0) - -#define BUS_DEVFN_TO_BDF(b, devfn) (devfn) -#define AMD_IOMMU_ALIAS_HASH_SZ (256) - -#define AMD_IOMMU_REG_ADDR_LOCKED (0x1) - -/* - * IOMMU Command bits - */ - -typedef enum { - AMD_IOMMU_CMD_INVAL = 0, - AMD_IOMMU_CMD_COMPL_WAIT, - AMD_IOMMU_CMD_INVAL_DEVTAB_ENTRY, - AMD_IOMMU_CMD_INVAL_IOMMU_PAGES, - AMD_IOMMU_CMD_INVAL_IOTLB_PAGES, - AMD_IOMMU_CMD_INVAL_INTR_TABLE, -} amd_iommu_cmd_t; - -typedef enum { - AMD_IOMMU_CMD_FLAGS_NONE = 0, - AMD_IOMMU_CMD_FLAGS_COMPL_WAIT = 1, - AMD_IOMMU_CMD_FLAGS_COMPL_WAIT_F = 2, - AMD_IOMMU_CMD_FLAGS_COMPL_WAIT_S = 4, - AMD_IOMMU_CMD_FLAGS_PAGE_PDE_INVAL = 8, - AMD_IOMMU_CMD_FLAGS_PAGE_INVAL_S = 16, - AMD_IOMMU_CMD_FLAGS_IOTLB_INVAL_S = 32 -} amd_iommu_cmd_flags_t; - -/* Common command bits */ -#define AMD_IOMMU_CMD_OPCODE (31 << 16 | 28) - -/* Completion Wait command bits */ -#define AMD_IOMMU_CMD_COMPL_WAIT_S (0 << 16 | 0) -#define AMD_IOMMU_CMD_COMPL_WAIT_I (1 << 16 | 1) -#define AMD_IOMMU_CMD_COMPL_WAIT_F (2 << 16 | 2) -#define AMD_IOMMU_CMD_COMPL_WAIT_STORE_ADDR_LO (31 << 16 | 3) -#define AMD_IOMMU_CMD_COMPL_WAIT_STORE_ADDR_HI (19 << 16 | 0) - -/* Invalidate Device Table entry command bits */ -#define AMD_IOMMU_CMD_INVAL_DEVTAB_DEVICEID (15 << 16 | 0) - -/* Invalidate IOMMU Pages command bits */ -#define AMD_IOMMU_CMD_INVAL_PAGES_DOMAINID (15 << 16 | 0) -#define AMD_IOMMU_CMD_INVAL_PAGES_S (0 << 16 | 0) -#define AMD_IOMMU_CMD_INVAL_PAGES_PDE (1 << 16 | 1) -#define AMD_IOMMU_CMD_INVAL_PAGES_ADDR_LO (31 << 16 | 12) -#define AMD_IOMMU_CMD_INVAL_PAGES_ADDR_HI (63 << 16 | 32) - - -/* Invalidate IOTLB command bits */ -#define AMD_IOMMU_CMD_INVAL_IOTLB_DEVICEID (15 << 16 | 0) -#define AMD_IOMMU_CMD_INVAL_IOTLB_MAXPEND (31 << 16 | 24) -#define AMD_IOMMU_CMD_INVAL_IOTLB_QUEUEID (15 << 16 | 0) -#define AMD_IOMMU_CMD_INVAL_IOTLB_S (0 << 16 | 0) -#define AMD_IOMMU_CMD_INVAL_IOTLB_ADDR_LO (31 << 16 | 12) -#define AMD_IOMMU_CMD_INVAL_IOTLB_ADDR_HI (31 << 16 | 0) - -#define AMD_IOMMU_DEFAULT_MAXPEND (10) - -/* Invalidate Interrupt Table bits */ -#define AMD_IOMMU_CMD_INVAL_INTR_DEVICEID (15 << 16 | 0) - -#if defined(__amd64) -#define dmac_cookie_addr dmac_laddress -#else -#define dmac_cookie_addr dmac_address -#endif - -#define AMD_IOMMU_TABLE_ALIGN ((1ULL << 12) - 1) - -#define AMD_IOMMU_MAX_DEVICEID (0xFFFF) - -/* - * DMA sync macros - * TODO: optimize sync only small ranges - */ -#define SYNC_FORDEV(h) (void) ddi_dma_sync(h, 0, 0, DDI_DMA_SYNC_FORDEV) -#define SYNC_FORKERN(h) (void) ddi_dma_sync(h, 0, 0, DDI_DMA_SYNC_FORKERNEL) - -#define WAIT_SEC(s) drv_usecwait(1000000*(s)) - -#define CMD2OFF(c) ((c) << 4) -#define OFF2CMD(o) ((o) >> 4) - -typedef union split { - uint64_t u64; - uint32_t u32[2]; -} split_t; - -#define BITPOS_START(b) ((b) >> 16) -#define BITPOS_END(b) ((b) & 0xFFFF) - -#define START_MASK64(s) (((s) == 63) ? ~((uint64_t)0) : \ - (uint64_t)((1ULL << ((s)+1)) - 1)) -#define START_MASK32(s) (((s) == 31) ? ~((uint32_t)0) : \ - (uint32_t)((1ULL << ((s)+1)) - 1)) -#define START_MASK16(s) (((s) == 15) ? ~((uint16_t)0) : \ - (uint16_t)((1ULL << ((s)+1)) - 1)) -#define START_MASK8(s) (((s) == 7) ? ~((uint8_t)0) : \ - (uint8_t)((1ULL << ((s)+1)) - 1)) - -#define END_MASK(e) ((1ULL << (e)) - 1) - -#define BIT_MASK64(s, e) (uint64_t)(START_MASK64(s) & ~END_MASK(e)) -#define BIT_MASK32(s, e) (uint32_t)(START_MASK32(s) & ~END_MASK(e)) -#define BIT_MASK16(s, e) (uint16_t)(START_MASK16(s) & ~END_MASK(e)) -#define BIT_MASK8(s, e) (uint8_t)(START_MASK8(s) & ~END_MASK(e)) - -#define AMD_IOMMU_REG_GET64_IMPL(rp, b) \ - (((*(rp)) & (START_MASK64(BITPOS_START(b)))) >> BITPOS_END(b)) -#define AMD_IOMMU_REG_GET64(rp, b) \ - ((amd_iommu_64bit_bug) ? amd_iommu_reg_get64_workaround(rp, b) : \ - AMD_IOMMU_REG_GET64_IMPL(rp, b)) -#define AMD_IOMMU_REG_GET32(rp, b) \ - (((*(rp)) & (START_MASK32(BITPOS_START(b)))) >> BITPOS_END(b)) -#define AMD_IOMMU_REG_GET16(rp, b) \ - (((*(rp)) & (START_MASK16(BITPOS_START(b)))) >> BITPOS_END(b)) -#define AMD_IOMMU_REG_GET8(rp, b) \ - (((*(rp)) & (START_MASK8(BITPOS_START(b)))) >> BITPOS_END(b)) - -#define AMD_IOMMU_REG_SET64_IMPL(rp, b, v) \ - ((*(rp)) = \ - (((uint64_t)(*(rp)) & ~(BIT_MASK64(BITPOS_START(b), BITPOS_END(b)))) \ - | ((uint64_t)(v) << BITPOS_END(b)))) - -#define AMD_IOMMU_REG_SET64(rp, b, v) \ - (void) ((amd_iommu_64bit_bug) ? \ - amd_iommu_reg_set64_workaround(rp, b, v) : \ - AMD_IOMMU_REG_SET64_IMPL(rp, b, v)) - -#define AMD_IOMMU_REG_SET32(rp, b, v) \ - ((*(rp)) = \ - (((uint32_t)(*(rp)) & ~(BIT_MASK32(BITPOS_START(b), BITPOS_END(b)))) \ - | ((uint32_t)(v) << BITPOS_END(b)))) - -#define AMD_IOMMU_REG_SET16(rp, b, v) \ - ((*(rp)) = \ - (((uint16_t)(*(rp)) & ~(BIT_MASK16(BITPOS_START(b), BITPOS_END(b)))) \ - | ((uint16_t)(v) << BITPOS_END(b)))) - -#define AMD_IOMMU_REG_SET8(rp, b, v) \ - ((*(rp)) = \ - (((uint8_t)(*(rp)) & ~(BIT_MASK8(BITPOS_START(b), BITPOS_END(b)))) \ - | ((uint8_t)(v) << BITPOS_END(b)))) - -/* - * Cast a 64 bit pointer to a uint64_t * - */ -#define REGADDR64(a) ((uint64_t *)(uintptr_t)(a)) - -typedef enum { - AMD_IOMMU_INTR_INVALID = 0, - AMD_IOMMU_INTR_TABLE, - AMD_IOMMU_INTR_ALLOCED, - AMD_IOMMU_INTR_HANDLER, - AMD_IOMMU_INTR_ENABLED -} amd_iommu_intr_state_t; - - -typedef struct amd_iommu { - kmutex_t aiomt_mutex; - kmutex_t aiomt_eventlock; - kmutex_t aiomt_cmdlock; - dev_info_t *aiomt_dip; - int aiomt_idx; - iommulib_handle_t aiomt_iommulib_handle; - iommulib_ops_t *aiomt_iommulib_ops; - uint32_t aiomt_cap_hdr; - uint8_t aiomt_npcache; - uint8_t aiomt_httun; - uint8_t aiomt_iotlb; - uint8_t aiomt_captype; - uint8_t aiomt_capid; - uint32_t aiomt_low_addr32; - uint32_t aiomt_hi_addr32; - uint64_t aiomt_reg_pa; - uint64_t aiomt_va; - uint64_t aiomt_reg_va; - uint32_t aiomt_range; - uint8_t aiomt_rng_bus; - uint8_t aiomt_first_devfn; - uint8_t aiomt_last_devfn; - uint8_t aiomt_rng_valid; - uint8_t aiomt_ht_unitid; - uint32_t aiomt_misc; - uint8_t aiomt_htatsresv; - uint8_t aiomt_vasize; - uint8_t aiomt_pasize; - uint8_t aiomt_msinum; - uint8_t aiomt_reg_pages; - uint32_t aiomt_reg_size; - uint32_t aiomt_devtbl_sz; - uint32_t aiomt_cmdbuf_sz; - uint32_t aiomt_eventlog_sz; - caddr_t aiomt_devtbl; - caddr_t aiomt_cmdbuf; - caddr_t aiomt_eventlog; - uint32_t *aiomt_cmd_tail; - uint32_t *aiomt_event_head; - ddi_dma_handle_t aiomt_dmahdl; - void *aiomt_dma_bufva; - uint64_t aiomt_dma_mem_realsz; - ddi_acc_handle_t aiomt_dma_mem_hdl; - ddi_dma_cookie_t aiomt_buf_dma_cookie; - uint_t aiomt_buf_dma_ncookie; - amd_iommu_intr_state_t aiomt_intr_state; - ddi_intr_handle_t *aiomt_intr_htable; - uint32_t aiomt_intr_htable_sz; - uint32_t aiomt_actual_intrs; - uint32_t aiomt_intr_cap; - uint64_t aiomt_reg_devtbl_va; - uint64_t aiomt_reg_cmdbuf_va; - uint64_t aiomt_reg_eventlog_va; - uint64_t aiomt_reg_ctrl_va; - uint64_t aiomt_reg_excl_base_va; - uint64_t aiomt_reg_excl_lim_va; - uint64_t aiomt_reg_cmdbuf_head_va; - uint64_t aiomt_reg_cmdbuf_tail_va; - uint64_t aiomt_reg_eventlog_head_va; - uint64_t aiomt_reg_eventlog_tail_va; - uint64_t aiomt_reg_status_va; - struct amd_iommu *aiomt_next; -} amd_iommu_t; - -typedef struct amd_iommu_dma_devtbl_ent { - uint16_t de_domainid; - uint8_t de_R; - uint8_t de_W; - caddr_t de_root_pgtbl; - uint8_t de_pgmode; -} amd_iommu_dma_devtbl_entry_t; - -typedef struct amd_iommu_alias { - uint16_t al_bdf; - uint16_t al_src_bdf; - struct amd_iommu_alias *al_next; -} amd_iommu_alias_t; - -typedef struct amd_iommu_cmdargs { - uint64_t ca_addr; - uint16_t ca_domainid; - uint16_t ca_deviceid; -} amd_iommu_cmdargs_t; - -struct amd_iommu_page_table; - -typedef struct amd_iommu_page_table_hash { - kmutex_t ampt_lock; - struct amd_iommu_page_table **ampt_hash; -} amd_iommu_page_table_hash_t; - -typedef enum { - AMD_IOMMU_LOG_INVALID_OP = 0, - AMD_IOMMU_LOG_DISPLAY, - AMD_IOMMU_LOG_DISCARD -} amd_iommu_log_op_t; - -typedef enum { - AMD_IOMMU_DEBUG_NONE = 0, - AMD_IOMMU_DEBUG_ALLOCHDL = 0x1, - AMD_IOMMU_DEBUG_FREEHDL = 0x2, - AMD_IOMMU_DEBUG_BIND = 0x4, - AMD_IOMMU_DEBUG_UNBIND = 0x8, - AMD_IOMMU_DEBUG_WIN = 0x10, - AMD_IOMMU_DEBUG_PAGE_TABLES = 0x20, - AMD_IOMMU_DEBUG_DEVTBL = 0x40, - AMD_IOMMU_DEBUG_CMDBUF = 0x80, - AMD_IOMMU_DEBUG_EVENTLOG = 0x100, - AMD_IOMMU_DEBUG_ACPI = 0x200, - AMD_IOMMU_DEBUG_PA2VA = 0x400, - AMD_IOMMU_DEBUG_TABLES = 0x800, - AMD_IOMMU_DEBUG_EXCL = 0x1000, - AMD_IOMMU_DEBUG_INTR = 0x2000 -} amd_iommu_debug_t; - -extern const char *amd_iommu_modname; -extern kmutex_t amd_iommu_global_lock; -extern amd_iommu_alias_t **amd_iommu_alias; -extern amd_iommu_page_table_hash_t amd_iommu_page_table_hash; -extern ddi_device_acc_attr_t amd_iommu_devacc; -extern amd_iommu_debug_t amd_iommu_debug; - -extern uint8_t amd_iommu_htatsresv; -extern uint8_t amd_iommu_vasize; -extern uint8_t amd_iommu_pasize; -extern int amd_iommu_64bit_bug; -extern int amd_iommu_unity_map; -extern int amd_iommu_no_RW_perms; -extern int amd_iommu_no_unmap; -extern int amd_iommu_pageva_inval_all; -extern int amd_iommu_disable; -extern char *amd_iommu_disable_list; - -extern uint64_t amd_iommu_reg_get64_workaround(uint64_t *regp, uint32_t bits); -extern uint64_t amd_iommu_reg_set64_workaround(uint64_t *regp, uint32_t bits, - uint64_t value); - -int amd_iommu_cmd(amd_iommu_t *iommu, amd_iommu_cmd_t cmd, - amd_iommu_cmdargs_t *cmdargs, amd_iommu_cmd_flags_t flags, int lock_held); -int amd_iommu_page_table_hash_init(amd_iommu_page_table_hash_t *ampt); -void amd_iommu_page_table_hash_fini(amd_iommu_page_table_hash_t *ampt); - -int amd_iommu_read_log(amd_iommu_t *iommu, amd_iommu_log_op_t op); -void amd_iommu_read_boot_props(void); -void amd_iommu_lookup_conf_props(dev_info_t *dip); - -#endif /* _KERNEL */ - -#ifdef __cplusplus -} -#endif - -#endif /* _AMD_IOMMU_IMPL_H */
--- a/usr/src/uts/intel/io/amd_iommu/amd_iommu_log.c Mon Sep 21 11:25:09 2009 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,582 +0,0 @@ -/* - * CDDL HEADER START - * - * The contents of this file are subject to the terms of the - * Common Development and Distribution License (the "License"). - * You may not use this file except in compliance with the License. - * - * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE - * or http://www.opensolaris.org/os/licensing. - * See the License for the specific language governing permissions - * and limitations under the License. - * - * When distributing Covered Code, include this CDDL HEADER in each - * file and include the License file at usr/src/OPENSOLARIS.LICENSE. - * If applicable, add the following below this CDDL HEADER, with the - * fields enclosed by brackets "[]" replaced with your own identifying - * information: Portions Copyright [yyyy] [name of copyright owner] - * - * CDDL HEADER END - */ - -/* - * Copyright 2008 Sun Microsystems, Inc. All rights reserved. - * Use is subject to license terms. - */ - -#include <sys/sunddi.h> -#include <sys/amd_iommu.h> -#include "amd_iommu_impl.h" -#include "amd_iommu_log.h" - - -static const char * -get_hw_error(uint8_t type) -{ - const char *hwerr; - - switch (type) { - case 0: - hwerr = "Reserved"; - break; - case 1: - hwerr = "Master Abort"; - break; - case 2: - hwerr = "Target Abort"; - break; - case 3: - hwerr = "Data Error"; - break; - default: - hwerr = "Unknown"; - break; - } - - return (hwerr); -} - -const char * -get_illegal_req(uint8_t type, uint8_t TR) -{ - const char *illreq; - - switch (type) { - case 0: - illreq = (TR == 1) ? "Translation I=0/V=0/V=1&&TV=0" : - "Read or Non-posted Write in INTR Range"; - break; - case 1: - illreq = (TR == 1) ? "Translation INTR/Port-IO/SysMgt; OR" - "Translation when SysMgt=11b/Port-IO when IOCTL=10b " - "while V=1 && TV=0" : - "Pre-translated transaction from device with I=0 or V=0"; - break; - case 2: - illreq = (TR == 1) ? "Reserved": - "Port-IO transaction for device with IoCtl = 00b"; - break; - case 3: - illreq = (TR == 1) ? "Reserved": - "Posted write to SysMgt with device SysMgt=00b " - "OR SysMgt=10b && message not INTx " - "OR Posted write to addr transaltion range with " - "HtAtsResv=1"; - break; - case 4: - illreq = (TR == 1) ? "Reserved": - "Read request or non-posted write in SysMgt with " - "device SysMgt=10b or 0xb" - "OR Read request or non-posted write in " - "addr translation range with HtAtsResv=1"; - break; - case 5: - illreq = (TR == 1) ? "Reserved": - "Posted write to Interrupt/EOI Range " - "for device that has IntCtl=00b"; - break; - case 6: - illreq = (TR == 1) ? "Reserved": - "Posted write to reserved Interrupt Address Range"; - break; - case 7: - illreq = (TR == 1) ? "Reserved": - "transaction to SysMgt when SysMgt=11b OR " - "transaction to Port-IO when IoCtl=10b while " - "while V=1 TV=0"; - break; - default: - illreq = "Unknown error"; - break; - } - return (illreq); -} - -static void -devtab_illegal_entry(amd_iommu_t *iommu, uint32_t *event) -{ - uint16_t deviceid; - uint8_t TR; - uint8_t RZ; - uint8_t RW; - uint8_t I; - uint32_t vaddr_lo; - uint32_t vaddr_hi; - const char *driver = ddi_driver_name(iommu->aiomt_dip); - int instance = ddi_get_instance(iommu->aiomt_dip); - const char *f = "devtab_illegal_entry"; - - ASSERT(AMD_IOMMU_REG_GET32(&event[1], AMD_IOMMU_EVENT_TYPE) == - AMD_IOMMU_EVENT_DEVTAB_ILLEGAL_ENTRY); - - deviceid = AMD_IOMMU_REG_GET32(&event[0], - AMD_IOMMU_EVENT_DEVTAB_ILL_DEVICEID); - - TR = AMD_IOMMU_REG_GET32(&event[1], - AMD_IOMMU_EVENT_DEVTAB_ILL_TR); - - RZ = AMD_IOMMU_REG_GET32(&event[1], - AMD_IOMMU_EVENT_DEVTAB_ILL_RZ); - - RW = AMD_IOMMU_REG_GET32(&event[1], - AMD_IOMMU_EVENT_DEVTAB_ILL_RW); - - I = AMD_IOMMU_REG_GET32(&event[1], - AMD_IOMMU_EVENT_DEVTAB_ILL_INTR); - - vaddr_lo = AMD_IOMMU_REG_GET32(&event[2], - AMD_IOMMU_EVENT_DEVTAB_ILL_VADDR_LO); - - vaddr_hi = event[3]; - - cmn_err(CE_WARN, "%s: %s%d: idx = %d. Illegal device table entry " - "deviceid=%u, %s request, %s %s transaction, %s request, " - "virtual address = %p", - f, driver, instance, iommu->aiomt_idx, - deviceid, - TR == 1 ? "Translation" : "Transaction", - RZ == 1 ? "Non-zero reserved bit" : "Illegal Level encoding", - RW == 1 ? "Write" : "Read", - I == 1 ? "Interrupt" : "Memory", - (void *)(uintptr_t)(((uint64_t)vaddr_hi) << 32 | vaddr_lo)); -} - -static void -io_page_fault(amd_iommu_t *iommu, uint32_t *event) -{ - uint16_t deviceid; - uint16_t domainid; - uint8_t TR; - uint8_t RZ; - uint8_t RW; - uint8_t PE; - uint8_t PR; - uint8_t I; - uint32_t vaddr_lo; - uint32_t vaddr_hi; - const char *driver = ddi_driver_name(iommu->aiomt_dip); - int instance = ddi_get_instance(iommu->aiomt_dip); - const char *f = "io_page_fault"; - - ASSERT(AMD_IOMMU_REG_GET32(&event[1], AMD_IOMMU_EVENT_TYPE) == - AMD_IOMMU_EVENT_IO_PAGE_FAULT); - - deviceid = AMD_IOMMU_REG_GET32(&event[0], - AMD_IOMMU_EVENT_IO_PGFAULT_DEVICEID); - - TR = AMD_IOMMU_REG_GET32(&event[1], AMD_IOMMU_EVENT_IO_PGFAULT_TR); - - RZ = AMD_IOMMU_REG_GET32(&event[1], AMD_IOMMU_EVENT_IO_PGFAULT_RZ); - - PE = AMD_IOMMU_REG_GET32(&event[1], AMD_IOMMU_EVENT_IO_PGFAULT_PE); - - RW = AMD_IOMMU_REG_GET32(&event[1], AMD_IOMMU_EVENT_IO_PGFAULT_RW); - - PR = AMD_IOMMU_REG_GET32(&event[1], AMD_IOMMU_EVENT_IO_PGFAULT_PR); - - I = AMD_IOMMU_REG_GET32(&event[1], AMD_IOMMU_EVENT_IO_PGFAULT_INTR); - - domainid = AMD_IOMMU_REG_GET32(&event[1], - AMD_IOMMU_EVENT_IO_PGFAULT_DOMAINID); - - vaddr_lo = event[2]; - - vaddr_hi = event[3]; - - cmn_err(CE_WARN, "%s: %s%d: idx = %d. IO Page Fault. " - "deviceid=%u, %s request, %s, %s permissions, %s transaction, " - "%s, %s request, domainid=%u, virtual address = %p", - f, driver, instance, iommu->aiomt_idx, - deviceid, - TR == 1 ? "Translation" : "Transaction", - RZ == 1 ? "Non-zero reserved bit" : "Illegal Level encoding", - PE == 1 ? "did not have" : "had", - RW == 1 ? "Write" : "Read", - PR == 1 ? "Page present or Interrupt Remapped" : - "Page not present or Interrupt Blocked", - I == 1 ? "Interrupt" : "Memory", - domainid, - (void *)(uintptr_t)(((uint64_t)vaddr_hi) << 32 | vaddr_lo)); -} - -static void -devtab_hw_error(amd_iommu_t *iommu, uint32_t *event) -{ - uint16_t deviceid; - uint8_t type; - uint8_t TR; - uint8_t RW; - uint8_t I; - uint32_t physaddr_lo; - uint32_t physaddr_hi; - const char *hwerr; - const char *driver = ddi_driver_name(iommu->aiomt_dip); - int instance = ddi_get_instance(iommu->aiomt_dip); - const char *f = "devtab_hw_error"; - - ASSERT(AMD_IOMMU_REG_GET32(&event[1], AMD_IOMMU_EVENT_TYPE) == - AMD_IOMMU_EVENT_DEVTAB_HW_ERROR); - - deviceid = AMD_IOMMU_REG_GET32(&event[0], - AMD_IOMMU_EVENT_DEVTAB_HWERR_DEVICEID); - - type = AMD_IOMMU_REG_GET32(&event[1], - AMD_IOMMU_EVENT_DEVTAB_HWERR_TYPE); - - hwerr = get_hw_error(type); - - TR = AMD_IOMMU_REG_GET32(&event[1], AMD_IOMMU_EVENT_DEVTAB_HWERR_TR); - - RW = AMD_IOMMU_REG_GET32(&event[1], AMD_IOMMU_EVENT_DEVTAB_HWERR_RW); - - I = AMD_IOMMU_REG_GET32(&event[1], AMD_IOMMU_EVENT_DEVTAB_HWERR_INTR); - - physaddr_lo = AMD_IOMMU_REG_GET32(&event[2], - AMD_IOMMU_EVENT_DEVTAB_HWERR_PHYSADDR_LO); - - physaddr_hi = event[3]; - - cmn_err(CE_WARN, "%s: %s%d: idx = %d. Device Table HW Error. " - "deviceid=%u, HW error type: %s, %s request, %s transaction, " - "%s request, physical address = %p", - f, driver, instance, iommu->aiomt_idx, - deviceid, hwerr, - TR == 1 ? "Translation" : "Transaction", - RW == 1 ? "Write" : "Read", - I == 1 ? "Interrupt" : "Memory", - (void *)(uintptr_t)(((uint64_t)physaddr_hi) << 32 | physaddr_lo)); -} - - -static void -pgtable_hw_error(amd_iommu_t *iommu, uint32_t *event) -{ - uint16_t deviceid; - uint16_t domainid; - uint8_t type; - uint8_t TR; - uint8_t RW; - uint8_t I; - uint32_t physaddr_lo; - uint32_t physaddr_hi; - const char *hwerr; - const char *driver = ddi_driver_name(iommu->aiomt_dip); - int instance = ddi_get_instance(iommu->aiomt_dip); - const char *f = "pgtable_hw_error"; - - ASSERT(AMD_IOMMU_REG_GET32(&event[1], AMD_IOMMU_EVENT_TYPE) == - AMD_IOMMU_EVENT_PGTABLE_HW_ERROR); - - deviceid = AMD_IOMMU_REG_GET32(&event[0], - AMD_IOMMU_EVENT_PGTABLE_HWERR_DEVICEID); - - type = AMD_IOMMU_REG_GET32(&event[1], - AMD_IOMMU_EVENT_DEVTAB_HWERR_TYPE); - - hwerr = get_hw_error(type); - - TR = AMD_IOMMU_REG_GET32(&event[1], AMD_IOMMU_EVENT_PGTABLE_HWERR_TR); - - RW = AMD_IOMMU_REG_GET32(&event[1], AMD_IOMMU_EVENT_PGTABLE_HWERR_RW); - - I = AMD_IOMMU_REG_GET32(&event[1], AMD_IOMMU_EVENT_PGTABLE_HWERR_INTR); - - domainid = AMD_IOMMU_REG_GET32(&event[1], - AMD_IOMMU_EVENT_PGTABLE_HWERR_DOMAINID); - - physaddr_lo = AMD_IOMMU_REG_GET32(&event[2], - AMD_IOMMU_EVENT_PGTABLE_HWERR_PHYSADDR_LO); - - physaddr_hi = event[3]; - - cmn_err(CE_WARN, "%s: %s%d: idx = %d. Page Table HW Error. " - "deviceid=%u, HW error type: %s, %s request, %s transaction, " - "%s request, domainid=%u, physical address = %p", - f, driver, instance, iommu->aiomt_idx, - deviceid, hwerr, - TR == 1 ? "Translation" : "Transaction", - RW == 1 ? "Write" : "Read", - I == 1 ? "Interrupt" : "Memory", - domainid, - (void *)(uintptr_t)(((uint64_t)physaddr_hi) << 32 | physaddr_lo)); -} - -static void -cmdbuf_illegal_cmd(amd_iommu_t *iommu, uint32_t *event) -{ - uint32_t physaddr_lo; - uint32_t physaddr_hi; - const char *driver = ddi_driver_name(iommu->aiomt_dip); - int instance = ddi_get_instance(iommu->aiomt_dip); - const char *f = "cmdbuf_illegal_cmd"; - - ASSERT(AMD_IOMMU_REG_GET32(&event[1], AMD_IOMMU_EVENT_TYPE) == - AMD_IOMMU_EVENT_CMDBUF_ILLEGAL_CMD); - - physaddr_lo = AMD_IOMMU_REG_GET32(&event[2], - AMD_IOMMU_EVENT_CMDBUF_ILLEGAL_CMD_PHYS_LO); - - physaddr_hi = event[3]; - - cmn_err(CE_WARN, "%s: %s%d: idx = %d. Illegal IOMMU command. " - "command physical address = %p", - f, driver, instance, iommu->aiomt_idx, - (void *)(uintptr_t)(((uint64_t)physaddr_hi) << 32 | physaddr_lo)); -} - -static void -cmdbuf_hw_error(amd_iommu_t *iommu, uint32_t *event) -{ - uint32_t physaddr_lo; - uint32_t physaddr_hi; - uint8_t type; - const char *hwerr; - const char *driver = ddi_driver_name(iommu->aiomt_dip); - int instance = ddi_get_instance(iommu->aiomt_dip); - const char *f = "cmdbuf_hw_error"; - - ASSERT(AMD_IOMMU_REG_GET32(&event[1], AMD_IOMMU_EVENT_TYPE) == - AMD_IOMMU_EVENT_CMDBUF_HW_ERROR); - - type = AMD_IOMMU_REG_GET32(&event[1], - AMD_IOMMU_EVENT_CMDBUF_HWERR_TYPE); - - hwerr = get_hw_error(type); - - physaddr_lo = AMD_IOMMU_REG_GET32(&event[2], - AMD_IOMMU_EVENT_CMDBUF_HWERR_PHYS_LO); - - physaddr_hi = event[3]; - - cmn_err(CE_WARN, "%s: %s%d: idx = %d. Command Buffer HW error. " - "HW error type = %s, command buffer physical address = %p", - f, driver, instance, iommu->aiomt_idx, - hwerr, - (void *)(uintptr_t)(((uint64_t)physaddr_hi) << 32 | physaddr_lo)); -} - -static void -iotlb_inval_to(amd_iommu_t *iommu, uint32_t *event) -{ - uint16_t deviceid; - uint32_t physaddr_lo; - uint32_t physaddr_hi; - uint8_t type; - const char *hwerr; - const char *driver = ddi_driver_name(iommu->aiomt_dip); - int instance = ddi_get_instance(iommu->aiomt_dip); - const char *f = "iotlb_inval_to"; - - ASSERT(AMD_IOMMU_REG_GET32(&event[1], AMD_IOMMU_EVENT_TYPE) == - AMD_IOMMU_EVENT_IOTLB_INVAL_TO); - - deviceid = AMD_IOMMU_REG_GET32(&event[0], - AMD_IOMMU_EVENT_IOTLB_INVAL_TO_DEVICEID); - - /* - * XXX bug in spec. Is the type field available +04 26:25 or is - * it reserved - */ - type = AMD_IOMMU_REG_GET32(&event[1], - AMD_IOMMU_EVENT_IOTLB_INVAL_TO_TYPE); - hwerr = get_hw_error(type); - - physaddr_lo = AMD_IOMMU_REG_GET32(&event[2], - AMD_IOMMU_EVENT_IOTLB_INVAL_TO_PHYS_LO); - - physaddr_hi = event[3]; - - cmn_err(CE_WARN, "%s: %s%d: idx = %d. deviceid = %u " - "IOTLB invalidation Timeout. " - "HW error type = %s, invalidation command physical address = %p", - f, driver, instance, iommu->aiomt_idx, deviceid, - hwerr, - (void *)(uintptr_t)(((uint64_t)physaddr_hi) << 32 | physaddr_lo)); -} - -static void -device_illegal_req(amd_iommu_t *iommu, uint32_t *event) -{ - uint16_t deviceid; - uint8_t TR; - uint32_t addr_lo; - uint32_t addr_hi; - uint8_t type; - const char *reqerr; - const char *driver = ddi_driver_name(iommu->aiomt_dip); - int instance = ddi_get_instance(iommu->aiomt_dip); - const char *f = "device_illegal_req"; - - ASSERT(AMD_IOMMU_REG_GET32(&event[1], AMD_IOMMU_EVENT_TYPE) == - AMD_IOMMU_EVENT_DEVICE_ILLEGAL_REQ); - - deviceid = AMD_IOMMU_REG_GET32(&event[0], - AMD_IOMMU_EVENT_DEVICE_ILLEGAL_REQ_DEVICEID); - - TR = AMD_IOMMU_REG_GET32(&event[1], - AMD_IOMMU_EVENT_DEVICE_ILLEGAL_REQ_TR); - - type = AMD_IOMMU_REG_GET32(&event[1], - AMD_IOMMU_EVENT_DEVICE_ILLEGAL_REQ_TYPE); - - reqerr = get_illegal_req(type, TR); - - - addr_lo = event[2]; - addr_hi = event[3]; - - cmn_err(CE_WARN, "%s: %s%d: idx = %d. deviceid = %d " - "Illegal Device Request. " - "Illegal Request type = %s, %s request, address accessed = %p", - f, driver, instance, iommu->aiomt_idx, deviceid, - reqerr, - TR == 1 ? "Translation" : "Transaction", - (void *)(uintptr_t)(((uint64_t)addr_hi) << 32 | addr_lo)); -} - -static void -amd_iommu_process_one_event(amd_iommu_t *iommu) -{ - uint32_t event[4]; - amd_iommu_event_t event_type; - int i; - const char *driver = ddi_driver_name(iommu->aiomt_dip); - int instance = ddi_get_instance(iommu->aiomt_dip); - const char *f = "amd_iommu_process_one_event"; - - ASSERT(MUTEX_HELD(&iommu->aiomt_eventlock)); - - SYNC_FORKERN(iommu->aiomt_dmahdl); - for (i = 0; i < 4; i++) { - event[i] = iommu->aiomt_event_head[i]; - } - - event_type = AMD_IOMMU_REG_GET32(&event[1], AMD_IOMMU_EVENT_TYPE); - - switch (event_type) { - case AMD_IOMMU_EVENT_DEVTAB_ILLEGAL_ENTRY: - devtab_illegal_entry(iommu, event); - break; - case AMD_IOMMU_EVENT_IO_PAGE_FAULT: - io_page_fault(iommu, event); - break; - case AMD_IOMMU_EVENT_DEVTAB_HW_ERROR: - devtab_hw_error(iommu, event); - break; - case AMD_IOMMU_EVENT_PGTABLE_HW_ERROR: - pgtable_hw_error(iommu, event); - break; - case AMD_IOMMU_EVENT_CMDBUF_HW_ERROR: - cmdbuf_hw_error(iommu, event); - break; - case AMD_IOMMU_EVENT_CMDBUF_ILLEGAL_CMD: - cmdbuf_illegal_cmd(iommu, event); - break; - case AMD_IOMMU_EVENT_IOTLB_INVAL_TO: - iotlb_inval_to(iommu, event); - break; - case AMD_IOMMU_EVENT_DEVICE_ILLEGAL_REQ: - device_illegal_req(iommu, event); - break; - default: - cmn_err(CE_WARN, "%s: %s%d: idx = %d. Unknown event: %u", - f, driver, instance, iommu->aiomt_idx, event_type); - break; - } -} - -int -amd_iommu_read_log(amd_iommu_t *iommu, amd_iommu_log_op_t op) -{ - caddr_t evtail; - uint64_t evtail_off; - uint64_t evhead_off; - - ASSERT(op != AMD_IOMMU_LOG_INVALID_OP); - - mutex_enter(&iommu->aiomt_eventlock); - - ASSERT(iommu->aiomt_event_head != NULL); - - /* XXX verify */ - evtail_off = AMD_IOMMU_REG_GET64( - REGADDR64(iommu->aiomt_reg_eventlog_tail_va), - AMD_IOMMU_EVENTTAILPTR); - - evtail_off = EV2OFF(evtail_off); - - ASSERT(evtail_off < iommu->aiomt_eventlog_sz); - - evtail = iommu->aiomt_eventlog + evtail_off; - - if (op == AMD_IOMMU_LOG_DISCARD) { - /*LINTED*/ - iommu->aiomt_event_head = (uint32_t *)evtail; - AMD_IOMMU_REG_SET64(REGADDR64( - iommu->aiomt_reg_eventlog_head_va), - AMD_IOMMU_EVENTHEADPTR, OFF2EV(evtail_off)); - cmn_err(CE_NOTE, "Discarded IOMMU event log"); - mutex_exit(&iommu->aiomt_eventlock); - return (DDI_SUCCESS); - } - - /*LINTED*/ - while (1) { - if ((caddr_t)iommu->aiomt_event_head == evtail) - break; - - cmn_err(CE_WARN, "evtail_off = %p, head = %p, tail = %p", - (void *)(uintptr_t)evtail_off, - (void *)iommu->aiomt_event_head, - (void *)evtail); - - amd_iommu_process_one_event(iommu); - - /* - * Update the head pointer in soft state - * and the head pointer register - */ - iommu->aiomt_event_head += 4; - if ((caddr_t)iommu->aiomt_event_head >= - iommu->aiomt_eventlog + iommu->aiomt_eventlog_sz) { - /* wraparound */ - iommu->aiomt_event_head = - /*LINTED*/ - (uint32_t *)iommu->aiomt_eventlog; - evhead_off = 0; - } else { - evhead_off = (caddr_t)iommu->aiomt_event_head - /*LINTED*/ - - iommu->aiomt_eventlog; - } - - ASSERT(evhead_off < iommu->aiomt_eventlog_sz); - - AMD_IOMMU_REG_SET64(REGADDR64( - iommu->aiomt_reg_eventlog_head_va), - AMD_IOMMU_EVENTHEADPTR, OFF2EV(evhead_off)); - } - mutex_exit(&iommu->aiomt_eventlock); - - return (DDI_SUCCESS); -}
--- a/usr/src/uts/intel/io/amd_iommu/amd_iommu_log.h Mon Sep 21 11:25:09 2009 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,116 +0,0 @@ -/* - * CDDL HEADER START - * - * The contents of this file are subject to the terms of the - * Common Development and Distribution License (the "License"). - * You may not use this file except in compliance with the License. - * - * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE - * or http://www.opensolaris.org/os/licensing. - * See the License for the specific language governing permissions - * and limitations under the License. - * - * When distributing Covered Code, include this CDDL HEADER in each - * file and include the License file at usr/src/OPENSOLARIS.LICENSE. - * If applicable, add the following below this CDDL HEADER, with the - * fields enclosed by brackets "[]" replaced with your own identifying - * information: Portions Copyright [yyyy] [name of copyright owner] - * - * CDDL HEADER END - */ -/* - * Copyright 2008 Sun Microsystems, Inc. All rights reserved. - * Use is subject to license terms. - */ - -#ifndef _AMD_IOMMU_LOG_H -#define _AMD_IOMMU_LOG_H - -#ifdef __cplusplus -extern "C" { -#endif - -#include <sys/amd_iommu.h> - -#ifdef _KERNEL - -#define EV2OFF(e) ((e) << 4) -#define OFF2EV(o) ((o) >> 4) - -typedef enum { - AMD_IOMMU_EVENT_INVALID = 0, - AMD_IOMMU_EVENT_DEVTAB_ILLEGAL_ENTRY = 1, - AMD_IOMMU_EVENT_IO_PAGE_FAULT = 2, - AMD_IOMMU_EVENT_DEVTAB_HW_ERROR = 3, - AMD_IOMMU_EVENT_PGTABLE_HW_ERROR = 4, - AMD_IOMMU_EVENT_CMDBUF_ILLEGAL_CMD = 5, - AMD_IOMMU_EVENT_CMDBUF_HW_ERROR = 6, - AMD_IOMMU_EVENT_IOTLB_INVAL_TO = 7, - AMD_IOMMU_EVENT_DEVICE_ILLEGAL_REQ = 8 -} amd_iommu_event_t; - -/* Common to all events */ -#define AMD_IOMMU_EVENT_TYPE (31 << 16 | 28) - -/* Illegal device Table Entry Event bits */ -#define AMD_IOMMU_EVENT_DEVTAB_ILL_DEVICEID (15 << 16 | 0) -#define AMD_IOMMU_EVENT_DEVTAB_ILL_TR (24 << 16 | 24) -#define AMD_IOMMU_EVENT_DEVTAB_ILL_RZ (23 << 16 | 23) -#define AMD_IOMMU_EVENT_DEVTAB_ILL_RW (21 << 16 | 21) -#define AMD_IOMMU_EVENT_DEVTAB_ILL_INTR (19 << 16 | 19) -#define AMD_IOMMU_EVENT_DEVTAB_ILL_VADDR_LO (31 << 16 | 2) - -/* IO Page Fault event bits */ -#define AMD_IOMMU_EVENT_IO_PGFAULT_DEVICEID (15 << 16 | 0) -#define AMD_IOMMU_EVENT_IO_PGFAULT_TR (24 << 16 | 24) -#define AMD_IOMMU_EVENT_IO_PGFAULT_RZ (23 << 16 | 23) -#define AMD_IOMMU_EVENT_IO_PGFAULT_PE (22 << 16 | 22) -#define AMD_IOMMU_EVENT_IO_PGFAULT_RW (21 << 16 | 21) -#define AMD_IOMMU_EVENT_IO_PGFAULT_PR (20 << 16 | 20) -#define AMD_IOMMU_EVENT_IO_PGFAULT_INTR (19 << 16 | 19) -#define AMD_IOMMU_EVENT_IO_PGFAULT_DOMAINID (15 << 16 | 0) - - -/* Device Table HW Error event bits */ -#define AMD_IOMMU_EVENT_DEVTAB_HWERR_DEVICEID (15 << 16 | 0) -#define AMD_IOMMU_EVENT_DEVTAB_HWERR_TYPE (26 << 16 | 25) -#define AMD_IOMMU_EVENT_DEVTAB_HWERR_TR (24 << 16 | 24) -#define AMD_IOMMU_EVENT_DEVTAB_HWERR_RW (21 << 16 | 21) -#define AMD_IOMMU_EVENT_DEVTAB_HWERR_INTR (19 << 16 | 19) -#define AMD_IOMMU_EVENT_DEVTAB_HWERR_PHYSADDR_LO (31 << 16 | 4) - - -/* Page Table HW Error event bits */ -#define AMD_IOMMU_EVENT_PGTABLE_HWERR_DEVICEID (15 << 16 | 0) -#define AMD_IOMMU_EVENT_DEVTAB_HWERR_TYPE (26 << 16 | 25) -#define AMD_IOMMU_EVENT_PGTABLE_HWERR_TR (24 << 16 | 24) -#define AMD_IOMMU_EVENT_PGTABLE_HWERR_RW (21 << 16 | 21) -#define AMD_IOMMU_EVENT_PGTABLE_HWERR_INTR (19 << 16 | 19) -#define AMD_IOMMU_EVENT_PGTABLE_HWERR_DOMAINID (15 << 16 | 0) -#define AMD_IOMMU_EVENT_PGTABLE_HWERR_PHYSADDR_LO (31 << 16 | 3) - -/* Illegal Command Error event bits */ -#define AMD_IOMMU_EVENT_CMDBUF_ILLEGAL_CMD_PHYS_LO (31 << 16 | 4) - -/* Command Buffer HW Error event bits */ -#define AMD_IOMMU_EVENT_CMDBUF_HWERR_TYPE (26 << 16 | 25) -#define AMD_IOMMU_EVENT_CMDBUF_HWERR_PHYS_LO (31 << 16 | 4) - - -/* IOTLB Invalidation TO event bits */ -#define AMD_IOMMU_EVENT_IOTLB_INVAL_TO_DEVICEID (15 << 16 | 0) -#define AMD_IOMMU_EVENT_IOTLB_INVAL_TO_TYPE (26 << 16 | 25) -#define AMD_IOMMU_EVENT_IOTLB_INVAL_TO_PHYS_LO (31 << 16 | 4) - -/* Illegal Device request event bits */ -#define AMD_IOMMU_EVENT_DEVICE_ILLEGAL_REQ_DEVICEID (15 << 16 | 0) -#define AMD_IOMMU_EVENT_DEVICE_ILLEGAL_REQ_TYPE (27 << 16 | 25) -#define AMD_IOMMU_EVENT_DEVICE_ILLEGAL_REQ_TR (24 << 16 | 24) - -#endif /* _KERNEL */ - -#ifdef __cplusplus -} -#endif - -#endif /* _AMD_IOMMU_LOG_H */
--- a/usr/src/uts/intel/io/amd_iommu/amd_iommu_page_tables.c Mon Sep 21 11:25:09 2009 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,1699 +0,0 @@ -/* - * CDDL HEADER START - * - * The contents of this file are subject to the terms of the - * Common Development and Distribution License (the "License"). - * You may not use this file except in compliance with the License. - * - * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE - * or http://www.opensolaris.org/os/licensing. - * See the License for the specific language governing permissions - * and limitations under the License. - * - * When distributing Covered Code, include this CDDL HEADER in each - * file and include the License file at usr/src/OPENSOLARIS.LICENSE. - * If applicable, add the following below this CDDL HEADER, with the - * fields enclosed by brackets "[]" replaced with your own identifying - * information: Portions Copyright [yyyy] [name of copyright owner] - * - * CDDL HEADER END - */ - -/* - * Copyright 2009 Sun Microsystems, Inc. All rights reserved. - * Use is subject to license terms. - */ - -#include <sys/sunddi.h> -#include <sys/sunndi.h> -#include <sys/acpi/acpi.h> -#include <sys/acpica.h> -#include <sys/amd_iommu.h> -#include <sys/bootconf.h> -#include <sys/sysmacros.h> -#include <sys/ddidmareq.h> - -#include "amd_iommu_impl.h" -#include "amd_iommu_acpi.h" -#include "amd_iommu_page_tables.h" - -ddi_dma_attr_t amd_iommu_pgtable_dma_attr = { - DMA_ATTR_V0, - 0U, /* dma_attr_addr_lo */ - 0xffffffffffffffffULL, /* dma_attr_addr_hi */ - 0xffffffffU, /* dma_attr_count_max */ - (uint64_t)4096, /* dma_attr_align */ - 1, /* dma_attr_burstsizes */ - 64, /* dma_attr_minxfer */ - 0xffffffffU, /* dma_attr_maxxfer */ - 0xffffffffU, /* dma_attr_seg */ - 1, /* dma_attr_sgllen, variable */ - 64, /* dma_attr_granular */ - 0 /* dma_attr_flags */ -}; - -static amd_iommu_domain_t **amd_iommu_domain_table; - -static struct { - int f_count; - amd_iommu_page_table_t *f_list; -} amd_iommu_pgtable_freelist; -int amd_iommu_no_pgtable_freelist; - -/*ARGSUSED*/ -static int -amd_iommu_get_src_bdf(amd_iommu_t *iommu, int32_t bdf, int32_t *src_bdfp) -{ - amd_iommu_acpi_ivhd_t *hinfop; - - hinfop = amd_iommu_lookup_ivhd(bdf); - if (hinfop == NULL || hinfop->ach_src_deviceid == -1) - *src_bdfp = bdf; - else - *src_bdfp = hinfop->ach_src_deviceid; - - return (DDI_SUCCESS); -} - -static dev_info_t * -amd_iommu_pci_dip(dev_info_t *rdip, const char *path) -{ - dev_info_t *pdip; - const char *driver = ddi_driver_name(rdip); - int instance = ddi_get_instance(rdip); - const char *f = "amd_iommu_pci_dip"; - - /* Hold rdip so it and its parents don't go away */ - ndi_hold_devi(rdip); - - if (ddi_is_pci_dip(rdip)) - return (rdip); - - pdip = rdip; - while (pdip = ddi_get_parent(pdip)) { - if (ddi_is_pci_dip(pdip)) { - ndi_hold_devi(pdip); - ndi_rele_devi(rdip); - return (pdip); - } - } - - cmn_err(CE_WARN, "%s: %s%d dip = %p has no PCI parent, path = %s", - f, driver, instance, (void *)rdip, path); - - ndi_rele_devi(rdip); - - ASSERT(0); - - return (NULL); -} - -/*ARGSUSED*/ -static int -amd_iommu_get_domain(amd_iommu_t *iommu, dev_info_t *rdip, int alias, - uint16_t deviceid, domain_id_t *domainid, const char *path) -{ - const char *f = "amd_iommu_get_domain"; - - *domainid = AMD_IOMMU_INVALID_DOMAIN; - - ASSERT(strcmp(ddi_driver_name(rdip), "agpgart") != 0); - - switch (deviceid) { - case AMD_IOMMU_INVALID_DOMAIN: - case AMD_IOMMU_IDENTITY_DOMAIN: - case AMD_IOMMU_PASSTHRU_DOMAIN: - case AMD_IOMMU_SYS_DOMAIN: - *domainid = AMD_IOMMU_SYS_DOMAIN; - break; - default: - *domainid = deviceid; - break; - } - - if (amd_iommu_debug & AMD_IOMMU_DEBUG_DEVTBL) { - cmn_err(CE_NOTE, "%s: domainid for %s = %d", - f, path, *domainid); - } - - return (DDI_SUCCESS); -} - -static uint16_t -hash_domain(domain_id_t domainid) -{ - return (domainid % AMD_IOMMU_DOMAIN_HASH_SZ); -} - -/*ARGSUSED*/ -void -amd_iommu_init_page_tables(amd_iommu_t *iommu) -{ - amd_iommu_domain_table = kmem_zalloc( - sizeof (amd_iommu_domain_t *) * AMD_IOMMU_DOMAIN_HASH_SZ, KM_SLEEP); -} - -/*ARGSUSED*/ -void -amd_iommu_fini_page_tables(amd_iommu_t *iommu) -{ - if (amd_iommu_domain_table) { - kmem_free(amd_iommu_domain_table, - sizeof (amd_iommu_domain_t *) * AMD_IOMMU_DOMAIN_HASH_SZ); - amd_iommu_domain_table = NULL; - } -} - -static amd_iommu_domain_t * -amd_iommu_lookup_domain(amd_iommu_t *iommu, domain_id_t domainid, - map_type_t type, int km_flags) -{ - uint16_t idx; - amd_iommu_domain_t *dp; - char name[AMD_IOMMU_VMEM_NAMELEN+1]; - - ASSERT(amd_iommu_domain_table); - - idx = hash_domain(domainid); - - for (dp = amd_iommu_domain_table[idx]; dp; dp = dp->d_next) { - if (dp->d_domainid == domainid) - return (dp); - } - - ASSERT(type != AMD_IOMMU_INVALID_MAP); - - dp = kmem_zalloc(sizeof (*dp), km_flags); - if (dp == NULL) - return (NULL); - dp->d_domainid = domainid; - dp->d_pgtable_root_4K = 0; /* make this explicit */ - - if (type == AMD_IOMMU_VMEM_MAP) { - uint64_t base; - uint64_t size; - (void) snprintf(name, sizeof (name), "dvma_idx%d_domain%d", - iommu->aiomt_idx, domainid); - base = MMU_PAGESIZE; - size = AMD_IOMMU_SIZE_4G - MMU_PAGESIZE; - dp->d_vmem = vmem_create(name, (void *)(uintptr_t)base, size, - MMU_PAGESIZE, NULL, NULL, NULL, 0, - km_flags == KM_SLEEP ? VM_SLEEP : VM_NOSLEEP); - if (dp->d_vmem == NULL) { - kmem_free(dp, sizeof (*dp)); - return (NULL); - } - } else { - dp->d_vmem = NULL; - } - - dp->d_next = amd_iommu_domain_table[idx]; - dp->d_prev = NULL; - amd_iommu_domain_table[idx] = dp; - if (dp->d_next) - dp->d_next->d_prev = dp; - dp->d_ref = 0; - - - return (dp); -} - -static void -amd_iommu_teardown_domain(amd_iommu_t *iommu, amd_iommu_domain_t *dp) -{ - uint16_t idx; - int flags; - amd_iommu_cmdargs_t cmdargs = {0}; - domain_id_t domainid = dp->d_domainid; - const char *f = "amd_iommu_teardown_domain"; - - ASSERT(dp->d_ref == 0); - - idx = hash_domain(dp->d_domainid); - - if (dp->d_prev == NULL) - amd_iommu_domain_table[idx] = dp->d_next; - else - dp->d_prev->d_next = dp->d_next; - - if (dp->d_next) - dp->d_next->d_prev = dp->d_prev; - - if (dp->d_vmem != NULL) { - vmem_destroy(dp->d_vmem); - dp->d_vmem = NULL; - } - - kmem_free(dp, sizeof (*dp)); - - cmdargs.ca_domainid = (uint16_t)domainid; - cmdargs.ca_addr = (uintptr_t)0x7FFFFFFFFFFFF000; - flags = AMD_IOMMU_CMD_FLAGS_PAGE_PDE_INVAL | - AMD_IOMMU_CMD_FLAGS_PAGE_INVAL_S; - - if (amd_iommu_cmd(iommu, AMD_IOMMU_CMD_INVAL_IOMMU_PAGES, - &cmdargs, flags, 0) != DDI_SUCCESS) { - cmn_err(CE_WARN, "%s: idx=%d: domainid=%d" - "Failed to invalidate domain in IOMMU HW cache", - f, iommu->aiomt_idx, cmdargs.ca_domainid); - } -} - -static int -amd_iommu_get_deviceid(amd_iommu_t *iommu, dev_info_t *rdip, int32_t *deviceid, - int *aliasp, const char *path) -{ - int bus = -1; - int device = -1; - int func = -1; - uint16_t bdf; - int32_t src_bdf; - dev_info_t *idip = iommu->aiomt_dip; - const char *driver = ddi_driver_name(idip); - int instance = ddi_get_instance(idip); - dev_info_t *pci_dip; - const char *f = "amd_iommu_get_deviceid"; - - /* be conservative. Always assume an alias */ - *aliasp = 1; - *deviceid = 0; - - /* Check for special special devices (rdip == NULL) */ - if (rdip == NULL) { - if (amd_iommu_get_src_bdf(iommu, -1, &src_bdf) != DDI_SUCCESS) { - cmn_err(CE_WARN, - "%s: %s%d: idx=%d, failed to get SRC BDF " - "for special-device", - f, driver, instance, iommu->aiomt_idx); - return (DDI_DMA_NOMAPPING); - } - *deviceid = src_bdf; - *aliasp = 1; - return (DDI_SUCCESS); - } - - if (amd_iommu_debug & AMD_IOMMU_DEBUG_DEVTBL) { - cmn_err(CE_NOTE, "%s: attempting to get deviceid for %s", - f, path); - } - - pci_dip = amd_iommu_pci_dip(rdip, path); - if (pci_dip == NULL) { - cmn_err(CE_WARN, "%s: %s%d: idx = %d, failed to get PCI dip " - "for rdip=%p, path = %s", - f, driver, instance, iommu->aiomt_idx, (void *)rdip, - path); - return (DDI_DMA_NOMAPPING); - } - - if (acpica_get_bdf(pci_dip, &bus, &device, &func) != DDI_SUCCESS) { - ndi_rele_devi(pci_dip); - cmn_err(CE_WARN, "%s: %s%d: idx=%d, failed to get BDF for " - "PCI dip (%p). rdip path = %s", - f, driver, instance, iommu->aiomt_idx, - (void *)pci_dip, path); - return (DDI_DMA_NOMAPPING); - } - - ndi_rele_devi(pci_dip); - - if (bus > UINT8_MAX || bus < 0 || - device > UINT8_MAX || device < 0 || - func > UINT8_MAX || func < 0) { - cmn_err(CE_WARN, "%s: %s%d: idx=%d, invalid BDF(%d,%d,%d) " - "for PCI dip (%p). rdip path = %s", f, driver, instance, - iommu->aiomt_idx, - bus, device, func, - (void *)pci_dip, path); - return (DDI_DMA_NOMAPPING); - } - - bdf = ((uint8_t)bus << 8) | ((uint8_t)device << 3) | (uint8_t)func; - - if (amd_iommu_get_src_bdf(iommu, bdf, &src_bdf) != DDI_SUCCESS) { - cmn_err(CE_WARN, "%s: %s%d: idx=%d, failed to get SRC BDF " - "for PCI dip (%p) rdip path = %s.", - f, driver, instance, iommu->aiomt_idx, (void *)pci_dip, - path); - return (DDI_DMA_NOMAPPING); - } - - if (amd_iommu_debug & AMD_IOMMU_DEBUG_DEVTBL) { - cmn_err(CE_NOTE, "%s: Deviceid = %u for path = %s", - f, src_bdf, path); - } - - *deviceid = src_bdf; - *aliasp = (src_bdf != bdf); - - return (DDI_SUCCESS); -} - -/*ARGSUSED*/ -static int -init_devtbl(amd_iommu_t *iommu, uint64_t *devtbl_entry, domain_id_t domainid, - amd_iommu_domain_t *dp) -{ - uint64_t entry[4] = {0}; - int i; - - /* If already passthru, don't touch */ - if (AMD_IOMMU_REG_GET64(&(devtbl_entry[0]), AMD_IOMMU_DEVTBL_V) == 0 && - AMD_IOMMU_REG_GET64(&(devtbl_entry[0]), AMD_IOMMU_DEVTBL_TV) == 0) { - return (0); - } - - if (AMD_IOMMU_REG_GET64(&(devtbl_entry[0]), AMD_IOMMU_DEVTBL_V) == 1 && - AMD_IOMMU_REG_GET64(&(devtbl_entry[0]), AMD_IOMMU_DEVTBL_TV) == 1) { - - ASSERT(dp->d_pgtable_root_4K == - AMD_IOMMU_REG_GET64(&(devtbl_entry[0]), - AMD_IOMMU_DEVTBL_ROOT_PGTBL)); - - ASSERT(dp->d_domainid == AMD_IOMMU_REG_GET64(&(devtbl_entry[1]), - AMD_IOMMU_DEVTBL_DOMAINID)); - - return (0); - } - - /* New devtbl entry for this domain. Bump up the domain ref-count */ - dp->d_ref++; - - entry[3] = 0; - entry[2] = 0; - AMD_IOMMU_REG_SET64(&(entry[1]), AMD_IOMMU_DEVTBL_SYSMGT, 1); - AMD_IOMMU_REG_SET64(&(entry[1]), AMD_IOMMU_DEVTBL_EX, 1); - AMD_IOMMU_REG_SET64(&(entry[1]), AMD_IOMMU_DEVTBL_SD, 0); - AMD_IOMMU_REG_SET64(&(entry[1]), AMD_IOMMU_DEVTBL_CACHE, 0); - AMD_IOMMU_REG_SET64(&(entry[1]), AMD_IOMMU_DEVTBL_IOCTL, 1); - AMD_IOMMU_REG_SET64(&(entry[1]), AMD_IOMMU_DEVTBL_SA, 0); - AMD_IOMMU_REG_SET64(&(entry[1]), AMD_IOMMU_DEVTBL_SE, 1); - AMD_IOMMU_REG_SET64(&(entry[1]), AMD_IOMMU_DEVTBL_IOTLB, 1); - AMD_IOMMU_REG_SET64(&(entry[1]), AMD_IOMMU_DEVTBL_DOMAINID, - (uint16_t)domainid); - AMD_IOMMU_REG_SET64(&(entry[0]), AMD_IOMMU_DEVTBL_IW, 1); - AMD_IOMMU_REG_SET64(&(entry[0]), AMD_IOMMU_DEVTBL_IR, 1); - AMD_IOMMU_REG_SET64(&(entry[0]), AMD_IOMMU_DEVTBL_ROOT_PGTBL, - dp->d_pgtable_root_4K); - AMD_IOMMU_REG_SET64(&(entry[0]), AMD_IOMMU_DEVTBL_PG_MODE, - AMD_IOMMU_PGTABLE_MAXLEVEL); - AMD_IOMMU_REG_SET64(&(entry[0]), AMD_IOMMU_DEVTBL_TV, - domainid == AMD_IOMMU_PASSTHRU_DOMAIN ? 0 : 1); - AMD_IOMMU_REG_SET64(&(entry[0]), AMD_IOMMU_DEVTBL_V, - domainid == AMD_IOMMU_PASSTHRU_DOMAIN ? 0 : 1); - - for (i = 1; i < 4; i++) { - devtbl_entry[i] = entry[i]; - } - devtbl_entry[0] = entry[0]; - - /* we did an actual init */ - return (1); -} - -void -amd_iommu_set_passthru(amd_iommu_t *iommu, dev_info_t *rdip) -{ - int32_t deviceid; - int alias; - uint64_t *devtbl_entry; - amd_iommu_cmdargs_t cmdargs = {0}; - char *path; - int pathfree; - int V; - int TV; - int instance; - const char *driver; - const char *f = "amd_iommu_set_passthru"; - - if (rdip) { - driver = ddi_driver_name(rdip); - instance = ddi_get_instance(rdip); - } else { - driver = "special-device"; - instance = 0; - } - - path = kmem_alloc(MAXPATHLEN, KM_NOSLEEP); - if (path) { - if (rdip) - (void) ddi_pathname(rdip, path); - else - (void) strcpy(path, "special-device"); - pathfree = 1; - } else { - pathfree = 0; - path = "<path-mem-alloc-failed>"; - } - - if (amd_iommu_get_deviceid(iommu, rdip, &deviceid, &alias, path) - != DDI_SUCCESS) { - cmn_err(CE_WARN, "%s: %s%d: idx=%d: rdip=%p. " - "Failed to get device ID for device %s.", f, driver, - instance, - iommu->aiomt_idx, (void *)rdip, path); - goto out; - } - - /* No deviceid */ - if (deviceid == -1) { - goto out; - } - - if ((deviceid + 1) * AMD_IOMMU_DEVTBL_ENTRY_SZ > - iommu->aiomt_devtbl_sz) { - cmn_err(CE_WARN, "%s: %s%d: IOMMU idx=%d, deviceid (%u) " - "for rdip (%p) exceeds device table size (%u), path=%s", - f, driver, - instance, iommu->aiomt_idx, deviceid, (void *)rdip, - iommu->aiomt_devtbl_sz, path); - goto out; - } - - /*LINTED*/ - devtbl_entry = (uint64_t *)&iommu->aiomt_devtbl - [deviceid * AMD_IOMMU_DEVTBL_ENTRY_SZ]; - - V = AMD_IOMMU_REG_GET64(&(devtbl_entry[0]), AMD_IOMMU_DEVTBL_V); - TV = AMD_IOMMU_REG_GET64(&(devtbl_entry[0]), AMD_IOMMU_DEVTBL_TV); - - /* Already passthru */ - if (V == 0 && TV == 0) { - goto out; - } - - /* Existing translations */ - if (V == 1 && TV == 1) { - goto out; - } - - /* Invalid setting */ - if (V == 0 && TV == 1) { - goto out; - } - - AMD_IOMMU_REG_SET64(&(devtbl_entry[0]), AMD_IOMMU_DEVTBL_V, 0); - - cmdargs.ca_deviceid = (uint16_t)deviceid; - (void) amd_iommu_cmd(iommu, AMD_IOMMU_CMD_INVAL_DEVTAB_ENTRY, - &cmdargs, 0, 0); - -out: - if (pathfree) - kmem_free(path, MAXPATHLEN); -} - -static int -amd_iommu_set_devtbl_entry(amd_iommu_t *iommu, dev_info_t *rdip, - domain_id_t domainid, uint16_t deviceid, amd_iommu_domain_t *dp, - const char *path) -{ - uint64_t *devtbl_entry; - amd_iommu_cmdargs_t cmdargs = {0}; - int error; - dev_info_t *idip = iommu->aiomt_dip; - const char *driver = ddi_driver_name(idip); - int instance = ddi_get_instance(idip); - const char *f = "amd_iommu_set_devtbl_entry"; - - if (amd_iommu_debug & AMD_IOMMU_DEBUG_DEVTBL) { - cmn_err(CE_WARN, "%s: attempting to set devtbl entry for %s", - f, path); - } - - if ((deviceid + 1) * AMD_IOMMU_DEVTBL_ENTRY_SZ > - iommu->aiomt_devtbl_sz) { - cmn_err(CE_WARN, "%s: %s%d: IOMMU idx=%d, deviceid (%u) " - "for rdip (%p) exceeds device table size (%u), path=%s", - f, driver, - instance, iommu->aiomt_idx, deviceid, (void *)rdip, - iommu->aiomt_devtbl_sz, path); - return (DDI_DMA_NOMAPPING); - } - - /*LINTED*/ - devtbl_entry = (uint64_t *)&iommu->aiomt_devtbl - [deviceid * AMD_IOMMU_DEVTBL_ENTRY_SZ]; - - if (amd_iommu_debug & AMD_IOMMU_DEBUG_DEVTBL) { - cmn_err(CE_WARN, "%s: deviceid=%u devtbl entry (%p) for %s", - f, deviceid, (void *)(uintptr_t)(*devtbl_entry), path); - } - - if (init_devtbl(iommu, devtbl_entry, domainid, dp)) { - cmdargs.ca_deviceid = deviceid; - error = amd_iommu_cmd(iommu, AMD_IOMMU_CMD_INVAL_DEVTAB_ENTRY, - &cmdargs, 0, 0); - } - - return (error); -} - -int -amd_iommu_clear_devtbl_entry(amd_iommu_t *iommu, dev_info_t *rdip, - domain_id_t domainid, uint16_t deviceid, amd_iommu_domain_t *dp, - int *domain_freed, char *path) -{ - uint64_t *devtbl_entry; - int error = DDI_SUCCESS; - amd_iommu_cmdargs_t cmdargs = {0}; - const char *driver = ddi_driver_name(iommu->aiomt_dip); - int instance = ddi_get_instance(iommu->aiomt_dip); - const char *f = "amd_iommu_clear_devtbl_entry"; - - if (amd_iommu_debug & AMD_IOMMU_DEBUG_DEVTBL) { - cmn_err(CE_NOTE, "%s: attempting to clear devtbl entry for " - "domainid = %d, deviceid = %u, path = %s", - f, domainid, deviceid, path); - } - - if ((deviceid + 1) * AMD_IOMMU_DEVTBL_ENTRY_SZ > - iommu->aiomt_devtbl_sz) { - cmn_err(CE_WARN, "%s: %s%d: IOMMU idx=%d, deviceid (%u) " - "for rdip (%p) exceeds device table size (%u), path = %s", - f, driver, instance, - iommu->aiomt_idx, deviceid, (void *)rdip, - iommu->aiomt_devtbl_sz, path); - return (DDI_FAILURE); - } - - /*LINTED*/ - devtbl_entry = (uint64_t *)&iommu->aiomt_devtbl - [deviceid * AMD_IOMMU_DEVTBL_ENTRY_SZ]; - - if (amd_iommu_debug & AMD_IOMMU_DEBUG_DEVTBL) { - cmn_err(CE_WARN, "%s: deviceid=%u devtbl entry (%p) for %s", - f, deviceid, (void *)(uintptr_t)(*devtbl_entry), path); - } - - if (AMD_IOMMU_REG_GET64(&(devtbl_entry[0]), AMD_IOMMU_DEVTBL_TV) == 0) { - /* Nothing to do */ - return (DDI_SUCCESS); - } - - ASSERT(dp->d_pgtable_root_4K == AMD_IOMMU_REG_GET64(&(devtbl_entry[0]), - AMD_IOMMU_DEVTBL_ROOT_PGTBL)); - - ASSERT(domainid == AMD_IOMMU_REG_GET64(&(devtbl_entry[1]), - AMD_IOMMU_DEVTBL_DOMAINID)); - - AMD_IOMMU_REG_SET64(&(devtbl_entry[0]), AMD_IOMMU_DEVTBL_TV, 0); - AMD_IOMMU_REG_SET64(&(devtbl_entry[0]), AMD_IOMMU_DEVTBL_ROOT_PGTBL, 0); - AMD_IOMMU_REG_SET64(&(devtbl_entry[0]), AMD_IOMMU_DEVTBL_V, 1); - - SYNC_FORDEV(iommu->aiomt_dmahdl); - - dp->d_ref--; - ASSERT(dp->d_ref >= 0); - - if (dp->d_ref == 0) { - *domain_freed = 1; - } - - cmdargs.ca_deviceid = deviceid; - error = amd_iommu_cmd(iommu, AMD_IOMMU_CMD_INVAL_DEVTAB_ENTRY, - &cmdargs, 0, 0); - if (error != DDI_SUCCESS) - error = DDI_FAILURE; - - return (error); -} - -int -amd_iommu_page_table_hash_init(amd_iommu_page_table_hash_t *ampt) -{ - ampt->ampt_hash = kmem_zalloc(sizeof (amd_iommu_page_table_t *) * - AMD_IOMMU_PGTABLE_HASH_SZ, KM_SLEEP); - return (DDI_SUCCESS); -} - -void -amd_iommu_page_table_hash_fini(amd_iommu_page_table_hash_t *ampt) -{ - kmem_free(ampt->ampt_hash, - sizeof (amd_iommu_page_table_t *) * AMD_IOMMU_PGTABLE_HASH_SZ); - ampt->ampt_hash = NULL; -} - -static uint32_t -pt_hashfn(uint64_t pa_4K) -{ - return (pa_4K % AMD_IOMMU_PGTABLE_HASH_SZ); -} - -static void -amd_iommu_insert_pgtable_hash(amd_iommu_page_table_t *pt) -{ - uint64_t pa_4K = ((uint64_t)pt->pt_cookie.dmac_cookie_addr) >> 12; - uint32_t idx = pt_hashfn(pa_4K); - - ASSERT((pt->pt_cookie.dmac_cookie_addr & AMD_IOMMU_PGTABLE_ALIGN) == 0); - - mutex_enter(&amd_iommu_page_table_hash.ampt_lock); - - pt->pt_next = amd_iommu_page_table_hash.ampt_hash[idx]; - pt->pt_prev = NULL; - amd_iommu_page_table_hash.ampt_hash[idx] = pt; - if (pt->pt_next) - pt->pt_next->pt_prev = pt; - - mutex_exit(&amd_iommu_page_table_hash.ampt_lock); -} - -static void -amd_iommu_remove_pgtable_hash(amd_iommu_page_table_t *pt) -{ - uint64_t pa_4K = (pt->pt_cookie.dmac_cookie_addr >> 12); - uint32_t idx = pt_hashfn(pa_4K); - - ASSERT((pt->pt_cookie.dmac_cookie_addr & AMD_IOMMU_PGTABLE_ALIGN) == 0); - - mutex_enter(&amd_iommu_page_table_hash.ampt_lock); - - if (pt->pt_next) - pt->pt_next->pt_prev = pt->pt_prev; - - if (pt->pt_prev) - pt->pt_prev->pt_next = pt->pt_next; - else - amd_iommu_page_table_hash.ampt_hash[idx] = pt->pt_next; - - pt->pt_next = NULL; - pt->pt_prev = NULL; - - mutex_exit(&amd_iommu_page_table_hash.ampt_lock); -} - -static amd_iommu_page_table_t * -amd_iommu_lookup_pgtable_hash(domain_id_t domainid, uint64_t pgtable_pa_4K) -{ - amd_iommu_page_table_t *pt; - uint32_t idx = pt_hashfn(pgtable_pa_4K); - - mutex_enter(&amd_iommu_page_table_hash.ampt_lock); - pt = amd_iommu_page_table_hash.ampt_hash[idx]; - for (; pt; pt = pt->pt_next) { - if (domainid != pt->pt_domainid) - continue; - ASSERT((pt->pt_cookie.dmac_cookie_addr & - AMD_IOMMU_PGTABLE_ALIGN) == 0); - if ((pt->pt_cookie.dmac_cookie_addr >> 12) == pgtable_pa_4K) { - break; - } - } - mutex_exit(&amd_iommu_page_table_hash.ampt_lock); - - return (pt); -} - -/*ARGSUSED*/ -static amd_iommu_page_table_t * -amd_iommu_lookup_pgtable(amd_iommu_t *iommu, amd_iommu_page_table_t *ppt, - amd_iommu_domain_t *dp, int level, uint16_t index) -{ - uint64_t *pdtep; - uint64_t pgtable_pa_4K; - - ASSERT(level > 0 && level <= AMD_IOMMU_PGTABLE_MAXLEVEL); - ASSERT(dp); - - if (level == AMD_IOMMU_PGTABLE_MAXLEVEL) { - ASSERT(ppt == NULL); - ASSERT(index == 0); - pgtable_pa_4K = dp->d_pgtable_root_4K; - } else { - ASSERT(ppt); - pdtep = &(ppt->pt_pgtblva[index]); - if (AMD_IOMMU_REG_GET64(pdtep, AMD_IOMMU_PTDE_PR) == 0) { - if (amd_iommu_debug & AMD_IOMMU_DEBUG_PAGE_TABLES) { - cmn_err(CE_NOTE, "Skipping PR=0 pdte: 0x%" - PRIx64, *pdtep); - } - return (NULL); - } - pgtable_pa_4K = AMD_IOMMU_REG_GET64(pdtep, AMD_IOMMU_PTDE_ADDR); - } - - return (amd_iommu_lookup_pgtable_hash(dp->d_domainid, pgtable_pa_4K)); -} - -static amd_iommu_page_table_t * -amd_iommu_alloc_from_freelist(void) -{ - int i; - uint64_t *pte_array; - amd_iommu_page_table_t *pt; - - if (amd_iommu_no_pgtable_freelist == 1) - return (NULL); - - if (amd_iommu_pgtable_freelist.f_count == 0) - return (NULL); - - pt = amd_iommu_pgtable_freelist.f_list; - amd_iommu_pgtable_freelist.f_list = pt->pt_next; - amd_iommu_pgtable_freelist.f_count--; - - pte_array = pt->pt_pgtblva; - for (i = 0; i < AMD_IOMMU_PGTABLE_SZ / (sizeof (*pte_array)); i++) { - ASSERT(pt->pt_pte_ref[i] == 0); - ASSERT(AMD_IOMMU_REG_GET64(&(pte_array[i]), - AMD_IOMMU_PTDE_PR) == 0); - } - - return (pt); -} - -static int -amd_iommu_alloc_pgtable(amd_iommu_t *iommu, domain_id_t domainid, - const char *path, amd_iommu_page_table_t **ptp, int km_flags) -{ - int err; - uint_t ncookies; - amd_iommu_page_table_t *pt; - dev_info_t *idip = iommu->aiomt_dip; - const char *driver = ddi_driver_name(idip); - int instance = ddi_get_instance(idip); - const char *f = "amd_iommu_alloc_pgtable"; - - *ptp = NULL; - - pt = amd_iommu_alloc_from_freelist(); - if (pt) - goto init_pgtable; - - pt = kmem_zalloc(sizeof (amd_iommu_page_table_t), km_flags); - if (pt == NULL) - return (DDI_DMA_NORESOURCES); - - /* - * Each page table is 4K in size - */ - pt->pt_mem_reqsz = AMD_IOMMU_PGTABLE_SZ; - - /* - * Alloc a DMA handle. Use the IOMMU dip as we want this DMA - * to *not* enter the IOMMU - no recursive entrance. - */ - err = ddi_dma_alloc_handle(idip, &amd_iommu_pgtable_dma_attr, - km_flags == KM_SLEEP ? DDI_DMA_SLEEP : DDI_DMA_DONTWAIT, - NULL, &pt->pt_dma_hdl); - if (err != DDI_SUCCESS) { - cmn_err(CE_WARN, "%s: %s%d: domainid = %d, path = %s. " - "Cannot alloc DMA handle for IO Page Table", - f, driver, instance, domainid, path); - kmem_free(pt, sizeof (amd_iommu_page_table_t)); - return (err == DDI_DMA_NORESOURCES ? err : DDI_DMA_NOMAPPING); - } - - /* - * Alloc memory for IO Page Table. - * XXX remove size_t cast kludge - */ - err = ddi_dma_mem_alloc(pt->pt_dma_hdl, pt->pt_mem_reqsz, - &amd_iommu_devacc, DDI_DMA_CONSISTENT|IOMEM_DATA_UNCACHED, - km_flags == KM_SLEEP ? DDI_DMA_SLEEP : DDI_DMA_DONTWAIT, - NULL, (caddr_t *)&pt->pt_pgtblva, - (size_t *)&pt->pt_mem_realsz, &pt->pt_mem_hdl); - if (err != DDI_SUCCESS) { - cmn_err(CE_WARN, "%s: %s%d: domainid=%d, path = %s. " - "Cannot allocate DMA memory for IO Page table", - f, driver, instance, domainid, path); - ddi_dma_free_handle(&pt->pt_dma_hdl); - kmem_free(pt, sizeof (amd_iommu_page_table_t)); - return (DDI_DMA_NORESOURCES); - } - - /* - * The Page table DMA VA must be 4K aligned and - * size >= than requested memory. - * - */ - ASSERT(((uint64_t)(uintptr_t)pt->pt_pgtblva & AMD_IOMMU_PGTABLE_ALIGN) - == 0); - ASSERT(pt->pt_mem_realsz >= pt->pt_mem_reqsz); - - /* - * Now bind the handle - */ - err = ddi_dma_addr_bind_handle(pt->pt_dma_hdl, NULL, - (caddr_t)pt->pt_pgtblva, pt->pt_mem_realsz, - DDI_DMA_READ | DDI_DMA_CONSISTENT, - km_flags == KM_SLEEP ? DDI_DMA_SLEEP : DDI_DMA_DONTWAIT, - NULL, &pt->pt_cookie, &ncookies); - if (err != DDI_DMA_MAPPED) { - cmn_err(CE_WARN, "%s: %s%d: domainid=%d, path = %s. " - "Cannot bind memory for DMA to IO Page Tables. " - "bufrealsz=%p", - f, driver, instance, domainid, path, - (void *)(uintptr_t)pt->pt_mem_realsz); - ddi_dma_mem_free(&pt->pt_mem_hdl); - ddi_dma_free_handle(&pt->pt_dma_hdl); - kmem_free(pt, sizeof (amd_iommu_page_table_t)); - return (err == DDI_DMA_PARTIAL_MAP ? DDI_DMA_NOMAPPING : - err); - } - - /* - * We assume the DMA engine on the IOMMU is capable of handling the - * whole page table in a single cookie. If not and multiple cookies - * are needed we fail. - */ - if (ncookies != 1) { - cmn_err(CE_WARN, "%s: %s%d: domainid = %d, path=%s " - "Cannot handle multiple " - "cookies for DMA to IO page Table, #cookies=%u", - f, driver, instance, domainid, path, ncookies); - (void) ddi_dma_unbind_handle(pt->pt_dma_hdl); - ddi_dma_mem_free(&pt->pt_mem_hdl); - ddi_dma_free_handle(&pt->pt_dma_hdl); - kmem_free(pt, sizeof (amd_iommu_page_table_t)); - return (DDI_DMA_NOMAPPING); - } - -init_pgtable: - /* - * The address in the cookie must be 4K aligned and >= table size - */ - ASSERT(pt->pt_cookie.dmac_cookie_addr != NULL); - ASSERT((pt->pt_cookie.dmac_cookie_addr & AMD_IOMMU_PGTABLE_ALIGN) == 0); - ASSERT(pt->pt_cookie.dmac_size >= pt->pt_mem_realsz); - ASSERT(pt->pt_cookie.dmac_size >= pt->pt_mem_reqsz); - ASSERT(pt->pt_mem_reqsz >= AMD_IOMMU_PGTABLE_SIZE); - ASSERT(pt->pt_mem_realsz >= pt->pt_mem_reqsz); - ASSERT(pt->pt_pgtblva); - - pt->pt_domainid = AMD_IOMMU_INVALID_DOMAIN; - pt->pt_level = 0x7; - pt->pt_index = 0; - pt->pt_ref = 0; - pt->pt_next = NULL; - pt->pt_prev = NULL; - pt->pt_parent = NULL; - - bzero(pt->pt_pgtblva, pt->pt_mem_realsz); - SYNC_FORDEV(pt->pt_dma_hdl); - - amd_iommu_insert_pgtable_hash(pt); - - *ptp = pt; - - return (DDI_SUCCESS); -} - -static int -amd_iommu_move_to_freelist(amd_iommu_page_table_t *pt) -{ - if (amd_iommu_no_pgtable_freelist == 1) - return (DDI_FAILURE); - - if (amd_iommu_pgtable_freelist.f_count == - AMD_IOMMU_PGTABLE_FREELIST_MAX) - return (DDI_FAILURE); - - pt->pt_next = amd_iommu_pgtable_freelist.f_list; - amd_iommu_pgtable_freelist.f_list = pt; - amd_iommu_pgtable_freelist.f_count++; - - return (DDI_SUCCESS); -} - -static void -amd_iommu_free_pgtable(amd_iommu_t *iommu, amd_iommu_page_table_t *pt) -{ - int i; - uint64_t *pte_array; - dev_info_t *dip = iommu->aiomt_dip; - int instance = ddi_get_instance(dip); - const char *driver = ddi_driver_name(dip); - const char *f = "amd_iommu_free_pgtable"; - - ASSERT(pt->pt_ref == 0); - - amd_iommu_remove_pgtable_hash(pt); - - pte_array = pt->pt_pgtblva; - for (i = 0; i < AMD_IOMMU_PGTABLE_SZ / (sizeof (*pte_array)); i++) { - ASSERT(pt->pt_pte_ref[i] == 0); - ASSERT(AMD_IOMMU_REG_GET64(&(pte_array[i]), - AMD_IOMMU_PTDE_PR) == 0); - } - - if (amd_iommu_move_to_freelist(pt) == DDI_SUCCESS) - return; - - /* Unbind the handle */ - if (ddi_dma_unbind_handle(pt->pt_dma_hdl) != DDI_SUCCESS) { - cmn_err(CE_WARN, "%s: %s%d: idx=%d, domainid=%d. " - "Failed to unbind handle: %p for IOMMU Page Table", - f, driver, instance, iommu->aiomt_idx, pt->pt_domainid, - (void *)pt->pt_dma_hdl); - } - /* Free the table memory allocated for DMA */ - ddi_dma_mem_free(&pt->pt_mem_hdl); - - /* Free the DMA handle */ - ddi_dma_free_handle(&pt->pt_dma_hdl); - - kmem_free(pt, sizeof (amd_iommu_page_table_t)); - -} - -static int -init_pde(amd_iommu_page_table_t *ppt, amd_iommu_page_table_t *pt) -{ - uint64_t *pdep = &(ppt->pt_pgtblva[pt->pt_index]); - uint64_t next_pgtable_pa_4K = (pt->pt_cookie.dmac_cookie_addr) >> 12; - - /* nothing to set. PDE is already set */ - if (AMD_IOMMU_REG_GET64(pdep, AMD_IOMMU_PTDE_PR) == 1) { - ASSERT(PT_REF_VALID(ppt)); - ASSERT(PT_REF_VALID(pt)); - ASSERT(ppt->pt_pte_ref[pt->pt_index] == 0); - ASSERT(AMD_IOMMU_REG_GET64(pdep, AMD_IOMMU_PTDE_ADDR) - == next_pgtable_pa_4K); - return (DDI_SUCCESS); - } - - ppt->pt_ref++; - ASSERT(PT_REF_VALID(ppt)); - - /* Page Directories are always RW */ - AMD_IOMMU_REG_SET64(pdep, AMD_IOMMU_PTDE_IW, 1); - AMD_IOMMU_REG_SET64(pdep, AMD_IOMMU_PTDE_IR, 1); - AMD_IOMMU_REG_SET64(pdep, AMD_IOMMU_PTDE_ADDR, - next_pgtable_pa_4K); - pt->pt_parent = ppt; - AMD_IOMMU_REG_SET64(pdep, AMD_IOMMU_PTDE_NXT_LVL, - pt->pt_level); - ppt->pt_pte_ref[pt->pt_index] = 0; - AMD_IOMMU_REG_SET64(pdep, AMD_IOMMU_PTDE_PR, 1); - SYNC_FORDEV(ppt->pt_dma_hdl); - ASSERT(AMD_IOMMU_REG_GET64(pdep, AMD_IOMMU_PTDE_PR) == 1); - - return (DDI_SUCCESS); -} - -static int -init_pte(amd_iommu_page_table_t *pt, uint64_t pa, uint16_t index, - struct ddi_dma_req *dmareq) -{ - uint64_t *ptep = &(pt->pt_pgtblva[index]); - uint64_t pa_4K = pa >> 12; - int R; - int W; - - /* nothing to set if PTE is already set */ - if (AMD_IOMMU_REG_GET64(ptep, AMD_IOMMU_PTDE_PR) == 1) { - /* - * Adjust current permissions - * DDI_DMA_WRITE means direction of DMA is MEM -> I/O - * so that requires Memory READ permissions i.e. sense - * is inverted. - * Note: either or both of DD_DMA_READ/WRITE may be set - */ - if (amd_iommu_no_RW_perms == 0) { - R = AMD_IOMMU_REG_GET64(ptep, AMD_IOMMU_PTDE_IR); - W = AMD_IOMMU_REG_GET64(ptep, AMD_IOMMU_PTDE_IW); - if (R == 0 && ((dmareq->dmar_flags & DDI_DMA_WRITE) || - (dmareq->dmar_flags & DDI_DMA_RDWR))) { - AMD_IOMMU_REG_SET64(ptep, AMD_IOMMU_PTDE_IR, 1); - } - if (W == 0 && ((dmareq->dmar_flags & DDI_DMA_READ) || - (dmareq->dmar_flags & DDI_DMA_RDWR))) { - AMD_IOMMU_REG_SET64(ptep, AMD_IOMMU_PTDE_IW, 1); - } - } - ASSERT(PT_REF_VALID(pt)); - pt->pt_pte_ref[index]++; - ASSERT(AMD_IOMMU_REG_GET64(ptep, AMD_IOMMU_PTDE_ADDR) - == pa_4K); - return (DDI_SUCCESS); - } - - pt->pt_ref++; - ASSERT(PT_REF_VALID(pt)); - - /* see comment above about inverting sense of RD/WR */ - if (amd_iommu_no_RW_perms == 0) { - AMD_IOMMU_REG_SET64(ptep, AMD_IOMMU_PTDE_IR, 0); - AMD_IOMMU_REG_SET64(ptep, AMD_IOMMU_PTDE_IW, 0); - if (dmareq->dmar_flags & DDI_DMA_RDWR) { - AMD_IOMMU_REG_SET64(ptep, AMD_IOMMU_PTDE_IW, 1); - AMD_IOMMU_REG_SET64(ptep, AMD_IOMMU_PTDE_IR, 1); - } else { - if (dmareq->dmar_flags & DDI_DMA_WRITE) { - AMD_IOMMU_REG_SET64(ptep, AMD_IOMMU_PTDE_IR, 1); - } - if (dmareq->dmar_flags & DDI_DMA_READ) { - AMD_IOMMU_REG_SET64(ptep, AMD_IOMMU_PTDE_IW, 1); - } - } - } else { - AMD_IOMMU_REG_SET64(ptep, AMD_IOMMU_PTDE_IR, 1); - AMD_IOMMU_REG_SET64(ptep, AMD_IOMMU_PTDE_IW, 1); - } - - /* TODO what is correct for FC and U */ - AMD_IOMMU_REG_SET64(ptep, AMD_IOMMU_PTE_FC, 0); - AMD_IOMMU_REG_SET64(ptep, AMD_IOMMU_PTE_U, 0); - AMD_IOMMU_REG_SET64(ptep, AMD_IOMMU_PTDE_ADDR, pa_4K); - AMD_IOMMU_REG_SET64(ptep, AMD_IOMMU_PTDE_NXT_LVL, 0); - ASSERT(pt->pt_pte_ref[index] == 0); - pt->pt_pte_ref[index] = 1; - AMD_IOMMU_REG_SET64(ptep, AMD_IOMMU_PTDE_PR, 1); - SYNC_FORDEV(pt->pt_dma_hdl); - ASSERT(AMD_IOMMU_REG_GET64(ptep, AMD_IOMMU_PTDE_PR) == 1); - - return (DDI_SUCCESS); -} - - -static void -init_pt(amd_iommu_page_table_t *pt, amd_iommu_domain_t *dp, - int level, uint16_t index) -{ - ASSERT(dp); - - if (level == AMD_IOMMU_PGTABLE_MAXLEVEL) { - dp->d_pgtable_root_4K = (pt->pt_cookie.dmac_cookie_addr) >> 12; - } else { - ASSERT(level >= 1 && level < AMD_IOMMU_PGTABLE_MAXLEVEL); - } - - pt->pt_domainid = dp->d_domainid; - pt->pt_level = level; - pt->pt_index = index; -} - -static int -amd_iommu_setup_1_pgtable(amd_iommu_t *iommu, dev_info_t *rdip, - struct ddi_dma_req *dmareq, - domain_id_t domainid, amd_iommu_domain_t *dp, - amd_iommu_page_table_t *ppt, - uint16_t index, int level, uint64_t va, uint64_t pa, - amd_iommu_page_table_t **ptp, uint16_t *next_idxp, const char *path, - int km_flags) -{ - int error; - amd_iommu_page_table_t *pt; - const char *driver = ddi_driver_name(rdip); - int instance = ddi_get_instance(rdip); - const char *f = "amd_iommu_setup_1_pgtable"; - - *ptp = NULL; - *next_idxp = 0; - error = DDI_SUCCESS; - - ASSERT(level > 0 && level <= AMD_IOMMU_PGTABLE_MAXLEVEL); - - ASSERT(dp); - if (level == AMD_IOMMU_PGTABLE_MAXLEVEL) { - ASSERT(ppt == NULL); - ASSERT(index == 0); - } else { - ASSERT(ppt); - } - - /* Check if page table is already allocated */ - if (pt = amd_iommu_lookup_pgtable(iommu, ppt, dp, level, index)) { - ASSERT(pt->pt_domainid == domainid); - ASSERT(pt->pt_level == level); - ASSERT(pt->pt_index == index); - goto out; - } - - if ((error = amd_iommu_alloc_pgtable(iommu, domainid, path, &pt, - km_flags)) != DDI_SUCCESS) { - cmn_err(CE_WARN, "%s: %s%d: idx = %u, domainid = %d, va = %p " - "path = %s", f, driver, instance, iommu->aiomt_idx, - domainid, (void *)(uintptr_t)va, path); - return (error); - } - - ASSERT(dp->d_domainid == domainid); - - init_pt(pt, dp, level, index); - -out: - if (level != AMD_IOMMU_PGTABLE_MAXLEVEL) { - error = init_pde(ppt, pt); - } - - if (level == 1) { - ASSERT(error == DDI_SUCCESS); - error = init_pte(pt, pa, AMD_IOMMU_VA_BITS(va, level), dmareq); - } else { - *next_idxp = AMD_IOMMU_VA_BITS(va, level); - *ptp = pt; - } - - return (error); -} - -typedef enum { - PDTE_NOT_TORN = 0x1, - PDTE_TORN_DOWN = 0x2, - PGTABLE_TORN_DOWN = 0x4 -} pdte_tear_t; - -static pdte_tear_t -amd_iommu_teardown_pdte(amd_iommu_t *iommu, - amd_iommu_page_table_t *pt, int index) -{ - uint8_t next_level; - pdte_tear_t retval; - uint64_t *ptdep = &(pt->pt_pgtblva[index]); - - next_level = AMD_IOMMU_REG_GET64(ptdep, - AMD_IOMMU_PTDE_NXT_LVL); - - if (AMD_IOMMU_REG_GET64(ptdep, AMD_IOMMU_PTDE_PR) == 1) { - if (pt->pt_level == 1) { - ASSERT(next_level == 0); - /* PTE */ - pt->pt_pte_ref[index]--; - if (pt->pt_pte_ref[index] != 0) { - return (PDTE_NOT_TORN); - } - } else { - ASSERT(next_level != 0 && next_level != 7); - } - ASSERT(pt->pt_pte_ref[index] == 0); - ASSERT(PT_REF_VALID(pt)); - - AMD_IOMMU_REG_SET64(ptdep, AMD_IOMMU_PTDE_PR, 0); - SYNC_FORDEV(pt->pt_dma_hdl); - ASSERT(AMD_IOMMU_REG_GET64(ptdep, - AMD_IOMMU_PTDE_PR) == 0); - pt->pt_ref--; - ASSERT(PT_REF_VALID(pt)); - retval = PDTE_TORN_DOWN; - } else { - ASSERT(0); - ASSERT(pt->pt_pte_ref[index] == 0); - ASSERT(PT_REF_VALID(pt)); - retval = PDTE_NOT_TORN; - } - - if (pt->pt_ref == 0) { - amd_iommu_free_pgtable(iommu, pt); - return (PGTABLE_TORN_DOWN); - } - - return (retval); -} - -static int -amd_iommu_create_pgtables(amd_iommu_t *iommu, dev_info_t *rdip, - struct ddi_dma_req *dmareq, uint64_t va, - uint64_t pa, uint16_t deviceid, domain_id_t domainid, - amd_iommu_domain_t *dp, const char *path, int km_flags) -{ - int level; - uint16_t index; - uint16_t next_idx; - amd_iommu_page_table_t *pt; - amd_iommu_page_table_t *ppt; - int error; - const char *driver = ddi_driver_name(rdip); - int instance = ddi_get_instance(rdip); - const char *f = "amd_iommu_create_pgtables"; - - if (amd_iommu_debug & AMD_IOMMU_DEBUG_PAGE_TABLES) { - cmn_err(CE_NOTE, "%s: %s%d: idx = %u, domainid = %d, " - "deviceid = %u, va = %p, pa = %p, path = %s", - f, driver, instance, - iommu->aiomt_idx, domainid, deviceid, - (void *)(uintptr_t)va, - (void *)(uintptr_t)pa, path); - } - - if (domainid == AMD_IOMMU_PASSTHRU_DOMAIN) { - /* No need for pagetables. Just set up device table entry */ - goto passthru; - } - - index = 0; - ppt = NULL; - for (level = AMD_IOMMU_PGTABLE_MAXLEVEL; level > 0; - level--, pt = NULL, next_idx = 0) { - if ((error = amd_iommu_setup_1_pgtable(iommu, rdip, dmareq, - domainid, dp, ppt, index, level, va, pa, &pt, - &next_idx, path, km_flags)) != DDI_SUCCESS) { - cmn_err(CE_WARN, "%s: %s%d: idx=%d: domainid=%d, " - "deviceid=%u, va= %p, pa = %p, Failed to setup " - "page table(s) at level = %d, path = %s.", - f, driver, instance, iommu->aiomt_idx, - domainid, deviceid, (void *)(uintptr_t)va, - (void *)(uintptr_t)pa, level, path); - return (error); - } - - if (level > 1) { - ASSERT(pt); - ASSERT(pt->pt_domainid == domainid); - ppt = pt; - index = next_idx; - } else { - ASSERT(level == 1); - ASSERT(pt == NULL); - ASSERT(next_idx == 0); - ppt = NULL; - index = 0; - } - } - -passthru: - if ((error = amd_iommu_set_devtbl_entry(iommu, rdip, domainid, deviceid, - dp, path)) != DDI_SUCCESS) { - cmn_err(CE_WARN, "%s: %s%d: idx=%d: rdip=%p, deviceid=%u, " - "domainid=%d." - "Failed to set device table entry for path %s.", - f, driver, instance, - iommu->aiomt_idx, (void *)rdip, deviceid, domainid, path); - return (error); - } - - SYNC_FORDEV(iommu->aiomt_dmahdl); - - return (DDI_SUCCESS); -} - -static int -amd_iommu_destroy_pgtables(amd_iommu_t *iommu, dev_info_t *rdip, - uint64_t pageva, uint16_t deviceid, domain_id_t domainid, - amd_iommu_domain_t *dp, map_type_t type, int *domain_freed, char *path) -{ - int level; - int flags; - amd_iommu_cmdargs_t cmdargs = {0}; - uint16_t index; - uint16_t prev_index; - amd_iommu_page_table_t *pt; - amd_iommu_page_table_t *ppt; - pdte_tear_t retval; - int tear_level; - int invalidate_pte; - int invalidate_pde; - int error = DDI_FAILURE; - const char *driver = ddi_driver_name(iommu->aiomt_dip); - int instance = ddi_get_instance(iommu->aiomt_dip); - const char *f = "amd_iommu_destroy_pgtables"; - - if (amd_iommu_debug & AMD_IOMMU_DEBUG_PAGE_TABLES) { - cmn_err(CE_NOTE, "%s: %s%d: idx = %u, domainid = %d, " - "deviceid = %u, va = %p, path = %s", - f, driver, instance, - iommu->aiomt_idx, domainid, deviceid, - (void *)(uintptr_t)pageva, path); - } - - if (domainid == AMD_IOMMU_PASSTHRU_DOMAIN) { - /* - * there are no pagetables for the passthru domain. - * Just the device table entry - */ - error = DDI_SUCCESS; - goto passthru; - } - - ppt = NULL; - index = 0; - for (level = AMD_IOMMU_PGTABLE_MAXLEVEL; level > 0; level--) { - pt = amd_iommu_lookup_pgtable(iommu, ppt, dp, level, index); - if (pt) { - ppt = pt; - index = AMD_IOMMU_VA_BITS(pageva, level); - continue; - } - break; - } - - if (level == 0) { - uint64_t *ptep; - uint64_t pa_4K; - - ASSERT(pt); - ASSERT(pt == ppt); - ASSERT(pt->pt_domainid == dp->d_domainid); - - ptep = &(pt->pt_pgtblva[index]); - - pa_4K = AMD_IOMMU_REG_GET64(ptep, AMD_IOMMU_PTDE_ADDR); - if (amd_iommu_unity_map || type == AMD_IOMMU_UNITY_MAP) { - ASSERT(pageva == (pa_4K << MMU_PAGESHIFT)); - } - } - - tear_level = -1; - invalidate_pde = 0; - invalidate_pte = 0; - for (++level; level <= AMD_IOMMU_PGTABLE_MAXLEVEL; level++) { - prev_index = pt->pt_index; - ppt = pt->pt_parent; - retval = amd_iommu_teardown_pdte(iommu, pt, index); - switch (retval) { - case PDTE_NOT_TORN: - goto invalidate; - case PDTE_TORN_DOWN: - invalidate_pte = 1; - goto invalidate; - case PGTABLE_TORN_DOWN: - invalidate_pte = 1; - invalidate_pde = 1; - tear_level = level; - break; - } - index = prev_index; - pt = ppt; - } - -invalidate: - /* - * Now teardown the IOMMU HW caches if applicable - */ - if (invalidate_pte) { - cmdargs.ca_domainid = (uint16_t)domainid; - if (amd_iommu_pageva_inval_all) { - cmdargs.ca_addr = (uintptr_t)0x7FFFFFFFFFFFF000; - flags = AMD_IOMMU_CMD_FLAGS_PAGE_PDE_INVAL | - AMD_IOMMU_CMD_FLAGS_PAGE_INVAL_S; - } else if (invalidate_pde) { - cmdargs.ca_addr = - (uintptr_t)AMD_IOMMU_VA_INVAL(pageva, tear_level); - flags = AMD_IOMMU_CMD_FLAGS_PAGE_PDE_INVAL | - AMD_IOMMU_CMD_FLAGS_PAGE_INVAL_S; - } else { - cmdargs.ca_addr = (uintptr_t)pageva; - flags = 0; - } - if (amd_iommu_cmd(iommu, AMD_IOMMU_CMD_INVAL_IOMMU_PAGES, - &cmdargs, flags, 0) != DDI_SUCCESS) { - cmn_err(CE_WARN, "%s: %s%d: idx=%d: domainid=%d, " - "rdip=%p. Failed to invalidate IOMMU HW cache " - "for %s", f, driver, instance, - iommu->aiomt_idx, domainid, (void *)rdip, path); - error = DDI_FAILURE; - goto out; - } - } - -passthru: - if (tear_level == AMD_IOMMU_PGTABLE_MAXLEVEL) { - error = amd_iommu_clear_devtbl_entry(iommu, rdip, domainid, - deviceid, dp, domain_freed, path); - } else { - error = DDI_SUCCESS; - } - -out: - SYNC_FORDEV(iommu->aiomt_dmahdl); - - return (error); -} - -static int -cvt_bind_error(int error) -{ - switch (error) { - case DDI_DMA_MAPPED: - case DDI_DMA_PARTIAL_MAP: - case DDI_DMA_NORESOURCES: - case DDI_DMA_NOMAPPING: - break; - default: - cmn_err(CE_PANIC, "Unsupported error code: %d", error); - /*NOTREACHED*/ - } - return (error); -} - -int -amd_iommu_map_pa2va(amd_iommu_t *iommu, dev_info_t *rdip, ddi_dma_attr_t *attrp, - struct ddi_dma_req *dmareq, uint64_t start_pa, uint64_t pa_sz, - map_type_t type, uint64_t *start_vap, int km_flags) -{ - pfn_t pfn_start; - pfn_t pfn_end; - pfn_t pfn; - int alias; - int32_t deviceid; - domain_id_t domainid; - amd_iommu_domain_t *dp; - uint64_t end_pa; - uint64_t start_va; - uint64_t end_va; - uint64_t pg_start; - uint64_t pg_end; - uint64_t pg; - uint64_t va_sz; - char *path; - int error = DDI_DMA_NOMAPPING; - const char *driver = ddi_driver_name(iommu->aiomt_dip); - int instance = ddi_get_instance(iommu->aiomt_dip); - const char *f = "amd_iommu_map_pa2va"; - - ASSERT(pa_sz != 0); - - *start_vap = 0; - - ASSERT(rdip); - - path = kmem_alloc(MAXPATHLEN, km_flags); - if (path == NULL) { - error = DDI_DMA_NORESOURCES; - goto out; - } - (void) ddi_pathname(rdip, path); - - /* - * First get deviceid - */ - if (amd_iommu_get_deviceid(iommu, rdip, &deviceid, &alias, path) - != DDI_SUCCESS) { - cmn_err(CE_WARN, "%s: %s%d: idx=%d: rdip=%p. " - "Failed to get device ID for %s.", f, driver, instance, - iommu->aiomt_idx, (void *)rdip, path); - error = DDI_DMA_NOMAPPING; - goto out; - } - - /* - * Next get the domain for this rdip - */ - if (amd_iommu_get_domain(iommu, rdip, alias, deviceid, &domainid, path) - != DDI_SUCCESS) { - cmn_err(CE_WARN, "%s: %s%d: idx=%d: rdip=%p, path=%s. " - "Failed to get domain.", f, driver, instance, - iommu->aiomt_idx, (void *)rdip, path); - error = DDI_DMA_NOMAPPING; - goto out; - } - - dp = amd_iommu_lookup_domain(iommu, domainid, type, km_flags); - if (dp == NULL) { - cmn_err(CE_WARN, "%s: %s%d: idx=%d: domainid=%d, rdip=%p. " - "Failed to get device ID for %s.", f, driver, instance, - iommu->aiomt_idx, domainid, (void *)rdip, path); - error = DDI_DMA_NORESOURCES; - goto out; - } - - ASSERT(dp->d_domainid == domainid); - - pfn_start = start_pa >> MMU_PAGESHIFT; - - if (amd_iommu_debug & AMD_IOMMU_DEBUG_PAGE_TABLES) { - cmn_err(CE_NOTE, "pa = %p, pfn_new = %p, pfn_start = %p, " - "pgshift = %d", - (void *)(uintptr_t)start_pa, - (void *)(uintptr_t)(start_pa >> MMU_PAGESHIFT), - (void *)(uintptr_t)pfn_start, MMU_PAGESHIFT); - } - - end_pa = start_pa + pa_sz - 1; - pfn_end = end_pa >> MMU_PAGESHIFT; - - if (amd_iommu_unity_map || type == AMD_IOMMU_UNITY_MAP) { - start_va = start_pa; - end_va = end_pa; - va_sz = pa_sz; - *start_vap = start_va; - } else { - va_sz = mmu_ptob(pfn_end - pfn_start + 1); - start_va = (uintptr_t)vmem_xalloc(dp->d_vmem, va_sz, - MAX(attrp->dma_attr_align, MMU_PAGESIZE), - 0, - attrp->dma_attr_seg + 1, - (void *)(uintptr_t)attrp->dma_attr_addr_lo, - (void *)(uintptr_t)MIN((attrp->dma_attr_addr_hi + 1), - AMD_IOMMU_SIZE_4G), /* XXX rollover */ - km_flags == KM_SLEEP ? VM_SLEEP : VM_NOSLEEP); - if (start_va == 0) { - cmn_err(CE_WARN, "%s: No VA resources", - amd_iommu_modname); - error = DDI_DMA_NORESOURCES; - goto out; - } - ASSERT((start_va & MMU_PAGEOFFSET) == 0); - end_va = start_va + va_sz - 1; - *start_vap = start_va + (start_pa & MMU_PAGEOFFSET); - } - - pg_start = start_va >> MMU_PAGESHIFT; - pg_end = end_va >> MMU_PAGESHIFT; - - pg = pg_start; - for (pfn = pfn_start; pfn <= pfn_end; pfn++, pg++) { - - if (amd_iommu_debug & AMD_IOMMU_DEBUG_PAGE_TABLES) { - cmn_err(CE_WARN, "%s: attempting to create page tables " - "for pfn = %p, va = %p, path = %s", - f, (void *)(uintptr_t)(pfn << MMU_PAGESHIFT), - (void *)(uintptr_t)(pg << MMU_PAGESHIFT), path); - - } - - if (amd_iommu_unity_map || type == AMD_IOMMU_UNITY_MAP) { - ASSERT(pfn == pg); - } - - if ((error = amd_iommu_create_pgtables(iommu, rdip, dmareq, - pg << MMU_PAGESHIFT, - pfn << MMU_PAGESHIFT, deviceid, domainid, dp, path, - km_flags)) != DDI_SUCCESS) { - cmn_err(CE_WARN, "Failed to create_pgtables"); - goto out; - } - - if (amd_iommu_debug & AMD_IOMMU_DEBUG_PAGE_TABLES) { - cmn_err(CE_WARN, "%s: successfuly created page tables " - "for pfn = %p, vapg = %p, path = %s", - f, (void *)(uintptr_t)pfn, - (void *)(uintptr_t)pg, path); - } - - } - ASSERT(pg == pg_end + 1); - - - if (amd_iommu_debug & AMD_IOMMU_DEBUG_PA2VA) { - cmn_err(CE_NOTE, "pa=%p, va=%p", - (void *)(uintptr_t)start_pa, - (void *)(uintptr_t)(*start_vap)); - } - error = DDI_DMA_MAPPED; - -out: - kmem_free(path, MAXPATHLEN); - return (cvt_bind_error(error)); -} - -int -amd_iommu_unmap_va(amd_iommu_t *iommu, dev_info_t *rdip, uint64_t start_va, - uint64_t va_sz, map_type_t type) -{ - uint64_t end_va; - uint64_t pg_start; - uint64_t pg_end; - uint64_t pg; - uint64_t actual_sz; - char *path; - int pathfree; - int alias; - int32_t deviceid; - domain_id_t domainid; - amd_iommu_domain_t *dp; - int error; - int domain_freed; - const char *driver = ddi_driver_name(iommu->aiomt_dip); - int instance = ddi_get_instance(iommu->aiomt_dip); - const char *f = "amd_iommu_unmap_va"; - - if (amd_iommu_no_unmap) - return (DDI_SUCCESS); - - path = kmem_alloc(MAXPATHLEN, KM_NOSLEEP); - if (path) { - (void) ddi_pathname(rdip, path); - pathfree = 1; - } else { - pathfree = 0; - path = "<path-mem-alloc-failed>"; - } - - /* - * First get deviceid - */ - if (amd_iommu_get_deviceid(iommu, rdip, &deviceid, &alias, path) - != DDI_SUCCESS) { - cmn_err(CE_WARN, "%s: %s%d: idx=%d: rdip=%p. " - "Failed to get device ID for %s.", f, driver, instance, - iommu->aiomt_idx, (void *)rdip, path); - error = DDI_FAILURE; - goto out; - } - - /* - * Next get the domain for this rdip - */ - if (amd_iommu_get_domain(iommu, rdip, alias, deviceid, &domainid, path) - != DDI_SUCCESS) { - cmn_err(CE_WARN, "%s: %s%d: idx=%d: rdip=%p, path=%s. " - "Failed to get domain.", f, driver, instance, - iommu->aiomt_idx, (void *)rdip, path); - error = DDI_FAILURE; - goto out; - } - - /* should never result in domain allocation/vmem_create */ - dp = amd_iommu_lookup_domain(iommu, domainid, AMD_IOMMU_INVALID_MAP, - KM_NOSLEEP); - if (dp == NULL) { - cmn_err(CE_WARN, "%s: %s%d: idx=%d: domainid=%d, rdip=%p. " - "Failed to get device ID for %s.", f, driver, instance, - iommu->aiomt_idx, domainid, (void *)rdip, path); - error = DDI_FAILURE; - goto out; - } - - ASSERT(dp->d_domainid == domainid); - - pg_start = start_va >> MMU_PAGESHIFT; - end_va = start_va + va_sz - 1; - pg_end = end_va >> MMU_PAGESHIFT; - actual_sz = (pg_end - pg_start + 1) << MMU_PAGESHIFT; - - domain_freed = 0; - for (pg = pg_start; pg <= pg_end; pg++) { - domain_freed = 0; - if (amd_iommu_destroy_pgtables(iommu, rdip, - pg << MMU_PAGESHIFT, deviceid, domainid, dp, type, - &domain_freed, path) != DDI_SUCCESS) { - error = DDI_FAILURE; - goto out; - } - if (domain_freed) { - ASSERT(pg == pg_end); - break; - } - } - - /* - * vmem_xalloc() must be paired with vmem_xfree - */ - if (type == AMD_IOMMU_VMEM_MAP && !amd_iommu_unity_map) { - vmem_xfree(dp->d_vmem, - (void *)(uintptr_t)(pg_start << MMU_PAGESHIFT), actual_sz); - } - - if (domain_freed) - amd_iommu_teardown_domain(iommu, dp); - - error = DDI_SUCCESS; -out: - if (pathfree) - kmem_free(path, MAXPATHLEN); - return (error); -}
--- a/usr/src/uts/intel/io/amd_iommu/amd_iommu_page_tables.h Mon Sep 21 11:25:09 2009 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,134 +0,0 @@ -/* - * CDDL HEADER START - * - * The contents of this file are subject to the terms of the - * Common Development and Distribution License (the "License"). - * You may not use this file except in compliance with the License. - * - * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE - * or http://www.opensolaris.org/os/licensing. - * See the License for the specific language governing permissions - * and limitations under the License. - * - * When distributing Covered Code, include this CDDL HEADER in each - * file and include the License file at usr/src/OPENSOLARIS.LICENSE. - * If applicable, add the following below this CDDL HEADER, with the - * fields enclosed by brackets "[]" replaced with your own identifying - * information: Portions Copyright [yyyy] [name of copyright owner] - * - * CDDL HEADER END - */ -/* - * Copyright 2008 Sun Microsystems, Inc. All rights reserved. - * Use is subject to license terms. - */ - -#ifndef _AMD_IOMMU_PAGE_TABLES_H -#define _AMD_IOMMU_PAGE_TABLES_H - -#ifdef __cplusplus -extern "C" { -#endif - -#ifdef _KERNEL - -/* Common to PTEs and PDEs */ -#define AMD_IOMMU_PTDE_IW (62 << 16 | 62) -#define AMD_IOMMU_PTDE_IR (61 << 16 | 61) -#define AMD_IOMMU_PTDE_ADDR (51 << 16 | 12) -#define AMD_IOMMU_PTDE_NXT_LVL (11 << 16 | 9) -#define AMD_IOMMU_PTDE_PR (0 << 16 | 0) - -#define AMD_IOMMU_PTE_FC (60 << 16 | 60) -#define AMD_IOMMU_PTE_U (59 << 16 | 59) - -#define AMD_IOMMU_VA_NBITS(l) ((l) == 6 ? 7 : 9) -#define AMD_IOMMU_VA_BITMASK(l) ((1 << AMD_IOMMU_VA_NBITS(l)) - 1) -#define AMD_IOMMU_VA_SHIFT(v, l) \ - ((v) >> (MMU_PAGESHIFT + (AMD_IOMMU_VA_NBITS(l - 1) * (l - 1)))) -#define AMD_IOMMU_VA_BITS(v, l) \ - (AMD_IOMMU_VA_SHIFT(v, l) & AMD_IOMMU_VA_BITMASK(l)) -#define AMD_IOMMU_VA_TOTBITS(l) \ - (((l) == 6 ? 7 + (l - 1) * 9: l*9) + MMU_PAGESHIFT) -#define AMD_IOMMU_VA_TOTMASK(l) ((1 << AMD_IOMMU_VA_TOTBITS(l)) - 1) -#define AMD_IOMMU_VA_INVAL_SETMASK(l) \ - (((1 << AMD_IOMMU_VA_TOTBITS(l)) - 1) >> 1) -#define AMD_IOMMU_VA_INVAL_CLRMASK(l) \ - (~(1 << (AMD_IOMMU_VA_TOTBITS(l) - 1))) -#define AMD_IOMMU_VA_INVAL(v, l) \ - (((v) & AMD_IOMMU_VA_INVAL_CLRMASK(l)) | AMD_IOMMU_VA_INVAL_SETMASK(l)) - -#define AMD_IOMMU_PGTABLE_SZ (4096) -#define AMD_IOMMU_PGTABLE_MAXLEVEL (6) -#define AMD_IOMMU_PGTABLE_HASH_SZ (256) - -#define AMD_IOMMU_PGTABLE_ALIGN ((1ULL << 12) - 1) -#define AMD_IOMMU_PGTABLE_SIZE (1ULL << 12) - -#define AMD_IOMMU_MAX_PDTE (1ULL << AMD_IOMMU_VA_NBITS(1)) -#define PT_REF_VALID(p) ((p)->pt_ref >= 0 && \ - (p)->pt_ref <= AMD_IOMMU_MAX_PDTE) - -#define AMD_IOMMU_DOMAIN_HASH_SZ (256) -#define AMD_IOMMU_PGTABLE_FREELIST_MAX (256) -#define AMD_IOMMU_PA2VA_HASH_SZ (256) - -#define AMD_IOMMU_SIZE_4G ((uint64_t)1 << 32) -#define AMD_IOMMU_VMEM_NAMELEN (30) - -typedef enum { - AMD_IOMMU_INVALID_DOMAIN = 0, - AMD_IOMMU_IDENTITY_DOMAIN = 0xFFFD, - AMD_IOMMU_PASSTHRU_DOMAIN = 0xFFFE, - AMD_IOMMU_SYS_DOMAIN = 0xFFFF -} domain_id_t; - -typedef enum { - AMD_IOMMU_INVALID_MAP = 0, - AMD_IOMMU_UNITY_MAP, - AMD_IOMMU_VMEM_MAP -} map_type_t; - -typedef struct amd_iommu_page_table { - domain_id_t pt_domainid; - int pt_level; - ddi_dma_handle_t pt_dma_hdl; - ddi_acc_handle_t pt_mem_hdl; - uint64_t pt_mem_reqsz; - uint64_t pt_mem_realsz; - uint64_t *pt_pgtblva; - uint64_t pt_pte_ref[AMD_IOMMU_MAX_PDTE]; - uint16_t pt_index; - int pt_ref; - ddi_dma_cookie_t pt_cookie; - struct amd_iommu_page_table *pt_next; - struct amd_iommu_page_table *pt_prev; - struct amd_iommu_page_table *pt_parent; -} amd_iommu_page_table_t; - -typedef struct amd_iommu_domain { - domain_id_t d_domainid; - uint64_t d_pgtable_root_4K; - int64_t d_ref; - vmem_t *d_vmem; - struct amd_iommu_domain *d_prev; - struct amd_iommu_domain *d_next; -} amd_iommu_domain_t; - -int amd_iommu_map_pa2va(amd_iommu_t *iommu, dev_info_t *rdip, - ddi_dma_attr_t *attrp, struct ddi_dma_req *dmareq, - uint64_t pa, uint64_t pa_sz, map_type_t type, - uint64_t *start_vap, int km_flags); -int amd_iommu_unmap_va(amd_iommu_t *iommu, dev_info_t *rdip, - uint64_t va, uint64_t va_sz, map_type_t type); -void amd_iommu_init_page_tables(amd_iommu_t *iommu); -void amd_iommu_fini_page_tables(amd_iommu_t *iommu); -void amd_iommu_set_passthru(amd_iommu_t *iommu, dev_info_t *rdip); - -#endif /* _KERNEL */ - -#ifdef __cplusplus -} -#endif - -#endif /* _AMD_IOMMU_PAGE_TABLES_H */
--- a/usr/src/uts/intel/io/intel_nhm/intel_nhm.h Mon Sep 21 11:25:09 2009 -0400 +++ b/usr/src/uts/intel/io/intel_nhm/intel_nhm.h Mon Sep 21 11:26:40 2009 -0400 @@ -34,6 +34,10 @@ #define NHM_EP_CPU 0x2c408086 #define NHM_WS_CPU 0x2c418086 #define NHM_CPU_RAS 0x2c1a8086 +#define NHM_JF_CPU 0x2c588086 +#define NHM_JF_CPU_RAS 0x2cda8086 +#define NHM_WM_CPU 0x2c708086 +#define NHM_WM_CPU_RAS 0x2d9a8086 #define NHM_INTERCONNECT "Intel QuickPath"
--- a/usr/src/uts/intel/io/intel_nhm/mem_addr.c Mon Sep 21 11:25:09 2009 -0400 +++ b/usr/src/uts/intel/io/intel_nhm/mem_addr.c Mon Sep 21 11:26:40 2009 -0400 @@ -749,9 +749,11 @@ first = 0; last = MAX_CPU_NODES; id = CPU_ID_RD(0); - if (id == NHM_EP_CPU || id == NHM_WS_CPU) { + if (id == NHM_EP_CPU || id == NHM_WS_CPU || id == NHM_JF_CPU || + id == NHM_WM_CPU) { id = CPU_ID_RD(1); - if (id != NHM_EP_CPU && id != NHM_WS_CPU) { + if (id != NHM_EP_CPU && id != NHM_WS_CPU && id != NHM_JF_CPU && + id != NHM_WM_CPU) { last = 1; } } else { @@ -829,10 +831,12 @@ uint32_t rir_limit; uint32_t rir_way; uint32_t mc_control; + uint32_t id; int nhm_slot; int nhm_lastslot; uint8_t rank; uint64_t base; + int ras_dev = 0; nhm_slot = choose_cpu(&nhm_lastslot); @@ -846,7 +850,10 @@ } for (i = nhm_slot; i < nhm_lastslot; i++) { - if (MC_CPU_RAS_RD(i) == NHM_CPU_RAS) { + id = MC_CPU_RAS_RD(i); + if (id == NHM_CPU_RAS || id == NHM_JF_CPU_RAS || + id == NHM_WM_CPU_RAS) { + ras_dev = 1; mc_ras_enables = MC_RAS_ENABLES_RD(i); if (RAS_LOCKSTEP_ENABLE(mc_ras_enables)) lockstep[i] = 1; @@ -923,7 +930,7 @@ } mc_control = MC_CONTROL_RD(nhm_slot); closed_page = MC_CONTROL_CLOSED_PAGE(mc_control); - if (MC_CPU_RAS_RD(nhm_slot) == NHM_CPU_RAS) + if (ras_dev) ecc_enabled = MC_CONTROL_ECCEN(mc_control); else if ((MC_STATUS_RD(nhm_slot) & WS_ECC_ENABLED) != 0) ecc_enabled = 1;
--- a/usr/src/uts/intel/io/intel_nhm/nhm_init.c Mon Sep 21 11:25:09 2009 -0400 +++ b/usr/src/uts/intel/io/intel_nhm/nhm_init.c Mon Sep 21 11:26:40 2009 -0400 @@ -234,13 +234,15 @@ uint32_t mc_dimm_clk_ratio_status; uint64_t cycle_time; uint32_t interval; + uint32_t id; int i; int hw_scrub = 0; if (ecc_enabled && (nhm_patrol_scrub || nhm_demand_scrub)) { for (i = 0; i < MAX_MEMORY_CONTROLLERS; i++) { - if (MC_CPU_RAS_RD(i) != NHM_CPU_RAS || - nhm_memory_on_ctl[i] == 0) + id = MC_CPU_RAS_RD(i); + if ((id != NHM_CPU_RAS && id != NHM_JF_CPU_RAS && + id != NHM_WM_CPU_RAS) || nhm_memory_on_ctl[i] == 0) continue; mc_ssrcontrol = MC_SSR_CONTROL_RD(i); if (nhm_demand_scrub && @@ -286,7 +288,8 @@ dimmpp = nhm_dimms; for (i = 0; i < MAX_MEMORY_CONTROLLERS; i++) { did = CPU_ID_RD(i); - if (did != NHM_EP_CPU && did != NHM_WS_CPU) { + if (did != NHM_EP_CPU && did != NHM_WS_CPU && + did != NHM_JF_CPU && did != NHM_WM_CPU) { dimmpp += CHANNELS_PER_MEMORY_CONTROLLER * MAX_DIMMS_PER_CHANNEL; continue; @@ -324,10 +327,11 @@ return (ENOTSUP); for (slot = 0; slot < MAX_CPU_NODES; slot++) { nhm_chipset = CPU_ID_RD(slot); - if (nhm_chipset == NHM_EP_CPU || nhm_chipset == NHM_WS_CPU) + if (nhm_chipset == NHM_EP_CPU || nhm_chipset == NHM_WS_CPU || + nhm_chipset == NHM_JF_CPU || nhm_chipset == NHM_WM_CPU) break; } - if (nhm_chipset != NHM_EP_CPU && nhm_chipset != NHM_WS_CPU) { + if (slot == MAX_CPU_NODES) { return (ENOTSUP); } mem_reg_init();
--- a/usr/src/uts/intel/io/pci/pci_boot.c Mon Sep 21 11:25:09 2009 -0400 +++ b/usr/src/uts/intel/io/pci/pci_boot.c Mon Sep 21 11:26:40 2009 -0400 @@ -44,6 +44,7 @@ #include <sys/acpica.h> #include <sys/intel_iommu.h> #include <sys/iommulib.h> +#include <sys/devcache.h> #define pci_getb (*pci_getb_func) #define pci_getw (*pci_getw_func) @@ -134,6 +135,15 @@ static void memlist_remove_list(struct memlist **list, struct memlist *remove_list); +static void pci_scan_bbn(void); +static int pci_unitaddr_cache_valid(void); +static int pci_bus_unitaddr(int); +static void pci_unitaddr_cache_create(void); + +static int pci_cache_unpack_nvlist(nvf_handle_t, nvlist_t *, char *); +static int pci_cache_pack_nvlist(nvf_handle_t, nvlist_t **); +static void pci_cache_free_list(nvf_handle_t); + extern int pci_slot_names_prop(int, char *, int); /* set non-zero to force PCI peer-bus renumbering */ @@ -149,10 +159,285 @@ } isa_res; /* + * PCI unit-address cache management + */ +static nvf_ops_t pci_unitaddr_cache_ops = { + "/etc/devices/pci_unitaddr_persistent", /* path to cache */ + pci_cache_unpack_nvlist, /* read in nvlist form */ + pci_cache_pack_nvlist, /* convert to nvlist form */ + pci_cache_free_list, /* free data list */ + NULL /* write complete callback */ +}; + +typedef struct { + list_node_t pua_nodes; + int pua_index; + int pua_addr; +} pua_node_t; + +nvf_handle_t puafd_handle; +int pua_cache_valid = 0; + + +/*ARGSUSED*/ +static ACPI_STATUS +pci_process_acpi_device(ACPI_HANDLE hdl, UINT32 level, void *ctx, void **rv) +{ + ACPI_BUFFER rb; + ACPI_OBJECT ro; + ACPI_DEVICE_INFO *adi; + + /* + * Use AcpiGetObjectInfo() to find the device _HID + * If not a PCI root-bus, ignore this device and continue + * the walk + */ + + rb.Length = ACPI_ALLOCATE_BUFFER; + if (ACPI_FAILURE(AcpiGetObjectInfo(hdl, &rb))) + return (AE_OK); + + adi = rb.Pointer; + if (!(adi->Valid & ACPI_VALID_HID)) { + AcpiOsFree(adi); + return (AE_OK); + } + + if (strncmp(adi->HardwareId.Value, PCI_ROOT_HID_STRING, + sizeof (PCI_ROOT_HID_STRING)) && + strncmp(adi->HardwareId.Value, PCI_EXPRESS_ROOT_HID_STRING, + sizeof (PCI_EXPRESS_ROOT_HID_STRING))) { + AcpiOsFree(adi); + return (AE_OK); + } + + AcpiOsFree(adi); + + /* + * XXX: ancient Big Bear broken _BBN will result in two + * bus 0 _BBNs being found, so we need to handle duplicate + * bus 0 gracefully. However, broken _BBN does not + * hide a childless root-bridge so no need to work-around it + * here + */ + rb.Pointer = &ro; + rb.Length = sizeof (ro); + if (ACPI_SUCCESS(AcpiEvaluateObjectTyped(hdl, "_BBN", + NULL, &rb, ACPI_TYPE_INTEGER))) { + /* PCI with _BBN, process it, go no deeper */ + if (pci_bus_res[ro.Integer.Value].par_bus == (uchar_t)-1 && + pci_bus_res[ro.Integer.Value].dip == NULL) + create_root_bus_dip((uchar_t)ro.Integer.Value); + return (AE_CTRL_DEPTH); + } + + /* PCI and no _BBN, continue walk */ + return (AE_OK); +} + +/* + * Scan the ACPI namespace for all top-level instances of _BBN + * in order to discover childless root-bridges (which enumeration + * may not find; root-bridges are inferred by the existence of + * children). This scan should find all root-bridges that have + * been enumerated, and any childless root-bridges not enumerated. + * Root-bridge for bus 0 may not have a _BBN object. + */ +static void +pci_scan_bbn() +{ + void *rv; + + (void) AcpiGetDevices(NULL, pci_process_acpi_device, NULL, &rv); +} + +static void +pci_unitaddr_cache_init(void) +{ + + puafd_handle = nvf_register_file(&pci_unitaddr_cache_ops); + ASSERT(puafd_handle); + + list_create(nvf_list(puafd_handle), sizeof (pua_node_t), + offsetof(pua_node_t, pua_nodes)); + + rw_enter(nvf_lock(puafd_handle), RW_WRITER); + (void) nvf_read_file(puafd_handle); + rw_exit(nvf_lock(puafd_handle)); +} + +/* + * Format of /etc/devices/pci_unitaddr_persistent: + * + * The persistent record of unit-address assignments contains + * a list of name/value pairs, where name is a string representation + * of the "index value" of the PCI root-bus and the value is + * the assigned unit-address. + * + * The "index value" is simply the zero-based index of the PCI + * root-buses ordered by physical bus number; first PCI bus is 0, + * second is 1, and so on. + */ + +/*ARGSUSED*/ +static int +pci_cache_unpack_nvlist(nvf_handle_t hdl, nvlist_t *nvl, char *name) +{ + long index; + int32_t value; + nvpair_t *np; + pua_node_t *node; + + np = NULL; + while ((np = nvlist_next_nvpair(nvl, np)) != NULL) { + /* name of nvpair is index value */ + if (ddi_strtol(nvpair_name(np), NULL, 10, &index) != 0) + continue; + + if (nvpair_value_int32(np, &value) != 0) + continue; + + node = kmem_zalloc(sizeof (pua_node_t), KM_SLEEP); + node->pua_index = index; + node->pua_addr = value; + list_insert_tail(nvf_list(hdl), node); + } + + pua_cache_valid = 1; + return (DDI_SUCCESS); +} + +static int +pci_cache_pack_nvlist(nvf_handle_t hdl, nvlist_t **ret_nvl) +{ + int rval; + nvlist_t *nvl, *sub_nvl; + list_t *listp; + pua_node_t *pua; + char buf[13]; + + ASSERT(RW_WRITE_HELD(nvf_lock(hdl))); + + rval = nvlist_alloc(&nvl, NV_UNIQUE_NAME, KM_SLEEP); + if (rval != DDI_SUCCESS) { + nvf_error("%s: nvlist alloc error %d\n", + nvf_cache_name(hdl), rval); + return (DDI_FAILURE); + } + + sub_nvl = NULL; + rval = nvlist_alloc(&sub_nvl, NV_UNIQUE_NAME, KM_SLEEP); + if (rval != DDI_SUCCESS) + goto error; + + listp = nvf_list(hdl); + for (pua = list_head(listp); pua != NULL; + pua = list_next(listp, pua)) { + (void) snprintf(buf, sizeof (buf), "%d", pua->pua_index); + rval = nvlist_add_int32(sub_nvl, buf, pua->pua_addr); + if (rval != DDI_SUCCESS) + goto error; + } + + rval = nvlist_add_nvlist(nvl, "table", sub_nvl); + if (rval != DDI_SUCCESS) + goto error; + nvlist_free(sub_nvl); + + *ret_nvl = nvl; + return (DDI_SUCCESS); + +error: + if (sub_nvl) + nvlist_free(sub_nvl); + ASSERT(nvl); + nvlist_free(nvl); + *ret_nvl = NULL; + return (DDI_FAILURE); +} + +static void +pci_cache_free_list(nvf_handle_t hdl) +{ + list_t *listp; + pua_node_t *pua; + + ASSERT(RW_WRITE_HELD(nvf_lock(hdl))); + + listp = nvf_list(hdl); + for (pua = list_head(listp); pua != NULL; + pua = list_next(listp, pua)) { + list_remove(listp, pua); + kmem_free(pua, sizeof (pua_node_t)); + } +} + + +static int +pci_unitaddr_cache_valid(void) +{ + + /* read only, no need for rw lock */ + return (pua_cache_valid); +} + + +static int +pci_bus_unitaddr(int index) +{ + pua_node_t *pua; + list_t *listp; + int addr; + + rw_enter(nvf_lock(puafd_handle), RW_READER); + + addr = -1; /* default return if no match */ + listp = nvf_list(puafd_handle); + for (pua = list_head(listp); pua != NULL; + pua = list_next(listp, pua)) { + if (pua->pua_index == index) { + addr = pua->pua_addr; + break; + } + } + + rw_exit(nvf_lock(puafd_handle)); + return (addr); +} + +static void +pci_unitaddr_cache_create(void) +{ + int i, index; + pua_node_t *node; + list_t *listp; + + rw_enter(nvf_lock(puafd_handle), RW_WRITER); + + index = 0; + listp = nvf_list(puafd_handle); + for (i = 0; i <= pci_bios_nbus; i++) { + /* skip non-root (peer) PCI busses */ + if ((pci_bus_res[i].par_bus != (uchar_t)-1) || + (pci_bus_res[i].dip == NULL)) + continue; + node = kmem_zalloc(sizeof (pua_node_t), KM_SLEEP); + node->pua_index = index++; + node->pua_addr = pci_bus_res[i].root_addr; + list_insert_tail(listp, node); + } + + (void) nvf_mark_dirty(puafd_handle); + rw_exit(nvf_lock(puafd_handle)); + nvf_wake_daemon(); +} + + +/* * Enumerate all PCI devices */ void -pci_setup_tree() +pci_setup_tree(void) { uint_t i, root_bus_addr = 0; @@ -978,9 +1263,42 @@ int bus; /* - * Excise phantom roots if possible + * Scan ACPI namespace for _BBN objects, make sure that + * childless root-bridges appear in devinfo tree + */ + pci_scan_bbn(); + pci_unitaddr_cache_init(); + + /* + * Fix-up unit-address assignments if cache is available */ - pci_renumber_root_busses(); + if (pci_unitaddr_cache_valid()) { + int pci_regs[] = {0, 0, 0}; + int new_addr; + int index = 0; + + for (bus = 0; bus <= pci_bios_nbus; bus++) { + /* skip non-root (peer) PCI busses */ + if ((pci_bus_res[bus].par_bus != (uchar_t)-1) || + (pci_bus_res[bus].dip == NULL)) + continue; + + new_addr = pci_bus_unitaddr(index); + if (pci_bus_res[bus].root_addr != new_addr) { + /* update reg property for node */ + pci_regs[0] = pci_bus_res[bus].root_addr = + new_addr; + (void) ndi_prop_update_int_array( + DDI_DEV_T_NONE, pci_bus_res[bus].dip, + "reg", (int *)pci_regs, 3); + } + index++; + } + } else { + /* perform legacy processing */ + pci_renumber_root_busses(); + pci_unitaddr_cache_create(); + } /* * Do root-bus resource discovery
--- a/usr/src/uts/intel/io/pci/pci_resource.c Mon Sep 21 11:25:09 2009 -0400 +++ b/usr/src/uts/intel/io/pci/pci_resource.c Mon Sep 21 11:26:40 2009 -0400 @@ -340,6 +340,12 @@ break; case ACPI_RESOURCE_TYPE_ADDRESS64: + /* + * We comment out this block because we currently cannot deal with + * PCI 64-bit addresses. Will revisit this when we add PCI 64-bit MMIO + * support. + */ +#if 0 if (rp->Data.Address64.AddressLength == 0) break; acpi_cb_cnt++; @@ -347,9 +353,11 @@ rp->Data.Address64.Info.TypeSpecific, bus), rp->Data.Address64.Minimum, rp->Data.Address64.AddressLength); +#endif break; case ACPI_RESOURCE_TYPE_EXTENDED_ADDRESS64: +#if 0 /* Will revisit this when we add PCI 64-bit MMIO support */ if (rp->Data.ExtAddress64.AddressLength == 0) break; acpi_cb_cnt++; @@ -357,6 +365,7 @@ rp->Data.ExtAddress64.Info.TypeSpecific, bus), rp->Data.ExtAddress64.Minimum, rp->Data.ExtAddress64.AddressLength); +#endif break; case ACPI_RESOURCE_TYPE_EXTENDED_IRQ:
--- a/usr/src/uts/intel/os/driver_aliases Mon Sep 21 11:25:09 2009 -0400 +++ b/usr/src/uts/intel/os/driver_aliases Mon Sep 21 11:26:40 2009 -0400 @@ -47,6 +47,7 @@ intel_nb5000 "pci8086,4003" intel_nb5000 "pci8086,65c0" intel_nhm "pci8086,3423" +intel_nhm "pci8086,372a" xpv "pci5853,1.1" amd_iommu "pci1022,11ff" amd_iommu "pci1002,5a23"
--- a/usr/src/uts/intel/sys/Makefile Mon Sep 21 11:25:09 2009 -0400 +++ b/usr/src/uts/intel/sys/Makefile Mon Sep 21 11:26:40 2009 -0400 @@ -31,7 +31,6 @@ # from being built, so these headers are not exported (installed). HDRS = \ - amd_iommu.h \ archsystm.h \ asm_linkage.h \ bootconf.h \
--- a/usr/src/uts/intel/sys/amd_iommu.h Mon Sep 21 11:25:09 2009 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,56 +0,0 @@ -/* - * CDDL HEADER START - * - * The contents of this file are subject to the terms of the - * Common Development and Distribution License (the "License"). - * You may not use this file except in compliance with the License. - * - * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE - * or http://www.opensolaris.org/os/licensing. - * See the License for the specific language governing permissions - * and limitations under the License. - * - * When distributing Covered Code, include this CDDL HEADER in each - * file and include the License file at usr/src/OPENSOLARIS.LICENSE. - * If applicable, add the following below this CDDL HEADER, with the - * fields enclosed by brackets "[]" replaced with your own identifying - * information: Portions Copyright [yyyy] [name of copyright owner] - * - * CDDL HEADER END - */ -/* - * Copyright 2008 Sun Microsystems, Inc. All rights reserved. - * Use is subject to license terms. - */ - -#ifndef _SYS_AMD_IOMMU_H -#define _SYS_AMD_IOMMU_H - -#ifdef __cplusplus -extern "C" { -#endif - -#include <sys/sunddi.h> -#include <sys/iommulib.h> - -#ifdef _KERNEL - -typedef struct amd_iommu_state { - int aioms_instance; /* instance */ - dev_info_t *aioms_devi; /* dip */ - struct amd_iommu *aioms_iommu_start; /* start of list of IOMMUs */ - struct amd_iommu *aioms_iommu_end; /* end of list of IOMMUs */ - int aioms_nunits; /* # of IOMMUs in function */ -} amd_iommu_state_t; - -int amd_iommu_setup(dev_info_t *dip, amd_iommu_state_t *statep); -int amd_iommu_teardown(dev_info_t *dip, amd_iommu_state_t *statep); -int amd_iommu_lookup_src_bdf(uint16_t bdf, uint16_t *src_bdfp); - -#endif /* _KERNEL */ - -#ifdef __cplusplus -} -#endif - -#endif /* _SYS_AMD_IOMMU_H */
--- a/usr/src/uts/intel/sys/mc_intel.h Mon Sep 21 11:25:09 2009 -0400 +++ b/usr/src/uts/intel/sys/mc_intel.h Mon Sep 21 11:26:40 2009 -0400 @@ -223,6 +223,23 @@ #define INTEL_QP_U2 0x34028086 #define INTEL_QP_U3 0x34048086 #define INTEL_QP_U4 0x34078086 +#define INTEL_QP_JF 0x37208086 +#define INTEL_QP_JF0 0x37008086 +#define INTEL_QP_JF1 0x37018086 +#define INTEL_QP_JF2 0x37028086 +#define INTEL_QP_JF3 0x37038086 +#define INTEL_QP_JF4 0x37048086 +#define INTEL_QP_JF5 0x37058086 +#define INTEL_QP_JF6 0x37068086 +#define INTEL_QP_JF7 0x37078086 +#define INTEL_QP_JF8 0x37088086 +#define INTEL_QP_JF9 0x37098086 +#define INTEL_QP_JFa 0x370a8086 +#define INTEL_QP_JFb 0x370b8086 +#define INTEL_QP_JFc 0x370c8086 +#define INTEL_QP_JFd 0x370d8086 +#define INTEL_QP_JFe 0x370e8086 +#define INTEL_QP_JFf 0x370f8086 /* Intel QuickPath Bus Interconnect Errors */
--- a/usr/src/uts/intel/sys/x86_archext.h Mon Sep 21 11:25:09 2009 -0400 +++ b/usr/src/uts/intel/sys/x86_archext.h Mon Sep 21 11:26:40 2009 -0400 @@ -550,6 +550,8 @@ #define X86_SOCKET_AM2R2 _X86_SOCKET_MKVAL(X86_VENDOR_AMD, 0x000200) #define X86_SOCKET_AM3 _X86_SOCKET_MKVAL(X86_VENDOR_AMD, 0x000400) #define X86_SOCKET_G34 _X86_SOCKET_MKVAL(X86_VENDOR_AMD, 0x000800) +#define X86_SOCKET_ASB2 _X86_SOCKET_MKVAL(X86_VENDOR_AMD, 0x001000) +#define X86_SOCKET_C32 _X86_SOCKET_MKVAL(X86_VENDOR_AMD, 0x002000) #if !defined(_ASM)
--- a/usr/src/uts/sun4u/io/pci/pci_pci.c Mon Sep 21 11:25:09 2009 -0400 +++ b/usr/src/uts/sun4u/io/pci/pci_pci.c Mon Sep 21 11:26:40 2009 -0400 @@ -1471,16 +1471,23 @@ base = PPB_16bit_IOADDR(io_base_lo); limit = PPB_16bit_IOADDR(io_limit_lo); + /* + * Check for 32-bit I/O support as per PCI-to-PCI Bridge Arch Spec + */ if ((io_base_lo & 0xf) == PPB_32BIT_IO) { base = PPB_LADDR(base, io_base_hi); - } - if ((io_limit_lo & 0xf) == PPB_32BIT_IO) { limit = PPB_LADDR(limit, io_limit_hi); } - if ((io_base_lo & PPB_32BIT_IO) && (io_limit_hi > 0)) { - base = PPB_LADDR(base, io_base_hi); - limit = PPB_LADDR(limit, io_limit_hi); + /* + * Check if the bridge implements an I/O address range as per + * PCI-to-PCI Bridge Arch Spec + */ + if ((io_base_lo != 0 || io_limit_lo != 0) && limit >= base) { + ranges[i].parent_low = ranges[i].child_low = + base; + ranges[i].size_low = limit - base + PPB_IOGRAIN; + i++; } /*