Mercurial > illumos > illumos-gate

changeset 9921:0c3d84a756da

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
6768098 system panics with PCIe fabric.(0x0)(0x43) due to masked errors. 6814026 PLX disable RO algorithm is incorrect 6813298 Legacy PCI Express Endpoint is not used correctly in pcie module 6841301 PCI ECS accesses with pcitool don't work on AMD processors 6813766 faulty EPKT FMA rules need to expect ereports coming from hostbridge 6841816 PCIe Error Handling's scan_fabric doesn't handle failed IO Addresses well 6798264 PCIe error handling doesn't handle zero bdf well 6802636 fault address may not be decoded correctly during PCIe error handling 6843716 suspicious definition of PCIE_REQ_ID_DEV_MASK 6831766 coredump in pci_bridge_declare()
author Krishna Elango <Krishna.Elango@Sun.COM>
date Sat, 20 Jun 2009 15:42:11 -0700
parents 9a117fecafb3
children bac6a4442a68
files usr/src/cmd/fm/eversholt/files/sparc/sun4/fire.esc usr/src/lib/fm/topo/modules/common/pcibus/pcibus.c usr/src/uts/common/io/pcie.c usr/src/uts/common/io/pcie_fault.c usr/src/uts/common/sys/pcie.h usr/src/uts/common/sys/pcie_impl.h usr/src/uts/i86pc/io/pci/pci_tools.c usr/src/uts/intel/io/pci/pci_pci.c usr/src/uts/sun4/io/px/px_dma.c usr/src/uts/sun4/io/px/px_fdvma.c usr/src/uts/sun4/io/px/px_fm.c usr/src/uts/sun4/io/px/px_ioapi.h usr/src/uts/sun4/io/px/px_pci.c usr/src/uts/sun4/io/px/pxb_plx.h usr/src/uts/sun4u/io/pci/pci_pci.c usr/src/uts/sun4u/io/px/px_err.c usr/src/uts/sun4u/io/px/px_lib4u.c usr/src/uts/sun4v/io/px/px_err.c usr/src/uts/sun4v/io/px/px_lib4v.c
diffstat 19 files changed, 318 insertions(+), 142 deletions(-) [+]
line wrap: on
line diff
--- a/usr/src/cmd/fm/eversholt/files/sparc/sun4/fire.esc	Sat Jun 20 12:12:10 2009 -0600
+++ b/usr/src/cmd/fm/eversholt/files/sparc/sun4/fire.esc	Sat Jun 20 15:42:11 2009 -0700
@@ -19,11 +19,9 @@
  * CDDL HEADER END
  */
 /* 
- * Copyright 2008 Sun Microsystems, Inc.  All rights reserved. 
+ * Copyright 2009 Sun Microsystems, Inc.  All rights reserved. 
  * Use is subject to license terms. 
  */ 
- 
-#pragma ident	"%Z%%M%	%I%	%E% SMI"
 
 #pragma dictionary "SUN4" 
 
@@ -174,6 +172,7 @@
  * o HV and Guest are out of sync.
  ***************/
 event ereport.io.fire.epkt@hostbridge/pciexrc {within(5s)};
+event ereport.io.fire.epkt@hostbridge {within(5s)};
 
 /******************************
  * Generic Rules Begin Here   *
@@ -211,6 +210,12 @@
     FITrate=HV_FIT;
 event fault.io.fire.sw-fw-mismatch@hostbridge/pciexrc, retire=0, response=0,
     FITrate=SW_HV_MISMATCH_FIT;
+event fault.io.fire.sw-epkt@hostbridge, retire=0, response=0,
+    FITrate=SW_FIT;
+event fault.io.fire.fw-epkt@hostbridge, retire=0, response=0,
+    FITrate=HV_FIT;
+event fault.io.fire.sw-fw-mismatch@hostbridge, retire=0, response=0,
+    FITrate=SW_HV_MISMATCH_FIT;
 event fault.io.fire.hb.sw-config@hostbridge/pciexrc, retire=0, response=0,
     FITrate=SW_FIT;
 event fault.io.fire.dmc.sw-algorithm@hostbridge/pciexrc, retire=0, response=0,
@@ -229,6 +234,11 @@
 	fault.io.fire.sw-fw-mismatch@hostbridge/pciexrc->(A)
     ereport.io.fire.epkt@hostbridge/pciexrc { MATCH_UNRECOGNIZED };
 
+prop fault.io.fire.sw-epkt@hostbridge,
+	fault.io.fire.fw-epkt@hostbridge,
+	fault.io.fire.sw-fw-mismatch@hostbridge->(A)
+    ereport.io.fire.epkt@hostbridge { MATCH_UNRECOGNIZED };
+
 prop fault.io.fire.hb.sw-config@hostbridge/pciexrc (0)->
     error.io.fire.jbc.driver@hostbridge/pciexrc;
 
@@ -658,4 +668,5 @@
     ereport.io.fire.pec.lup@hostbridge/pciexrc,
     error.io.fire.dmc.nodiag@hostbridge/pciexrc,
     error.io.fire.dmc.secondary@hostbridge/pciexrc,
+    ereport.io.fire.epkt@hostbridge { !MATCH_UNRECOGNIZED },
     ereport.io.fire.epkt@hostbridge/pciexrc { !MATCH_UNRECOGNIZED };
--- a/usr/src/lib/fm/topo/modules/common/pcibus/pcibus.c	Sat Jun 20 12:12:10 2009 -0600
+++ b/usr/src/lib/fm/topo/modules/common/pcibus/pcibus.c	Sat Jun 20 15:42:11 2009 -0700
@@ -20,7 +20,7 @@
  */
 
 /*
- * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
@@ -413,7 +413,7 @@
 
 	devtyp = pci_devtype_get(mod, din);
 	/* Check if the children are PCI or PCIe */
-	if (strcmp(devtyp, "pciex") == 0)
+	if (devtyp && (strcmp(devtyp, "pciex") == 0))
 		err = pci_children_instantiate(mod, fn, din, board, bridge,
 		    rc, TRUST_BDF, depth + 1);
 	else
--- a/usr/src/uts/common/io/pcie.c	Sat Jun 20 12:12:10 2009 -0600
+++ b/usr/src/uts/common/io/pcie.c	Sat Jun 20 15:42:11 2009 -0700
@@ -333,8 +333,10 @@
 	pfd_p->pe_valid = B_FALSE;
 
 	/* Allocate the root fault struct for both RC and RP */
-	if (PCIE_IS_ROOT(bus_p))
+	if (PCIE_IS_ROOT(bus_p)) {
 		PCIE_ROOT_FAULT(pfd_p) = PCIE_ZALLOC(pf_root_fault_t);
+		PCIE_ROOT_FAULT(pfd_p)->scan_bdf = PCIE_INVALID_BDF;
+	}
 
 	PCI_ERR_REG(pfd_p) = PCIE_ZALLOC(pf_pci_err_regs_t);
 
@@ -349,13 +351,21 @@
 			    PCIE_ZALLOC(pf_pcie_rp_err_regs_t);
 
 		PCIE_ADV_REG(pfd_p) = PCIE_ZALLOC(pf_pcie_adv_err_regs_t);
+		PCIE_ADV_REG(pfd_p)->pcie_ue_tgt_bdf = PCIE_INVALID_BDF;
 
-		if (PCIE_IS_RP(bus_p))
+		if (PCIE_IS_RP(bus_p)) {
 			PCIE_ADV_RP_REG(pfd_p) =
 			    PCIE_ZALLOC(pf_pcie_adv_rp_err_regs_t);
-		else if (PCIE_IS_PCIE_BDG(bus_p))
+			PCIE_ADV_RP_REG(pfd_p)->pcie_rp_ce_src_id =
+			    PCIE_INVALID_BDF;
+			PCIE_ADV_RP_REG(pfd_p)->pcie_rp_ue_src_id =
+			    PCIE_INVALID_BDF;
+		} else if (PCIE_IS_PCIE_BDG(bus_p)) {
 			PCIE_ADV_BDG_REG(pfd_p) =
 			    PCIE_ZALLOC(pf_pcie_adv_bdg_err_regs_t);
+			PCIE_ADV_BDG_REG(pfd_p)->pcie_sue_tgt_bdf =
+			    PCIE_INVALID_BDF;
+		}
 
 		if (PCIE_IS_PCIE_BDG(bus_p) && PCIE_IS_PCIX(bus_p)) {
 			PCIX_BDG_ERR_REG(pfd_p) =
@@ -472,6 +482,7 @@
 	pfd_p->pe_valid = B_FALSE;
 
 	PCIE_ROOT_FAULT(pfd_p) = PCIE_ZALLOC(pf_root_fault_t);
+	PCIE_ROOT_FAULT(pfd_p)->scan_bdf = PCIE_INVALID_BDF;
 	PCI_ERR_REG(pfd_p) = PCIE_ZALLOC(pf_pci_err_regs_t);
 	PCI_BDG_ERR_REG(pfd_p) = PCIE_ZALLOC(pf_pci_bdg_err_regs_t);
 	PCIE_ERR_REG(pfd_p) = PCIE_ZALLOC(pf_pcie_err_regs_t);
@@ -581,7 +592,7 @@
 			bus_p->bus_aer_off = NULL;
 	} else {
 		bus_p->bus_pcie_off = NULL;
-		bus_p->bus_dev_type = PCIE_PCIECAP_DEV_TYPE_PCI_DEV;
+		bus_p->bus_dev_type = PCIE_PCIECAP_DEV_TYPE_PCI_PSEUDO;
 	}
 
 	if ((PCI_CAP_LOCATE(eh, PCI_CAP_ID_PCIX, &bus_p->bus_pcix_off))
@@ -1017,11 +1028,11 @@
 	 * As part of the probing, the PCI fcode interpreter may setup a DMA
 	 * request if a given card has a fcode on it using dip and rdip of the
 	 * AP (attachment point) i.e, dip and rdip of px/px_pci driver. In this
-	 * case, return zero for the bdf since we cannot get to the bdf value
-	 * of the actual device which will be initiating this DMA.
+	 * case, return a invalid value for the bdf since we cannot get to the
+	 * bdf value of the actual device which will be initiating this DMA.
 	 */
 	if (rdip == dip)
-		return (0);
+		return (PCIE_INVALID_BDF);
 
 	cdip = pcie_get_my_childs_dip(dip, rdip);
 
@@ -1029,10 +1040,11 @@
 	 * For a given rdip, return the bdf value of dip's (px or px_pci)
 	 * immediate child or secondary bus-id if dip is a PCIe2PCI bridge.
 	 *
-	 * XXX - For now, return bdf value of zero for all PCI and PCI-X devices
-	 * since this needs more work.
+	 * XXX - For now, return a invalid bdf value for all PCI and PCI-X
+	 * devices since this needs more work.
 	 */
-	return (PCI_GET_PCIE2PCI_SECBUS(cdip) ? 0 : PCI_GET_BDF(cdip));
+	return (PCI_GET_PCIE2PCI_SECBUS(cdip) ?
+	    PCIE_INVALID_BDF : PCI_GET_BDF(cdip));
 }
 
 uint32_t
--- a/usr/src/uts/common/io/pcie_fault.c	Sat Jun 20 12:12:10 2009 -0600
+++ b/usr/src/uts/common/io/pcie_fault.c	Sat Jun 20 15:42:11 2009 -0700
@@ -212,8 +212,8 @@
 	}
 
 	/*
-	 * Scan the fabric using the fault_bdf and fault_addr in error q.
-	 * fault_bdf will be valid in the following cases:
+	 * Scan the fabric using the scan_bdf and scan_addr in error q.
+	 * scan_bdf will be valid in the following cases:
 	 *	- Fabric message
 	 *	- Poisoned TLP
 	 *	- Signaled UR/CA
@@ -227,8 +227,9 @@
 		if (impl.pf_fault->full_scan)
 			full_scan = B_TRUE;
 
-		if (full_scan || impl.pf_fault->fault_bdf ||
-		    impl.pf_fault->fault_addr)
+		if (full_scan ||
+		    PCIE_CHECK_VALID_BDF(impl.pf_fault->scan_bdf) ||
+		    impl.pf_fault->scan_addr)
 			scan_flag |= pf_dispatch(rdip, &impl, full_scan);
 
 		if (full_scan)
@@ -277,7 +278,7 @@
 pf_dispatch(dev_info_t *pdip, pf_impl_t *impl, boolean_t full_scan)
 {
 	dev_info_t	*dip;
-	pcie_req_id_t	rid = impl->pf_fault->fault_bdf;
+	pcie_req_id_t	rid = impl->pf_fault->scan_bdf;
 	pcie_bus_t	*bus_p;
 	int		scan_flag = PF_SCAN_SUCCESS;
 
@@ -300,7 +301,7 @@
 		if (full_scan ||
 		    (bus_p->bus_bdf == rid) ||
 		    pf_in_bus_range(bus_p, rid) ||
-		    pf_in_addr_range(bus_p, impl->pf_fault->fault_addr)) {
+		    pf_in_addr_range(bus_p, impl->pf_fault->scan_addr)) {
 			int hdl_flag = pf_default_hdl(dip, impl);
 			scan_flag |= hdl_flag;
 
@@ -348,8 +349,13 @@
 			break;
 		}
 		case PCIE_PCIECAP_DEV_TYPE_PCIE_DEV:
+		case PCIE_PCIECAP_DEV_TYPE_PCI_DEV:
+			/*
+			 * Reached a PCIe end point so stop. Note dev_type
+			 * PCI_DEV is just a PCIe device that requires IO Space
+			 */
 			break;
-		case PCIE_PCIECAP_DEV_TYPE_PCI_DEV:
+		case PCIE_PCIECAP_DEV_TYPE_PCI_PSEUDO:
 			if (PCIE_IS_BDG(bus_p))
 				scan_flag |= pf_dispatch(dip, impl, B_TRUE);
 			break;
@@ -402,6 +408,7 @@
 
 	for (i = 0; i < bus_p->bus_addr_entries; i++, ranges_p++) {
 		switch (ranges_p->child_high & PCI_ADDR_MASK) {
+		case PCI_ADDR_IO:
 		case PCI_ADDR_MEM32:
 			low = ranges_p->child_low;
 			hi = ranges_p->size_low + low;
@@ -767,8 +774,8 @@
 
 	/* Since this data structure is reused, make sure to reset it */
 	root_fault->full_scan = B_FALSE;
-	root_fault->fault_bdf = 0;
-	root_fault->fault_addr = 0;
+	root_fault->scan_bdf = PCIE_INVALID_BDF;
+	root_fault->scan_addr = 0;
 
 	if (!PCIE_HAS_AER(bus_p) &&
 	    (PCI_BDG_ERR_REG(pfd_p)->pci_bdg_sec_stat & PF_PCI_BDG_ERR)) {
@@ -811,19 +818,17 @@
 	}
 
 	/* By this point, there is only 1 fault detected */
-	if ((root_err & PCIE_AER_RE_STS_CE_RCVD) &&
-	    rp_regs->pcie_rp_ce_src_id) {
-		PCIE_ROOT_FAULT(pfd_p)->fault_bdf = rp_regs->pcie_rp_ce_src_id;
+	if (root_err & PCIE_AER_RE_STS_CE_RCVD) {
+		PCIE_ROOT_FAULT(pfd_p)->scan_bdf = rp_regs->pcie_rp_ce_src_id;
 		num_faults--;
-	} else if ((root_err & PCIE_AER_RE_STS_FE_NFE_RCVD) &&
-	    rp_regs->pcie_rp_ue_src_id) {
-		PCIE_ROOT_FAULT(pfd_p)->fault_bdf = rp_regs->pcie_rp_ue_src_id;
-			num_faults--;
+	} else if (root_err & PCIE_AER_RE_STS_FE_NFE_RCVD) {
+		PCIE_ROOT_FAULT(pfd_p)->scan_bdf = rp_regs->pcie_rp_ue_src_id;
+		num_faults--;
 	} else if ((HAS_AER_LOGS(pfd_p, PCIE_AER_UCE_CA) ||
 	    HAS_AER_LOGS(pfd_p, PCIE_AER_UCE_UR)) &&
 	    (pf_tlp_decode(PCIE_PFD2BUS(pfd_p), PCIE_ADV_REG(pfd_p)) ==
 	    DDI_SUCCESS)) {
-		PCIE_ROOT_FAULT(pfd_p)->fault_bdf =
+		PCIE_ROOT_FAULT(pfd_p)->scan_addr =
 		    PCIE_ADV_REG(pfd_p)->pcie_ue_tgt_addr;
 		num_faults--;
 	}
@@ -1171,6 +1176,12 @@
 	NULL,			NULL
 };
 
+#define	PF_MASKED_AER_ERR(pfd_p) \
+	(PCIE_ADV_REG(pfd_p)->pcie_ue_status & \
+	    ((PCIE_ADV_REG(pfd_p)->pcie_ue_mask) ^ 0xFFFFFFFF))
+#define	PF_MASKED_SAER_ERR(pfd_p) \
+	(PCIE_ADV_BDG_REG(pfd_p)->pcie_sue_status & \
+	    ((PCIE_ADV_BDG_REG(pfd_p)->pcie_sue_mask) ^ 0xFFFFFFFF))
 /*
  * Analyse all the PCIe Fault Data (erpt) gathered during dispatch in the erpt
  * Queue.
@@ -1186,25 +1197,24 @@
 
 		switch (PCIE_PFD2BUS(pfd_p)->bus_dev_type) {
 		case PCIE_PCIECAP_DEV_TYPE_PCIE_DEV:
+		case PCIE_PCIECAP_DEV_TYPE_PCI_DEV:
 			if (PCIE_DEVSTS_CE_DETECTED &
 			    PCIE_ERR_REG(pfd_p)->pcie_err_status)
 				sts_flags |= PF_ERR_CE;
 
 			pf_adjust_for_no_aer(pfd_p);
 			sts_flags |= pf_analyse_error_tbl(derr, impl,
-			    pfd_p, pcie_pcie_tbl,
-			    PCIE_ADV_REG(pfd_p)->pcie_ue_status);
+			    pfd_p, pcie_pcie_tbl, PF_MASKED_AER_ERR(pfd_p));
 			break;
 		case PCIE_PCIECAP_DEV_TYPE_ROOT:
 			pf_adjust_for_no_aer(pfd_p);
 			sts_flags |= pf_analyse_error_tbl(derr, impl,
-			    pfd_p, pcie_rp_tbl,
-			    PCIE_ADV_REG(pfd_p)->pcie_ue_status);
+			    pfd_p, pcie_rp_tbl, PF_MASKED_AER_ERR(pfd_p));
 			break;
 		case PCIE_PCIECAP_DEV_TYPE_RC_PSEUDO:
 			/* no adjust_for_aer for pseudo RC */
 			sts_flags |= pf_analyse_error_tbl(derr, impl, pfd_p,
-			    pcie_rp_tbl, PCIE_ADV_REG(pfd_p)->pcie_ue_status);
+			    pcie_rp_tbl, PF_MASKED_AER_ERR(pfd_p));
 			break;
 		case PCIE_PCIECAP_DEV_TYPE_UP:
 		case PCIE_PCIECAP_DEV_TYPE_DOWN:
@@ -1214,8 +1224,7 @@
 
 			pf_adjust_for_no_aer(pfd_p);
 			sts_flags |= pf_analyse_error_tbl(derr, impl,
-			    pfd_p, pcie_sw_tbl,
-			    PCIE_ADV_REG(pfd_p)->pcie_ue_status);
+			    pfd_p, pcie_sw_tbl, PF_MASKED_AER_ERR(pfd_p));
 			break;
 		case PCIE_PCIECAP_DEV_TYPE_PCIE2PCI:
 			if (PCIE_DEVSTS_CE_DETECTED &
@@ -1226,10 +1235,10 @@
 			pf_adjust_for_no_saer(pfd_p);
 			sts_flags |= pf_analyse_error_tbl(derr,
 			    impl, pfd_p, pcie_pcie_tbl,
-			    PCIE_ADV_REG(pfd_p)->pcie_ue_status);
+			    PF_MASKED_AER_ERR(pfd_p));
 			sts_flags |= pf_analyse_error_tbl(derr,
 			    impl, pfd_p, pcie_pcie_bdg_tbl,
-			    PCIE_ADV_BDG_REG(pfd_p)->pcie_sue_status);
+			    PF_MASKED_SAER_ERR(pfd_p));
 			/*
 			 * Some non-compliant PCIe devices do not utilize PCIe
 			 * error registers.  So fallthrough and rely on legacy
@@ -1239,7 +1248,7 @@
 			    & PCIE_ERR_REG(pfd_p)->pcie_err_status)
 				break;
 			/* FALLTHROUGH */
-		case PCIE_PCIECAP_DEV_TYPE_PCI_DEV:
+		case PCIE_PCIECAP_DEV_TYPE_PCI_PSEUDO:
 			sts_flags |= pf_analyse_error_tbl(derr, impl,
 			    pfd_p, pcie_pci_tbl,
 			    PCI_ERR_REG(pfd_p)->pci_err_status);
@@ -1612,7 +1621,7 @@
 
 		secbus = PCIE_ADV_REG(pfd_p)->pcie_ue_tgt_bdf;
 
-		if (secbus & 0xFF)
+		if (!PCIE_CHECK_VALID_BDF(secbus) || (secbus & 0xFF))
 			goto done;
 
 		bdg_pfd_p = pf_get_pcie_bridge(pfd_p, secbus);
@@ -1906,7 +1915,7 @@
 		if (!(PCI_BDG_ERR_REG(rc_pfd_p)->pci_bdg_sec_stat & abort_type))
 			continue;
 
-		fault_bdf = PCIE_ROOT_FAULT(rc_pfd_p)->fault_bdf;
+		fault_bdf = PCIE_ROOT_FAULT(rc_pfd_p)->scan_bdf;
 
 		/* The Fault BDF = Device's BDF */
 		if (fault_bdf == bus_p->bus_bdf)
@@ -1914,7 +1923,7 @@
 
 		/* The Fault Addr is in device's address range */
 		if (pf_in_addr_range(bus_p,
-		    PCIE_ROOT_FAULT(rc_pfd_p)->fault_addr))
+		    PCIE_ROOT_FAULT(rc_pfd_p)->scan_addr))
 			return (B_TRUE);
 
 		/* The Fault BDF is from PCIe-PCI Bridge's secondary bus */
@@ -1943,7 +1952,7 @@
 		return;
 	case PCIE_AER_SUCE_RCVD_TA:
 	case PCIE_AER_SUCE_RCVD_MA:
-		*bdf = 0;
+		*bdf = PCIE_INVALID_BDF;
 		*trans_type = PF_ADDR_PIO;
 		return;
 	case PCIE_AER_SUCE_USC_ERR:
@@ -1952,12 +1961,12 @@
 		break;
 	default:
 		*addr = 0;
-		*bdf = 0;
+		*bdf = PCIE_INVALID_BDF;
 		*trans_type = 0;
 		return;
 	}
 
-	*bdf = 0;
+	*bdf = PCIE_INVALID_BDF;
 	*trans_type = PF_ADDR_PIO;
 	for (rc_pfd_p = pfd_p->pe_prev; rc_pfd_p;
 	    rc_pfd_p = rc_pfd_p->pe_prev) {
@@ -1987,7 +1996,7 @@
 	pcix_attr_t	*attr;
 	uint64_t	addr;
 	uint32_t	trans_type;
-	pcie_req_id_t	bdf;
+	pcie_req_id_t	bdf = PCIE_INVALID_BDF;
 
 	attr = (pcix_attr_t *)&PCIE_ADV_BDG_HDR(pfd_p, 0);
 	*cmd = GET_SAER_CMD(pfd_p);
@@ -2045,7 +2054,7 @@
 		/* FALLTHROUGH */
 	default:
 		PCIE_ADV_BDG_REG(pfd_p)->pcie_sue_tgt_trans = 0;
-		PCIE_ADV_BDG_REG(pfd_p)->pcie_sue_tgt_bdf = 0;
+		PCIE_ADV_BDG_REG(pfd_p)->pcie_sue_tgt_bdf = PCIE_INVALID_BDF;
 		PCIE_ADV_BDG_REG(pfd_p)->pcie_sue_tgt_addr = 0;
 		return (DDI_FAILURE);
 	}
@@ -2066,7 +2075,7 @@
 	ddi_fm_error_t		derr;
 
 	/* If we don't know the addr or rid just return with NOTFOUND */
-	if (addr == NULL && bdf == NULL)
+	if ((addr == NULL) && !PCIE_CHECK_VALID_BDF(bdf))
 		return (PF_HDL_NOTFOUND);
 
 	if (!(flag & (PF_ADDR_DMA | PF_ADDR_PIO | PF_ADDR_CFG))) {
@@ -2110,7 +2119,8 @@
 	dip_bdf = PCI_GET_BDF(dip);
 
 	/* Check if dip and BDF match, if not recurse to it's children. */
-	if (!PCIE_IS_RC(bus_p) && (bdf == NULL || dip_bdf == bdf)) {
+	if (!PCIE_IS_RC(bus_p) && (!PCIE_CHECK_VALID_BDF(bdf) ||
+	    dip_bdf == bdf)) {
 		if ((flag & PF_ADDR_DMA) && DDI_FM_DMA_ERR_CAP(fmhdl->fh_cap))
 			fcp = fmhdl->fh_dma_cache;
 		else
@@ -2227,7 +2237,7 @@
 	 * If a handler isn't found and we know this is the right device mark
 	 * them all failed.
 	 */
-	if ((addr != NULL) && (bdf != NULL) && (found == 0)) {
+	if ((addr != NULL) && PCIE_CHECK_VALID_BDF(bdf) && (found == 0)) {
 		status = pf_hdl_compare(dip, derr, flag, addr, bdf, fcp);
 		if (status == PF_HDL_FOUND)
 			found++;
@@ -2300,12 +2310,12 @@
 int
 pf_tlp_decode(pcie_bus_t *bus_p, pf_pcie_adv_err_regs_t *adv_reg_p) {
 	pcie_tlp_hdr_t	*tlp_hdr = (pcie_tlp_hdr_t *)adv_reg_p->pcie_ue_hdr;
-	pcie_req_id_t	my_bdf, tlp_bdf, flt_bdf = 0xFFFF;
+	pcie_req_id_t	my_bdf, tlp_bdf, flt_bdf = PCIE_INVALID_BDF;
 	uint64_t	flt_addr = 0;
 	uint32_t	flt_trans_type = 0;
 
 	adv_reg_p->pcie_ue_tgt_addr = 0;
-	adv_reg_p->pcie_ue_tgt_bdf = 0;
+	adv_reg_p->pcie_ue_tgt_bdf = PCIE_INVALID_BDF;
 	adv_reg_p->pcie_ue_tgt_trans = 0;
 
 	my_bdf = bus_p->bus_bdf;
@@ -2338,7 +2348,7 @@
 			flt_bdf = tlp_bdf;
 		} else {
 			flt_trans_type = PF_ADDR_PIO;
-			flt_bdf = 0;
+			flt_bdf = PCIE_INVALID_BDF;
 		}
 		break;
 	case PCIE_TLP_TYPE_CFG0:
@@ -2614,7 +2624,7 @@
 			/* Clear these values as they no longer valid */
 			PCIE_ADV_REG(pfd_p)->pcie_ue_tgt_trans = 0;
 			PCIE_ADV_REG(pfd_p)->pcie_ue_tgt_addr = 0;
-			PCIE_ADV_REG(pfd_p)->pcie_ue_tgt_bdf = 0;
+			PCIE_ADV_REG(pfd_p)->pcie_ue_tgt_bdf = PCIE_INVALID_BDF;
 		}
 
 		/* PCIe BDG AER registers */
@@ -2653,7 +2663,8 @@
 			/* Clear these values as they no longer valid */
 			PCIE_ADV_BDG_REG(pfd_p)->pcie_sue_tgt_trans = 0;
 			PCIE_ADV_BDG_REG(pfd_p)->pcie_sue_tgt_addr = 0;
-			PCIE_ADV_BDG_REG(pfd_p)->pcie_sue_tgt_bdf = 0;
+			PCIE_ADV_BDG_REG(pfd_p)->pcie_sue_tgt_bdf =
+			    PCIE_INVALID_BDF;
 		}
 
 		/* PCIe RP registers */
--- a/usr/src/uts/common/sys/pcie.h	Sat Jun 20 12:12:10 2009 -0600
+++ b/usr/src/uts/common/sys/pcie.h	Sat Jun 20 15:42:11 2009 -0700
@@ -19,7 +19,7 @@
  * CDDL HEADER END
  */
 /*
- * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
@@ -62,7 +62,7 @@
 #define	PCIE_PCIECAP_VER_1_0		0x1	/* PCI-E spec 1.0 */
 #define	PCIE_PCIECAP_VER_MASK		0xF	/* Version Mask */
 #define	PCIE_PCIECAP_DEV_TYPE_PCIE_DEV	0x00	/* PCI-E Endpont Device */
-#define	PCIE_PCIECAP_DEV_TYPE_PCI_DEV	0x10	/* Leg PCI Endpont Device */
+#define	PCIE_PCIECAP_DEV_TYPE_PCI_DEV	0x10	/* "Leg PCI" Endpont Device */
 #define	PCIE_PCIECAP_DEV_TYPE_ROOT	0x40	/* Root Port of Root Complex */
 #define	PCIE_PCIECAP_DEV_TYPE_UP	0x50	/* Upstream Port of Switch */
 #define	PCIE_PCIECAP_DEV_TYPE_DOWN	0x60	/* Downstream Port of Switch */
@@ -216,7 +216,9 @@
 #define	PCIE_LINKCAP_DLL_ACTIVE_REP_CAPABLE	0x100000    /* DLL Active */
 							    /* Capable bit */
 
-#define	PCIE_LINKCAP_PORT_NUMBER	0xF0000000	/* Port Number */
+#define	PCIE_LINKCAP_PORT_NUMBER	0xFF000000	/* Port Number */
+#define	PCIE_LINKCAP_PORT_NUMBER_SHIFT	24	/* Port Number Shift */
+#define	PCIE_LINKCAP_PORT_NUMBER_MASK	0xFF	/* Port Number Mask */
 
 /*
  * Link Control Register (2 bytes)
@@ -590,7 +592,7 @@
 #define	PCIE_REQ_ID_BUS_SHIFT	8
 #define	PCIE_REQ_ID_BUS_MASK	0xFF00
 #define	PCIE_REQ_ID_DEV_SHIFT	3
-#define	PCIE_REQ_ID_DEV_MASK	0x00F1
+#define	PCIE_REQ_ID_DEV_MASK	0x00F8
 #define	PCIE_REQ_ID_FUNC_SHIFT	0
 #define	PCIE_REQ_ID_FUNC_MASK	0x0007
 
--- a/usr/src/uts/common/sys/pcie_impl.h	Sat Jun 20 12:12:10 2009 -0600
+++ b/usr/src/uts/common/sys/pcie_impl.h	Sat Jun 20 15:42:11 2009 -0700
@@ -61,8 +61,7 @@
 
 #define	PCIE_IS_PCIE(bus_p) (bus_p->bus_pcie_off)
 #define	PCIE_IS_PCIX(bus_p) (bus_p->bus_pcix_off)
-#define	PCIE_IS_PCI(bus_p) \
-	(bus_p->bus_dev_type == PCIE_PCIECAP_DEV_TYPE_PCI_DEV)
+#define	PCIE_IS_PCI(bus_p) (!PCIE_IS_PCIE(bus_p))
 #define	PCIE_HAS_AER(bus_p) (bus_p->bus_aer_off)
 /* IS_ROOT = is RC or RP */
 #define	PCIE_IS_ROOT(bus_p) (PCIE_IS_RC(bus_p) || PCIE_IS_RP(bus_p))
@@ -76,13 +75,14 @@
 #define	PCIE_IS_RP(bus_p) \
 	((bus_p->bus_dev_type == PCIE_PCIECAP_DEV_TYPE_ROOT) && \
 	    PCIE_IS_PCIE(bus_p))
+#define	PCIE_IS_SWU(bus_p) \
+	(bus_p->bus_dev_type == PCIE_PCIECAP_DEV_TYPE_UP)
+#define	PCIE_IS_SWD(bus_p) \
+	(bus_p->bus_dev_type == PCIE_PCIECAP_DEV_TYPE_DOWN)
 #define	PCIE_IS_SW(bus_p) \
-	((bus_p->bus_dev_type == PCIE_PCIECAP_DEV_TYPE_UP) || \
-	    (bus_p->bus_dev_type == PCIE_PCIECAP_DEV_TYPE_DOWN))
+	(PCIE_IS_SWU(bus_p) || PCIE_IS_SWD(bus_p))
 #define	PCIE_IS_BDG(bus_p)  (bus_p->bus_hdr_type == PCI_HEADER_ONE)
-#define	PCIE_IS_PCI_BDG(bus_p) \
-	((bus_p->bus_dev_type == PCIE_PCIECAP_DEV_TYPE_PCI_DEV) && \
-	    PCIE_IS_BDG(bus_p))
+#define	PCIE_IS_PCI_BDG(bus_p) (PCIE_IS_PCI(bus_p) && PCIE_IS_BDG(bus_p))
 #define	PCIE_IS_PCIE_BDG(bus_p) \
 	(bus_p->bus_dev_type == PCIE_PCIECAP_DEV_TYPE_PCIE2PCI)
 #define	PCIE_IS_PCI2PCIE(bus_p) \
@@ -216,7 +216,7 @@
 	} pcie_ext;
 	uint32_t pcie_ue_tgt_trans;	/* Fault trans type from AER Logs */
 	uint64_t pcie_ue_tgt_addr;	/* Fault addr from AER Logs */
-	pcie_req_id_t pcie_ue_tgt_bdf;	/* Fault bdf from SAER Logs */
+	pcie_req_id_t pcie_ue_tgt_bdf;	/* Fault bdf from AER Logs */
 } pf_pcie_adv_err_regs_t;
 
 typedef struct pf_pcie_rp_err_regs {
@@ -233,8 +233,8 @@
 } pf_pcie_err_regs_t;
 
 typedef struct pf_root_fault {
-	pcie_req_id_t	fault_bdf;	/* Fault BDF of error */
-	uint64_t	fault_addr;	/* Fault Addr of error */
+	pcie_req_id_t	scan_bdf;	/* BDF from error logs */
+	uint64_t	scan_addr;	/* Addr from error logs */
 	boolean_t	full_scan;	/* Option to do a full scan */
 } pf_root_fault_t;
 
@@ -337,7 +337,19 @@
 #define	PF_HDL_FOUND		1
 #define	PF_HDL_NOTFOUND		2
 
-#define	PCIE_PCIECAP_DEV_TYPE_RC_PSEUDO	0x100
+/*
+ * PCIe Capability Device Type Pseudo Definitions.
+ *
+ * PCI_PSEUDO is used on real PCI devices.  The Legacy PCI definition in the
+ * PCIe spec really refers to PCIe devices that *require* IO Space access.  IO
+ * Space access is usually frowned upon now in PCIe, but there for legacy
+ * purposes.
+ */
+#define	PCIE_PCIECAP_DEV_TYPE_RC_PSEUDO		0x100
+#define	PCIE_PCIECAP_DEV_TYPE_PCI_PSEUDO	0x101
+
+#define	PCIE_INVALID_BDF	0xFFFF
+#define	PCIE_CHECK_VALID_BDF(x)	(x != PCIE_INVALID_BDF)
 
 typedef struct {
 	dev_info_t	*dip;
--- a/usr/src/uts/i86pc/io/pci/pci_tools.c	Sat Jun 20 12:12:10 2009 -0600
+++ b/usr/src/uts/i86pc/io/pci/pci_tools.c	Sat Jun 20 15:42:11 2009 -0700
@@ -975,7 +975,7 @@
 
 			/*
 			 * Access device.  prg is modified.
-			 * First, check for AMD northbridges for I/O access
+			 * First, check for AMD K8 northbridges for I/O access
 			 * (This fix will move in future to pcitool user-land)
 			 * Next, check for PCIe devices and do
 			 * memory-mapped access
@@ -983,8 +983,10 @@
 			 */
 			if ((prg.bus_no == 0) &&
 			    (prg.dev_no >= 0x18) &&
-			    (prg.dev_no < (0x18 + ncpus)) &&
-			    (cpuid_getvendor(CPU) == X86_VENDOR_AMD)) {
+			    (prg.dev_no <
+			    (0x18 + ncpus/cpuid_get_ncpu_per_chip(CPU))) &&
+			    (cpuid_getvendor(CPU) == X86_VENDOR_AMD) &&
+			    (cpuid_getfamily(CPU) == 0xf)) {
 				rval = pcitool_cfg_access(dip, &prg,
 				    write_flag);
 			} else if (max_cfg_size == PCIE_CONF_HDR_SIZE) {
--- a/usr/src/uts/intel/io/pci/pci_pci.c	Sat Jun 20 12:12:10 2009 -0600
+++ b/usr/src/uts/intel/io/pci/pci_pci.c	Sat Jun 20 15:42:11 2009 -0700
@@ -19,7 +19,7 @@
  * CDDL HEADER END
  */
 /*
- * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
@@ -219,7 +219,7 @@
 		ushort_t bridge_control;
 	} config_state[PCI_MAX_CHILDREN];
 
-	uint8_t parent_bus;
+	uint16_t parent_bus;
 } ppb_devstate_t;
 
 
@@ -339,7 +339,7 @@
 			return (DDI_FAILURE);
 		}
 
-		ppb->parent_bus = PCIE_PCIECAP_DEV_TYPE_PCI_DEV;
+		ppb->parent_bus = PCIE_PCIECAP_DEV_TYPE_PCI_PSEUDO;
 		for (pdip = ddi_get_parent(devi); pdip && (pdip != root) &&
 		    (ppb->parent_bus != PCIE_PCIECAP_DEV_TYPE_PCIE_DEV);
 		    pdip = ddi_get_parent(pdip)) {
--- a/usr/src/uts/sun4/io/px/px_dma.c	Sat Jun 20 12:12:10 2009 -0600
+++ b/usr/src/uts/sun4/io/px/px_dma.c	Sat Jun 20 15:42:11 2009 -0700
@@ -118,8 +118,8 @@
 	 * XXX No IOMMU protection for broken devices.
 	 */
 	ASSERT((intptr_t)ddi_get_parent_data(rdip) >> 1 == 0);
-	mp->dmai_bdf = ((intptr_t)ddi_get_parent_data(rdip) == 1) ? 0 :
-	    pcie_get_bdf_for_dma_xfer(dip, rdip);
+	mp->dmai_bdf = ((intptr_t)ddi_get_parent_data(rdip) == 1) ?
+	    PCIE_INVALID_BDF : pcie_get_bdf_for_dma_xfer(dip, rdip);
 
 	return (mp);
 }
--- a/usr/src/uts/sun4/io/px/px_fdvma.c	Sat Jun 20 12:12:10 2009 -0600
+++ b/usr/src/uts/sun4/io/px/px_fdvma.c	Sat Jun 20 15:42:11 2009 -0700
@@ -211,8 +211,8 @@
 	 * XXX No IOMMU protection for broken devices.
 	 */
 	ASSERT((intptr_t)ddi_get_parent_data(rdip) >> 1 == 0);
-	mp->dmai_bdf = ((intptr_t)ddi_get_parent_data(rdip) == 1) ? 0 :
-	    pcie_get_bdf_for_dma_xfer(dip, rdip);
+	mp->dmai_bdf = ((intptr_t)ddi_get_parent_data(rdip) == 1) ?
+	    PCIE_INVALID_BDF : pcie_get_bdf_for_dma_xfer(dip, rdip);
 
 	DBG(DBG_DMA_CTL, dip,
 	    "DDI_DMA_RESERVE: mp=%p dvma=%x npages=%x private=%p\n",
--- a/usr/src/uts/sun4/io/px/px_fm.c	Sat Jun 20 12:12:10 2009 -0600
+++ b/usr/src/uts/sun4/io/px/px_fm.c	Sat Jun 20 15:42:11 2009 -0700
@@ -19,7 +19,7 @@
  * CDDL HEADER END
  */
 /*
- * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
@@ -299,7 +299,7 @@
 	int		lookup, rc_err, fab_err;
 	uint64_t	addr, base_addr;
 	uint64_t	fault_addr = (uint64_t)derr->fme_bus_specific;
-	pcie_req_id_t	bdf;
+	pcie_req_id_t	bdf = PCIE_INVALID_BDF;
 	px_ranges_t	*ranges_p;
 	int		range_len;
 
@@ -339,7 +339,7 @@
 			case PCI_ADDR_MEM32:
 				acc_type = PF_ADDR_PIO;
 				addr = fault_addr - base_addr;
-				bdf = NULL;
+				bdf = PCIE_INVALID_BDF;
 				break;
 			}
 			break;
@@ -591,15 +591,15 @@
 
 	if (regs->primary_ue & (PCIE_AER_UCE_UR | PCIE_AER_UCE_CA)) {
 		if (pf_tlp_decode(PCIE_DIP2BUS(dip), adv_reg) == DDI_SUCCESS)
-			PCIE_ROOT_FAULT(pfd_p)->fault_bdf =
+			PCIE_ROOT_FAULT(pfd_p)->scan_bdf =
 			    adv_reg->pcie_ue_tgt_bdf;
 	} else if (regs->primary_ue & PCIE_AER_UCE_PTLP) {
 		if (pf_tlp_decode(PCIE_DIP2BUS(dip), adv_reg) == DDI_SUCCESS) {
-			PCIE_ROOT_FAULT(pfd_p)->fault_bdf =
+			PCIE_ROOT_FAULT(pfd_p)->scan_bdf =
 			    adv_reg->pcie_ue_tgt_bdf;
 			if (adv_reg->pcie_ue_tgt_trans ==
 			    PF_ADDR_PIO)
-				PCIE_ROOT_FAULT(pfd_p)->fault_addr =
+				PCIE_ROOT_FAULT(pfd_p)->scan_addr =
 				    adv_reg->pcie_ue_tgt_addr;
 		}
 
@@ -708,8 +708,8 @@
 	pf_data_t	*pfd_p = &px_p->px_pfd_arr[idx];
 
 	/* Clear Old Data */
-	PCIE_ROOT_FAULT(pfd_p)->fault_bdf = 0;
-	PCIE_ROOT_FAULT(pfd_p)->fault_addr = 0;
+	PCIE_ROOT_FAULT(pfd_p)->scan_bdf = PCIE_INVALID_BDF;
+	PCIE_ROOT_FAULT(pfd_p)->scan_addr = 0;
 	PCI_BDG_ERR_REG(pfd_p)->pci_bdg_sec_stat = 0;
 	PCIE_ADV_REG(pfd_p)->pcie_ce_status = 0;
 	PCIE_ADV_REG(pfd_p)->pcie_ue_status = 0;
@@ -734,26 +734,26 @@
  * o errs rcvd in RC, that may have been propagated to/from the fabric
  * o the fabric scan code should scan the device path of fault bdf/addr
  *
- * fault_bdf: The bdf that caused the fault, which may have error bits set.
- * fault_addr: The PIO addr that caused the fault, such as failed PIO, but not
+ * scan_bdf: The bdf that caused the fault, which may have error bits set.
+ * scan_addr: The PIO addr that caused the fault, such as failed PIO, but not
  *	       failed DMAs.
  * s_status: Secondary Status equivalent to why the fault occured.
  *	     (ie S-TA/MA, R-TA)
- * Either the fault bdf or addr may be NULL, but not both.
+ * Either the scan bdf or addr may be NULL, but not both.
  */
 void
-px_rp_en_q(px_t *px_p, pcie_req_id_t fault_bdf, uint32_t fault_addr,
+px_rp_en_q(px_t *px_p, pcie_req_id_t scan_bdf, uint32_t scan_addr,
     uint16_t s_status)
 {
 	pf_data_t	*pfd_p;
 
-	if (!fault_bdf && !fault_addr)
+	if (!PCIE_CHECK_VALID_BDF(scan_bdf) && !scan_addr)
 		return;
 
 	pfd_p = px_get_pfd(px_p);
 
-	PCIE_ROOT_FAULT(pfd_p)->fault_bdf = fault_bdf;
-	PCIE_ROOT_FAULT(pfd_p)->fault_addr = (uint64_t)fault_addr;
+	PCIE_ROOT_FAULT(pfd_p)->scan_bdf = scan_bdf;
+	PCIE_ROOT_FAULT(pfd_p)->scan_addr = (uint64_t)scan_addr;
 	PCI_BDG_ERR_REG(pfd_p)->pci_bdg_sec_stat = s_status;
 }
 
@@ -784,8 +784,8 @@
 	 * only mark the device as "Non-Fatal" if the addr == NULL and bdf !=
 	 * NULL.
 	 */
-	status = (!addr && (bus_p->bus_bdf == bdf)) ? DDI_FM_NONFATAL :
-	    DDI_FM_FATAL;
+	status = (!addr && (PCIE_CHECK_VALID_BDF(bdf) &&
+	    (bus_p->bus_bdf == bdf))) ? DDI_FM_NONFATAL : DDI_FM_FATAL;
 
 	return (status);
 }
@@ -840,7 +840,8 @@
 	 */
 	size = hp->ah_len;
 	if (((fault_addr >= base_addr) && (fault_addr < (base_addr + size))) ||
-	    ((fault_addr == NULL) && (bdf == PCIE_DIP2BUS(dip)->bus_bdf)))
+	    ((fault_addr == NULL) && (PCIE_CHECK_VALID_BDF(bdf) &&
+	    (bdf == PCIE_DIP2BUS(dip)->bus_bdf))))
 		status = DDI_FM_NONFATAL;
 
 	return (status);
@@ -876,7 +877,7 @@
 	 * know the BDF and ADDR == 0.
 	 */
 	if (((addr >= base_addr) && (addr < (base_addr + size))) ||
-	    ((addr == NULL) && (bdf != NULL)))
+	    ((addr == NULL) && PCIE_CHECK_VALID_BDF(bdf)))
 		status = DDI_FM_NONFATAL;
 
 	return (status);
--- a/usr/src/uts/sun4/io/px/px_ioapi.h	Sat Jun 20 12:12:10 2009 -0600
+++ b/usr/src/uts/sun4/io/px/px_ioapi.h	Sat Jun 20 15:42:11 2009 -0700
@@ -225,7 +225,7 @@
 #define	PCI_MAP_ATTR_BDF_MASK	0xffff0000
 
 #define	PX_ADD_ATTR_EXTNS(attr, bdf) \
-	(attr | (bdf << PCI_MAP_ATTR_BDF))
+	(attr | (PCIE_CHECK_VALID_BDF(bdf) ? (bdf << PCI_MAP_ATTR_BDF) : 0))
 
 typedef enum io_sync_direction {
 	IO_SYNC_DEVICE		= (uint32_t)0x01,
--- a/usr/src/uts/sun4/io/px/px_pci.c	Sat Jun 20 12:12:10 2009 -0600
+++ b/usr/src/uts/sun4/io/px/px_pci.c	Sat Jun 20 15:42:11 2009 -0700
@@ -2119,31 +2119,159 @@
 }
 #endif
 
+
 #ifdef PX_PLX
 /*
- * Disable PLX specific relaxed ordering mode.  Due to PLX
+ * Disable PLX specific relaxed ordering mode.	Due to PLX
  * erratum #6, use of this mode with Cut-Through Cancellation
  * can result in dropped Completion type packets.
+ *
+ * Clear the Relaxed Ordering Mode on 8533 and 8548 switches.
+ * To disable RO, clear bit 5 in offset 0x664, an undocumented
+ * bit in the PLX spec, on Ports 0, 8 and 12.  Proprietary PLX
+ * registers are normally accessible only via memspace from Port
+ * 0.  If port 0 is attached go ahead and disable RO on Port 0,
+ * 8 and 12, if they exist.
  */
 static void
 plx_ro_disable(pxb_devstate_t *pxb)
 {
-	uint32_t		val;
-	ddi_acc_handle_t	hdl = pxb->pxb_config_handle;
+	pcie_bus_t		*bus_p = PCIE_DIP2BUS(pxb->pxb_dip);
+	dev_info_t		*dip = pxb->pxb_dip;
+	pci_regspec_t		*reg_spec, *addr_spec;
+	int			rlen, alen;
+	int			orig_rsize, new_rsize;
+	uint_t			rnum, anum;
+	ddi_device_acc_attr_t	attr;
+	ddi_acc_handle_t	hdl;
+	caddr_t			regsp;
+	uint32_t		val, port_enable;
+	char			*offset;
+	char			*port_offset;
+
+	if (!((pxb->pxb_device_id == PXB_DEVICE_PLX_8533) ||
+	    (pxb->pxb_device_id == PXB_DEVICE_PLX_8548)))
+		return;
+
+	/* You can also only do this on Port 0 */
+	val = PCIE_CAP_GET(32, bus_p, PCIE_LINKCAP);
+	val = (val >> PCIE_LINKCAP_PORT_NUMBER_SHIFT) &
+	    PCIE_LINKCAP_PORT_NUMBER_MASK;
+
+	DBG(DBG_ATTACH, dip, "PLX RO Disable : bdf=0x%x port=%d\n",
+	    bus_p->bus_bdf, val);
+
+	if (val != 0)
+		return;
+
+	/*
+	 * Read the reg property, but allocate extra space incase we need to add
+	 * a new entry later.
+	 */
+	if (ddi_getproplen(DDI_DEV_T_ANY, dip, DDI_PROP_DONTPASS, "reg",
+	    &orig_rsize) != DDI_SUCCESS)
+		return;
+
+	new_rsize = orig_rsize + sizeof (pci_regspec_t);
+	reg_spec = kmem_alloc(new_rsize, KM_SLEEP);
+
+	if (ddi_getlongprop_buf(DDI_DEV_T_ANY, dip, DDI_PROP_DONTPASS, "reg",
+	    (caddr_t)reg_spec, &orig_rsize) != DDI_SUCCESS)
+		goto fail;
+
+	/* Find the mem32 reg property */
+	rlen = orig_rsize / sizeof (pci_regspec_t);
+	for (rnum = 0; rnum < rlen; rnum++) {
+		if ((reg_spec[rnum].pci_phys_hi & PCI_ADDR_MASK) ==
+		    PCI_ADDR_MEM32)
+			goto fix;
+	}
+
+	/*
+	 * Mem32 reg property was not found.
+	 * Look for it in assign-address property.
+	 */
+	addr_spec = bus_p->bus_assigned_addr;
+	alen = bus_p->bus_assigned_entries;
+	for (anum = 0; anum < alen; anum++) {
+		if ((addr_spec[anum].pci_phys_hi & PCI_ADDR_MASK) ==
+		    PCI_ADDR_MEM32)
+			goto update;
+	}
+
+	/* Unable to find mem space assigned address, give up. */
+	goto fail;
 
-	switch (pxb->pxb_device_id) {
-	case PXB_DEVICE_PLX_8533:
-	case PXB_DEVICE_PLX_8548:
-		/*
-		 * Clear the Relaxed Ordering Mode bit of the Egress
-		 * Performance Counter register on 8533 and 8548 switches.
-		 */
-		val = pci_config_get32(hdl, PLX_EGRESS_PERFCTR_OFFSET);
-		if (val & PLX_RO_MODE_BIT) {
-			val ^= PLX_RO_MODE_BIT;
-			pci_config_put32(hdl, PLX_EGRESS_PERFCTR_OFFSET, val);
-		}
-		break;
-	}
+update:
+	/*
+	 * Add the mem32 access to the reg spec.
+	 * Use the last entry which was previously allocated.
+	 */
+	reg_spec[rnum].pci_phys_hi = (addr_spec[anum].pci_phys_hi &
+	    ~PCI_REG_REL_M);
+	reg_spec[rnum].pci_phys_mid = 0;
+	reg_spec[rnum].pci_phys_low = 0;
+	reg_spec[rnum].pci_size_hi = addr_spec[anum].pci_size_hi;
+	reg_spec[rnum].pci_size_low = addr_spec[anum].pci_size_low;
+
+	/* Create the new reg_spec data and update the property */
+	if (ddi_prop_update_int_array(DDI_DEV_T_NONE, dip, "reg",
+	    (int *)reg_spec, (new_rsize / sizeof (int))) != DDI_SUCCESS)
+		goto fail;
+
+fix:
+	attr.devacc_attr_version = DDI_DEVICE_ATTR_V0;
+	attr.devacc_attr_endian_flags  = DDI_STRUCTURE_LE_ACC;
+	attr.devacc_attr_dataorder = DDI_STRICTORDER_ACC;
+
+	if (ddi_regs_map_setup(dip, rnum, &regsp, 0, 0, &attr,
+	    &hdl) != DDI_SUCCESS)
+		goto fail;
+
+	/* Grab register which shows which ports are enabled */
+	offset = (char *)regsp + PLX_INGRESS_PORT_ENABLE;
+	port_enable = ddi_get32(hdl, (uint32_t *)offset);
+
+	if ((port_enable == 0xFFFFFFFF) || (port_enable == 0))
+		goto done;
+
+	offset = (char *)regsp + PLX_INGRESS_CONTROL_SHADOW;
+
+	/* Disable RO on Port 0 */
+	port_offset = 0x0 + offset;
+	val = ddi_get32(hdl, (uint32_t *)port_offset);
+	if (val & PLX_RO_MODE_BIT)
+		val ^= PLX_RO_MODE_BIT;
+	ddi_put32(hdl, (uint32_t *)port_offset, val);
+
+	/* Disable RO on Port 8, but make sure its enabled */
+	if (!(port_enable & (1 << 8)))
+		goto port12;
+
+	port_offset = (8 * 0x1000) + offset;
+	val = ddi_get32(hdl, (uint32_t *)port_offset);
+	if (val & PLX_RO_MODE_BIT)
+		val ^= PLX_RO_MODE_BIT;
+	ddi_put32(hdl, (uint32_t *)port_offset, val);
+
+port12:
+	/* Disable RO on Port 12, but make sure it exists */
+	if (!(port_enable & (1 << 12)))
+		goto done;
+
+	port_offset = (12 * 0x1000) + offset;
+	val = ddi_get32(hdl, (uint32_t *)port_offset);
+	if (val & PLX_RO_MODE_BIT)
+		val ^= PLX_RO_MODE_BIT;
+	ddi_put32(hdl, (uint32_t *)port_offset, val);
+
+	goto done;
+
+fail:
+	DBG(DBG_ATTACH, dip, "PLX RO Disable failed.\n");
+
+done:
+	ddi_regs_map_free(&hdl);
+	kmem_free(reg_spec, new_rsize);
 }
 #endif /* PX_PLX */
--- a/usr/src/uts/sun4/io/px/pxb_plx.h	Sat Jun 20 12:12:10 2009 -0600
+++ b/usr/src/uts/sun4/io/px/pxb_plx.h	Sat Jun 20 15:42:11 2009 -0700
@@ -19,15 +19,13 @@
  * CDDL HEADER END
  */
 /*
- * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
 #ifndef	_SYS_PXB_PLX_H
 #define	_SYS_PXB_PLX_H
 
-#pragma ident	"%Z%%M%	%I%	%E% SMI"
-
 #ifdef	__cplusplus
 extern "C" {
 #endif
@@ -47,8 +45,11 @@
 #define	PXB_DEVICE_PLX_AA_REV	0xAA
 
 /* Register offsets and bits specific to the 8548 and 8533 */
-#define	PLX_EGRESS_PERFCTR_OFFSET	0x1F0
-#define	PLX_RO_MODE_BIT			0x100000
+#define	PLX_INGRESS_CONTROL_SHADOW	0x664
+#define	PLX_INGRESS_PORT_ENABLE		0x668
+#define	PLX_CAM_PORT_8			0x2e8
+#define	PLX_CAM_PORT_12			0x2f8
+#define	PLX_RO_MODE_BIT			0x20
 
 #ifdef	__cplusplus
 }
--- a/usr/src/uts/sun4u/io/pci/pci_pci.c	Sat Jun 20 12:12:10 2009 -0600
+++ b/usr/src/uts/sun4u/io/pci/pci_pci.c	Sat Jun 20 15:42:11 2009 -0700
@@ -19,7 +19,7 @@
  * CDDL HEADER END
  */
 /*
- * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
@@ -244,7 +244,7 @@
 	int fm_cap;
 	ddi_iblock_cookie_t fm_ibc;
 
-	uint8_t parent_bus;
+	uint16_t parent_bus;
 } ppb_devstate_t;
 
 /*
@@ -1711,7 +1711,7 @@
 	 */
 	ddi_fm_handler_register(ppb_p->dip, ppb_err_callback, NULL);
 
-	ppb_p->parent_bus = PCIE_PCIECAP_DEV_TYPE_PCI_DEV;
+	ppb_p->parent_bus = PCIE_PCIECAP_DEV_TYPE_PCI_PSEUDO;
 	for (pdip = ddi_get_parent(ppb_p->dip); pdip && (pdip != root) &&
 	    (ppb_p->parent_bus != PCIE_PCIECAP_DEV_TYPE_PCIE_DEV);
 	    pdip = ddi_get_parent(pdip)) {
--- a/usr/src/uts/sun4u/io/px/px_err.c	Sat Jun 20 12:12:10 2009 -0600
+++ b/usr/src/uts/sun4u/io/px/px_err.c	Sat Jun 20 15:42:11 2009 -0700
@@ -19,12 +19,10 @@
  * CDDL HEADER END
  */
 /*
- * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
-#pragma ident	"%Z%%M%	%I%	%E% SMI"
-
 /*
  * sun4u Fire Error Handling
  */
@@ -1714,7 +1712,7 @@
 	char		buf[FM_MAX_CLASS];
 	boolean_t	pri = PX_ERR_IS_PRI(bit);
 	px_t		*px_p = DIP_TO_STATE(rpdip);
-	pcie_req_id_t	fault_bdf = 0;
+	pcie_req_id_t	fault_bdf = PCIE_INVALID_BDF;
 	uint16_t	s_status = 0;
 
 	if (pri) {
--- a/usr/src/uts/sun4u/io/px/px_lib4u.c	Sat Jun 20 12:12:10 2009 -0600
+++ b/usr/src/uts/sun4u/io/px/px_lib4u.c	Sat Jun 20 15:42:11 2009 -0700
@@ -1498,7 +1498,7 @@
 	px_ranges_t	*ranges_p;
 	int		range_len;
 	uint32_t	addr_high, addr_low;
-	pcie_req_id_t	bdf = 0;
+	pcie_req_id_t	bdf = PCIE_INVALID_BDF;
 
 	/* Create the derr */
 	bzero(&derr, sizeof (ddi_fm_error_t));
@@ -1533,7 +1533,7 @@
 				if (rdip)
 					bdf = PCI_GET_BDF(rdip);
 				else
-					bdf = NULL;
+					bdf = PCIE_INVALID_BDF;
 				break;
 			}
 			break;
--- a/usr/src/uts/sun4v/io/px/px_err.c	Sat Jun 20 12:12:10 2009 -0600
+++ b/usr/src/uts/sun4v/io/px/px_err.c	Sat Jun 20 15:42:11 2009 -0700
@@ -19,12 +19,10 @@
  * CDDL HEADER END
  */
 /*
- * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
-#pragma ident	"%Z%%M%	%I%	%E% SMI"
-
 /*
  * sun4v Fire Error Handling
  */
@@ -139,7 +137,7 @@
 px_err_fill_pfd(dev_info_t *dip, px_t *px_p, px_rc_err_t *epkt) {
 	pf_pcie_adv_err_regs_t adv_reg;
 	int		sts = DDI_SUCCESS;
-	pcie_req_id_t	fault_bdf = 0;
+	pcie_req_id_t	fault_bdf = PCIE_INVALID_BDF;
 	uint64_t	fault_addr = 0;
 	uint16_t	s_status = 0;
 
@@ -179,7 +177,7 @@
 
 		sts = pf_tlp_decode(PCIE_DIP2BUS(dip), &adv_reg);
 		fault_bdf = adv_reg.pcie_ue_tgt_bdf;
-		fault_addr = adv_reg.pcie_ue_tgt_bdf;
+		fault_addr = adv_reg.pcie_ue_tgt_addr;
 	}
 
 	if (sts == DDI_SUCCESS)
@@ -671,7 +669,7 @@
 px_mmu_handle_lookup(dev_info_t *dip, ddi_fm_error_t *derr, px_rc_err_t *epkt)
 {
 	uint64_t addr = (uint64_t)epkt->addr;
-	pcie_req_id_t bdf = NULL;
+	pcie_req_id_t bdf = PCIE_INVALID_BDF;
 
 	if (epkt->rc_descr.H) {
 		bdf = (uint32_t)((epkt->hdr[0] >> 16) && 0xFFFF);
--- a/usr/src/uts/sun4v/io/px/px_lib4v.c	Sat Jun 20 12:12:10 2009 -0600
+++ b/usr/src/uts/sun4v/io/px/px_lib4v.c	Sat Jun 20 15:42:11 2009 -0700
@@ -1527,7 +1527,7 @@
     r_addr_t addr)
 {
 	uint32_t	addr_high, addr_low;
-	pcie_req_id_t	bdf;
+	pcie_req_id_t	bdf = PCIE_INVALID_BDF;
 	px_ranges_t	*ranges_p;
 	int		range_len, i;
 	ddi_acc_impl_t *hp = (ddi_acc_impl_t *)handle;
@@ -1557,7 +1557,7 @@
 				bdf = (pcie_req_id_t)(addr_low >> 12);
 				break;
 			default:
-				bdf = NULL;
+				bdf = PCIE_INVALID_BDF;
 				break;
 			}
 			break;