view usr/src/cmd/fm/modules/common/sw-diag-response/subsidiary/smf/smf_diag.c @ 12979:ab9ae749152f

PSARC/2009/617 Software Events Notification Parameters CLI PSARC/2009/618 snmp-notify: SNMP Notification Daemon for Software Events PSARC/2009/619 smtp-notify: Email Notification Daemon for Software Events PSARC/2010/225 fmd for non-global Solaris zones PSARC/2010/226 Solaris Instance UUID PSARC/2010/227 nvlist_nvflag(3NVPAIR) PSARC/2010/228 libfmevent additions PSARC/2010/257 sysevent_evc_setpropnvl and sysevent_evc_getpropnvl PSARC/2010/265 FMRI and FMA Event Stabilty, 'ireport' category 1 event class, and the 'sw' FMRI scheme PSARC/2010/278 FMA/SMF integration: instance state transitions PSARC/2010/279 Modelling panics within FMA PSARC/2010/290 logadm.conf upgrade 6392476 fmdump needs to pretty-print 6393375 userland ereport/ireport event generation interfaces 6445732 Add email notification agent for FMA and software events 6804168 RFE: Allow an efficient means to monitor SMF services status changes 6866661 scf_values_destroy(3SCF) will segfault if is passed NULL 6884709 Add snmp notification agent for FMA and software events 6884712 Add private interface to tap into libfmd_msg macro expansion capabilities 6897919 fmd to run in a non-global zone 6897937 fmd use of non-private doors is not safe 6900081 add a UUID to Solaris kernel image for use in crashdump identification 6914884 model panic events as a defect diagnosis in FMA 6944862 fmd_case_open_uuid, fmd_case_uuisresolved, fmd_nvl_create_defect 6944866 log legacy sysevents in fmd 6944867 enumerate svc scheme in topo 6944868 software-diagnosis and software-response fmd modules 6944870 model SMF maintenance state as a defect diagnosis in FMA 6944876 savecore runs in foreground for systems with zfs root and dedicated dump 6965796 Implement notification parameters for SMF state transitions and FMA events 6968287 SUN-FM-MIB.mib needs to be updated to reflect Oracle information 6972331 logadm.conf upgrade PSARC/2010/290
author Gavin Maltby <gavin.maltby@oracle.com>
date Fri, 30 Jul 2010 17:04:17 +1000
parents
children
line wrap: on
line source

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
 */

/*
 * SMF software-diagnosis subsidiary
 *
 * We model service instances in maintenance state as a defect diagnosis
 * in FMA.  When an instance transitions to maintenance state the SMF
 * graph engine publishes an event which we subscribe to here, and diagnose
 * a corresponding defect.
 *
 * We always solve a case immediately after opening it.  But we leave the
 * case close action to the response agent which needs to cache case UUIDs.
 * So in the normal case, where software-response is loaded and operational,
 * our cases will transition to CLOSED state moments after we solve them.
 * But if fmd restarts in the interim or if software-response is not loaded
 * then our cases may hang around in SOLVED state for a while, which means
 * we could iterate over them on receipt of new events.  But we don't -
 * we blindly solve a new case for every new maintenance event received,
 * and leave it to the fmd duplicate detection and history-based diagnosis
 * logic to do the right thing.
 *
 * Our sibling SMF response subsidiary propogates fmadm-initiated repairs
 * into SMF, and svcadm-initiated clears back into FMA.  In both cases
 * the case is moved on to the RESOLVED state, even if fmd is unable to
 * verify that the service is out of maintenance state (i.e., no longer
 * isolated).  If the service immediately re-enters maintenance state then
 * we diagnose a fresh case.  The history-based diagnosis changes in fmd
 * "do the right thing" and avoid throwing away new cases as duplicates
 * of old ones hanging around in the "resolved but not all usable again"
 * state.
 */

#include <strings.h>
#include <fm/libtopo.h>
#include <fm/fmd_fmri.h>

#include "../../common/sw.h"
#include "smf.h"

static id_t myid;

static struct {
	fmd_stat_t swde_smf_diagnosed;
	fmd_stat_t swde_smf_bad_class;
	fmd_stat_t swde_smf_no_attr;
	fmd_stat_t swde_smf_bad_attr;
	fmd_stat_t swde_smf_bad_fmri;
	fmd_stat_t swde_smf_no_uuid;
	fmd_stat_t swde_smf_no_reason_short;
	fmd_stat_t swde_smf_no_reason_long;
	fmd_stat_t swde_smf_no_svcname;
	fmd_stat_t swde_smf_admin_maint_drop;
	fmd_stat_t swde_smf_bad_nvlist_pack;
	fmd_stat_t swde_smf_dupuuid;
} swde_smf_stats = {
	{ "swde_smf_diagnosed", FMD_TYPE_UINT64,
	    "maintenance state defects published" },
	{ "swde_smf_bad_class", FMD_TYPE_UINT64,
	    "incorrect event class received" },
	{ "swde_smf_no_attr", FMD_TYPE_UINT64,
	    "malformed event - missing attr nvlist" },
	{ "swde_smf_bad_attr", FMD_TYPE_UINT64,
	    "malformed event - invalid attr list" },
	{ "swde_smf_bad_fmri", FMD_TYPE_UINT64,
	    "malformed event - fmri2str fails" },
	{ "swde_smf_no_uuid", FMD_TYPE_UINT64,
	    "malformed event - missing uuid" },
	{ "swde_smf_no_reason_short", FMD_TYPE_UINT64,
	    "SMF transition event had no reason-short" },
	{ "swde_smf_no_reason_long", FMD_TYPE_UINT64,
	    "SMF transition event had no reason-long" },
	{ "swde_smf_no_svcname", FMD_TYPE_UINT64,
	    "SMF transition event had no svc-string" },
	{ "swde_smf_admin_maint_drop", FMD_TYPE_UINT64,
	    "maintenance transitions requested by admin - no diagnosis" },
	{ "swde_smf_bad_nvlist_pack", FMD_TYPE_UINT64,
	    "failed nvlist_size or nvlist_pack" },
	{ "swde_smf_dupuuid", FMD_TYPE_UINT64,
	    "duplicate events received" },
};

#define	SWDE_SMF_CASEDATA_VERS		1

typedef struct swde_smf_casedata {
	uint32_t scd_vers;		/* must be first member */
	size_t scd_nvlbufsz;		/* size of following buffer */
					/* packed fmri nvlist follows */
} swde_smf_casedata_t;

#define	BUMPSTAT(stat)		swde_smf_stats.stat.fmds_value.ui64++

/*ARGSUSED*/
void
swde_smf_recv(fmd_hdl_t *hdl, fmd_event_t *ep, nvlist_t *nvl,
    const char *class, void *arg)
{
	char *rsn = NULL, *rsnl = NULL, *svcname = NULL;
	nvlist_t *attr, *svcfmri, *defect;
	swde_smf_casedata_t *cdp;
	fmd_case_t *cp;
	char *fmribuf;
	char *uuid;
	size_t sz;

	if (!fmd_nvl_class_match(hdl, nvl, TRANCLASS("maintenance"))) {
		BUMPSTAT(swde_smf_bad_class);
		return;
	}

	if (nvlist_lookup_nvlist(nvl, FM_IREPORT_ATTRIBUTES, &attr) != 0) {
		BUMPSTAT(swde_smf_no_attr);
		return;
	}

	if (nvlist_lookup_string(nvl, FM_IREPORT_UUID, &uuid) != 0) {
		BUMPSTAT(swde_smf_no_uuid);
		return;
	}

	if (nvlist_lookup_nvlist(attr, "svc", &svcfmri) != 0) {
		BUMPSTAT(swde_smf_bad_attr);
		return;
	}

	if (nvlist_lookup_string(attr, "reason-short", &rsn) != 0) {
		BUMPSTAT(swde_smf_no_reason_short);
		return;
	}

	if (nvlist_lookup_string(attr, "reason-long", &rsnl) != 0) {
		BUMPSTAT(swde_smf_no_reason_long);
		return;
	}

	if (nvlist_lookup_string(attr, "svc-string", &svcname) != 0) {
		BUMPSTAT(swde_smf_no_svcname);
		return;
	}

	if (strcmp(rsn, "administrative_request") == 0) {
		BUMPSTAT(swde_smf_admin_maint_drop);
		return;
	}

	/*
	 * Our case checkpoint data, version 1.
	 */
	if (nvlist_size(svcfmri, &sz, NV_ENCODE_NATIVE) != 0) {
		BUMPSTAT(swde_smf_bad_nvlist_pack);
		return;
	}
	cdp = fmd_hdl_zalloc(hdl, sizeof (*cdp) + sz, FMD_SLEEP);
	cdp->scd_vers = SWDE_SMF_CASEDATA_VERS;
	fmribuf = (char *)cdp + sizeof (*cdp);
	cdp->scd_nvlbufsz = sz;
	(void) nvlist_pack(svcfmri, &fmribuf, &sz, NV_ENCODE_NATIVE, 0);

	/*
	 * Open a case with UUID matching the originating event, and no
	 * associated serialization data.  Create a defect and add it to
	 * the case, and link the originating event to the case.  This
	 * call will return NULL if a case with the requested UUID already
	 * exists, which would mean we are processing an event twice so
	 * we can discard.
	 */
	if ((cp = swde_case_open(hdl, myid, uuid, SWDE_SMF_CASEDATA_VERS,
	    (void *)cdp, sizeof (*cdp) + sz)) == NULL) {
		BUMPSTAT(swde_smf_dupuuid);
		fmd_hdl_free(hdl, cdp, sizeof (*cdp) + sz);
		return;
	}

	defect = fmd_nvl_create_defect(hdl, SW_SMF_MAINT_DEFECT,
	    100, svcfmri, NULL, svcfmri);
	if (rsn != NULL)
		(void) nvlist_add_string(defect, "reason-short", rsn);
	if (rsnl != NULL)
		(void) nvlist_add_string(defect, "reason-long", rsnl);
	if (svcname != NULL)
		(void) nvlist_add_string(defect, "svc-string", svcname);
	fmd_case_add_suspect(hdl, cp, defect);
	fmd_case_add_ereport(hdl, cp, ep);

	/*
	 * Now solve the case, and immediately close it.  Although the
	 * resource is already isolated (SMF put it in maintenance state)
	 * we do not immediately close the case here - our sibling response
	 * logic will do that after caching the case UUID.
	 */
	fmd_case_solve(hdl, cp);
	BUMPSTAT(swde_smf_diagnosed);
}

/*
 * In the normal course of events we keep in sync with SMF through the
 * maintenance enter/clear events it raises.  Even if a maintenance
 * state is cleared using svcadm while fmd is not running, the event
 * will pend and be consumed when fmd does start and we'll close the
 * case (in the response agent).
 *
 * But is is possible for discontinuities to produce some confusion:
 *
 *	- if an instance is in maintenance state (and so shown in svcs -x
 *	  and fmadm faulty output) at the time we clone a new boot
 *	  environment then when we boot the new BE we can be out of
 *	  sync if the instance is cleared when we boot there
 *
 *	- meddling with /var/fm state - eg manual clear of files there,
 *	  or restore of old state
 *
 * So as an extra guard we have a case verify function which is called
 * at fmd restart (module load for software-diagnosis).  We must
 * return 0 to close the case, non-zero to retain it.
 */
int
swde_smf_vrfy(fmd_hdl_t *hdl, fmd_case_t *cp)
{
	swde_smf_casedata_t *cdp;
	nvlist_t *svcfmri;
	uint32_t v;
	int rv;

	cdp = swde_case_data(hdl, cp, &v);

	if (cdp == NULL || v != 1)
		return (0);	/* bad or damaged - just close */

	if (nvlist_unpack((char *)cdp + sizeof (*cdp),
	    cdp->scd_nvlbufsz, &svcfmri, 0) != 0)
		return (0);	/* ditto */

	switch (fmd_nvl_fmri_service_state(hdl, svcfmri)) {
	case FMD_SERVICE_STATE_UNUSABLE:
		/*
		 * Keep case iff in maintenance state
		 */
		rv = 1;
		break;

	default:
		/*
		 * Discard the case for all other states - cleared,
		 * service no longer exists, ... whatever.
		 */
		rv = 0;
		break;
	}

	nvlist_free(svcfmri);
	return (rv);
}

const struct sw_disp swde_smf_disp[] = {
	{ TRANCLASS("maintenance"), swde_smf_recv, NULL },
	{ NULL, NULL, NULL }
};

/*ARGSUSED*/
int
swde_smf_init(fmd_hdl_t *hdl, id_t id, const struct sw_disp **dpp, int *nelemp)
{
	myid = id;

	(void) fmd_stat_create(hdl, FMD_STAT_NOALLOC, sizeof (swde_smf_stats) /
	    sizeof (fmd_stat_t), (fmd_stat_t *)&swde_smf_stats);

	fmd_hdl_subscribe(hdl, TRANCLASS("maintenance"));

	*dpp = &swde_smf_disp[0];
	*nelemp = sizeof (swde_smf_disp) / sizeof (swde_smf_disp[0]);
	return (SW_SUB_INIT_SUCCESS);
}

const struct sw_subinfo smf_diag_info = {
	"smf diagnosis",		/* swsub_name */
	SW_CASE_SMF,			/* swsub_casetype */
	swde_smf_init,			/* swsub_init */
	NULL,				/* swsub_fini */
	NULL,				/* swsub_timeout */
	NULL,				/* swsub_case_close */
	swde_smf_vrfy,			/* swsub_case_vrfy */
};