Monitor and emit events for XenStore changes to XenBus trees

of the devices we manage.  These changes can be due to writes
we make ourselves or due to changes made by the control domain.
The goal of these changes is to insure that all state transitions
can be detected regardless of their source and to allow common
device policies (e.g. "onlined" backend devices) to be centralized
in the XenBus bus code.

sys/xen/xenbus/xenbusvar.h:
sys/xen/xenbus/xenbus.c:
sys/xen/xenbus/xenbus_if.m:
	Add a new method for XenBus drivers "localend_changed".
	This method is invoked whenever a write is detected to
	a device's XenBus tree.  The default implementation of
	this method is a no-op.

sys/xen/xenbus/xenbus_if.m:
sys/dev/xen/netfront/netfront.c:
sys/dev/xen/blkfront/blkfront.c:
sys/dev/xen/blkback/blkback.c:
	Change the signature of the "otherend_changed" method.
	This notification cannot fail, so it should return void.

sys/xen/xenbus/xenbusb_back.c:
	Add "online" device handling to the XenBus Back Bus
	support code.  An online backend device remains active
	after a front-end detaches as a reconnect is expected
	to occur in the near future.

sys/xen/interface/io/xenbus.h:
	Add comment block further explaining the meaning and
	driver responsibilities associated with the XenBus
	Closed state.

sys/xen/xenbus/xenbusb.c:
sys/xen/xenbus/xenbusb.h:
sys/xen/xenbus/xenbusb_back.c:
sys/xen/xenbus/xenbusb_front.c:
sys/xen/xenbus/xenbusb_if.m:
	o Register a XenStore watch against the local XenBus tree
	  for all devices.
	o Cache the string length of the path to our local tree.
	o Allow the xenbus front and back drivers to hook/filter both
	  local and otherend watch processing.
	o Update the device ivar version of "state" when we detect
	  a XenStore update of that node.

sys/dev/xen/control/control.c:
sys/xen/xenbus/xenbus.c:
sys/xen/xenbus/xenbusb.c:
sys/xen/xenbus/xenbusb.h:
sys/xen/xenbus/xenbusvar.h:
sys/xen/xenstore/xenstorevar.h:
	Allow clients of the XenStore watch mechanism to attach
	a single uintptr_t worth of client data to the watch.
	This removes the need to carefully place client watch
	data within enclosing objects so that a cast or offsetof
	calculation can be used to convert from watch to enclosing
	object.

Sponsored by:	Spectra Logic Corporation
MFC after:	1 week

10 files changed, 292 insertions, 85 deletions

diff --git a/sys/xen/interface/io/xenbus.h b/sys/xen/interface/io/xenbus.h
index 5e24f31..672c7d4 100644
--- a/sys/xen/interface/io/xenbus.h
+++ b/sys/xen/interface/io/xenbus.h
@@ -64,6 +64,15 @@ enum xenbus_state {
 
     /*
      * Closed: No connection exists between front and back end.
+     *
+     * For backend devices with the "online" attribute, the front can
+     * request a reconnect at any time.  To handle this transition
+     * gracefully, backend devices must reinitialize any XenStore data
+     * used to negotiate features with a peer before transitioning to
+     * the closed state.  When a reconnect request occurs, the
+     * XenBus backend support code will automatically transition the
+     * backend device from Closed to InitWait, kicking off the ring
+     * and feature negotiation process.
      */
     XenbusStateClosed        = 6,
 
diff --git a/sys/xen/xenbus/xenbus.c b/sys/xen/xenbus/xenbus.c
index c3e5fee..8887066 100644
--- a/sys/xen/xenbus/xenbus.c
+++ b/sys/xen/xenbus/xenbus.c
@@ -103,12 +103,13 @@ xenbus_strstate(XenbusState state)
 
 int 
 xenbus_watch_path(device_t dev, char *path, struct xs_watch *watch, 
-    xs_watch_cb_t *callback)
+    xs_watch_cb_t *callback, uintptr_t callback_data)
 {
 	int error;
 
 	watch->node = path;
 	watch->callback = callback;
+	watch->callback_data = callback_data;
 
 	error = xs_register_watch(watch);
 
@@ -124,7 +125,7 @@ xenbus_watch_path(device_t dev, char *path, struct xs_watch *watch,
 int
 xenbus_watch_path2(device_t dev, const char *path,
     const char *path2, struct xs_watch *watch, 
-    xs_watch_cb_t *callback)
+    xs_watch_cb_t *callback, uintptr_t callback_data)
 {
 	int error;
 	char *state = malloc(strlen(path) + 1 + strlen(path2) + 1,
@@ -134,7 +135,7 @@ xenbus_watch_path2(device_t dev, const char *path,
 	strcat(state, "/");
 	strcat(state, path2);
 
-	error = xenbus_watch_path(dev, state, watch, callback);
+	error = xenbus_watch_path(dev, state, watch, callback, callback_data);
 	if (error) {
 		free(state,M_XENBUS);
 	}
@@ -286,3 +287,8 @@ xenbus_dev_is_online(device_t dev)
 
 	return (value);
 }
+
+void
+xenbus_localend_changed(device_t dev, const char *path)
+{
+}
diff --git a/sys/xen/xenbus/xenbus_if.m b/sys/xen/xenbus/xenbus_if.m
index d671418..87d7c7f 100644
--- a/sys/xen/xenbus/xenbus_if.m
+++ b/sys/xen/xenbus/xenbus_if.m
@@ -27,7 +27,11 @@
 #
 
 #include <sys/bus.h>
-#include <xen/interface/io/xenbus.h>
+
+#include <machine/atomic.h>
+#include <machine/xen/xen-os.h>
+#include <xen/evtchn.h>
+#include <xen/xenbus/xenbusvar.h>
 
 INTERFACE xenbus;
 
@@ -39,7 +43,21 @@ INTERFACE xenbus;
  *                   state has changed..
  * \param _newstate  The new state of the otherend device.
  */
-METHOD int otherend_changed {
+METHOD void otherend_changed {
 	device_t _dev;
 	enum xenbus_state _newstate;
 };
+
+/**
+ * \brief Callback triggered when the XenStore tree of the local end
+ *        of a split device changes.
+ *
+ * \param _dev   NewBus device_t for this XenBus device whose otherend's
+ *               state has changed..
+ * \param _path  The tree relative sub-path to the modified node.  The empty
+ *               string indicates the root of the tree was destroyed.
+ */
+METHOD void localend_changed {
+	device_t _dev;
+	const char * _path;
+} DEFAULT xenbus_localend_changed;
diff --git a/sys/xen/xenbus/xenbusb.c b/sys/xen/xenbus/xenbusb.c
index 4bc86aa..cc519c5 100644
--- a/sys/xen/xenbus/xenbusb.c
+++ b/sys/xen/xenbus/xenbusb.c
@@ -90,10 +90,16 @@ xenbusb_free_child_ivars(struct xenbus_device_ivars *ivars)
 		ivars->xd_otherend_watch.node = NULL;
 	}
 
+	if (ivars->xd_local_watch.node != NULL) {
+		xs_unregister_watch(&ivars->xd_local_watch);
+		ivars->xd_local_watch.node = NULL;
+	}
+
 	if (ivars->xd_node != NULL) {
 		free(ivars->xd_node, M_XENBUS);
 		ivars->xd_node = NULL;
 	}
+	ivars->xd_node_len = 0;
 
 	if (ivars->xd_type != NULL) {
 		free(ivars->xd_type, M_XENBUS);
@@ -104,6 +110,7 @@ xenbusb_free_child_ivars(struct xenbus_device_ivars *ivars)
 		free(ivars->xd_otherend_path, M_XENBUS);
 		ivars->xd_otherend_path = NULL;
 	}
+	ivars->xd_otherend_path_len = 0;
 
 	free(ivars, M_XENBUS);
 }
@@ -121,30 +128,64 @@ xenbusb_free_child_ivars(struct xenbus_device_ivars *ivars)
  *                   watch event data.  The vector should be indexed via the
  *                   xs_watch_type enum in xs_wire.h.
  * \param vec_size   The number of elements in vec.
- *
- * \return  The device_t of the found device if any, or NULL.
- *
- * \note device_t is a pointer type, so it can be compared against
- *       NULL for validity. 
  */
 static void
-xenbusb_otherend_changed(struct xs_watch *watch, const char **vec,
+xenbusb_otherend_watch_cb(struct xs_watch *watch, const char **vec,
     unsigned int vec_size __unused)
 {
 	struct xenbus_device_ivars *ivars;
-	device_t dev;
+	device_t child;
+	device_t bus;
+	const char *path;
 	enum xenbus_state newstate;
 
-	ivars = (struct xenbus_device_ivars *) watch;
-	dev = ivars->xd_dev;
+	ivars = (struct xenbus_device_ivars *)watch->callback_data;
+	child = ivars->xd_dev;
+	bus = device_get_parent(child);
 
-	if (!ivars->xd_otherend_path
-	 || strncmp(ivars->xd_otherend_path, vec[XS_WATCH_PATH],
-		    strlen(ivars->xd_otherend_path)))
+	path = vec[XS_WATCH_PATH];
+	if (ivars->xd_otherend_path == NULL
+	 || strncmp(ivars->xd_otherend_path, path, ivars->xd_otherend_path_len))
 		return;
 
 	newstate = xenbus_read_driver_state(ivars->xd_otherend_path);
-	XENBUS_OTHEREND_CHANGED(dev, newstate);
+	XENBUSB_OTHEREND_CHANGED(bus, child, newstate);
+}
+
+/**
+ * XenBus watch callback registered against the XenStore sub-tree
+ * represnting the local half of a split device connection.
+ *
+ * This callback is invoked whenever any XenStore data in the subtree
+ * is modified, either by us or another privledged domain.
+ *
+ * \param watch      The xs_watch object used to register this callback
+ *                   function.
+ * \param vec        An array of pointers to NUL terminated strings containing
+ *                   watch event data.  The vector should be indexed via the
+ *                   xs_watch_type enum in xs_wire.h.
+ * \param vec_size   The number of elements in vec.
+ *
+ */
+static void
+xenbusb_local_watch_cb(struct xs_watch *watch, const char **vec,
+    unsigned int vec_size __unused)
+{
+	struct xenbus_device_ivars *ivars;
+	device_t child;
+	device_t bus;
+	const char *path;
+
+	ivars = (struct xenbus_device_ivars *)watch->callback_data;
+	child = ivars->xd_dev;
+	bus = device_get_parent(child);
+
+	path = vec[XS_WATCH_PATH];
+	if (ivars->xd_node == NULL
+	 || strncmp(ivars->xd_node, path, ivars->xd_node_len))
+		return;
+
+	XENBUSB_LOCALEND_CHANGED(bus, child, &path[ivars->xd_node_len]);
 }
 
 /**
@@ -193,12 +234,14 @@ xenbusb_delete_child(device_t dev, device_t child)
 
 	/*
 	 * We no longer care about the otherend of the
-	 * connection.  Cancel the watch now so that we
+	 * connection.  Cancel the watches now so that we
 	 * don't try to handle an event for a partially
 	 * detached child.
 	 */
 	if (ivars->xd_otherend_watch.node != NULL)
 		xs_unregister_watch(&ivars->xd_otherend_watch);
+	if (ivars->xd_local_watch.node != NULL)
+		xs_unregister_watch(&ivars->xd_local_watch);
 	
 	device_delete_child(dev, child);
 	xenbusb_free_child_ivars(ivars);
@@ -421,6 +464,7 @@ xenbusb_probe_children(device_t dev)
 			 */
 			ivars = device_get_ivars(kids[i]);
 			xs_register_watch(&ivars->xd_otherend_watch);
+			xs_register_watch(&ivars->xd_local_watch);
 		}
 		free(kids, M_TEMP);
 	}
@@ -475,7 +519,7 @@ xenbusb_devices_changed(struct xs_watch *watch, const char **vec,
 	char *p;
 	u_int component;
 
-	xbs = (struct xenbusb_softc *)watch;
+	xbs = (struct xenbusb_softc *)watch->callback_data;
 	dev = xbs->xbs_dev;
 
 	if (len <= XS_WATCH_PATH) {
@@ -620,6 +664,7 @@ xenbusb_add_device(device_t dev, const char *type, const char *id)
 		sx_init(&ivars->xd_lock, "xdlock");
 		ivars->xd_flags = XDF_CONNECTING;
 		ivars->xd_node = strdup(devpath, M_XENBUS);
+		ivars->xd_node_len = strlen(devpath);
 		ivars->xd_type  = strdup(type, M_XENBUS);
 		ivars->xd_state = XenbusStateInitialising;
 
@@ -630,12 +675,16 @@ xenbusb_add_device(device_t dev, const char *type, const char *id)
 			goto out;
 		}
 
-		statepath = malloc(strlen(ivars->xd_otherend_path)
+		statepath = malloc(ivars->xd_otherend_path_len
 		    + strlen("/state") + 1, M_XENBUS, M_WAITOK);
 		sprintf(statepath, "%s/state", ivars->xd_otherend_path);
-
 		ivars->xd_otherend_watch.node = statepath;
-		ivars->xd_otherend_watch.callback = xenbusb_otherend_changed;
+		ivars->xd_otherend_watch.callback = xenbusb_otherend_watch_cb;
+		ivars->xd_otherend_watch.callback_data = (uintptr_t)ivars;
+
+		ivars->xd_local_watch.node = ivars->xd_node;
+		ivars->xd_local_watch.callback = xenbusb_local_watch_cb;
+		ivars->xd_local_watch.callback_data = (uintptr_t)ivars;
 
 		mtx_lock(&xbs->xbs_lock);
 		xbs->xbs_connecting_children++;
@@ -693,6 +742,7 @@ xenbusb_attach(device_t dev, char *bus_node, u_int id_components)
 
 	xbs->xbs_device_watch.node = bus_node;
 	xbs->xbs_device_watch.callback = xenbusb_devices_changed;
+	xbs->xbs_device_watch.callback_data = (uintptr_t)xbs;
 
 	TASK_INIT(&xbs->xbs_probe_children, 0, xenbusb_probe_children_cb, dev);
 
@@ -735,7 +785,7 @@ xenbusb_resume(device_t dev)
 
 			DEVICE_RESUME(kids[i]);
 
-			statepath = malloc(strlen(ivars->xd_otherend_path)
+			statepath = malloc(ivars->xd_otherend_path_len
 			    + strlen("/state") + 1, M_XENBUS, M_WAITOK);
 			sprintf(statepath, "%s/state", ivars->xd_otherend_path);
 
@@ -819,7 +869,7 @@ xenbusb_write_ivar(device_t dev, device_t child, int index, uintptr_t value)
 	{
 		int error;
 
-		newstate = (enum xenbus_state) value;
+		newstate = (enum xenbus_state)value;
 		sx_xlock(&ivars->xd_lock);
 		if (ivars->xd_state == newstate) {
 			error = 0;
@@ -876,3 +926,24 @@ xenbusb_write_ivar(device_t dev, device_t child, int index, uintptr_t value)
 
 	return (ENOENT);
 }
+
+void
+xenbusb_otherend_changed(device_t bus, device_t child, enum xenbus_state state)
+{
+	XENBUS_OTHEREND_CHANGED(child, state);
+}
+
+void
+xenbusb_localend_changed(device_t bus, device_t child, const char *path)
+{
+
+	if (strcmp(path, "/state") != 0) {
+		struct xenbus_device_ivars *ivars;
+
+		ivars = device_get_ivars(child);
+		sx_xlock(&ivars->xd_lock);
+		ivars->xd_state = xenbus_read_driver_state(ivars->xd_node);
+		sx_xunlock(&ivars->xd_lock);
+	}
+	XENBUS_LOCALEND_CHANGED(child, path);
+}
diff --git a/sys/xen/xenbus/xenbusb.h b/sys/xen/xenbus/xenbusb.h
index 75abb98..33008f7 100644
--- a/sys/xen/xenbus/xenbusb.h
+++ b/sys/xen/xenbus/xenbusb.h
@@ -41,7 +41,6 @@
  * Datastructures and function declarations for use in implementing
  * bus attachements (e.g. frontend and backend device busses) for XenBus.
  */
-#include "xenbusb_if.h"
 
 /**
  * Enumeration of state flag values for the xbs_flags field of
@@ -61,10 +60,6 @@ struct xenbusb_softc {
 	 * XenStore watch used to monitor the subtree of the
 	 * XenStore where devices for this bus attachment arrive	
 	 * and depart.
-	 *
-	 * \note This field must be the first in the softc structure
-	 *       so that a simple cast can be used to retrieve the
-	 *	 softc from within a XenStore watch event callback.
 	 */
 	struct xs_watch	        xbs_device_watch;
 
@@ -129,14 +124,17 @@ struct xenbus_device_ivars {
 	 * XenStore watch used to monitor the subtree of the
 	 * XenStore where information about the otherend of
 	 * the split Xen device this device instance represents.
-	 *
-	 * \note This field must be the first in the instance
-	 *	 variable structure so that a simple cast can be
-	 *	 used to retrieve ivar data from within a XenStore
-	 *	 watch event callback.
 	 */
 	struct xs_watch		xd_otherend_watch;
 
+	/**
+	 * XenStore watch used to monitor the XenStore sub-tree
+	 * associated with this device.  This watch will fire
+	 * for modifications that we make from our domain as
+	 * well as for those made by the control domain.
+	 */
+	struct xs_watch		xd_local_watch;
+
 	/** Sleepable lock used to protect instance data. */
 	struct sx		xd_lock;
 
@@ -152,6 +150,9 @@ struct xenbus_device_ivars {
 	 */
 	char		       *xd_node;
 
+	/** The length of xd_node.  */
+	int			xd_node_len;
+
 	/** XenBus device type ("vbd", "vif", etc.). */
 	char		       *xd_type;
 
@@ -168,6 +169,9 @@ struct xenbus_device_ivars {
 	 * about the otherend of this split device instance.
 	 */
 	char		       *xd_otherend_path;
+
+	/** The length of xd_otherend_path.  */
+	int			xd_otherend_path_len;
 };
 
 /**
@@ -247,6 +251,26 @@ int xenbusb_write_ivar(device_t dev, device_t child, int index,
 		       uintptr_t value);
 
 /**
+ * \brief Common XenBus method implementing responses to peer state changes.
+ * 
+ * \param bus       The XenBus bus parent of child.
+ * \param child     The XenBus child whose peer stat has changed.
+ * \param state     The current state of the peer.
+ */
+void xenbusb_otherend_changed(device_t bus, device_t child,
+			      enum xenbus_state state);
+
+/**
+ * \brief Common XenBus method implementing responses to local XenStore changes.
+ * 
+ * \param bus    The XenBus bus parent of child.
+ * \param child  The XenBus child whose peer stat has changed.
+ * \param path   The tree relative sub-path to the modified node.  The empty
+ *               string indicates the root of the tree was destroyed.
+ */
+void xenbusb_localend_changed(device_t bus, device_t child, const char *path);
+
+/**
  * \brief Attempt to add a XenBus device instance to this XenBus bus.
  *
  * \param dev   The NewBus device representing this XenBus bus.
@@ -269,4 +293,6 @@ int xenbusb_write_ivar(device_t dev, device_t child, int index,
  */
 int xenbusb_add_device(device_t dev, const char *type, const char *id);
 
+#include "xenbusb_if.h"
+
 #endif /* _XEN_XENBUS_XENBUSB_H */
diff --git a/sys/xen/xenbus/xenbusb_back.c b/sys/xen/xenbus/xenbusb_back.c
index 32bbc04..1252abe 100644
--- a/sys/xen/xenbus/xenbusb_back.c
+++ b/sys/xen/xenbus/xenbusb_back.c
@@ -208,57 +208,79 @@ xenbusb_back_get_otherend_node(device_t dev, struct xenbus_device_ivars *ivars)
 
 	if (error == 0) {
 		ivars->xd_otherend_path = strdup(otherend_path, M_XENBUS);
+		ivars->xd_otherend_path_len = strlen(otherend_path);
 		free(otherend_path, M_XENSTORE);
 	}
 	return (error);
 }
 
 /**
- * \brief Backend XenBus child instance variable write access method.
- *
- * \param dev    The NewBus device representing this XenBus bus.
- * \param child	 The NewBus device representing a child of dev%'s XenBus bus.
- * \param index	 The index of the instance variable to access.
- * \param value  The new value to set in the instance variable accessed.
- *
- * \return  On success, 0. Otherwise an errno value indicating the
- *          type of failure.
- *
- * Xenbus_back overrides this method so that it can trap state transitions
- * of local backend devices and clean up their XenStore entries as necessary
- * during device instance teardown.
+ * \brief Backend XenBus method implementing responses to peer state changes.
+ * 
+ * \param bus       The XenBus bus parent of child.
+ * \param child     The XenBus child whose peer stat has changed.
+ * \param state     The current state of the peer.
  */
-static int
-xenbusb_back_write_ivar(device_t dev, device_t child, int index,
-			uintptr_t value)
+static void
+xenbusb_back_otherend_changed(device_t bus, device_t child,
+			      enum xenbus_state peer_state)
 {
-	int error;
+	/* Perform default processing of state. */
+	xenbusb_otherend_changed(bus, child, peer_state);
 
-	error = xenbusb_write_ivar(dev, child, index, value); 
+	/*
+	 * "Online" devices are never fully detached in the
+	 * newbus sense.  Only the front<->back connection is
+	 * torn down.  If the front returns to the initialising
+	 * state after closing a previous connection, signal
+	 * our willingness to reconnect and that all necessary
+	 * XenStore data for feature negotiation is present.
+	 */
+	if (peer_state == XenbusStateInitialising
+	 && xenbus_dev_is_online(child) != 0
+	 && xenbus_get_state(child) == XenbusStateClosed)
+		xenbus_set_state(child, XenbusStateInitWait);
+}
 
-	if (index == XENBUS_IVAR_STATE
-	 && (enum xenbus_state)value == XenbusStateClosed
-	 && xenbus_dev_is_online(child) == 0) {
+/**
+ * \brief Backend XenBus method implementing responses to local
+ *        XenStore changes.
+ * 
+ * \param bus    The XenBus bus parent of child.
+ * \param child  The XenBus child whose peer stat has changed.
+ * \param_path   The tree relative sub-path to the modified node.  The empty
+ *               string indicates the root of the tree was destroyed.
+ */
+static void
+xenbusb_back_localend_changed(device_t bus, device_t child, const char *path)
+{
 
-		/*
-		 * Cleanup the hotplug entry in the XenStore if
-		 * present.  The control domain expects any userland
-		 * component associated with this device to destroy
-		 * this node in order to signify it is safe to 
-		 * teardown the device.  However, not all backends
-		 * rely on userland components, and those that
-		 * do should either use a communication channel
-		 * other than the XenStore, or ensure the hotplug
-		 * data is already cleaned up.
-		 *
-		 * This removal ensures that no matter what path
-		 * is taken to mark a back-end closed, the control
-		 * domain will understand that it is closed.
-		 */
-		xs_rm(XST_NIL, xenbus_get_node(child), "hotplug-status");
-	}
+	xenbusb_localend_changed(bus, child, path);
 
-	return (error);
+	if (strcmp(path, "/state") != 0
+	 && strcmp(path, "/online") != 0)
+		return;
+
+	if (xenbus_get_state(child) != XenbusStateClosed
+	 || xenbus_dev_is_online(child) != 0)
+		return;
+
+	/*
+	 * Cleanup the hotplug entry in the XenStore if
+	 * present.  The control domain expects any userland
+	 * component associated with this device to destroy
+	 * this node in order to signify it is safe to 
+	 * teardown the device.  However, not all backends
+	 * rely on userland components, and those that
+	 * do should either use a communication channel
+	 * other than the XenStore, or ensure the hotplug
+	 * data is already cleaned up.
+	 *
+	 * This removal ensures that no matter what path
+	 * is taken to mark a back-end closed, the control
+	 * domain will understand that it is closed.
+	 */
+	xs_rm(XST_NIL, xenbus_get_node(child), "hotplug-status");
 }
 
 /*-------------------- Private Device Attachment Data  -----------------------*/
@@ -275,7 +297,7 @@ static device_method_t xenbusb_back_methods[] = {
 	/* Bus Interface */ 
 	DEVMETHOD(bus_print_child,      xenbusb_print_child),
 	DEVMETHOD(bus_read_ivar,        xenbusb_read_ivar), 
-	DEVMETHOD(bus_write_ivar,       xenbusb_back_write_ivar), 
+	DEVMETHOD(bus_write_ivar,       xenbusb_write_ivar), 
 	DEVMETHOD(bus_alloc_resource,   bus_generic_alloc_resource),
 	DEVMETHOD(bus_release_resource, bus_generic_release_resource),
 	DEVMETHOD(bus_activate_resource, bus_generic_activate_resource),
@@ -284,6 +306,8 @@ static device_method_t xenbusb_back_methods[] = {
 	/* XenBus Bus Interface */
 	DEVMETHOD(xenbusb_enumerate_type, xenbusb_back_enumerate_type),
 	DEVMETHOD(xenbusb_get_otherend_node, xenbusb_back_get_otherend_node),
+	DEVMETHOD(xenbusb_otherend_changed, xenbusb_back_otherend_changed),
+	DEVMETHOD(xenbusb_localend_changed, xenbusb_back_localend_changed),
 	{ 0, 0 } 
 }; 
 
diff --git a/sys/xen/xenbus/xenbusb_front.c b/sys/xen/xenbus/xenbusb_front.c
index 0bc06a4..b4e470e 100644
--- a/sys/xen/xenbus/xenbusb_front.c
+++ b/sys/xen/xenbus/xenbusb_front.c
@@ -156,6 +156,7 @@ xenbusb_front_get_otherend_node(device_t dev, struct xenbus_device_ivars *ivars)
 
 	if (error == 0) {
 		ivars->xd_otherend_path = strdup(otherend_path, M_XENBUS);
+		ivars->xd_otherend_path_len = strlen(otherend_path);
 		free(otherend_path, M_XENSTORE);
 	}
 	return (error);
diff --git a/sys/xen/xenbus/xenbusb_if.m b/sys/xen/xenbus/xenbusb_if.m
index a32e3f6..c49f333 100644
--- a/sys/xen/xenbus/xenbusb_if.m
+++ b/sys/xen/xenbus/xenbusb_if.m
@@ -31,10 +31,12 @@
 #
 
 #include <sys/bus.h>
+#include <sys/lock.h>
+#include <sys/sx.h>
+#include <sys/taskqueue.h>
 
-HEADER {
-struct xenbus_device_ivars;
-}
+#include <xen/xenstore/xenstorevar.h>
+#include <xen/xenbus/xenbusb.h>
 
 INTERFACE xenbusb;
 
@@ -76,3 +78,34 @@ METHOD int get_otherend_node {
 	device_t _dev;
 	struct xenbus_device_ivars *_ivars;
 }
+
+/**
+ * \brief Handle a XenStore change detected in the peer tree of a child
+ *        device of the bus.
+ *
+ * \param _bus       NewBus device_t for this XenBus (front/back) bus instance.
+ * \param _child     NewBus device_t for the child device whose peer XenStore
+ *                   tree has changed.
+ * \param _state     The current state of the peer.
+ */
+METHOD void otherend_changed {
+	device_t _bus;
+	device_t _child;
+	enum xenbus_state _state;
+} DEFAULT xenbusb_otherend_changed;
+
+/**
+ * \brief Handle a XenStore change detected in the local tree of a child
+ *        device of the bus.
+ *
+ * \param _bus    NewBus device_t for this XenBus (front/back) bus instance.
+ * \param _child  NewBus device_t for the child device whose peer XenStore
+ *                tree has changed.
+ * \param _path   The tree relative sub-path to the modified node.  The empty
+ *                string indicates the root of the tree was destroyed.
+ */
+METHOD void localend_changed {
+	device_t _bus;
+	device_t _child;
+	const char * _path;
+} DEFAULT xenbusb_localend_changed;
diff --git a/sys/xen/xenbus/xenbusvar.h b/sys/xen/xenbus/xenbusvar.h
index 55d7f29..bf2a342 100644
--- a/sys/xen/xenbus/xenbusvar.h
+++ b/sys/xen/xenbus/xenbusvar.h
@@ -51,8 +51,6 @@
 
 #include <xen/xenstore/xenstorevar.h>
 
-#include "xenbus_if.h"
-
 /* XenBus allocations including XenStore data returned to clients. */
 MALLOC_DECLARE(M_XENBUS);
 
@@ -116,6 +114,8 @@ XenbusState xenbus_read_driver_state(const char *path);
  *                  must be stable for the lifetime of the watch.
  * \param callback  The function to call when XenStore objects at or below
  *                  path are modified.
+ * \param cb_data   Client data that can be retrieved from the watch object
+ *                  during the callback.
  *
  * \return  On success, 0. Otherwise an errno value indicating the
  *          type of failure.
@@ -126,7 +126,8 @@ XenbusState xenbus_read_driver_state(const char *path);
  */
 int xenbus_watch_path(device_t dev, char *path,
 		      struct xs_watch *watch, 
-		      xs_watch_cb_t *callback);
+		      xs_watch_cb_t *callback,
+		      uintptr_t cb_data);
 
 /**
  * Initialize and register a watch at path/path2 in the XenStore.
@@ -138,6 +139,8 @@ int xenbus_watch_path(device_t dev, char *path,
  *                  must be stable for the lifetime of the watch.
  * \param callback  The function to call when XenStore objects at or below
  *                  path are modified.
+ * \param cb_data   Client data that can be retrieved from the watch object
+ *                  during the callback.
  *
  * \return  On success, 0. Otherwise an errno value indicating the
  *          type of failure.
@@ -153,7 +156,8 @@ int xenbus_watch_path(device_t dev, char *path,
  */
 int xenbus_watch_path2(device_t dev, const char *path,
 		       const char *path2, struct xs_watch *watch, 
-		       xs_watch_cb_t *callback);
+		       xs_watch_cb_t *callback,
+		       uintptr_t cb_data);
 
 /**
  * Grant access to the given ring_mfn to the peer of the given device.
@@ -275,4 +279,16 @@ const char *xenbus_strstate(enum xenbus_state state);
  */
 int xenbus_dev_is_online(device_t dev);
 
+/**
+ * Default callback invoked when a change to the local XenStore sub-tree
+ * for a device is modified.
+ * 
+ * \param dev   The XenBus device whose tree was modified.
+ * \param path  The tree relative sub-path to the modified node.  The empty
+ *              string indicates the root of the tree was destroyed.
+ */
+void xenbus_localend_changed(device_t dev, const char *path);
+
+#include "xenbus_if.h"
+
 #endif /* _XEN_XENBUS_XENBUSVAR_H */
diff --git a/sys/xen/xenstore/xenstorevar.h b/sys/xen/xenstore/xenstorevar.h
index df41e31..4a1382d 100644
--- a/sys/xen/xenstore/xenstorevar.h
+++ b/sys/xen/xenstore/xenstorevar.h
@@ -56,8 +56,8 @@ struct xenstore_domain_interface;
 struct xs_watch;
 extern struct xenstore_domain_interface *xen_store;
 
-typedef	void (xs_watch_cb_t)(struct xs_watch *,
-				   const char **vec, unsigned int len);
+typedef	void (xs_watch_cb_t)(struct xs_watch *, const char **vec,
+    unsigned int len);
 
 /* Register callback to watch subtree (node) in the XenStore. */
 struct xs_watch
@@ -69,6 +69,9 @@ struct xs_watch
 
 	/* Callback (executed in a process context with no locks held). */
 	xs_watch_cb_t *callback;
+
+	/* Callback client data untouched by the XenStore watch mechanism. */
+	uintptr_t callback_data;
 };
 LIST_HEAD(xs_watch_list, xs_watch);
 
@@ -301,7 +304,7 @@ int xs_gather(struct xs_transaction t, const char *dir, ...);
  * XenStore watches allow a client to be notified via a callback (embedded
  * within the watch object) of changes to an object in the XenStore.
  *
- * \param watch  A xenbus_watch struct with it's node and callback fields
+ * \param watch  An xs_watch struct with it's node and callback fields
  *               properly initialized.
  *
  * \return  On success, 0. Otherwise an errno value indicating the