Lots of improvement to the bridging code.

In order of importance:

 + each cluster now uses private data structures (filtering and
   local address tables) so you can treat them as fully independent
   switches. This part of the work was supported by:
        Cisco Systems, Inc. - NSITE lab, RTP, NC.

 + cleaned up the handling of configuration, so the system will behave
   much better when real or pseudo devices are dynamically attached
   or detached. It should also not panic anymore on systems with large
   number of devices, closing a few existings PRs on the topic.

 + while at it, add support for VLAN. This means that a FreeBSD box
   can now work as a real VLAN switch, with trunk interfaces etc.
   As an example:
        ifconfig vlan0 vlan 3 vlandev dc0
        ifconfig vlan1 vlan 4 vlandev dc0
        net.link.ether.bridge_cfg="vlan0:3,dc1:3,vlan1:4,dc1:4"
   uses dc0 as a trunk interface, and dc1 and dc3 as ports on vlans 3 and 4
   You get the idea...
   NOTA BENE: by default bridge_cfg is initialised to "" so even if
   you enable bridging, no packets will be bridged until you set the
   list of interfaces on which you want this to happen.

 + large restructuring of the code, moving private vars and types from
   bridge.h to bridge.c.

 + added a lot of comments to the code to explain how to use it.

1 files changed, 378 insertions, 242 deletions

diff --git a/sys/net/bridge.c b/sys/net/bridge.c
index 98dfde9..f05221c 100644
--- a/sys/net/bridge.c
+++ b/sys/net/bridge.c
@@ -1,5 +1,7 @@
 /*
- * Copyright (c) 1998-2001 Luigi Rizzo
+ * Copyright (c) 1998-2002 Luigi Rizzo
+ *
+ * Work partly supported by: Cisco Systems, Inc. - NSITE lab, RTP, NC
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
@@ -27,19 +29,37 @@
 
 /*
  * This code implements bridging in FreeBSD. It only acts on ethernet
- * type of interfaces (others are still usable for routing).
- * A bridging table holds the source MAC address/dest. interface for each
- * known node. The table is indexed using an hash of the source address.
+ * interfaces, including VLANs (others are still usable for routing).
+ * A FreeBSD host can implement multiple logical bridges, called
+ * "clusters". Each cluster is made of a set of interfaces, and
+ * identified by a "cluster-id" which is a number in the range 1..2^16-1.
+ * 
+ * Bridging is enabled by the sysctl variable
+ *	net.link.ether.bridge
+ * the grouping of interfaces into clusters is done with
+ *	net.link.ether.bridge_cfg
+ * containing a list of interfaces each optionally followed by
+ * a colon and the cluster it belongs to (1 is the default).
+ * Separators can be * spaces, commas or tabs, e.g.
+ *	net.link.ether.bridge_cfg="fxp0:2 fxp1:2 dc0 dc1:1"
+ * Optionally bridged packets can be passed through the firewall,
+ * this is controlled by the variable
+ *	net.link.ether.bridge_ipfw
+ *
+ * For each cluster there is a descriptor (cluster_softc) storing
+ * the following data structures:
+ * - a hash table with the MAC address and destination interface for each
+ *   known node. The table is indexed using a hash of the source address.
+ * - an array with the MAC addresses of the interfaces used in the cluster.
  *
  * Input packets are tapped near the beginning of ether_input(), and
- * analysed by calling bridge_in(). Depending on the result, the packet
+ * analysed by bridge_in(). Depending on the result, the packet
  * can be forwarded to one or more output interfaces using bdg_forward(),
  * and/or sent to the upper layer (e.g. in case of multicast).
  *
- * Output packets are intercepted near the end of ether_output(),
- * the correct destination is selected calling bridge_dst_lookup(),
- * and then forwarding is done using bdg_forward().
- * Bridging is controlled by the sysctl variable net.link.ether.bridge
+ * Output packets are intercepted near the end of ether_output().
+ * The correct destination is selected by bridge_dst_lookup(),
+ * and then forwarding is done by bdg_forward().
  *
  * The arp code is also modified to let a machine answer to requests
  * irrespective of the port the request came from.
@@ -49,24 +69,23 @@
  * Periodically, interfaces are unmuted by bdg_timeout().
  * Muting is only implemented as a safety measure, and also as
  * a mechanism to support a user-space implementation of the spanning
- * tree algorithm. In the final release, unmuting will only occur
- * because of explicit action of the user-level daemon.
+ * tree algorithm.
  *
  * To build a bridging kernel, use the following option
  *    option BRIDGE
  * and then at runtime set the sysctl variable to enable bridging.
  *
- * Only one interface is supposed to have addresses set (but
- * there are no problems in practice if you set addresses for more
- * than one interface).
+ * Only one interface per cluster is supposed to have addresses set (but
+ * there are no substantial problems if you set addresses for none or
+ * for more than one interface).
  * Bridging will act before routing, but nothing prevents a machine
  * from doing both (modulo bugs in the implementation...).
  *
  * THINGS TO REMEMBER
  *  - bridging is incompatible with multicast routing on the same
  *    machine. There is not an easy fix to this.
+ *  - be very careful when bridging VLANs
  *  - loop detection is still not very robust.
- *  - the interface of bdg_forward() could be improved.
  */
 
 #include <sys/param.h>
@@ -92,13 +111,74 @@
 #include <netinet/ip_dummynet.h>
 #include <net/bridge.h>
 
-static struct ifnet *bridge_in(struct ifnet *, struct ether_header *);
-static struct mbuf *bdg_forward(struct mbuf *,
-	struct ether_header *const, struct ifnet *);
-static void bdgtakeifaces(void);
+/*--------------------*/
+
+/*
+ * For each cluster, source MAC addresses are stored into a hash
+ * table which locates the port they reside on.
+ */
+#define HASH_SIZE 8192	/* Table size, must be a power of 2 */
+
+typedef struct hash_table {		/* each entry.		*/
+    struct ifnet *	name;
+    u_char		etheraddr[6];
+    u_int16_t		used;		/* also, padding	*/
+} bdg_hash_table ;
+
+/*
+ * The hash function applied to MAC addresses. Out of the 6 bytes,
+ * the last ones tend to vary more. Since we are on a little endian machine,
+ * we have to do some gimmick...
+ */
+#define HASH_FN(addr)   (	\
+    ntohs( ((u_int16_t *)addr)[1] ^ ((u_int16_t *)addr)[2] ) & (HASH_SIZE -1))
+
+/*
+ * This is the data structure where local addresses are stored.
+ */
+struct bdg_addr {
+    u_char	etheraddr[6] ;
+    u_int16_t	_padding ;
+};
+
+/*
+ * The configuration of each cluster includes the cluster id, a pointer to
+ * the hash table, and an array of local MAC addresses (of size "ports").
+ */
+struct cluster_softc {
+    u_int16_t	cluster_id;
+    u_int16_t	ports;
+    bdg_hash_table *ht;
+    struct bdg_addr	*my_macs;	/* local MAC addresses */
+};
+
+
+static int n_clusters;				/* number of clusters */
+static struct cluster_softc *clusters;
+
+#define BDG_MUTED(ifp) (ifp2sc[ifp->if_index].flags & IFF_MUTE)
+#define BDG_MUTE(ifp) ifp2sc[ifp->if_index].flags |= IFF_MUTE
+#define BDG_CLUSTER(ifp) (ifp2sc[ifp->if_index].cluster)
+
+#define BDG_SAMECLUSTER(ifp,src) \
+	(src == NULL || BDG_CLUSTER(ifp) == BDG_CLUSTER(src) )
+
+#ifdef __i386__
+#define BDG_MATCH(a,b) ( \
+    ((u_int16_t *)(a))[2] == ((u_int16_t *)(b))[2] && \
+    *((u_int32_t *)(a)) == *((u_int32_t *)(b)) )
+#define IS_ETHER_BROADCAST(a) ( \
+	*((u_int32_t *)(a)) == 0xffffffff && \
+	((u_int16_t *)(a))[2] == 0xffff )
+#else
+/* for machines that do not support unaligned access */
+#define	BDG_MATCH(a,b)		(!bcmp(a, b, ETHER_ADDR_LEN) )
+#define	IS_ETHER_BROADCAST(a)	(!bcmp(a, "\377\377\377\377\377\377", 6))
+#endif
+
 
 /*
- * For debugging, you can use the following macros.
+ * For timing-related debugging, you can use the following macros.
  * remember, rdtsc() only works on Pentium-class machines
 
     quad_t ticks;
@@ -113,13 +193,11 @@ static void bdgtakeifaces(void);
 #define DEB(x)
 
 static int bdginit(void);
-static void flush_table(void);
-static void bdg_promisc_on(void);
 static void parse_bdg_cfg(void);
 
 static int bdg_ipfw = 0 ;
-static bdg_hash_table *bdg_table = NULL ;
 
+#if 0 /* debugging only */
 static char *bdg_dst_names[] = {
 	"BDG_NULL    ",
 	"BDG_BCAST   ",
@@ -130,7 +208,7 @@ static char *bdg_dst_names[] = {
 	"BDG_IN      ",
 	"BDG_OUT     ",
 	"BDG_FORWARD " };
-
+#endif
 /*
  * System initialization
  */
@@ -138,147 +216,201 @@ static char *bdg_dst_names[] = {
 static struct bdg_stats bdg_stats ;
 static struct callout_handle bdg_timeout_h ;
 
-#define	IFP_CHK(ifp, x) \
-	if (ifp2sc[ifp->if_index].magic != 0xDEADBEEF) { x ; }
-
 /*
- * Find the right pkt destination:
- *	BDG_BCAST	is a broadcast
- *	BDG_MCAST	is a multicast
- *	BDG_LOCAL	is for a local address
- *	BDG_DROP	must be dropped
- *	other		ifp of the dest. interface (incl.self)
- *
- * We assume this is only called for interfaces for which bridging
- * is enabled, i.e. BDG_USED(ifp) is true.
+ * Add an interface to a cluster, possibly creating a new entry in
+ * the cluster table. This requires reallocation of the table and
+ * updating pointers in ifp2sc.
  */
-static __inline
-struct ifnet *
-bridge_dst_lookup(struct ether_header *eh)
+static struct cluster_softc *
+add_cluster(u_int16_t cluster_id, struct arpcom *ac)
 {
-    struct ifnet *dst ;
-    int index ;
-    bdg_addr *p ;
+    struct cluster_softc *c = NULL;
+    int i;
 
-    if (IS_ETHER_BROADCAST(eh->ether_dhost))
-	return BDG_BCAST ;
-    if (eh->ether_dhost[0] & 1)
-	return BDG_MCAST ;
-    /*
-     * Lookup local addresses in case one matches.
-     */
-    for (index = bdg_ports, p = bdg_addresses ; index ; index--, p++ )
-	if (BDG_MATCH(p->etheraddr, eh->ether_dhost) )
-	    return BDG_LOCAL ;
+    for (i = 0; i < n_clusters ; i++)
+	if (clusters[i].cluster_id == cluster_id)
+	    goto found;
+
+    /* Not found, need to reallocate */
+    c = malloc((1+n_clusters) * sizeof (*c), M_IFADDR, M_DONTWAIT | M_ZERO);
+    if (c == NULL) {/* malloc failure */
+	printf("-- bridge: cannot add new cluster\n");
+	return NULL;
+    }
+    c[n_clusters].ht = (struct hash_table *)
+	    malloc(HASH_SIZE * sizeof(struct hash_table),
+		M_IFADDR, M_WAITOK | M_ZERO);
+    if (c[n_clusters].ht == NULL) {
+	printf("-- bridge: cannot allocate hash table for new cluster\n");
+	free(c, M_IFADDR);
+	return NULL;
+    }
+    c[n_clusters].my_macs = (struct bdg_addr *)
+	    malloc(BDG_MAX_PORTS * sizeof(struct bdg_addr),
+		M_IFADDR, M_WAITOK | M_ZERO);
+    if (c[n_clusters].my_macs == NULL) {
+        printf("-- bridge: cannot allocate mac addr table for new cluster\n");
+	free(c[n_clusters].ht, M_IFADDR);
+        free(c, M_IFADDR);
+        return NULL;
+    }
+
+    c[n_clusters].cluster_id = cluster_id;
+    c[n_clusters].ports = 0;
     /*
-     * Look for a possible destination in table
+     * now copy old descriptors here
      */
-    index= HASH_FN( eh->ether_dhost );
-    dst = bdg_table[index].name;
-    if ( dst && BDG_MATCH( bdg_table[index].etheraddr, eh->ether_dhost) )
-	return dst ;
-    else
-	return BDG_UNKNOWN ;
+    if (n_clusters > 0) {
+	for (i=0; i < n_clusters; i++)
+	    c[i] = clusters[i];
+	/*
+	 * and finally update pointers in ifp2sc
+	 */
+	for (i = 0 ; i < if_index && i < BDG_MAX_PORTS; i++)
+	    if (ifp2sc[i].cluster != NULL)
+		ifp2sc[i].cluster = c + (ifp2sc[i].cluster - clusters);
+	free(clusters, M_IFADDR);
+    }
+    clusters = c;
+    i = n_clusters;		/* index of cluster entry */
+    n_clusters++;
+found:
+    c = clusters + i;		/* the right cluster ... */
+    bcopy(ac->ac_enaddr, &(c->my_macs[c->ports]), 6);
+    c->ports++;
+    return c;
 }
+
+
 /*
- * turn off promisc mode, optionally clear the IFF_USED flag.
- * The flag is turned on by parse_bdg_config
+ * Turn off bridging, by clearing promisc mode on the interface,
+ * marking the interface as unused, and clearing the name in the
+ * stats entry.
+ * Also dispose the hash tables associated with the clusters.
  */
 static void
-bdg_promisc_off(int clear_used)
+bridge_off(void)
 {
     struct ifnet *ifp ;
+    int i, s;
+
+    DEB(printf("bridge_off: n_clusters %d\n", n_clusters);)
     TAILQ_FOREACH(ifp, &ifnet, if_link) {
-	if ( (ifp2sc[ifp->if_index].flags & IFF_BDG_PROMISC) ) {
-	    int s, ret ;
+	struct bdg_softc *b;
+
+	if (ifp->if_index >= BDG_MAX_PORTS)
+	    continue;	/* make sure we do not go beyond the end */
+	b = &(ifp2sc[ifp->if_index]);
+
+	if ( b->flags & IFF_BDG_PROMISC ) {
 	    s = splimp();
-	    ret = ifpromisc(ifp, 0);
+	    ifpromisc(ifp, 0);
 	    splx(s);
-	    ifp2sc[ifp->if_index].flags &= ~(IFF_BDG_PROMISC|IFF_MUTE) ;
+	    b->flags &= ~(IFF_BDG_PROMISC|IFF_MUTE) ;
 	    DEB(printf(">> now %s%d promisc OFF if_flags 0x%x bdg_flags 0x%x\n",
 		    ifp->if_name, ifp->if_unit,
-		    ifp->if_flags, ifp2sc[ifp->if_index].flags);)
-	}
-	if (clear_used) {
-	    ifp2sc[ifp->if_index].flags &= ~(IFF_USED) ;
-	    bdg_stats.s[ifp->if_index].name[0] = '\0';
+		    ifp->if_flags, b->flags);)
 	}
+	b->flags &= ~(IFF_USED) ;
+	b->cluster = NULL;
+	bdg_stats.s[ifp->if_index].name[0] = '\0';
+    }
+    /* flush_tables */
+
+    s = splimp();
+    for (i=0; i < n_clusters; i++) {
+	free(clusters[i].ht, M_IFADDR);
+	free(clusters[i].my_macs, M_IFADDR);
     }
+    if (clusters != NULL)
+	free(clusters, M_IFADDR);
+    clusters = NULL;
+    n_clusters =0;
+    splx(s);
 }
 
 /*
  * set promisc mode on the interfaces we use.
  */
 static void
-bdg_promisc_on()
+bridge_on(void)
 {
     struct ifnet *ifp ;
     int s ;
 
     TAILQ_FOREACH(ifp, &ifnet, if_link) {
-	if ( !BDG_USED(ifp) )
+	struct bdg_softc *b = &ifp2sc[ifp->if_index];
+
+	if ( !(b->flags & IFF_USED) )
 	    continue ;
-	if ( 0 == ( ifp->if_flags & IFF_UP) ) {
+	if ( !( ifp->if_flags & IFF_UP) ) {
 	    s = splimp();
 	    if_up(ifp);
 	    splx(s);
 	}
-	if ( !(ifp2sc[ifp->if_index].flags & IFF_BDG_PROMISC) ) {
+	if ( !(b->flags & IFF_BDG_PROMISC) ) {
 	    int ret ;
 	    s = splimp();
 	    ret = ifpromisc(ifp, 1);
 	    splx(s);
-	    ifp2sc[ifp->if_index].flags |= IFF_BDG_PROMISC ;
-	    printf(">> now %s%d promisc ON if_flags 0x%x bdg_flags 0x%x\n",
+	    b->flags |= IFF_BDG_PROMISC ;
+	    DEB(printf(">> now %s%d promisc ON if_flags 0x%x bdg_flags 0x%x\n",
 		    ifp->if_name, ifp->if_unit,
-		    ifp->if_flags, ifp2sc[ifp->if_index].flags);
+		    ifp->if_flags, b->flags);)
 	}
-	if (BDG_MUTED(ifp)) {
-	    printf(">> unmuting %s%d\n", ifp->if_name, ifp->if_unit);
-	    BDG_UNMUTE(ifp) ;
+	if (b->flags & IFF_MUTE) {
+	    DEB(printf(">> unmuting %s%d\n", ifp->if_name, ifp->if_unit);)
+	    b->flags &= ~IFF_MUTE;
 	}
     }
 }
 
-static int
-sysctl_bdg(SYSCTL_HANDLER_ARGS)
+/**
+ * reconfigure bridge.
+ * This is also done every time we attach or detach an interface.
+ * Main use is to make sure that we do not bridge on some old
+ * (ejected) device. So, it would be really useful to have a
+ * pointer to the modified device as an argument. Without it, we
+ * have to scan all interfaces.
+ */
+static void
+reconfigure_bridge(void)
 {
-    int error, oldval = do_bridge ;
-
-    error = sysctl_handle_int(oidp,
-	oidp->oid_arg1, oidp->oid_arg2, req);
-    DEB( printf("called sysctl for bridge name %s arg2 %d val %d->%d\n",
-	oidp->oid_name, oidp->oid_arg2,
-	oldval, do_bridge); )
-
-    if (oldval != do_bridge) {
-	bdg_promisc_off( 1 ); /* reset previously used interfaces */
-	flush_table();
-	if (do_bridge) {
-	    parse_bdg_cfg();
-	    bdg_promisc_on();
+    bridge_off();
+    if (do_bridge) {
+	if (if_index >= BDG_MAX_PORTS) {
+	    printf("-- sorry too many interfaces (%d, max is %d),"
+		" disabling bridging\n", if_index, BDG_MAX_PORTS);
+	    do_bridge=0;
+	    return;
 	}
+	parse_bdg_cfg();
+	bridge_on();
     }
-    return error ;
 }
 
-static char bridge_cfg[256] = { "" } ;
+static char bridge_cfg[1024] = { "" } ;
 
 /*
  * parse the config string, set IFF_USED, name and cluster_id
  * for all interfaces found.
  * The config string is a list of "if[:cluster]" with
- * a number of possible separators (see "sep").
+ * a number of possible separators (see "sep"). In particular the
+ * use of the space lets you set bridge_cfg with the output from
+ * "ifconfig -l"
  */
 static void
 parse_bdg_cfg()
 {
     char *p, *beg ;
-    int i, l, cluster;
-    struct bdg_softc *b;
+    int l, cluster;
     static char *sep = ", \t";
 
     for (p = bridge_cfg; *p ; p++) {
+	struct ifnet *ifp;
+	int found = 0;
+	char c;
+
 	if (index(sep, *p))	/* skip separators */
 	    continue ;
 	/* names are lowercase and digits */
@@ -291,53 +423,83 @@ parse_bdg_cfg()
 	    cluster = 1 ;
 	else			/* fetch cluster */
 	    cluster = strtoul( p+1, &p, 10);
+	c = *p;
+	*p = '\0';
 	/*
-	 * now search in bridge strings
+	 * now search in interface list for a matching name
 	 */
-	for (i=0, b = ifp2sc ; i < if_index ; i++, b++) {
+	TAILQ_FOREACH(ifp, &ifnet, if_link) {
 	    char buf[32];
-	    struct ifnet *ifp = b->ifp ;
 
-	    if (ifp == NULL)
-		continue;
 	    sprintf(buf, "%s%d", ifp->if_name, ifp->if_unit);
 	    if (!strncmp(beg, buf, l)) {
-		b->cluster_id = htons(cluster) ;
+		struct bdg_softc *b = &ifp2sc[ifp->if_index];
+		if (ifp->if_type != IFT_ETHER && ifp->if_type != IFT_L2VLAN) {
+		    printf("%s is not an ethernet, continue\n", buf);
+		    continue;
+		}
+		if (b->flags & IFF_USED) {
+		    printf("%s already used, skipping\n", buf);
+		    break;
+		}
+		b->cluster = add_cluster(htons(cluster), (struct arpcom *)ifp);
 		b->flags |= IFF_USED ;
 		sprintf(bdg_stats.s[ifp->if_index].name,
 			"%s%d:%d", ifp->if_name, ifp->if_unit, cluster);
 
-		DEB(printf("--++  found %s\n",
-		    bdg_stats.s[ifp->if_index].name);)
+		DEB(printf("--++  found %s next c %d\n",
+		    bdg_stats.s[ifp->if_index].name, c);)
+		found = 1;
 		break ;
 	    }
 	}
+	if (!found)
+	    printf("interface %s Not found in bridge\n", beg);
+	*p = c;
+	if (c == '\0')
+	    break; /* no more */
     }
 }
 
+
+/*
+ * handler for net.link.ether.bridge
+ */
+static int
+sysctl_bdg(SYSCTL_HANDLER_ARGS)
+{
+    int error, oldval = do_bridge ;
+
+    error = sysctl_handle_int(oidp, oidp->oid_arg1, oidp->oid_arg2, req);
+    DEB( printf("called sysctl for bridge name %s arg2 %d val %d->%d\n",
+	oidp->oid_name, oidp->oid_arg2,
+	oldval, do_bridge); )
+
+    if (oldval != do_bridge)
+	reconfigure_bridge();
+    return error ;
+}
+
+/*
+ * handler for net.link.ether.bridge_cfg
+ */
 static int
 sysctl_bdg_cfg(SYSCTL_HANDLER_ARGS)
 {
     int error = 0 ;
-    char oldval[256] ;
+    char old_cfg[1024] ;
 
-    strcpy(oldval, bridge_cfg) ;
+    strcpy(old_cfg, bridge_cfg) ;
 
-    error = sysctl_handle_string(oidp,
-	    bridge_cfg, oidp->oid_arg2, req);
+    error = sysctl_handle_string(oidp, bridge_cfg, oidp->oid_arg2, req);
     DEB(
 	printf("called sysctl for bridge name %s arg2 %d err %d val %s->%s\n",
 		oidp->oid_name, oidp->oid_arg2,
 		error,
-		oldval, bridge_cfg);
+		old_cfg, bridge_cfg);
 	)
-    if (strcmp(oldval, bridge_cfg)) {
-	bdg_promisc_off( 1 );  /* reset previously-used interfaces */
-	flush_table();
-	parse_bdg_cfg();        /* and set new ones... */
-	if (do_bridge)
-	    bdg_promisc_on();   /* re-enable interfaces */
-    }
+    if (strcmp(old_cfg, bridge_cfg))
+	reconfigure_bridge();
     return error ;
 }
 
@@ -345,7 +507,7 @@ static int
 sysctl_refresh(SYSCTL_HANDLER_ARGS)
 {
     if (req->newptr)
-	    bdgtakeifaces();
+	reconfigure_bridge();
     
     return 0;
 }
@@ -404,20 +566,6 @@ SYSCTL_STRUCT(_net_link_ether, PF_BDG, bdgstats,
 static int bdg_loops ;
 
 /*
- * completely flush the bridge table.
- */
-static void
-flush_table()
-{   
-    int s,i;
-
-    s = splimp();
-    for (i=0; i< HASH_SIZE; i++)
-	bdg_table[i].name= NULL; /* clear table */
-    splx(s);
-}
-
-/*
  * called periodically to flush entries etc.
  */
 static void
@@ -428,11 +576,15 @@ bdg_timeout(void *dummy)
     if (do_bridge) {
 	static int age_index = 0 ; /* index of table position to age */
 	int l = age_index + HASH_SIZE/4 ;
+	int i;
 	/*
 	 * age entries in the forwarding table.
 	 */
 	if (l > HASH_SIZE)
 	    l = HASH_SIZE ;
+
+    for (i=0; i<n_clusters; i++) {
+	bdg_hash_table *bdg_table = clusters[i].ht;
 	for (; age_index < l ; age_index++)
 	    if (bdg_table[age_index].used)
 		bdg_table[age_index].used = 0 ;
@@ -440,13 +592,14 @@ bdg_timeout(void *dummy)
 		/* printf("xx flushing stale entry %d\n", age_index); */
 		bdg_table[age_index].name = NULL ;
 	    }
+    }
 	if (age_index >= HASH_SIZE)
 	    age_index = 0 ;
 
 	if (--slowtimer <= 0 ) {
 	    slowtimer = 5 ;
 
-	    bdg_promisc_on() ; /* we just need unmute, really */
+	    bridge_on() ; /* we just need unmute, really */
 	    bdg_loops = 0 ;
 	}
     }
@@ -454,96 +607,53 @@ bdg_timeout(void *dummy)
 }
 
 /*
- * local MAC addresses are held in a small array. This makes comparisons
- * much faster.
- */
-bdg_addr bdg_addresses[BDG_MAX_PORTS];
-static int bdg_ports ;
-static int bdg_max_ports = BDG_MAX_PORTS ;
-
-/*
- * initialization of bridge code. This needs to be done after all
- * interfaces have been configured.
- */
-static int
-bdginit(void)
-{
-    bdg_table = (struct hash_table *)
-	    malloc(HASH_SIZE * sizeof(struct hash_table),
-		M_IFADDR, M_WAITOK | M_ZERO);
-    if (bdg_table == NULL)
-	return ENOMEM;
-    ifp2sc = malloc(BDG_MAX_PORTS * sizeof(struct bdg_softc),
-		M_IFADDR, M_WAITOK | M_ZERO );
-    if (ifp2sc == NULL) {
-	free(bdg_table, M_IFADDR);
-	bdg_table = NULL ;
-	return ENOMEM ;
-    }
-
-    bridge_in_ptr = bridge_in;
-    bdg_forward_ptr = bdg_forward;
-    bdgtakeifaces_ptr = bdgtakeifaces;
-
-    flush_table();
-
-    bzero(&bdg_stats, sizeof(bdg_stats) );
-    bdgtakeifaces();
-    bdg_timeout(0);
-    do_bridge=0;
-    return 0 ;
-}
-   
-/**
- * fetch interfaces that can do bridging.
- * This is re-done every time we attach or detach an interface.
+ * Find the right pkt destination:
+ *	BDG_BCAST	is a broadcast
+ *	BDG_MCAST	is a multicast
+ *	BDG_LOCAL	is for a local address
+ *	BDG_DROP	must be dropped
+ *	other		ifp of the dest. interface (incl.self)
+ *
+ * We assume this is only called for interfaces for which bridging
+ * is enabled, i.e. BDG_USED(ifp) is true.
  */
-static void
-bdgtakeifaces(void)
+static __inline
+struct ifnet *
+bridge_dst_lookup(struct ether_header *eh, struct cluster_softc *c)
 {
-    struct ifnet *ifp;
-    struct arpcom *ac ;
-    bdg_addr *p = bdg_addresses ;
-    struct bdg_softc *bp;
-
-    bdg_ports = 0 ;
-    *bridge_cfg = '\0';
-    printf("BRIDGE 011031, have %d interfaces\n", if_index);
-    TAILQ_FOREACH(ifp, &ifnet, if_link)
-	if (ifp->if_type == IFT_ETHER) { /* ethernet ? */
-	    /*
-	     * XXX should try to grow the arrays as needed.
-	     */
-	    bp = &ifp2sc[ifp->if_index] ;
-	    ac = (struct arpcom *)ifp;
-	    sprintf(bridge_cfg + strlen(bridge_cfg),
-		"%s%d:1,", ifp->if_name, ifp->if_unit);
-	    printf("-- index %d %s type %d phy %d addrl %d addr %6D\n",
-		    ifp->if_index,
-		    bdg_stats.s[ifp->if_index].name,
-		    (int)ifp->if_type, (int) ifp->if_physical,
-		    (int)ifp->if_addrlen,
-		    ac->ac_enaddr, "." );
-	    bcopy(ac->ac_enaddr, p->etheraddr, 6);
-	    p++ ;
-	    bp->ifp = ifp ;
-	    bp->flags = IFF_USED ;
-	    bp->cluster_id = htons(1) ;
-	    bp->magic = 0xDEADBEEF ;
-
-	    sprintf(bdg_stats.s[ifp->if_index].name,
-		"%s%d:%d", ifp->if_name, ifp->if_unit,
-		ntohs(bp->cluster_id));
-	    bdg_ports ++ ;
-	}
+    struct ifnet *dst ;
+    int index ;
+    struct bdg_addr *p ;
+    bdg_hash_table *bt;		/* pointer to entry in hash table */
 
+    if (IS_ETHER_BROADCAST(eh->ether_dhost))
+	return BDG_BCAST ;
+    if (eh->ether_dhost[0] & 1)
+	return BDG_MCAST ;
+    /*
+     * Lookup local addresses in case one matches.
+     */
+    for (index = c->ports, p = c->my_macs; index ; index--, p++ )
+	if (BDG_MATCH(p->etheraddr, eh->ether_dhost) )
+	    return BDG_LOCAL ;
+    /*
+     * Look for a possible destination in table
+     */
+    index= HASH_FN( eh->ether_dhost );
+    bt = &(c->ht[index]);
+    dst = bt->name;
+    if ( dst && BDG_MATCH( bt->etheraddr, eh->ether_dhost) )
+	return dst ;
+    else
+	return BDG_UNKNOWN ;
 }
 
 /**
  * bridge_in() is invoked to perform bridging decision on input packets.
  *
  * On Input:
- *   eh		Ethernet header of the incoming packet. We only need this.
+ *   eh		Ethernet header of the incoming packet.
+ *   ifp	interface the packet is coming from.
  *
  * On Return: destination of packet, one of
  *   BDG_BCAST	broadcast
@@ -561,20 +671,20 @@ bridge_in(struct ifnet *ifp, struct ether_header *eh)
 {
     int index;
     struct ifnet *dst , *old ;
+    bdg_hash_table *bt;			/* location in hash table */
     int dropit = BDG_MUTED(ifp) ;
 
     /*
      * hash the source address
      */
     index= HASH_FN(eh->ether_shost);
-    bdg_table[index].used = 1 ;
-    old = bdg_table[index].name ;
+    bt = &(ifp2sc[ifp->if_index].cluster->ht[index]);
+    bt->used = 1 ;
+    old = bt->name ;
     if ( old ) { /* the entry is valid. */
-	IFP_CHK(old, printf("bridge_in-- reading table\n") );
-
-	if (!BDG_MATCH( eh->ether_shost, bdg_table[index].etheraddr) ) {
+	if (!BDG_MATCH( eh->ether_shost, bt->etheraddr) ) {
 	    bdg_ipfw_colls++ ;
-	    bdg_table[index].name = NULL ;
+	    bt->name = NULL ;
 	} else if (old != ifp) {
 	    /*
 	     * Found a loop. Either a machine has moved, or there
@@ -584,7 +694,7 @@ bridge_in(struct ifnet *ifp, struct ether_header *eh)
 	     * suspect a reconfiguration and disable forwarding
 	     * from the old interface.
 	     */
-	    bdg_table[index].name = ifp ; /* relocate address */
+	    bt->name = ifp ; /* relocate address */
 	    printf("-- loop (%d) %6D to %s%d from %s%d (%s)\n",
 			bdg_loops, eh->ether_shost, ".",
 			ifp->if_name, ifp->if_unit,
@@ -601,13 +711,13 @@ bridge_in(struct ifnet *ifp, struct ether_header *eh)
     /*
      * now write the source address into the table
      */
-    if (bdg_table[index].name == NULL) {
+    if (bt->name == NULL) {
 	DEB(printf("new addr %6D at %d for %s%d\n",
 	    eh->ether_shost, ".", index, ifp->if_name, ifp->if_unit);)
-	bcopy(eh->ether_shost, bdg_table[index].etheraddr, 6);
-	bdg_table[index].name = ifp ;
+	bcopy(eh->ether_shost, bt->etheraddr, 6);
+	bt->name = ifp ;
     }
-    dst = bridge_dst_lookup(eh);
+    dst = bridge_dst_lookup(eh, ifp2sc[ifp->if_index].cluster);
     /*
      * bridge_dst_lookup can return the following values:
      *   BDG_BCAST, BDG_MCAST, BDG_LOCAL, BDG_UNKNOWN, BDG_DROP, ifp.
@@ -619,11 +729,11 @@ bridge_in(struct ifnet *ifp, struct ether_header *eh)
      */
     BDG_STAT(ifp, BDG_IN);
     switch ((uintptr_t)dst) {
-    case (uintptr_t) BDG_BCAST:
-    case (uintptr_t) BDG_MCAST:
-    case (uintptr_t) BDG_LOCAL:
-    case (uintptr_t) BDG_UNKNOWN:
-    case (uintptr_t) BDG_DROP:
+    case (uintptr_t)BDG_BCAST:
+    case (uintptr_t)BDG_MCAST:
+    case (uintptr_t)BDG_LOCAL:
+    case (uintptr_t)BDG_UNKNOWN:
+    case (uintptr_t)BDG_DROP:
 	BDG_STAT(ifp, dst);
 	break ;
     default :
@@ -699,7 +809,7 @@ bdg_forward(struct mbuf *m0, struct ether_header *const eh, struct ifnet *dst)
 	bdg_thru++; /* count packet, only once */
 
     if (src == NULL) /* packet from ether_output */
-	dst = bridge_dst_lookup(eh);
+	dst = bridge_dst_lookup(eh, ifp2sc[real_dst->if_index].cluster);
 
     if (dst == BDG_DROP) { /* this should not happen */
 	printf("xx bdg_forward for BDG_DROP\n");
@@ -719,7 +829,7 @@ bdg_forward(struct mbuf *m0, struct ether_header *const eh, struct ifnet *dst)
 	ifp = dst ;
 	once = 1 ;
     }
-    if (ifp <= BDG_FORWARD)
+    if ( (u_int)(ifp) <= (u_int)BDG_FORWARD )
 	panic("bdg_forward: bad dst");
 
     /*
@@ -863,13 +973,13 @@ forward:
 		bdg_predict++;
 	    } else {
 		M_PREPEND(m, ETHER_HDR_LEN, M_DONTWAIT);
-		if (!m && verbose)
+	        if (!m && verbose)
 		    printf("M_PREPEND failed\n");
 		if (m == NULL)
 		    return m0;
 		bcopy(&save_eh, mtod(m, struct ether_header *), ETHER_HDR_LEN);
 	    }
-	    if (! IF_HANDOFF(&last->if_snd, m, last)) {
+	    if (!IF_HANDOFF(&last->if_snd, m, last)) {
 #if 0
 		BDG_MUTE(last); /* should I also mute ? */
 #endif
@@ -900,13 +1010,40 @@ forward:
 }
 
 /*
+ * initialization of bridge code.
+ */
+static int
+bdginit(void)
+{
+    printf("BRIDGE 020214 loaded\n");
+
+    ifp2sc = malloc(BDG_MAX_PORTS * sizeof(struct bdg_softc),
+		M_IFADDR, M_WAITOK | M_ZERO );
+    if (ifp2sc == NULL)
+	return ENOMEM ;
+
+    bridge_in_ptr = bridge_in;
+    bdg_forward_ptr = bdg_forward;
+    bdgtakeifaces_ptr = reconfigure_bridge;
+
+    n_clusters = 0;
+    clusters = NULL;
+    do_bridge=0;
+
+    bzero(&bdg_stats, sizeof(bdg_stats) );
+    bdgtakeifaces_ptr();
+    bdg_timeout(0);
+    return 0 ;
+}
+
+/*
  * initialization code, both for static and dynamic loading.
  */
 static int
 bridge_modevent(module_t mod, int type, void *unused)
 {
 	int s;
-	int err;
+	int err = 0 ;
 
 	switch (type) {
 	case MOD_LOAD:
@@ -929,8 +1066,7 @@ bridge_modevent(module_t mod, int type, void *unused)
 		bdg_forward_ptr = NULL;
 		bdgtakeifaces_ptr = NULL;
 		untimeout(bdg_timeout, NULL, bdg_timeout_h);
-		free(bdg_table, M_IFADDR);
-		bdg_table = NULL ;
+		free_table();
 		free(ifp2sc, M_IFADDR);
 		ifp2sc = NULL ;
 		splx(s);