Improve performance and functionality of the bitstring(3) api

Two new functions are provided, bit_ffs_at() and bit_ffc_at(), which allow
for efficient searching of set or cleared bits starting from any bit offset
within the bit string.

Performance is improved by operating on longs instead of bytes and using
ffsl() for searches within a long. ffsl() is a compiler builtin in both
clang and gcc for most architectures, converting what was a brute force
while loop search into a couple of instructions.

All of the bitstring(3) API continues to be contained in the header file.
Some of the functions are large enough that perhaps they should be uninlined
and moved to a library, but that is beyond the scope of this commit.

sys/sys/bitstring.h:
        Convert the majority of the existing bit string implementation from
        macros to inline functions.

        Properly protect the implementation from inadvertant macro expansion
        when included in a user's program by prefixing all private
        macros/functions and local variables with '_'.

        Add bit_ffs_at() and bit_ffc_at(). Implement bit_ffs() and
        bit_ffc() in terms of their "at" counterparts.

        Provide a kernel implementation of bit_alloc(), making the full API
        usable in the kernel.

        Improve code documenation.

share/man/man3/bitstring.3:
        Add pre-exisiting API bit_ffc() to the synopsis.

        Document new APIs.

        Document the initialization state of the bit strings
        allocated/declared by bit_alloc() and bit_decl().

        Correct documentation for bitstr_size(). The original code comments
        indicate the size is in bytes, not "elements of bitstr_t". The new
        implementation follows this lead. Only hastd assumed "elements"
        rather than bytes and it has been corrected.

etc/mtree/BSD.tests.dist:
tests/sys/Makefile:
tests/sys/sys/Makefile:
tests/sys/sys/bitstring.c:
        Add tests for all existing and new functionality.

include/bitstring.h
	Include all headers needed by sys/bitstring.h

lib/libbluetooth/bluetooth.h:
usr.sbin/bluetooth/hccontrol/le.c:
        Include bitstring.h instead of sys/bitstring.h.

sbin/hastd/activemap.c:
        Correct usage of bitstr_size().

sys/dev/xen/blkback/blkback.c
        Use new bit_alloc.

sys/kern/subr_unit.c:
        Remove hard-coded assumption that sizeof(bitstr_t) is 1.  Get rid of
        unrb.busy, which caches the number of bits set in unrb.map.  When
        INVARIANTS are disabled, nothing needs to know that information.
        callapse_unr can be adapted to use bit_ffs and bit_ffc instead.
        Eliminating unrb.busy saves memory, simplifies the code, and
        provides a slight speedup when INVARIANTS are disabled.

sys/net/flowtable.c:
        Use the new kernel implementation of bit-alloc, instead of hacking
        the old libc-dependent macro.

sys/sys/param.h
        Update __FreeBSD_version to indicate availability of new API

Submitted by:   gibbs, asomers
Reviewed by:    gibbs, ngie
MFC after:      4 weeks
Sponsored by:   Spectra Logic Corp
Differential Revision:  https://reviews.freebsd.org/D6004

1 files changed, 217 insertions, 104 deletions

diff --git a/sys/sys/bitstring.h b/sys/sys/bitstring.h
index 125ef51..a8d5652 100644
--- a/sys/sys/bitstring.h
+++ b/sys/sys/bitstring.h
@@ -29,118 +29,231 @@
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
+ * Copyright (c) 2014 Spectra Logic Corporation
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions, and the following disclaimer,
+ *    without modification.
+ * 2. Redistributions in binary form must reproduce at minimum a disclaimer
+ *    substantially similar to the "NO WARRANTY" disclaimer below
+ *    ("Disclaimer") and any redistribution must be conditioned upon
+ *    including a substantially similar Disclaimer requirement for further
+ *    binary redistribution.
+ *
+ * NO WARRANTY
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTIBILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * HOLDERS OR CONTRIBUTORS BE LIABLE FOR SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
+ * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGES.
+ *
  * $FreeBSD$
  */
-
 #ifndef _SYS_BITSTRING_H_
 #define	_SYS_BITSTRING_H_
 
-typedef	unsigned char bitstr_t;
+#ifdef _KERNEL
+#include <sys/libkern.h>
+#include <sys/malloc.h>
+#endif
+
+typedef	unsigned long bitstr_t;
+
+/*---------------------- Private Implementation Details ----------------------*/
+#define	_BITSTR_MASK (~0UL)
+#define	_BITSTR_BITS (sizeof(bitstr_t) * 8)
 
-/* internal macros */
-				/* byte of the bitstring bit is in */
-#define	_bit_byte(bit) \
-	((bit) >> 3)
+/* bitstr_t in bit string containing the bit. */
+static inline int
+_bit_idx(int _bit)
+{
+	return (_bit / _BITSTR_BITS);
+}
 
-				/* mask for the bit within its byte */
-#define	_bit_mask(bit) \
-	(1 << ((bit)&0x7))
+/* bit number within bitstr_t at _bit_idx(_bit). */
+static inline int
+_bit_offset(int _bit)
+{
+	return (_bit % _BITSTR_BITS);
+}
 
-/* external macros */
-				/* bytes in a bitstring of nbits bits */
-#define	bitstr_size(nbits) \
-	(((nbits) + 7) >> 3)
+/* Mask for the bit within its long. */
+static inline bitstr_t
+_bit_mask(int _bit)
+{
+	return (1UL << _bit_offset(_bit));
+}
 
-				/* allocate a bitstring */
-#define	bit_alloc(nbits) \
-	(bitstr_t *)calloc((size_t)bitstr_size(nbits), sizeof(bitstr_t))
+static inline bitstr_t
+_bit_make_mask(int _start, int _stop)
+{
+	return ((_BITSTR_MASK << _bit_offset(_start)) &
+	    (_BITSTR_MASK >> (_BITSTR_BITS - _bit_offset(_stop) - 1)));
+}
 
-				/* allocate a bitstring on the stack */
+/*----------------------------- Public Interface -----------------------------*/
+/* Number of bytes consumed by a bit string of nbits bits */
+#define	bitstr_size(_nbits) \
+	(((_nbits) + _BITSTR_BITS - 1) / 8)
+
+/* Allocate a bit string initialized with no bits set. */
+#ifdef _KERNEL
+static inline bitstr_t *
+bit_alloc(int _nbits, struct malloc_type *type, int flags)
+{
+	return ((bitstr_t *)malloc(bitstr_size(_nbits), type, flags | M_ZERO));
+}
+#else
+static inline bitstr_t *
+bit_alloc(int _nbits)
+{
+	return ((bitstr_t *)calloc(bitstr_size(_nbits), 1));
+}
+#endif
+
+/* Allocate a bit string on the stack with no bits set. */
 #define	bit_decl(name, nbits) \
-	((name)[bitstr_size(nbits)])
-
-				/* is bit N of bitstring name set? */
-#define	bit_test(name, bit) \
-	((name)[_bit_byte(bit)] & _bit_mask(bit))
-
-				/* set bit N of bitstring name */
-#define	bit_set(name, bit) \
-	((name)[_bit_byte(bit)] |= _bit_mask(bit))
-
-				/* clear bit N of bitstring name */
-#define	bit_clear(name, bit) \
-	((name)[_bit_byte(bit)] &= ~_bit_mask(bit))
-
-				/* clear bits start ... stop in bitstring */
-#define	bit_nclear(name, start, stop) do { \
-	register bitstr_t *_name = (name); \
-	register int _start = (start), _stop = (stop); \
-	register int _startbyte = _bit_byte(_start); \
-	register int _stopbyte = _bit_byte(_stop); \
-	if (_startbyte == _stopbyte) { \
-		_name[_startbyte] &= ((0xff >> (8 - (_start&0x7))) | \
-				      (0xff << ((_stop&0x7) + 1))); \
-	} else { \
-		_name[_startbyte] &= 0xff >> (8 - (_start&0x7)); \
-		while (++_startbyte < _stopbyte) \
-			_name[_startbyte] = 0; \
-		_name[_stopbyte] &= 0xff << ((_stop&0x7) + 1); \
-	} \
-} while (0)
-
-				/* set bits start ... stop in bitstring */
-#define	bit_nset(name, start, stop) do { \
-	register bitstr_t *_name = (name); \
-	register int _start = (start), _stop = (stop); \
-	register int _startbyte = _bit_byte(_start); \
-	register int _stopbyte = _bit_byte(_stop); \
-	if (_startbyte == _stopbyte) { \
-		_name[_startbyte] |= ((0xff << (_start&0x7)) & \
-				    (0xff >> (7 - (_stop&0x7)))); \
-	} else { \
-		_name[_startbyte] |= 0xff << ((_start)&0x7); \
-		while (++_startbyte < _stopbyte) \
-	    		_name[_startbyte] = 0xff; \
-		_name[_stopbyte] |= 0xff >> (7 - (_stop&0x7)); \
-	} \
-} while (0)
-
-				/* find first bit clear in name */
-#define	bit_ffc(name, nbits, value) do { \
-	register bitstr_t *_name = (name); \
-	register int _byte, _nbits = (nbits); \
-	register int _stopbyte = _bit_byte(_nbits - 1), _value = -1; \
-	if (_nbits > 0) \
-		for (_byte = 0; _byte <= _stopbyte; ++_byte) \
-			if (_name[_byte] != 0xff) { \
-				bitstr_t _lb; \
-				_value = _byte << 3; \
-				for (_lb = _name[_byte]; (_lb&0x1); \
-				    ++_value, _lb >>= 1); \
-				break; \
-			} \
-	if (_value >= nbits) \
-		_value = -1; \
-	*(value) = _value; \
-} while (0)
-
-				/* find first bit set in name */
-#define	bit_ffs(name, nbits, value) do { \
-	register bitstr_t *_name = (name); \
-	register int _byte, _nbits = (nbits); \
-	register int _stopbyte = _bit_byte(_nbits - 1), _value = -1; \
-	if (_nbits > 0) \
-		for (_byte = 0; _byte <= _stopbyte; ++_byte) \
-			if (_name[_byte]) { \
-				bitstr_t _lb; \
-				_value = _byte << 3; \
-				for (_lb = _name[_byte]; !(_lb&0x1); \
-				    ++_value, _lb >>= 1); \
-				break; \
-			} \
-	if (_value >= nbits) \
-		_value = -1; \
-	*(value) = _value; \
-} while (0)
-
-#endif /* !_SYS_BITSTRING_H_ */
+	((name)[bitstr_size(nbits) / sizeof(bitstr_t)])
+
+/* Is bit N of bit string set? */
+static inline int
+bit_test(const bitstr_t *_bitstr, int _bit)
+{
+	return ((_bitstr[_bit_idx(_bit)] & _bit_mask(_bit)) != 0);
+}
+
+/* Set bit N of bit string. */
+static inline void
+bit_set(bitstr_t *_bitstr, int _bit)
+{
+	_bitstr[_bit_idx(_bit)] |= _bit_mask(_bit);
+}
+
+/* clear bit N of bit string name */
+static inline void
+bit_clear(bitstr_t *_bitstr, int _bit)
+{
+	_bitstr[_bit_idx(_bit)] &= ~_bit_mask(_bit);
+}
+
+/* Set bits start ... stop inclusive in bit string. */
+static inline void
+bit_nset(bitstr_t *_bitstr, int _start, int _stop)
+{
+	bitstr_t *_stopbitstr;
+
+	_stopbitstr = _bitstr + _bit_idx(_stop);
+	_bitstr += _bit_idx(_start);
+
+	if (_bitstr == _stopbitstr) {
+		*_bitstr |= _bit_make_mask(_start, _stop);
+	} else {
+		*_bitstr |= _bit_make_mask(_start, _BITSTR_BITS - 1);
+		while (++_bitstr < _stopbitstr)
+	    		*_bitstr = _BITSTR_MASK;
+		*_stopbitstr |= _bit_make_mask(0, _stop);
+	}
+}
+
+/* Clear bits start ... stop inclusive in bit string. */
+static inline void
+bit_nclear(bitstr_t *_bitstr, int _start, int _stop)
+{
+	bitstr_t *_stopbitstr;
+
+	_stopbitstr = _bitstr + _bit_idx(_stop);
+	_bitstr += _bit_idx(_start);
+
+	if (_bitstr == _stopbitstr) {
+		*_bitstr &= ~_bit_make_mask(_start, _stop);
+	} else {
+		*_bitstr &= ~_bit_make_mask(_start, _BITSTR_BITS - 1);
+		while (++_bitstr < _stopbitstr)
+			*_bitstr = 0;
+		*_stopbitstr &= ~_bit_make_mask(0, _stop);
+	}
+}
+
+/* Find the first bit set in bit string at or after bit start. */
+static inline void
+bit_ffs_at(bitstr_t *_bitstr, int _start, int _nbits, int *_result)
+{
+	bitstr_t *_curbitstr;
+	bitstr_t *_stopbitstr;
+	bitstr_t _test;
+	int _value, _offset;
+
+	if (_nbits > 0) {
+		_curbitstr = _bitstr + _bit_idx(_start);
+		_stopbitstr = _bitstr + _bit_idx(_nbits - 1);
+
+		_test = *_curbitstr;
+		if (_bit_offset(_start) != 0)
+			_test &= _bit_make_mask(_start, _BITSTR_BITS - 1);
+		while (_test == 0 && _curbitstr < _stopbitstr)
+			_test = *(++_curbitstr);
+		
+		_offset = ffsl(_test);
+		_value = ((_curbitstr - _bitstr) * _BITSTR_BITS) + _offset - 1;
+		if (_offset == 0 || _value >= _nbits)
+			_value = -1;
+	} else {
+		_value = -1;
+	}
+	*_result = _value;
+}
+
+/* Find the first bit clear in bit string at or after bit start. */
+static inline void
+bit_ffc_at(bitstr_t *_bitstr, int _start, int _nbits, int *_result)
+{
+	bitstr_t *_curbitstr;
+	bitstr_t *_stopbitstr;
+	bitstr_t _test;
+	int _value, _offset;
+
+	if (_nbits > 0) {
+		_curbitstr = _bitstr + _bit_idx(_start);
+		_stopbitstr = _bitstr + _bit_idx(_nbits - 1);
+
+		_test = *_curbitstr;
+		if (_bit_offset(_start) != 0)
+			_test |= _bit_make_mask(0, _start - 1);
+		while (_test == _BITSTR_MASK && _curbitstr < _stopbitstr)
+			_test = *(++_curbitstr);
+		
+		_offset = ffsl(~_test);
+		_value = ((_curbitstr - _bitstr) * _BITSTR_BITS) + _offset - 1;
+		if (_offset == 0 || _value >= _nbits)
+			_value = -1;
+	} else {
+		_value = -1;
+	}
+	*_result = _value;
+}
+
+/* Find the first bit set in bit string. */
+static inline void
+bit_ffs(bitstr_t *_bitstr, int _nbits, int *_result)
+{
+	bit_ffs_at(_bitstr, /*start*/0, _nbits, _result);
+}
+
+/* Find the first bit clear in bit string. */
+static inline void
+bit_ffc(bitstr_t *_bitstr, int _nbits, int *_result)
+{
+	bit_ffc_at(_bitstr, /*start*/0, _nbits, _result);
+}
+
+#endif	/* _SYS_BITSTRING_H_ */