path: root/sys/contrib/octeon-sdk/cvmx-atomic.h

/***********************license start***************
 * Copyright (c) 2003-2010  Cavium Inc. (support@cavium.com). All rights
 * reserved.
 *
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are
 * met:
 *
 *   * Redistributions of source code must retain the above copyright
 *     notice, this list of conditions and the following disclaimer.
 *
 *   * Redistributions in binary form must reproduce the above
 *     copyright notice, this list of conditions and the following
 *     disclaimer in the documentation and/or other materials provided
 *     with the distribution.

 *   * Neither the name of Cavium Inc. nor the names of
 *     its contributors may be used to endorse or promote products
 *     derived from this software without specific prior written
 *     permission.

 * This Software, including technical data, may be subject to U.S. export  control
 * laws, including the U.S. Export Administration Act and its  associated
 * regulations, and may be subject to export or import  regulations in other
 * countries.

 * TO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS"
 * AND WITH ALL FAULTS AND CAVIUM INC. MAKES NO PROMISES, REPRESENTATIONS OR
 * WARRANTIES, EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH RESPECT TO
 * THE SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY REPRESENTATION OR
 * DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT DEFECTS, AND CAVIUM
 * SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) WARRANTIES OF TITLE,
 * MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A PARTICULAR PURPOSE, LACK OF
 * VIRUSES, ACCURACY OR COMPLETENESS, QUIET ENJOYMENT, QUIET POSSESSION OR
 * CORRESPONDENCE TO DESCRIPTION. THE ENTIRE  RISK ARISING OUT OF USE OR
 * PERFORMANCE OF THE SOFTWARE LIES WITH YOU.
 ***********************license end**************************************/







/**
 * @file
 *
 * This file provides atomic operations
 *
 * <hr>$Revision: 70030 $<hr>
 *
 *
 */


#ifndef __CVMX_ATOMIC_H__
#define __CVMX_ATOMIC_H__

#ifdef	__cplusplus
extern "C" {
#endif


/**
 * Atomically adds a signed value to a 32 bit (aligned) memory location.
 *
 * This version does not perform 'sync' operations to enforce memory
 * operations.  This should only be used when there are no memory operation
 * ordering constraints.  (This should NOT be used for reference counting -
 * use the standard version instead.)
 *
 * @param ptr    address in memory to add incr to
 * @param incr   amount to increment memory location by (signed)
 */
static inline void cvmx_atomic_add32_nosync(int32_t *ptr, int32_t incr)
{
    if (OCTEON_IS_MODEL(OCTEON_CN3XXX))
    {
	uint32_t tmp;

        __asm__ __volatile__(
        ".set noreorder         \n"
        "1: ll   %[tmp], %[val] \n"
        "   addu %[tmp], %[inc] \n"
        "   sc   %[tmp], %[val] \n"
        "   beqz %[tmp], 1b     \n"
        "   nop                 \n"
        ".set reorder           \n"
        : [val] "+m" (*ptr), [tmp] "=&r" (tmp)
        : [inc] "r" (incr)
        : "memory");
    }
    else
    {
        __asm__ __volatile__(
        "   saa %[inc], (%[base]) \n"
        : "+m" (*ptr)
        : [inc] "r" (incr), [base] "r" (ptr)
        : "memory");
    }
}

/**
 * Atomically adds a signed value to a 32 bit (aligned) memory location.
 *
 * Memory access ordering is enforced before/after the atomic operation,
 * so no additional 'sync' instructions are required.
 *
 *
 * @param ptr    address in memory to add incr to
 * @param incr   amount to increment memory location by (signed)
 */
static inline void cvmx_atomic_add32(int32_t *ptr, int32_t incr)
{
    CVMX_SYNCWS;
    cvmx_atomic_add32_nosync(ptr, incr);
    CVMX_SYNCWS;
}

/**
 * Atomically sets a 32 bit (aligned) memory location to a value
 *
 * @param ptr    address of memory to set
 * @param value  value to set memory location to.
 */
static inline void cvmx_atomic_set32(int32_t *ptr, int32_t value)
{
    CVMX_SYNCWS;
    *ptr = value;
    CVMX_SYNCWS;
}

/**
 * Returns the current value of a 32 bit (aligned) memory
 * location.
 *
 * @param ptr    Address of memory to get
 * @return Value of the memory
 */
static inline int32_t cvmx_atomic_get32(int32_t *ptr)
{
    return *(volatile int32_t *)ptr;
}

/**
 * Atomically adds a signed value to a 64 bit (aligned) memory location.
 *
 * This version does not perform 'sync' operations to enforce memory
 * operations.  This should only be used when there are no memory operation
 * ordering constraints.  (This should NOT be used for reference counting -
 * use the standard version instead.)
 *
 * @param ptr    address in memory to add incr to
 * @param incr   amount to increment memory location by (signed)
 */
static inline void cvmx_atomic_add64_nosync(int64_t *ptr, int64_t incr)
{
    if (OCTEON_IS_MODEL(OCTEON_CN3XXX))
    {
	uint64_t tmp;
        __asm__ __volatile__(
        ".set noreorder         \n"
        "1: lld  %[tmp], %[val] \n"
        "   daddu %[tmp], %[inc] \n"
        "   scd  %[tmp], %[val] \n"
        "   beqz %[tmp], 1b     \n"
        "   nop                 \n"
        ".set reorder           \n"
        : [val] "+m" (*ptr), [tmp] "=&r" (tmp)
        : [inc] "r" (incr)
        : "memory");
    }
    else
    {
        __asm__ __volatile__(
        "   saad %[inc], (%[base])  \n"
        : "+m" (*ptr)
        : [inc] "r" (incr), [base] "r" (ptr)
        : "memory");
    }
}

/**
 * Atomically adds a signed value to a 64 bit (aligned) memory location.
 *
 * Memory access ordering is enforced before/after the atomic operation,
 * so no additional 'sync' instructions are required.
 *
 *
 * @param ptr    address in memory to add incr to
 * @param incr   amount to increment memory location by (signed)
 */
static inline void cvmx_atomic_add64(int64_t *ptr, int64_t incr)
{
    CVMX_SYNCWS;
    cvmx_atomic_add64_nosync(ptr, incr);
    CVMX_SYNCWS;
}

/**
 * Atomically sets a 64 bit (aligned) memory location to a value
 *
 * @param ptr    address of memory to set
 * @param value  value to set memory location to.
 */
static inline void cvmx_atomic_set64(int64_t *ptr, int64_t value)
{
    CVMX_SYNCWS;
    *ptr = value;
    CVMX_SYNCWS;
}

/**
 * Returns the current value of a 64 bit (aligned) memory
 * location.
 *
 * @param ptr    Address of memory to get
 * @return Value of the memory
 */
static inline int64_t cvmx_atomic_get64(int64_t *ptr)
{
    return *(volatile int64_t *)ptr;
}

/**
 * Atomically compares the old value with the value at ptr, and if they match,
 * stores new_val to ptr.
 * If *ptr and old don't match, function returns failure immediately.
 * If *ptr and old match, function spins until *ptr updated to new atomically, or
 *  until *ptr and old no longer match
 *
 * Does no memory synchronization.
 *
 * @return 1 on success (match and store)
 *         0 on no match
 */
static inline uint32_t cvmx_atomic_compare_and_store32_nosync(uint32_t *ptr, uint32_t old_val, uint32_t new_val)
{
    uint32_t tmp, ret;

    __asm__ __volatile__(
    ".set noreorder         \n"
    "1: ll   %[tmp], %[val] \n"
    "   li   %[ret], 0     \n"
    "   bne  %[tmp], %[old], 2f \n"
    "   move %[tmp], %[new_val] \n"
    "   sc   %[tmp], %[val] \n"
    "   beqz %[tmp], 1b     \n"
    "   li   %[ret], 1      \n"
    "2: nop               \n"
    ".set reorder           \n"
    : [val] "+m" (*ptr), [tmp] "=&r" (tmp), [ret] "=&r" (ret)
    : [old] "r" (old_val), [new_val] "r" (new_val)
    : "memory");

    return(ret);

}

/**
 * Atomically compares the old value with the value at ptr, and if they match,
 * stores new_val to ptr.
 * If *ptr and old don't match, function returns failure immediately.
 * If *ptr and old match, function spins until *ptr updated to new atomically, or
 *  until *ptr and old no longer match
 *
 * Does memory synchronization that is required to use this as a locking primitive.
 *
 * @return 1 on success (match and store)
 *         0 on no match
 */
static inline uint32_t cvmx_atomic_compare_and_store32(uint32_t *ptr, uint32_t old_val, uint32_t new_val)
{
    uint32_t ret;
    CVMX_SYNCWS;
    ret = cvmx_atomic_compare_and_store32_nosync(ptr, old_val, new_val);
    CVMX_SYNCWS;
    return ret;


}

/**
 * Atomically compares the old value with the value at ptr, and if they match,
 * stores new_val to ptr.
 * If *ptr and old don't match, function returns failure immediately.
 * If *ptr and old match, function spins until *ptr updated to new atomically, or
 *  until *ptr and old no longer match
 *
 * Does no memory synchronization.
 *
 * @return 1 on success (match and store)
 *         0 on no match
 */
static inline uint64_t cvmx_atomic_compare_and_store64_nosync(uint64_t *ptr, uint64_t old_val, uint64_t new_val)
{
    uint64_t tmp, ret;

    __asm__ __volatile__(
    ".set noreorder         \n"
    "1: lld  %[tmp], %[val] \n"
    "   li   %[ret], 0     \n"
    "   bne  %[tmp], %[old], 2f \n"
    "   move %[tmp], %[new_val] \n"
    "   scd  %[tmp], %[val] \n"
    "   beqz %[tmp], 1b     \n"
    "   li   %[ret], 1      \n"
    "2: nop               \n"
    ".set reorder           \n"
    : [val] "+m" (*ptr), [tmp] "=&r" (tmp), [ret] "=&r" (ret)
    : [old] "r" (old_val), [new_val] "r" (new_val)
    : "memory");

    return(ret);

}

/**
 * Atomically compares the old value with the value at ptr, and if they match,
 * stores new_val to ptr.
 * If *ptr and old don't match, function returns failure immediately.
 * If *ptr and old match, function spins until *ptr updated to new atomically, or
 *  until *ptr and old no longer match
 *
 * Does memory synchronization that is required to use this as a locking primitive.
 *
 * @return 1 on success (match and store)
 *         0 on no match
 */
static inline uint64_t cvmx_atomic_compare_and_store64(uint64_t *ptr, uint64_t old_val, uint64_t new_val)
{
    uint64_t ret;
    CVMX_SYNCWS;
    ret = cvmx_atomic_compare_and_store64_nosync(ptr, old_val, new_val);
    CVMX_SYNCWS;
    return ret;
}

/**
 * Atomically adds a signed value to a 64 bit (aligned) memory location,
 * and returns previous value.
 *
 * This version does not perform 'sync' operations to enforce memory
 * operations.  This should only be used when there are no memory operation
 * ordering constraints.  (This should NOT be used for reference counting -
 * use the standard version instead.)
 *
 * @param ptr    address in memory to add incr to
 * @param incr   amount to increment memory location by (signed)
 *
 * @return Value of memory location before increment
 */
static inline int64_t cvmx_atomic_fetch_and_add64_nosync(int64_t *ptr, int64_t incr)
{
    uint64_t tmp, ret;

#if !defined(__FreeBSD__) || !defined(_KERNEL)
    if (OCTEON_IS_MODEL(OCTEON_CN6XXX) || OCTEON_IS_MODEL(OCTEON_CNF7XXX))
    {
	CVMX_PUSH_OCTEON2;
	if (__builtin_constant_p(incr) && incr == 1)
	{
	    __asm__ __volatile__(
		"laid  %0,(%2)"
		: "=r" (ret), "+m" (ptr) : "r" (ptr) : "memory");
	}
	else if (__builtin_constant_p(incr) && incr == -1)
        {
	    __asm__ __volatile__(
		"ladd  %0,(%2)"
		: "=r" (ret), "+m" (ptr) : "r" (ptr) : "memory");
        }
        else
        {
	    __asm__ __volatile__(
		"laad  %0,(%2),%3"
		: "=r" (ret), "+m" (ptr) : "r" (ptr), "r" (incr) : "memory");
        }
	CVMX_POP_OCTEON2;
    }
    else
    {
#endif
        __asm__ __volatile__(
            ".set noreorder          \n"
            "1: lld   %[tmp], %[val] \n"
            "   move  %[ret], %[tmp] \n"
            "   daddu %[tmp], %[inc] \n"
            "   scd   %[tmp], %[val] \n"
            "   beqz  %[tmp], 1b     \n"
            "   nop                  \n"
            ".set reorder            \n"
            : [val] "+m" (*ptr), [tmp] "=&r" (tmp), [ret] "=&r" (ret)
            : [inc] "r" (incr)
            : "memory");
#if !defined(__FreeBSD__) || !defined(_KERNEL)
    }
#endif

    return (ret);
}

/**
 * Atomically adds a signed value to a 64 bit (aligned) memory location,
 * and returns previous value.
 *
 * Memory access ordering is enforced before/after the atomic operation,
 * so no additional 'sync' instructions are required.
 *
 * @param ptr    address in memory to add incr to
 * @param incr   amount to increment memory location by (signed)
 *
 * @return Value of memory location before increment
 */
static inline int64_t cvmx_atomic_fetch_and_add64(int64_t *ptr, int64_t incr)
{
    uint64_t ret;
    CVMX_SYNCWS;
    ret = cvmx_atomic_fetch_and_add64_nosync(ptr, incr);
    CVMX_SYNCWS;
    return ret;
}

/**
 * Atomically adds a signed value to a 32 bit (aligned) memory location,
 * and returns previous value.
 *
 * This version does not perform 'sync' operations to enforce memory
 * operations.  This should only be used when there are no memory operation
 * ordering constraints.  (This should NOT be used for reference counting -
 * use the standard version instead.)
 *
 * @param ptr    address in memory to add incr to
 * @param incr   amount to increment memory location by (signed)
 *
 * @return Value of memory location before increment
 */
static inline int32_t cvmx_atomic_fetch_and_add32_nosync(int32_t *ptr, int32_t incr)
{
    uint32_t tmp, ret;

#if !defined(__FreeBSD__) || !defined(_KERNEL)
    if (OCTEON_IS_MODEL(OCTEON_CN6XXX) || OCTEON_IS_MODEL(OCTEON_CNF7XXX))
    {
	CVMX_PUSH_OCTEON2;
	if (__builtin_constant_p(incr) && incr == 1)
	{
	    __asm__ __volatile__(
		"lai  %0,(%2)"
		: "=r" (ret), "+m" (ptr) : "r" (ptr) : "memory");
	}
	else if (__builtin_constant_p(incr) && incr == -1)
        {
	    __asm__ __volatile__(
		"lad  %0,(%2)"
		: "=r" (ret), "+m" (ptr) : "r" (ptr) : "memory");
        }
        else
        {
	    __asm__ __volatile__(
		"laa  %0,(%2),%3"
		: "=r" (ret), "+m" (ptr) : "r" (ptr), "r" (incr) : "memory");
        }
	CVMX_POP_OCTEON2;
    }
    else
    {
#endif
        __asm__ __volatile__(
            ".set noreorder         \n"
            "1: ll   %[tmp], %[val] \n"
            "   move %[ret], %[tmp] \n"
            "   addu %[tmp], %[inc] \n"
            "   sc   %[tmp], %[val] \n"
            "   beqz %[tmp], 1b     \n"
            "   nop                 \n"
            ".set reorder           \n"
            : [val] "+m" (*ptr), [tmp] "=&r" (tmp), [ret] "=&r" (ret)
            : [inc] "r" (incr)
            : "memory");
#if !defined(__FreeBSD__) || !defined(_KERNEL)
    }
#endif

    return (ret);
}

/**
 * Atomically adds a signed value to a 32 bit (aligned) memory location,
 * and returns previous value.
 *
 * Memory access ordering is enforced before/after the atomic operation,
 * so no additional 'sync' instructions are required.
 *
 * @param ptr    address in memory to add incr to
 * @param incr   amount to increment memory location by (signed)
 *
 * @return Value of memory location before increment
 */
static inline int32_t cvmx_atomic_fetch_and_add32(int32_t *ptr, int32_t incr)
{
    uint32_t ret;
    CVMX_SYNCWS;
    ret = cvmx_atomic_fetch_and_add32_nosync(ptr, incr);
    CVMX_SYNCWS;
    return ret;
}

/**
 * Atomically set bits in a 64 bit (aligned) memory location,
 * and returns previous value.
 *
 * This version does not perform 'sync' operations to enforce memory
 * operations.  This should only be used when there are no memory operation
 * ordering constraints.
 *
 * @param ptr    address in memory
 * @param mask   mask of bits to set
 *
 * @return Value of memory location before setting bits
 */
static inline uint64_t cvmx_atomic_fetch_and_bset64_nosync(uint64_t *ptr, uint64_t mask)
{
    uint64_t tmp, ret;

    __asm__ __volatile__(
    ".set noreorder         \n"
    "1: lld  %[tmp], %[val] \n"
    "   move %[ret], %[tmp] \n"
    "   or   %[tmp], %[msk] \n"
    "   scd  %[tmp], %[val] \n"
    "   beqz %[tmp], 1b     \n"
    "   nop                 \n"
    ".set reorder           \n"
    : [val] "+m" (*ptr), [tmp] "=&r" (tmp), [ret] "=&r" (ret)
    : [msk] "r" (mask)
    : "memory");

    return (ret);
}

/**
 * Atomically set bits in a 32 bit (aligned) memory location,
 * and returns previous value.
 *
 * This version does not perform 'sync' operations to enforce memory
 * operations.  This should only be used when there are no memory operation
 * ordering constraints.
 *
 * @param ptr    address in memory
 * @param mask   mask of bits to set
 *
 * @return Value of memory location before setting bits
 */
static inline uint32_t cvmx_atomic_fetch_and_bset32_nosync(uint32_t *ptr, uint32_t mask)
{
    uint32_t tmp, ret;

    __asm__ __volatile__(
    ".set noreorder         \n"
    "1: ll   %[tmp], %[val] \n"
    "   move %[ret], %[tmp] \n"
    "   or   %[tmp], %[msk] \n"
    "   sc   %[tmp], %[val] \n"
    "   beqz %[tmp], 1b     \n"
    "   nop                 \n"
    ".set reorder           \n"
    : [val] "+m" (*ptr), [tmp] "=&r" (tmp), [ret] "=&r" (ret)
    : [msk] "r" (mask)
    : "memory");

    return (ret);
}

/**
 * Atomically clear bits in a 64 bit (aligned) memory location,
 * and returns previous value.
 *
 * This version does not perform 'sync' operations to enforce memory
 * operations.  This should only be used when there are no memory operation
 * ordering constraints.
 *
 * @param ptr    address in memory
 * @param mask   mask of bits to clear
 *
 * @return Value of memory location before clearing bits
 */
static inline uint64_t cvmx_atomic_fetch_and_bclr64_nosync(uint64_t *ptr, uint64_t mask)
{
    uint64_t tmp, ret;

    __asm__ __volatile__(
    ".set noreorder         \n"
    "   nor  %[msk], 0      \n"
    "1: lld  %[tmp], %[val] \n"
    "   move %[ret], %[tmp] \n"
    "   and  %[tmp], %[msk] \n"
    "   scd  %[tmp], %[val] \n"
    "   beqz %[tmp], 1b     \n"
    "   nop                 \n"
    ".set reorder           \n"
    : [val] "+m" (*ptr), [tmp] "=&r" (tmp), [ret] "=&r" (ret), [msk] "+r" (mask)
    : : "memory");

    return (ret);
}

/**
 * Atomically clear bits in a 32 bit (aligned) memory location,
 * and returns previous value.
 *
 * This version does not perform 'sync' operations to enforce memory
 * operations.  This should only be used when there are no memory operation
 * ordering constraints.
 *
 * @param ptr    address in memory
 * @param mask   mask of bits to clear
 *
 * @return Value of memory location before clearing bits
 */
static inline uint32_t cvmx_atomic_fetch_and_bclr32_nosync(uint32_t *ptr, uint32_t mask)
{
    uint32_t tmp, ret;

    __asm__ __volatile__(
    ".set noreorder         \n"
    "   nor  %[msk], 0      \n"
    "1: ll   %[tmp], %[val] \n"
    "   move %[ret], %[tmp] \n"
    "   and  %[tmp], %[msk] \n"
    "   sc   %[tmp], %[val] \n"
    "   beqz %[tmp], 1b     \n"
    "   nop                 \n"
    ".set reorder           \n"
    : [val] "+m" (*ptr), [tmp] "=&r" (tmp), [ret] "=&r" (ret), [msk] "+r" (mask)
    : : "memory");

    return (ret);
}

/**
 * Atomically swaps value in 64 bit (aligned) memory location,
 * and returns previous value.
 *
 * This version does not perform 'sync' operations to enforce memory
 * operations.  This should only be used when there are no memory operation
 * ordering constraints.
 *
 * @param ptr       address in memory
 * @param new_val   new value to write
 *
 * @return Value of memory location before swap operation
 */
static inline uint64_t cvmx_atomic_swap64_nosync(uint64_t *ptr, uint64_t new_val)
{
    uint64_t tmp, ret;

#if !defined(__FreeBSD__) || !defined(_KERNEL)
    if (OCTEON_IS_MODEL(OCTEON_CN6XXX) || OCTEON_IS_MODEL(OCTEON_CNF7XXX))
    {
	CVMX_PUSH_OCTEON2;
	if (__builtin_constant_p(new_val) && new_val == 0)
	{
	    __asm__ __volatile__(
		"lacd  %0,(%1)"
		: "=r" (ret) : "r" (ptr) : "memory");
	}
	else if (__builtin_constant_p(new_val) && new_val == ~0ull)
        {
	    __asm__ __volatile__(
		"lasd  %0,(%1)"
		: "=r" (ret) : "r" (ptr) : "memory");
        }
        else
        {
	    __asm__ __volatile__(
		"lawd  %0,(%1),%2"
		: "=r" (ret) : "r" (ptr), "r" (new_val) : "memory");
        }
	CVMX_POP_OCTEON2;
    }
    else
    {
#endif
        __asm__ __volatile__(
            ".set noreorder         \n"
            "1: lld  %[ret], %[val] \n"
            "   move %[tmp], %[new_val] \n"
            "   scd  %[tmp], %[val] \n"
            "   beqz %[tmp],  1b    \n"
            "   nop                 \n"
            ".set reorder           \n"
            : [val] "+m" (*ptr), [tmp] "=&r" (tmp), [ret] "=&r" (ret)
            : [new_val] "r"  (new_val)
            : "memory");
#if !defined(__FreeBSD__) || !defined(_KERNEL)
    }
#endif

    return (ret);
}

/**
 * Atomically swaps value in 32 bit (aligned) memory location,
 * and returns previous value.
 *
 * This version does not perform 'sync' operations to enforce memory
 * operations.  This should only be used when there are no memory operation
 * ordering constraints.
 *
 * @param ptr       address in memory
 * @param new_val   new value to write
 *
 * @return Value of memory location before swap operation
 */
static inline uint32_t cvmx_atomic_swap32_nosync(uint32_t *ptr, uint32_t new_val)
{
    uint32_t tmp, ret;

#if !defined(__FreeBSD__) || !defined(_KERNEL)
    if (OCTEON_IS_MODEL(OCTEON_CN6XXX) || OCTEON_IS_MODEL(OCTEON_CNF7XXX))
    {
	CVMX_PUSH_OCTEON2;
	if (__builtin_constant_p(new_val) && new_val == 0)
	{
	    __asm__ __volatile__(
		"lac  %0,(%1)"
		: "=r" (ret) : "r" (ptr) : "memory");
	}
	else if (__builtin_constant_p(new_val) && new_val == ~0u)
        {
	    __asm__ __volatile__(
		"las  %0,(%1)"
		: "=r" (ret) : "r" (ptr) : "memory");
        }
        else
        {
	    __asm__ __volatile__(
		"law  %0,(%1),%2"
		: "=r" (ret) : "r" (ptr), "r" (new_val) : "memory");
        }
	CVMX_POP_OCTEON2;
    }
    else
    {
#endif
        __asm__ __volatile__(
        ".set noreorder         \n"
        "1: ll   %[ret], %[val] \n"
        "   move %[tmp], %[new_val] \n"
        "   sc   %[tmp], %[val] \n"
        "   beqz %[tmp],  1b    \n"
        "   nop                 \n"
        ".set reorder           \n"
        : [val] "+m" (*ptr), [tmp] "=&r" (tmp), [ret] "=&r" (ret)
        : [new_val] "r"  (new_val)
        : "memory");
#if !defined(__FreeBSD__) || !defined(_KERNEL)
    }
#endif

    return (ret);
}

#ifdef	__cplusplus
}
#endif

#endif /* __CVMX_ATOMIC_H__ */