From 443b9a14ecbe811071467d54d6f2f1182835cc4d Mon Sep 17 00:00:00 2001 From: Serge Semin Date: Wed, 11 Jan 2017 03:11:33 +0300 Subject: NTB: Alter MW API to support multi-ports devices Multi-port NTB devices permit to share a memory between all accessible peers. Memory Windows API is altered to correspondingly initialize and map memory windows for such devices: ntb_mw_count(pidx); - number of inbound memory windows, which can be allocated for shared buffer with specified peer device. ntb_mw_get_align(pidx, widx); - get alignment and size restriction parameters to properly allocate inbound memory region. ntb_peer_mw_count(); - get number of outbound memory windows. ntb_peer_mw_get_addr(widx); - get mapping address of an outbound memory window If hardware supports inbound translation configured on the local ntb port: ntb_mw_set_trans(pidx, widx); - set translation address of allocated inbound memory window so a peer device could access it. ntb_mw_clear_trans(pidx, widx); - clear the translation address of an inbound memory window. If hardware supports outbound translation configured on the peer ntb port: ntb_peer_mw_set_trans(pidx, widx); - set translation address of a memory window retrieved from a peer device ntb_peer_mw_clear_trans(pidx, widx); - clear the translation address of an outbound memory window Signed-off-by: Serge Semin Acked-by: Allen Hubbe Signed-off-by: Jon Mason --- include/linux/ntb.h | 208 ++++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 163 insertions(+), 45 deletions(-) (limited to 'include/linux/ntb.h') diff --git a/include/linux/ntb.h b/include/linux/ntb.h index b2b2924..2ea83f9 100644 --- a/include/linux/ntb.h +++ b/include/linux/ntb.h @@ -5,6 +5,7 @@ * GPL LICENSE SUMMARY * * Copyright (C) 2015 EMC Corporation. All Rights Reserved. + * Copyright (C) 2016 T-Platforms. All Rights Reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of version 2 of the GNU General Public License as @@ -18,6 +19,7 @@ * BSD LICENSE * * Copyright (C) 2015 EMC Corporation. All Rights Reserved. + * Copyright (C) 2016 T-Platforms. All Rights Reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -201,9 +203,13 @@ static inline int ntb_ctx_ops_is_valid(const struct ntb_ctx_ops *ops) * @link_enable: See ntb_link_enable(). * @link_disable: See ntb_link_disable(). * @mw_count: See ntb_mw_count(). - * @mw_get_range: See ntb_mw_get_range(). + * @mw_get_align: See ntb_mw_get_align(). * @mw_set_trans: See ntb_mw_set_trans(). * @mw_clear_trans: See ntb_mw_clear_trans(). + * @peer_mw_count: See ntb_peer_mw_count(). + * @peer_mw_get_addr: See ntb_peer_mw_get_addr(). + * @peer_mw_set_trans: See ntb_peer_mw_set_trans(). + * @peer_mw_clear_trans:See ntb_peer_mw_clear_trans(). * @db_is_unsafe: See ntb_db_is_unsafe(). * @db_valid_mask: See ntb_db_valid_mask(). * @db_vector_count: See ntb_db_vector_count(). @@ -241,13 +247,20 @@ struct ntb_dev_ops { enum ntb_speed max_speed, enum ntb_width max_width); int (*link_disable)(struct ntb_dev *ntb); - int (*mw_count)(struct ntb_dev *ntb); - int (*mw_get_range)(struct ntb_dev *ntb, int idx, - phys_addr_t *base, resource_size_t *size, - resource_size_t *align, resource_size_t *align_size); - int (*mw_set_trans)(struct ntb_dev *ntb, int idx, + int (*mw_count)(struct ntb_dev *ntb, int pidx); + int (*mw_get_align)(struct ntb_dev *ntb, int pidx, int widx, + resource_size_t *addr_align, + resource_size_t *size_align, + resource_size_t *size_max); + int (*mw_set_trans)(struct ntb_dev *ntb, int pidx, int widx, dma_addr_t addr, resource_size_t size); - int (*mw_clear_trans)(struct ntb_dev *ntb, int idx); + int (*mw_clear_trans)(struct ntb_dev *ntb, int pidx, int widx); + int (*peer_mw_count)(struct ntb_dev *ntb); + int (*peer_mw_get_addr)(struct ntb_dev *ntb, int widx, + phys_addr_t *base, resource_size_t *size); + int (*peer_mw_set_trans)(struct ntb_dev *ntb, int pidx, int widx, + u64 addr, resource_size_t size); + int (*peer_mw_clear_trans)(struct ntb_dev *ntb, int pidx, int widx); int (*db_is_unsafe)(struct ntb_dev *ntb); u64 (*db_valid_mask)(struct ntb_dev *ntb); @@ -295,9 +308,13 @@ static inline int ntb_dev_ops_is_valid(const struct ntb_dev_ops *ops) ops->link_enable && ops->link_disable && ops->mw_count && - ops->mw_get_range && - ops->mw_set_trans && + ops->mw_get_align && + (ops->mw_set_trans || + ops->peer_mw_set_trans) && /* ops->mw_clear_trans && */ + ops->peer_mw_count && + ops->peer_mw_get_addr && + /* ops->peer_mw_clear_trans && */ /* ops->db_is_unsafe && */ ops->db_valid_mask && @@ -655,79 +672,180 @@ static inline int ntb_link_disable(struct ntb_dev *ntb) } /** - * ntb_mw_count() - get the number of memory windows + * ntb_mw_count() - get the number of inbound memory windows, which could + * be created for a specified peer device * @ntb: NTB device context. + * @pidx: Port index of peer device. * * Hardware and topology may support a different number of memory windows. + * Moreover different peer devices can support different number of memory + * windows. Simply speaking this method returns the number of possible inbound + * memory windows to share with specified peer device. * * Return: the number of memory windows. */ -static inline int ntb_mw_count(struct ntb_dev *ntb) +static inline int ntb_mw_count(struct ntb_dev *ntb, int pidx) { - return ntb->ops->mw_count(ntb); + return ntb->ops->mw_count(ntb, pidx); } /** - * ntb_mw_get_range() - get the range of a memory window + * ntb_mw_get_align() - get the restriction parameters of inbound memory window * @ntb: NTB device context. - * @idx: Memory window number. - * @base: OUT - the base address for mapping the memory window - * @size: OUT - the size for mapping the memory window - * @align: OUT - the base alignment for translating the memory window - * @align_size: OUT - the size alignment for translating the memory window - * - * Get the range of a memory window. NULL may be given for any output - * parameter if the value is not needed. The base and size may be used for - * mapping the memory window, to access the peer memory. The alignment and - * size may be used for translating the memory window, for the peer to access - * memory on the local system. - * - * Return: Zero on success, otherwise an error number. + * @pidx: Port index of peer device. + * @widx: Memory window index. + * @addr_align: OUT - the base alignment for translating the memory window + * @size_align: OUT - the size alignment for translating the memory window + * @size_max: OUT - the maximum size of the memory window + * + * Get the alignments of an inbound memory window with specified index. + * NULL may be given for any output parameter if the value is not needed. + * The alignment and size parameters may be used for allocation of proper + * shared memory. + * + * Return: Zero on success, otherwise a negative error number. */ -static inline int ntb_mw_get_range(struct ntb_dev *ntb, int idx, - phys_addr_t *base, resource_size_t *size, - resource_size_t *align, resource_size_t *align_size) +static inline int ntb_mw_get_align(struct ntb_dev *ntb, int pidx, int widx, + resource_size_t *addr_align, + resource_size_t *size_align, + resource_size_t *size_max) { - return ntb->ops->mw_get_range(ntb, idx, base, size, - align, align_size); + return ntb->ops->mw_get_align(ntb, pidx, widx, addr_align, size_align, + size_max); } /** - * ntb_mw_set_trans() - set the translation of a memory window + * ntb_mw_set_trans() - set the translation of an inbound memory window * @ntb: NTB device context. - * @idx: Memory window number. - * @addr: The dma address local memory to expose to the peer. + * @pidx: Port index of peer device. + * @widx: Memory window index. + * @addr: The dma address of local memory to expose to the peer. * @size: The size of the local memory to expose to the peer. * * Set the translation of a memory window. The peer may access local memory * through the window starting at the address, up to the size. The address - * must be aligned to the alignment specified by ntb_mw_get_range(). The size - * must be aligned to the size alignment specified by ntb_mw_get_range(). + * and size must be aligned in compliance with restrictions of + * ntb_mw_get_align(). The region size should not exceed the size_max parameter + * of that method. + * + * This method may not be implemented due to the hardware specific memory + * windows interface. * * Return: Zero on success, otherwise an error number. */ -static inline int ntb_mw_set_trans(struct ntb_dev *ntb, int idx, +static inline int ntb_mw_set_trans(struct ntb_dev *ntb, int pidx, int widx, dma_addr_t addr, resource_size_t size) { - return ntb->ops->mw_set_trans(ntb, idx, addr, size); + if (!ntb->ops->mw_set_trans) + return 0; + + return ntb->ops->mw_set_trans(ntb, pidx, widx, addr, size); } /** - * ntb_mw_clear_trans() - clear the translation of a memory window + * ntb_mw_clear_trans() - clear the translation address of an inbound memory + * window * @ntb: NTB device context. - * @idx: Memory window number. + * @pidx: Port index of peer device. + * @widx: Memory window index. * - * Clear the translation of a memory window. The peer may no longer access - * local memory through the window. + * Clear the translation of an inbound memory window. The peer may no longer + * access local memory through the window. * * Return: Zero on success, otherwise an error number. */ -static inline int ntb_mw_clear_trans(struct ntb_dev *ntb, int idx) +static inline int ntb_mw_clear_trans(struct ntb_dev *ntb, int pidx, int widx) { if (!ntb->ops->mw_clear_trans) - return ntb->ops->mw_set_trans(ntb, idx, 0, 0); + return ntb_mw_set_trans(ntb, pidx, widx, 0, 0); + + return ntb->ops->mw_clear_trans(ntb, pidx, widx); +} + +/** + * ntb_peer_mw_count() - get the number of outbound memory windows, which could + * be mapped to access a shared memory + * @ntb: NTB device context. + * + * Hardware and topology may support a different number of memory windows. + * This method returns the number of outbound memory windows supported by + * local device. + * + * Return: the number of memory windows. + */ +static inline int ntb_peer_mw_count(struct ntb_dev *ntb) +{ + return ntb->ops->peer_mw_count(ntb); +} + +/** + * ntb_peer_mw_get_addr() - get map address of an outbound memory window + * @ntb: NTB device context. + * @widx: Memory window index (within ntb_peer_mw_count() return value). + * @base: OUT - the base address of mapping region. + * @size: OUT - the size of mapping region. + * + * Get base and size of memory region to map. NULL may be given for any output + * parameter if the value is not needed. The base and size may be used for + * mapping the memory window, to access the peer memory. + * + * Return: Zero on success, otherwise a negative error number. + */ +static inline int ntb_peer_mw_get_addr(struct ntb_dev *ntb, int widx, + phys_addr_t *base, resource_size_t *size) +{ + return ntb->ops->peer_mw_get_addr(ntb, widx, base, size); +} + +/** + * ntb_peer_mw_set_trans() - set a translation address of a memory window + * retrieved from a peer device + * @ntb: NTB device context. + * @pidx: Port index of peer device the translation address received from. + * @widx: Memory window index. + * @addr: The dma address of the shared memory to access. + * @size: The size of the shared memory to access. + * + * Set the translation of an outbound memory window. The local device may + * access shared memory allocated by a peer device sent the address. + * + * This method may not be implemented due to the hardware specific memory + * windows interface, so a translation address can be only set on the side, + * where shared memory (inbound memory windows) is allocated. + * + * Return: Zero on success, otherwise an error number. + */ +static inline int ntb_peer_mw_set_trans(struct ntb_dev *ntb, int pidx, int widx, + u64 addr, resource_size_t size) +{ + if (!ntb->ops->peer_mw_set_trans) + return 0; + + return ntb->ops->peer_mw_set_trans(ntb, pidx, widx, addr, size); +} + +/** + * ntb_peer_mw_clear_trans() - clear the translation address of an outbound + * memory window + * @ntb: NTB device context. + * @pidx: Port index of peer device. + * @widx: Memory window index. + * + * Clear the translation of a outbound memory window. The local device may no + * longer access a shared memory through the window. + * + * This method may not be implemented due to the hardware specific memory + * windows interface. + * + * Return: Zero on success, otherwise an error number. + */ +static inline int ntb_peer_mw_clear_trans(struct ntb_dev *ntb, int pidx, + int widx) +{ + if (!ntb->ops->peer_mw_clear_trans) + return ntb_peer_mw_set_trans(ntb, pidx, widx, 0, 0); - return ntb->ops->mw_clear_trans(ntb, idx); + return ntb->ops->peer_mw_clear_trans(ntb, pidx, widx); } /** -- cgit v1.1