summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--mm/percpu.c108
1 files changed, 60 insertions, 48 deletions
diff --git a/mm/percpu.c b/mm/percpu.c
index f1d0e90..7d9bc35 100644
--- a/mm/percpu.c
+++ b/mm/percpu.c
@@ -307,6 +307,50 @@ static void pcpu_chunk_addr_insert(struct pcpu_chunk *new)
}
/**
+ * pcpu_extend_area_map - extend area map for allocation
+ * @chunk: target chunk
+ *
+ * Extend area map of @chunk so that it can accomodate an allocation.
+ * A single allocation can split an area into three areas, so this
+ * function makes sure that @chunk->map has at least two extra slots.
+ *
+ * RETURNS:
+ * 0 if noop, 1 if successfully extended, -errno on failure.
+ */
+static int pcpu_extend_area_map(struct pcpu_chunk *chunk)
+{
+ int new_alloc;
+ int *new;
+ size_t size;
+
+ /* has enough? */
+ if (chunk->map_alloc >= chunk->map_used + 2)
+ return 0;
+
+ new_alloc = PCPU_DFL_MAP_ALLOC;
+ while (new_alloc < chunk->map_used + 2)
+ new_alloc *= 2;
+
+ new = pcpu_mem_alloc(new_alloc * sizeof(new[0]));
+ if (!new)
+ return -ENOMEM;
+
+ size = chunk->map_alloc * sizeof(chunk->map[0]);
+ memcpy(new, chunk->map, size);
+
+ /*
+ * map_alloc < PCPU_DFL_MAP_ALLOC indicates that the chunk is
+ * one of the first chunks and still using static map.
+ */
+ if (chunk->map_alloc >= PCPU_DFL_MAP_ALLOC)
+ pcpu_mem_free(chunk->map, size);
+
+ chunk->map_alloc = new_alloc;
+ chunk->map = new;
+ return 0;
+}
+
+/**
* pcpu_split_block - split a map block
* @chunk: chunk of interest
* @i: index of map block to split
@@ -321,44 +365,16 @@ static void pcpu_chunk_addr_insert(struct pcpu_chunk *new)
* depending on @head, is reduced by @tail bytes and @tail byte block
* is inserted after the target block.
*
- * RETURNS:
- * 0 on success, -errno on failure.
+ * @chunk->map must have enough free slots to accomodate the split.
*/
-static int pcpu_split_block(struct pcpu_chunk *chunk, int i, int head, int tail)
+static void pcpu_split_block(struct pcpu_chunk *chunk, int i,
+ int head, int tail)
{
int nr_extra = !!head + !!tail;
- int target = chunk->map_used + nr_extra;
-
- /* reallocation required? */
- if (chunk->map_alloc < target) {
- int new_alloc;
- int *new;
- size_t size;
-
- new_alloc = PCPU_DFL_MAP_ALLOC;
- while (new_alloc < target)
- new_alloc *= 2;
-
- new = pcpu_mem_alloc(new_alloc * sizeof(new[0]));
- if (!new)
- return -ENOMEM;
-
- size = chunk->map_alloc * sizeof(chunk->map[0]);
- memcpy(new, chunk->map, size);
-
- /*
- * map_alloc < PCPU_DFL_MAP_ALLOC indicates that the
- * chunk is one of the first chunks and still using
- * static map.
- */
- if (chunk->map_alloc >= PCPU_DFL_MAP_ALLOC)
- pcpu_mem_free(chunk->map, size);
- chunk->map_alloc = new_alloc;
- chunk->map = new;
- }
+ BUG_ON(chunk->map_alloc < chunk->map_used + nr_extra);
- /* insert a new subblock */
+ /* insert new subblocks */
memmove(&chunk->map[i + nr_extra], &chunk->map[i],
sizeof(chunk->map[0]) * (chunk->map_used - i));
chunk->map_used += nr_extra;
@@ -371,7 +387,6 @@ static int pcpu_split_block(struct pcpu_chunk *chunk, int i, int head, int tail)
chunk->map[i++] -= tail;
chunk->map[i] = tail;
}
- return 0;
}
/**
@@ -384,8 +399,11 @@ static int pcpu_split_block(struct pcpu_chunk *chunk, int i, int head, int tail)
* Note that this function only allocates the offset. It doesn't
* populate or map the area.
*
+ * @chunk->map must have at least two free slots.
+ *
* RETURNS:
- * Allocated offset in @chunk on success, -errno on failure.
+ * Allocated offset in @chunk on success, -1 if no matching area is
+ * found.
*/
static int pcpu_alloc_area(struct pcpu_chunk *chunk, int size, int align)
{
@@ -433,8 +451,7 @@ static int pcpu_alloc_area(struct pcpu_chunk *chunk, int size, int align)
/* split if warranted */
if (head || tail) {
- if (pcpu_split_block(chunk, i, head, tail))
- return -ENOMEM;
+ pcpu_split_block(chunk, i, head, tail);
if (head) {
i++;
off += head;
@@ -461,14 +478,8 @@ static int pcpu_alloc_area(struct pcpu_chunk *chunk, int size, int align)
chunk->contig_hint = max_contig; /* fully scanned */
pcpu_chunk_relocate(chunk, oslot);
- /*
- * Tell the upper layer that this chunk has no area left.
- * Note that this is not an error condition but a notification
- * to upper layer that it needs to look at other chunks.
- * -ENOSPC is chosen as it isn't used in memory subsystem and
- * matches the meaning in a way.
- */
- return -ENOSPC;
+ /* tell the upper layer that this chunk has no matching area */
+ return -1;
}
/**
@@ -755,7 +766,8 @@ static void *pcpu_alloc(size_t size, size_t align, bool reserved)
/* serve reserved allocations from the reserved chunk if available */
if (reserved && pcpu_reserved_chunk) {
chunk = pcpu_reserved_chunk;
- if (size > chunk->contig_hint)
+ if (size > chunk->contig_hint ||
+ pcpu_extend_area_map(chunk) < 0)
goto out_unlock;
off = pcpu_alloc_area(chunk, size, align);
if (off >= 0)
@@ -768,11 +780,11 @@ static void *pcpu_alloc(size_t size, size_t align, bool reserved)
list_for_each_entry(chunk, &pcpu_slot[slot], list) {
if (size > chunk->contig_hint)
continue;
+ if (pcpu_extend_area_map(chunk) < 0)
+ goto out_unlock;
off = pcpu_alloc_area(chunk, size, align);
if (off >= 0)
goto area_found;
- if (off != -ENOSPC)
- goto out_unlock;
}
}
OpenPOWER on IntegriCloud