MFC 265440, 265441, 265444, 265445, 265446, 265447:

  Move the pl310.enabled tunable to hw.pl310.enabled.  Clean up a few minor
  style(9) nits.  Use DEVMETHOD_END.

  Break out the code that figures out the L2 cache geometry to its own
  routine, so that it can be called from multiple places in upcoming changes.

  Call platform_pl310_init() before enabling the controller, and handle the
  case where the controller is already enabled.

  Add defines for the bits in the PL310 debug control register.

  Add a public routine to set the L2 cache ram latencies.  This can be
  called by platform init routines to fine-tune cache performance.

  Enable PL310 power-saving modes and tune the cache ram latencies for imx6.

3 files changed, 117 insertions, 44 deletions

diff --git a/sys/arm/arm/pl310.c b/sys/arm/arm/pl310.c
index b087365..83e6907 100644
--- a/sys/arm/arm/pl310.c
+++ b/sys/arm/arm/pl310.c
@@ -71,7 +71,7 @@ __FBSDID("$FreeBSD$");
 } while(0);
 
 static int pl310_enabled = 1;
-TUNABLE_INT("pl310.enabled", &pl310_enabled);
+TUNABLE_INT("hw.pl310.enabled", &pl310_enabled);
 
 static uint32_t g_l2cache_way_mask;
 
@@ -125,6 +125,35 @@ pl310_print_config(struct pl310_softc *sc)
 		(prefetch & PREFETCH_CTRL_OFFSET_MASK));
 }
 
+void
+pl310_set_ram_latency(struct pl310_softc *sc, uint32_t which_reg,
+   uint32_t read, uint32_t write, uint32_t setup)
+{
+	uint32_t v;
+
+	KASSERT(which_reg == PL310_TAG_RAM_CTRL || 
+	    which_reg == PL310_DATA_RAM_CTRL,
+	    ("bad pl310 ram latency register address"));
+
+	v = pl310_read4(sc, which_reg);
+	if (setup != 0) {
+		KASSERT(setup <= 8, ("bad pl310 setup latency: %d", setup));
+		v &= ~RAM_CTRL_SETUP_MASK;
+		v |= (setup - 1) << RAM_CTRL_SETUP_SHIFT;
+	}
+	if (read != 0) {
+		KASSERT(read <= 8, ("bad pl310 read latency: %d", read));
+		v &= ~RAM_CTRL_READ_MASK;
+		v |= (read - 1) << RAM_CTRL_READ_SHIFT;
+	}
+	if (write != 0) {
+		KASSERT(write <= 8, ("bad pl310 write latency: %d", write));
+		v &= ~RAM_CTRL_WRITE_MASK;
+		v |= (write - 1) << RAM_CTRL_WRITE_SHIFT;
+	}
+	pl310_write4(sc, which_reg, v);
+}
+
 static int
 pl310_filter(void *arg)
 {
@@ -149,7 +178,8 @@ static __inline void
 pl310_wait_background_op(uint32_t off, uint32_t mask)
 {
 
-	while (pl310_read4(pl310_softc, off) & mask);
+	while (pl310_read4(pl310_softc, off) & mask)
+		continue;
 }
 
 
@@ -167,6 +197,7 @@ pl310_wait_background_op(uint32_t off, uint32_t mask)
 static __inline void
 pl310_cache_sync(void)
 {
+
 	if ((pl310_softc == NULL) || !pl310_softc->sc_enabled)
 		return;
 
@@ -318,6 +349,23 @@ pl310_inv_range(vm_paddr_t start, vm_size_t size)
 	PL310_UNLOCK(pl310_softc);
 }
 
+static void
+pl310_set_way_sizes(struct pl310_softc *sc)
+{
+	uint32_t aux_value;
+
+	aux_value = pl310_read4(sc, PL310_AUX_CTRL);
+	g_way_size = (aux_value & AUX_CTRL_WAY_SIZE_MASK) >>
+	    AUX_CTRL_WAY_SIZE_SHIFT;
+	g_way_size = 1 << (g_way_size + 13);
+	if (aux_value & (1 << AUX_CTRL_ASSOCIATIVITY_SHIFT))
+		g_ways_assoc = 16;
+	else
+		g_ways_assoc = 8;
+	g_l2cache_way_mask = (1 << g_ways_assoc) - 1;
+	g_l2cache_size = g_way_size * g_ways_assoc;
+}
+
 static int
 pl310_probe(device_t dev)
 {
@@ -335,12 +383,11 @@ static int
 pl310_attach(device_t dev)
 {
 	struct pl310_softc *sc = device_get_softc(dev);
-	int rid = 0;
-	uint32_t aux_value;
-	uint32_t ctrl_value;
-	uint32_t cache_id;
+	int rid;
+	uint32_t cache_id, debug_ctrl;
 
 	sc->sc_dev = dev;
+	rid = 0;
 	sc->sc_mem_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, 
 	    RF_ACTIVE);
 	if (sc->sc_mem_res == NULL)
@@ -356,7 +403,6 @@ pl310_attach(device_t dev)
 
 	pl310_softc = sc;
 	mtx_init(&sc->sc_mtx, "pl310lock", NULL, MTX_SPIN);
-	sc->sc_enabled = pl310_enabled;
 
 	/* activate the interrupt */
 	bus_setup_intr(dev, sc->sc_irq_res, INTR_TYPE_MISC | INTR_MPSAFE,
@@ -368,39 +414,49 @@ pl310_attach(device_t dev)
 	device_printf(dev, "Part number: 0x%x, release: 0x%x\n",
 	    (cache_id >> CACHE_ID_PARTNUM_SHIFT) & CACHE_ID_PARTNUM_MASK,
 	    (cache_id >> CACHE_ID_RELEASE_SHIFT) & CACHE_ID_RELEASE_MASK);
-	aux_value = pl310_read4(sc, PL310_AUX_CTRL);
-	g_way_size = (aux_value & AUX_CTRL_WAY_SIZE_MASK) >>
-	    AUX_CTRL_WAY_SIZE_SHIFT;
-	g_way_size = 1 << (g_way_size + 13);
-	if (aux_value & (1 << AUX_CTRL_ASSOCIATIVITY_SHIFT))
-		g_ways_assoc = 16;
-	else
-		g_ways_assoc = 8;
-	g_l2cache_way_mask = (1 << g_ways_assoc) - 1;
-	g_l2cache_size = g_way_size * g_ways_assoc;
-	/* Print the information */
-	device_printf(dev, "L2 Cache: %uKB/%dB %d ways\n", (g_l2cache_size / 1024),
-	       g_l2cache_line_size, g_ways_assoc);
 
-	ctrl_value = pl310_read4(sc, PL310_CTRL);
+	/*
+	 * If L2 cache is already enabled then something has violated the rules,
+	 * because caches are supposed to be off at kernel entry.  The cache
+	 * must be disabled to write the configuration registers without
+	 * triggering an access error (SLVERR), but there's no documented safe
+	 * procedure for disabling the L2 cache in the manual.  So we'll try to
+	 * invent one:
+	 *  - Use the debug register to force write-through mode and prevent
+	 *    linefills (allocation of new lines on read); now anything we do
+	 *    will not cause new data to come into the L2 cache.
+	 *  - Writeback and invalidate the current contents.
+	 *  - Disable the controller.
+	 *  - Restore the original debug settings.
+	 */
+	if (pl310_read4(sc, PL310_CTRL) & CTRL_ENABLED) {
+		device_printf(dev, "Warning: L2 Cache should not already be "
+		    "active; trying to de-activate and re-initialize...\n");
+		sc->sc_enabled = 1;
+		debug_ctrl = pl310_read4(sc, PL310_DEBUG_CTRL);
+		platform_pl310_write_debug(sc, debug_ctrl |
+		    DEBUG_CTRL_DISABLE_WRITEBACK | DEBUG_CTRL_DISABLE_LINEFILL);
+		pl310_set_way_sizes(sc);
+		pl310_wbinv_all();
+		platform_pl310_write_ctrl(sc, CTRL_DISABLED);
+		platform_pl310_write_debug(sc, debug_ctrl);
+	}
+	sc->sc_enabled = pl310_enabled;
 
-	if (sc->sc_enabled && !(ctrl_value & CTRL_ENABLED)) {
-		/* invalidate current content */
+	if (sc->sc_enabled) {
+		platform_pl310_init(sc);
+		pl310_set_way_sizes(sc); /* platform init might change these */
 		pl310_write4(pl310_softc, PL310_INV_WAY, 0xffff);
 		pl310_wait_background_op(PL310_INV_WAY, 0xffff);
-
-		/* Enable the L2 cache if disabled */
 		platform_pl310_write_ctrl(sc, CTRL_ENABLED);
-		device_printf(dev, "L2 Cache enabled\n");
+		device_printf(dev, "L2 Cache enabled: %uKB/%dB %d ways\n", 
+		    (g_l2cache_size / 1024), g_l2cache_line_size, g_ways_assoc);
 		if (bootverbose)
 			pl310_print_config(sc);
-	} 
-
-	if (!sc->sc_enabled && (ctrl_value & CTRL_ENABLED)) {
+	} else {
 		/*
-		 * Set counters so when cache event happens
-		 * we'll get interrupt and be warned that something 
-		 * is off
+		 * Set counters so when cache event happens we'll get interrupt
+		 * and be warned that something is off.
 		 */
 
 		/* Cache Line Eviction for Counter 0 */
@@ -410,12 +466,6 @@ pl310_attach(device_t dev)
 		pl310_write4(sc, PL310_EVENT_COUNTER1_CONF, 
 		    EVENT_COUNTER_CONF_INCR | EVENT_COUNTER_CONF_DRREQ);
 
-		/* Temporary switch on for final flush*/
-		sc->sc_enabled = 1;
-		pl310_wbinv_all();
-		sc->sc_enabled = 0;
-		platform_pl310_write_ctrl(sc, CTRL_DISABLED);
-
 		/* Enable and clear pending interrupts */
 		pl310_write4(sc, PL310_INTR_CLEAR, INTR_MASK_ECNTR);
 		pl310_write4(sc, PL310_INTR_MASK, INTR_MASK_ALL);
@@ -429,11 +479,6 @@ pl310_attach(device_t dev)
 		device_printf(dev, "L2 Cache disabled\n");
 	}
 
-	if (sc->sc_enabled)
-		platform_pl310_init(sc);
-
-	pl310_wbinv_all();
-
 	/* Set the l2 functions in the set of cpufuncs */
 	cpufuncs.cf_l2cache_wbinv_all = pl310_wbinv_all;
 	cpufuncs.cf_l2cache_wbinv_range = pl310_wbinv_range;
@@ -446,7 +491,7 @@ pl310_attach(device_t dev)
 static device_method_t pl310_methods[] = {
 	DEVMETHOD(device_probe, pl310_probe),
 	DEVMETHOD(device_attach, pl310_attach),
-	{0, 0},
+	DEVMETHOD_END
 };
 
 static driver_t pl310_driver = {
diff --git a/sys/arm/freescale/imx/imx6_pl310.c b/sys/arm/freescale/imx/imx6_pl310.c
index a4e383e..9e0427f 100644
--- a/sys/arm/freescale/imx/imx6_pl310.c
+++ b/sys/arm/freescale/imx/imx6_pl310.c
@@ -44,6 +44,19 @@ __FBSDID("$FreeBSD$");
 void
 platform_pl310_init(struct pl310_softc *sc)
 {
+	uint32_t reg;
+
+	/*
+	 * Enable power saving modes:
+	 *  - Dynamic Gating stops the clock when the controller is idle.
+	 *  - Standby stops the clock when the cores are in WFI mode.
+	 */
+	reg = pl310_read4(sc, PL310_POWER_CTRL);
+	reg |= POWER_CTRL_ENABLE_GATING | POWER_CTRL_ENABLE_STANDBY;
+	pl310_write4(sc, PL310_POWER_CTRL, reg);
+
+	pl310_set_ram_latency(sc, PL310_TAG_RAM_CTRL,  4, 2, 3);
+	pl310_set_ram_latency(sc, PL310_DATA_RAM_CTRL, 4, 2, 3);
 }
 
 void
diff --git a/sys/arm/include/pl310.h b/sys/arm/include/pl310.h
index b4200a8..8730877 100644
--- a/sys/arm/include/pl310.h
+++ b/sys/arm/include/pl310.h
@@ -62,6 +62,14 @@
 #define 	AUX_CTRL_DATA_PREFETCH		(1 << 28)
 #define 	AUX_CTRL_INSTR_PREFETCH		(1 << 29)
 #define 	AUX_CTRL_EARLY_BRESP		(1 << 30)
+#define PL310_TAG_RAM_CTRL			0x108
+#define PL310_DATA_RAM_CTRL			0x10C
+#define		RAM_CTRL_WRITE_SHIFT		8
+#define		RAM_CTRL_WRITE_MASK		(0x7 << 8)
+#define		RAM_CTRL_READ_SHIFT		4
+#define		RAM_CTRL_READ_MASK		(0x7 << 4)
+#define		RAM_CTRL_SETUP_SHIFT		0
+#define		RAM_CTRL_SETUP_MASK		(0x7 << 0)
 #define PL310_EVENT_COUNTER_CTRL	0x200
 #define		EVENT_COUNTER_CTRL_ENABLED	(1 << 0)
 #define		EVENT_COUNTER_CTRL_C0_RESET	(1 << 1)
@@ -113,6 +121,9 @@
 #define PL310_ADDR_FILTER_STAR		0xC00
 #define PL310_ADDR_FILTER_END		0xC04
 #define PL310_DEBUG_CTRL		0xF40
+#define		DEBUG_CTRL_DISABLE_LINEFILL	(1 << 0)
+#define		DEBUG_CTRL_DISABLE_WRITEBACK	(1 << 1)
+#define		DEBUG_CTRL_SPNIDEN		(1 << 2)
 #define PL310_PREFETCH_CTRL		0xF60
 #define		PREFETCH_CTRL_OFFSET_MASK	(0x1f)
 #define		PREFETCH_CTRL_NOTSAMEID		(1 << 21)
@@ -123,6 +134,8 @@
 #define		PREFETCH_CTRL_INSTR_PREFETCH	(1 << 29)
 #define		PREFETCH_CTRL_DL		(1 << 30)
 #define PL310_POWER_CTRL		0xF60
+#define		POWER_CTRL_ENABLE_GATING	(1 << 0)
+#define		POWER_CTRL_ENABLE_STANDBY	(1 << 1)
 
 struct pl310_softc {
 	device_t	sc_dev;
@@ -162,6 +175,8 @@ pl310_write4(struct pl310_softc *sc, bus_size_t off, uint32_t val)
 }
 
 void pl310_print_config(struct pl310_softc *sc);
+void pl310_set_ram_latency(struct pl310_softc *sc, uint32_t which_reg,
+    uint32_t read, uint32_t write, uint32_t setup);
 
 void platform_pl310_init(struct pl310_softc *);
 void platform_pl310_write_ctrl(struct pl310_softc *, uint32_t);