vt(4): Fix cursor handling in vt_flush()

There were situations where the cursor was not erased/redrawn or its
position was marked as dirty even though it's not displayed. The code is
now more straightforward.

At the same, add a function to determine if the cursor covers a given
area. This is used by backends to know if they need to draw the cursor.

This new function should be paired with a new state in struct vt_device,
called vd_mshown, which indicates if the cursor should be displayed.
This again simplifies vd_bitblt_text_t callback's API.

MFC after:	1 week

3 files changed, 98 insertions, 56 deletions

diff --git a/sys/dev/vt/hw/vga/vt_vga.c b/sys/dev/vt/hw/vga/vt_vga.c
index 2e28f85..18f2464 100644
--- a/sys/dev/vt/hw/vga/vt_vga.c
+++ b/sys/dev/vt/hw/vga/vt_vga.c
@@ -521,8 +521,7 @@ vga_bitblt_pixels_block_ncolors(struct vt_device *vd, const uint8_t *masks,
 
 static void
 vga_bitblt_one_text_pixels_block(struct vt_device *vd,
-    const struct vt_window *vw, unsigned int x, unsigned int y,
-    int cursor_displayed)
+    const struct vt_window *vw, unsigned int x, unsigned int y)
 {
 	const struct vt_buf *vb;
 	const struct vt_font *vf;
@@ -533,10 +532,6 @@ vga_bitblt_one_text_pixels_block(struct vt_device *vd,
 	term_char_t c;
 	term_color_t fg, bg;
 	const uint8_t *src;
-#ifndef SC_NO_CUTPASTE
-	struct vt_mouse_cursor *cursor;
-	unsigned int mx, my;
-#endif
 
 	vb = &vw->vw_buf;
 	vf = vw->vw_font;
@@ -631,16 +626,21 @@ vga_bitblt_one_text_pixels_block(struct vt_device *vd,
 	 * the current position could be different than the one used
 	 * to mark the area dirty.
 	 */
-	cursor = vd->vd_mcursor;
-	mx = vd->vd_moldx + vw->vw_offset.tp_col;
-	my = vd->vd_moldy + vw->vw_offset.tp_row;
-	if (cursor_displayed &&
-	    ((mx >= x && x + VT_VGA_PIXELS_BLOCK - 1 >= mx) ||
-	     (mx < x && mx + cursor->width >= x)) &&
-	    ((my >= y && y + vf->vf_height - 1 >= my) ||
-	     (my < y && my + cursor->height >= y))) {
+	term_rect_t drawn_area;
+
+	drawn_area.tr_begin.tp_col = x;
+	drawn_area.tr_begin.tp_row = y;
+	drawn_area.tr_end.tp_col = x + VT_VGA_PIXELS_BLOCK;
+	drawn_area.tr_end.tp_row = y + vf->vf_height;
+	if (vd->vd_mshown && vt_is_cursor_in_area(vd, &drawn_area)) {
+		struct vt_mouse_cursor *cursor;
+		unsigned int mx, my;
 		unsigned int dst_x, src_y, dst_y, y_count;
 
+		cursor = vd->vd_mcursor;
+		mx = vd->vd_mx_drawn + vw->vw_offset.tp_col;
+		my = vd->vd_my_drawn + vw->vw_offset.tp_row;
+
 		/* Compute the portion of the cursor we want to copy. */
 		src_x = x > mx ? x - mx : 0;
 		dst_x = mx > x ? mx - x : 0;
@@ -686,7 +686,7 @@ vga_bitblt_one_text_pixels_block(struct vt_device *vd,
 
 static void
 vga_bitblt_text_gfxmode(struct vt_device *vd, const struct vt_window *vw,
-    const term_rect_t *area, int cursor_displayed)
+    const term_rect_t *area)
 {
 	const struct vt_font *vf;
 	unsigned int col, row;
@@ -741,7 +741,11 @@ vga_bitblt_text_gfxmode(struct vt_device *vd, const struct vt_window *vw,
 	    * VT_VGA_PIXELS_BLOCK;
 	y2 = row * vf->vf_height + vw->vw_offset.tp_row;
 
-	/* Clip the area to the screen size. */
+	/*
+	 * Clip the area to the screen size.
+	 *
+	 * FIXME: Take vw_offset into account.
+	 */
 	x2 = min(x2, vd->vd_width - 1);
 	y2 = min(y2, vd->vd_height - 1);
 
@@ -762,15 +766,14 @@ vga_bitblt_text_gfxmode(struct vt_device *vd, const struct vt_window *vw,
 
 	for (y = y1; y < y2; y += vf->vf_height) {
 		for (x = x1; x < x2; x += VT_VGA_PIXELS_BLOCK) {
-			vga_bitblt_one_text_pixels_block(vd, vw, x, y,
-			    cursor_displayed);
+			vga_bitblt_one_text_pixels_block(vd, vw, x, y);
 		}
 	}
 }
 
 static void
 vga_bitblt_text_txtmode(struct vt_device *vd, const struct vt_window *vw,
-    const term_rect_t *area, int cursor_displayed)
+    const term_rect_t *area)
 {
 	struct vga_softc *sc;
 	const struct vt_buf *vb;
@@ -814,13 +817,13 @@ vga_bitblt_text_txtmode(struct vt_device *vd, const struct vt_window *vw,
 
 static void
 vga_bitblt_text(struct vt_device *vd, const struct vt_window *vw,
-    const term_rect_t *area, int cursor_displayed)
+    const term_rect_t *area)
 {
 
 	if (!(vd->vd_flags & VDF_TEXTMODE)) {
-		vga_bitblt_text_gfxmode(vd, vw, area, cursor_displayed);
+		vga_bitblt_text_gfxmode(vd, vw, area);
 	} else {
-		vga_bitblt_text_txtmode(vd, vw, area, cursor_displayed);
+		vga_bitblt_text_txtmode(vd, vw, area);
 	}
 }
 
diff --git a/sys/dev/vt/vt.h b/sys/dev/vt/vt.h
index 0f1f0b0..e179722 100644
--- a/sys/dev/vt/vt.h
+++ b/sys/dev/vt/vt.h
@@ -130,11 +130,12 @@ struct vt_device {
 	struct vt_mouse_cursor	*vd_mcursor;	/* (?) Cursor bitmap. */
 	term_color_t		 vd_mcursor_fg;	/* (?) Cursor fg color. */
 	term_color_t		 vd_mcursor_bg;	/* (?) Cursor bg color. */
-#endif
+	vt_axis_t		 vd_mx_drawn;	/* (?) Mouse X and Y      */
+	vt_axis_t		 vd_my_drawn;	/*     as of last redraw. */
+	int			 vd_mshown;	/* (?) Mouse shown during */
+#endif						/*     last redrawn.      */
 	uint16_t		 vd_mx;		/* (?) Current mouse X. */
 	uint16_t		 vd_my;		/* (?) current mouse Y. */
-	vt_axis_t		 vd_moldx;	/* (?) Mouse X as of last redraw. */
-	vt_axis_t		 vd_moldy;	/* (?) Mouse Y as of last redraw. */
 	uint32_t		 vd_mstate;	/* (?) Mouse state. */
 	vt_axis_t		 vd_width;	/* (?) Screen width. */
 	vt_axis_t		 vd_height;	/* (?) Screen height. */
@@ -303,7 +304,7 @@ typedef void vd_bitbltchr_t(struct vt_device *vd, const uint8_t *src,
 typedef void vd_putchar_t(struct vt_device *vd, term_char_t,
     vt_axis_t top, vt_axis_t left, term_color_t fg, term_color_t bg);
 typedef void vd_bitblt_text_t(struct vt_device *vd, const struct vt_window *vw,
-    const term_rect_t *area, int cursor_displayed);
+    const term_rect_t *area);
 typedef int vd_fb_ioctl_t(struct vt_device *, u_long, caddr_t, struct thread *);
 typedef int vd_fb_mmap_t(struct vt_device *, vm_ooffset_t, vm_paddr_t *, int,
     vm_memattr_t *);
@@ -415,6 +416,8 @@ void vt_mouse_state(int show);
 /* Utilities. */
 void	vt_determine_colors(term_char_t c, int cursor,
 	    term_color_t *fg, term_color_t *bg);
+int	vt_is_cursor_in_area(const struct vt_device *vd,
+	    const term_rect_t *area);
 
 #endif /* !_DEV_VT_VT_H_ */
 
diff --git a/sys/dev/vt/vt_core.c b/sys/dev/vt/vt_core.c
index 66f41c3..0c40e63 100644
--- a/sys/dev/vt/vt_core.c
+++ b/sys/dev/vt/vt_core.c
@@ -819,16 +819,46 @@ vt_determine_colors(term_char_t c, int cursor,
 }
 
 #ifndef SC_NO_CUTPASTE
+int
+vt_is_cursor_in_area(const struct vt_device *vd, const term_rect_t *area)
+{
+	unsigned int mx, my, x1, y1, x2, y2;
+
+	/*
+	 * We use the cursor position saved during the current refresh,
+	 * in case the cursor moved since.
+	 */
+	mx = vd->vd_mx_drawn;
+	my = vd->vd_my_drawn;
+
+	x1 = area->tr_begin.tp_col;
+	y1 = area->tr_begin.tp_row;
+	x2 = area->tr_end.tp_col;
+	y2 = area->tr_end.tp_row;
+
+	if (((mx >= x1 && x2 - 1 >= mx) ||
+	     (mx < x1 && mx + vd->vd_mcursor->width >= x1)) &&
+	    ((my >= y1 && y2 - 1 >= my) ||
+	     (my < y1 && my + vd->vd_mcursor->height >= y1)))
+		return (1);
+
+	return (0);
+}
+
 static void
-vt_mark_mouse_position_as_dirty(struct vt_device *vd, int x, int y)
+vt_mark_mouse_position_as_dirty(struct vt_device *vd)
 {
 	term_rect_t area;
 	struct vt_window *vw;
 	struct vt_font *vf;
+	int x, y;
 
 	vw = vd->vd_curwindow;
 	vf = vw->vw_font;
 
+	x = vd->vd_mx_drawn;
+	y = vd->vd_my_drawn;
+
 	if (vf != NULL) {
 		area.tr_begin.tp_col = (x - vw->vw_offset.tp_col) /
 		    vf->vf_width;
@@ -897,9 +927,8 @@ vt_flush(struct vt_device *vd)
 	term_rect_t tarea;
 	term_pos_t size;
 	term_char_t *r;
-	int cursor_displayed;
 #ifndef SC_NO_CUTPASTE
-	int bpl, h, w;
+	int cursor_was_shown, cursor_moved, bpl, h, w;
 #endif
 
 	vw = vd->vd_curwindow;
@@ -913,34 +942,42 @@ vt_flush(struct vt_device *vd)
 	if (((vd->vd_flags & VDF_TEXTMODE) == 0) && (vf == NULL))
 		return;
 
-	cursor_displayed = 0;
-
 #ifndef SC_NO_CUTPASTE
+	cursor_was_shown = vd->vd_mshown;
+	cursor_moved = (vd->vd_mx != vd->vd_mx_drawn ||
+	    vd->vd_my != vd->vd_my_drawn);
+
+	/* Check if the cursor should be displayed or not. */
 	if ((vd->vd_flags & VDF_MOUSECURSOR) && /* Mouse support enabled. */
-	    !(vw->vw_flags & VWF_MOUSE_HIDE)) { /* Cursor displayed.      */
-		if (vd->vd_moldx != vd->vd_mx ||
-		    vd->vd_moldy != vd->vd_my) {
-			/* Mark last mouse position as dirty to erase. */
-			vt_mark_mouse_position_as_dirty(vd,
-			    vd->vd_moldx, vd->vd_moldy);
+	    !(vw->vw_flags & VWF_MOUSE_HIDE) && /* Cursor displayed.      */
+	    !kdb_active && panicstr == NULL) {  /* DDB inactive.          */
+		vd->vd_mshown = 1;
+	} else {
+		vd->vd_mshown = 0;
+	}
 
-			/*
-			 * Save point of last mouse cursor to erase it
-			 * later.
-			 */
-			vd->vd_moldx = vd->vd_mx;
-			vd->vd_moldy = vd->vd_my;
-		}
+	/*
+	 * If the cursor changed display state or moved, we must mark
+	 * the old position as dirty, so that it's erased.
+	 */
+	if (cursor_was_shown != vd->vd_mshown ||
+	    (vd->vd_mshown && cursor_moved))
+		vt_mark_mouse_position_as_dirty(vd);
 
-		if (!kdb_active && panicstr == NULL) {
-			/* Mouse enabled, and DDB isn't active. */
-			cursor_displayed = 1;
+	/*
+         * Save position of the mouse cursor. It's used by backends to
+         * know where to draw the cursor and during the next refresh to
+         * erase the previous position.
+	 */
+	vd->vd_mx_drawn = vd->vd_mx;
+	vd->vd_my_drawn = vd->vd_my;
 
-			/* Mark new mouse position as dirty. */
-			vt_mark_mouse_position_as_dirty(vd,
-			    vd->vd_mx, vd->vd_my);
-		}
-	}
+	/*
+	 * If the cursor is displayed and has moved since last refresh,
+	 * mark the new position as dirty.
+	 */
+	if (vd->vd_mshown && cursor_moved)
+		vt_mark_mouse_position_as_dirty(vd);
 #endif
 
 	vtbuf_undirty(&vw->vw_buf, &tarea, &tmask);
@@ -957,8 +994,7 @@ vt_flush(struct vt_device *vd)
 
 	if (vd->vd_driver->vd_bitblt_text != NULL) {
 		if (tarea.tr_begin.tp_col < tarea.tr_end.tp_col) {
-			vd->vd_driver->vd_bitblt_text(vd, vw, &tarea,
-			    cursor_displayed);
+			vd->vd_driver->vd_bitblt_text(vd, vw, &tarea);
 		}
 	} else {
 		/*
@@ -980,7 +1016,7 @@ vt_flush(struct vt_device *vd)
 		}
 
 #ifndef SC_NO_CUTPASTE
-		if (cursor_displayed) {
+		if (vd->vd_mshown) {
 			/* Bytes per source line. */
 			bpl = (vd->vd_mcursor->width + 7) >> 3;
 			w = vd->vd_mcursor->width;
@@ -1640,7 +1676,7 @@ vt_mouse_state(int show)
 	}
 
 	/* Mark mouse position as dirty. */
-	vt_mark_mouse_position_as_dirty(vd, vd->vd_mx, vd->vd_my);
+	vt_mark_mouse_position_as_dirty(vd);
 }
 #endif