vnc: return the number of rectangles

Some encodings like tight supports tiling (spliting in
multiple sub-rectangles). So we needed a way to tell
vnc_update_client() how much rectangles are in the buffer.

zlib, raw and hextile always send a full rectangle.

Signed-off-by: Corentin Chary <corentincj@iksaif.net>
Signed-off-by: Anthony Liguori <aliguori@us.ibm.com>

4 files changed, 26 insertions, 16 deletions

diff --git a/vnc-encoding-hextile.c b/vnc-encoding-hextile.c
index a01c5e2..728f25e 100644
--- a/vnc-encoding-hextile.c
+++ b/vnc-encoding-hextile.c
@@ -62,8 +62,8 @@ static void hextile_enc_cord(uint8_t *ptr, int x, int y, int w, int h)
 #undef BPP
 #undef GENERIC
 
-void vnc_hextile_send_framebuffer_update(VncState *vs, int x,
-                                         int y, int w, int h)
+int vnc_hextile_send_framebuffer_update(VncState *vs, int x,
+                                        int y, int w, int h)
 {
     int i, j;
     int has_fg, has_bg;
@@ -83,6 +83,7 @@ void vnc_hextile_send_framebuffer_update(VncState *vs, int x,
     free(last_fg);
     free(last_bg);
 
+    return 1;
 }
 
 void vnc_hextile_set_pixel_conversion(VncState *vs, int generic)
diff --git a/vnc-encoding-zlib.c b/vnc-encoding-zlib.c
index 1d4dd1a..88ac863 100644
--- a/vnc-encoding-zlib.c
+++ b/vnc-encoding-zlib.c
@@ -116,7 +116,7 @@ static int vnc_zlib_stop(VncState *vs)
     return zstream->total_out - previous_out;
 }
 
-void vnc_zlib_send_framebuffer_update(VncState *vs, int x, int y, int w, int h)
+int vnc_zlib_send_framebuffer_update(VncState *vs, int x, int y, int w, int h)
 {
     int old_offset, new_offset, bytes_written;
 
@@ -132,13 +132,15 @@ void vnc_zlib_send_framebuffer_update(VncState *vs, int x, int y, int w, int h)
     bytes_written = vnc_zlib_stop(vs);
 
     if (bytes_written == -1)
-        return;
+        return 0;
 
     // hack in the size
     new_offset = vs->output.offset;
     vs->output.offset = old_offset;
     vnc_write_u32(vs, bytes_written);
     vs->output.offset = new_offset;
+
+    return 1;
 }
 
 void vnc_zlib_clear(VncState *vs)
diff --git a/vnc.c b/vnc.c
index f48f253..626f173 100644
--- a/vnc.c
+++ b/vnc.c
@@ -652,7 +652,7 @@ static void vnc_write_pixels_generic(VncState *vs, struct PixelFormat *pf,
     }
 }
 
-void vnc_raw_send_framebuffer_update(VncState *vs, int x, int y, int w, int h)
+int vnc_raw_send_framebuffer_update(VncState *vs, int x, int y, int w, int h)
 {
     int i;
     uint8_t *row;
@@ -663,23 +663,27 @@ void vnc_raw_send_framebuffer_update(VncState *vs, int x, int y, int w, int h)
         vs->write_pixels(vs, &vd->server->pf, row, w * ds_get_bytes_per_pixel(vs->ds));
         row += ds_get_linesize(vs->ds);
     }
+    return 1;
 }
 
-static void send_framebuffer_update(VncState *vs, int x, int y, int w, int h)
+static int send_framebuffer_update(VncState *vs, int x, int y, int w, int h)
 {
+    int n = 0;
+
     switch(vs->vnc_encoding) {
         case VNC_ENCODING_ZLIB:
-            vnc_zlib_send_framebuffer_update(vs, x, y, w, h);
+            n = vnc_zlib_send_framebuffer_update(vs, x, y, w, h);
             break;
         case VNC_ENCODING_HEXTILE:
             vnc_framebuffer_update(vs, x, y, w, h, VNC_ENCODING_HEXTILE);
-            vnc_hextile_send_framebuffer_update(vs, x, y, w, h);
+            n = vnc_hextile_send_framebuffer_update(vs, x, y, w, h);
             break;
         default:
             vnc_framebuffer_update(vs, x, y, w, h, VNC_ENCODING_RAW);
-            vnc_raw_send_framebuffer_update(vs, x, y, w, h);
+            n = vnc_raw_send_framebuffer_update(vs, x, y, w, h);
             break;
     }
+    return n;
 }
 
 static void vnc_copy(VncState *vs, int src_x, int src_y, int dst_x, int dst_y, int w, int h)
@@ -834,6 +838,7 @@ static int vnc_update_client(VncState *vs, int has_dirty)
         int y;
         int n_rectangles;
         int saved_offset;
+        int n;
 
         if (vs->output.offset && !vs->audio_cap && !vs->force_update)
             /* kernel send buffers are full -> drop frames to throttle */
@@ -866,16 +871,18 @@ static int vnc_update_client(VncState *vs, int has_dirty)
                 } else {
                     if (last_x != -1) {
                         int h = find_and_clear_dirty_height(vs, y, last_x, x);
-                        send_framebuffer_update(vs, last_x * 16, y, (x - last_x) * 16, h);
-                        n_rectangles++;
+                        n = send_framebuffer_update(vs, last_x * 16, y,
+                                                    (x - last_x) * 16, h);
+                        n_rectangles += n;
                     }
                     last_x = -1;
                 }
             }
             if (last_x != -1) {
                 int h = find_and_clear_dirty_height(vs, y, last_x, x);
-                send_framebuffer_update(vs, last_x * 16, y, (x - last_x) * 16, h);
-                n_rectangles++;
+                n = send_framebuffer_update(vs, last_x * 16, y,
+                                            (x - last_x) * 16, h);
+                n_rectangles += n;
             }
         }
         vs->output.buffer[saved_offset] = (n_rectangles >> 8) & 0xFF;
diff --git a/vnc.h b/vnc.h
index 2fbe614..cd6495f 100644
--- a/vnc.h
+++ b/vnc.h
@@ -398,13 +398,13 @@ void vnc_framebuffer_update(VncState *vs, int x, int y, int w, int h,
 void vnc_convert_pixel(VncState *vs, uint8_t *buf, uint32_t v);
 
 /* Encodings */
-void vnc_raw_send_framebuffer_update(VncState *vs, int x, int y, int w, int h);
+int vnc_raw_send_framebuffer_update(VncState *vs, int x, int y, int w, int h);
 
-void vnc_hextile_send_framebuffer_update(VncState *vs, int x,
+int vnc_hextile_send_framebuffer_update(VncState *vs, int x,
                                          int y, int w, int h);
 void vnc_hextile_set_pixel_conversion(VncState *vs, int generic);
 
-void vnc_zlib_send_framebuffer_update(VncState *vs, int x, int y, int w, int h);
+int vnc_zlib_send_framebuffer_update(VncState *vs, int x, int y, int w, int h);
 void vnc_zlib_clear(VncState *vs);
 
 #endif /* __QEMU_VNC_H */