[Xen-changelog] [qemu-xen stable-4.6] ui: vnc: avoid floating point exception

[patchbot@xxxxxxx]

commit b3286c9f3861eb2504f8bf461a721cfd22619a0c
Author:     Prasad J Pandit <pjp@xxxxxxxxxxxxxxxxx>
AuthorDate: Thu Dec 3 18:54:17 2015 +0530
Commit:     Stefano Stabellini <stefano.stabellini@xxxxxxxxxxxxx>
CommitDate: Fri Feb 5 12:43:51 2016 +0000

    ui: vnc: avoid floating point exception
    
    While sending 'SetPixelFormat' messages to a VNC server,
    the client could set the 'red-max', 'green-max' and 'blue-max'
    values to be zero. This leads to a floating point exception in
    write_png_palette while doing frame buffer updates.
    
    Reported-by: Lian Yihan <lianyihan@xxxxxx>
    Signed-off-by: Prasad J Pandit <pjp@xxxxxxxxxxxxxxxxx>
    Reviewed-by: Gerd Hoffmann <kraxel@xxxxxxxxxx>
    Signed-off-by: Peter Maydell <peter.maydell@xxxxxxxxxx>
---
 ui/vnc.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/ui/vnc.c b/ui/vnc.c
index db4d97e..76caa897 100644
--- a/ui/vnc.c
+++ b/ui/vnc.c
@@ -2036,15 +2036,15 @@ static void set_pixel_format(VncState *vs,
         return;
     }
 
-    vs->client_pf.rmax = red_max;
+    vs->client_pf.rmax = red_max ? red_max : 0xFF;
     vs->client_pf.rbits = hweight_long(red_max);
     vs->client_pf.rshift = red_shift;
     vs->client_pf.rmask = red_max << red_shift;
-    vs->client_pf.gmax = green_max;
+    vs->client_pf.gmax = green_max ? green_max : 0xFF;
     vs->client_pf.gbits = hweight_long(green_max);
     vs->client_pf.gshift = green_shift;
     vs->client_pf.gmask = green_max << green_shift;
-    vs->client_pf.bmax = blue_max;
+    vs->client_pf.bmax = blue_max ? blue_max : 0xFF;
     vs->client_pf.bbits = hweight_long(blue_max);
     vs->client_pf.bshift = blue_shift;
     vs->client_pf.bmask = blue_max << blue_shift;
--
generated by git-patchbot for /home/xen/git/qemu-xen.git#stable-4.6

_______________________________________________
Xen-changelog mailing list
Xen-changelog@xxxxxxxxxxxxx
http://lists.xensource.com/xen-changelog