vnc: tight: add JPEG and gradient subencoding with smooth image detection

Add gradient filter and JPEG compression with an heuristic to detect how
lossy the comppression will be. This code has been adapted from
libvncserver/tight.c.

JPEG support can be enabled/disabled at compile time with --enable-vnc-jpeg
and --disable-vnc-jpeg.

Signed-off-by: Corentin Chary <corentincj@iksaif.net>
Signed-off-by: Anthony Liguori <aliguori@us.ibm.com>

Makefile.target

@@ -177,6 +177,7 @@ LIBS+=-lz
 
 QEMU_CFLAGS += $(VNC_TLS_CFLAGS)
 QEMU_CFLAGS += $(VNC_SASL_CFLAGS)
+QEMU_CFLAGS += $(VNC_JPEG_CFLAGS)
 
 # xen backend driver support
 obj-$(CONFIG_XEN) += xen_machine_pv.o xen_domainbuild.o

configure

@@ -268,6 +268,7 @@ uuid=""
 vde=""
 vnc_tls=""
 vnc_sasl=""
+vnc_jpeg=""
 xen=""
 linux_aio=""
 attr=""
@@ -575,6 +576,10 @@ for opt do
   ;;
   --enable-vnc-sasl) vnc_sasl="yes"
   ;;
+  --disable-vnc-jpeg) vnc_jpeg="no"
+  ;;
+  --enable-vnc-jpeg) vnc_jpeg="yes"
+  ;;
   --disable-slirp) slirp="no"
   ;;
   --disable-uuid) uuid="no"
@@ -825,6 +830,8 @@ echo "  --disable-vnc-tls        disable TLS encryption for VNC server"
 echo "  --enable-vnc-tls         enable TLS encryption for VNC server"
 echo "  --disable-vnc-sasl       disable SASL encryption for VNC server"
 echo "  --enable-vnc-sasl        enable SASL encryption for VNC server"
+echo "  --disable-vnc-jpeg       disable JPEG lossy compression for VNC server"
+echo "  --enable-vnc-jpeg        enable JPEG lossy compression for VNC server"
 echo "  --disable-curses         disable curses output"
 echo "  --enable-curses          enable curses output"
 echo "  --disable-curl           disable curl connectivity"
@@ -1245,6 +1252,27 @@ EOF
   fi
 fi
 
+##########################################
+# VNC JPEG detection
+if test "$vnc_jpeg" = "yes" ; then
+cat > $TMPC <<EOF
+#include <stdio.h>
+#include <jpeglib.h>
+int main(void) { struct jpeg_compress_struct s; jpeg_create_compress(&s); return 0; }
+EOF
+    vnc_jpeg_cflags=""
+    vnc_jpeg_libs="-ljpeg"
+  if compile_prog "$vnc_jpeg_cflags" "$vnc_jpeg_libs" ; then
+    vnc_jpeg=yes
+    libs_softmmu="$vnc_jpeg_libs $libs_softmmu"
+  else
+    if test "$vnc_jpeg" = "yes" ; then
+      feature_not_found "vnc-jpeg"
+    fi
+    vnc_jpeg=no
+  fi
+fi
+
 ##########################################
 # fnmatch() probe, used for ACL routines
 fnmatch="no"
@@ -2094,6 +2122,7 @@ echo "Block whitelist   $block_drv_whitelist"
 echo "Mixer emulation   $mixemu"
 echo "VNC TLS support   $vnc_tls"
 echo "VNC SASL support  $vnc_sasl"
+echo "VNC JPEG support  $vnc_jpeg"
 if test -n "$sparc_cpu"; then
     echo "Target Sparc Arch $sparc_cpu"
 fi
@@ -2231,6 +2260,10 @@ if test "$vnc_sasl" = "yes" ; then
   echo "CONFIG_VNC_SASL=y" >> $config_host_mak
   echo "VNC_SASL_CFLAGS=$vnc_sasl_cflags" >> $config_host_mak
 fi
+if test "$vnc_jpeg" = "yes" ; then
+  echo "CONFIG_VNC_JPEG=y" >> $config_host_mak
+  echo "VNC_JPEG_CFLAGS=$vnc_jpeg_cflags" >> $config_host_mak
+fi
 if test "$fnmatch" = "yes" ; then
   echo "CONFIG_FNMATCH=y" >> $config_host_mak
 fi

vnc-encoding-tight.h

@@ -173,4 +173,9 @@
 #define VNC_TIGHT_MIN_SOLID_SUBRECT_SIZE  2048
 #define VNC_TIGHT_MAX_SPLIT_TILE_SIZE       16
 
+#define VNC_TIGHT_JPEG_MIN_RECT_SIZE      4096
+#define VNC_TIGHT_DETECT_SUBROW_WIDTH        7
+#define VNC_TIGHT_DETECT_MIN_WIDTH           8
+#define VNC_TIGHT_DETECT_MIN_HEIGHT          8
+
 #endif /* VNC_ENCODING_TIGHT_H */

vnc.h

@@ -176,6 +176,10 @@ struct VncState
     Buffer tight;
     Buffer tight_tmp;
     Buffer tight_zlib;
+    Buffer tight_gradient;
+#ifdef CONFIG_VNC_JPEG
+    Buffer tight_jpeg;
+#endif
     int tight_levels[4];
     z_stream tight_stream[4];