From 613fd51f26315d8212662c46f8e625c541e4bb59 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Lureau?= Date: Wed, 5 Mar 2025 15:10:02 +0400 Subject: [PATCH] feat: add QOI image codec MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Quite OK Image format ([1]) losslessly compresses images to a similar size of PNG, while offering 20x-50x faster encoding and 3x-4x faster decoding. Add a new QOI codec (UUID 4dae9af8-b399-4df6-b43a-662fd9c0f5d6) for SetSurface command. The PDU data contains the QOI header (14 bytes) + data "chunks" and the end marker (8 bytes). Some benchmarks showing interesting results (using ironrdp/perfenc) Bitmap: 74s user CPU, 92.5% compression RemoteFx (lossy): 201s user CPU, 96.72% compression QOI: 10s user CPU, 96.20% compression Note: the "qoicoubeh" crate is my own fork of "qoi-rust" project. The plan is to switch back to it as soon as the maintainer resume its activites (https://github.com/aldanor/qoi-rust/issues/14). [1]: https://qoiformat.org/ Signed-off-by: Marc-André Lureau --- Cargo.lock | 11 ++++ benches/Cargo.toml | 4 ++ benches/src/perfenc.rs | 8 ++- crates/ironrdp-client/Cargo.toml | 1 + crates/ironrdp-connector/Cargo.toml | 2 + crates/ironrdp-pdu/Cargo.toml | 1 + crates/ironrdp-pdu/src/rdp/capability_sets.rs | 2 +- .../src/rdp/capability_sets/bitmap_codecs.rs | 41 +++++++++++++ crates/ironrdp-server/Cargo.toml | 4 +- crates/ironrdp-server/src/encoder/mod.rs | 61 ++++++++++++++++++- crates/ironrdp-server/src/server.rs | 12 ++++ crates/ironrdp-session/Cargo.toml | 6 +- crates/ironrdp-session/src/fast_path.rs | 17 ++++++ crates/ironrdp-testsuite-core/Cargo.toml | 2 +- .../tests/session/mod.rs | 18 +++++- crates/ironrdp-web/Cargo.toml | 3 +- crates/ironrdp/Cargo.toml | 1 + 17 files changed, 184 insertions(+), 10 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 267f3bdf..12028e37 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2715,6 +2715,7 @@ dependencies = [ "ironrdp-rdpsnd", "ironrdp-svc", "ironrdp-tokio", + "qoicoubeh", "rayon", "rustls-pemfile", "tokio", @@ -2736,6 +2737,7 @@ dependencies = [ "ironrdp-graphics", "ironrdp-pdu", "ironrdp-svc", + "qoicoubeh", "tracing", ] @@ -4161,6 +4163,15 @@ dependencies = [ "unarray", ] +[[package]] +name = "qoicoubeh" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9b82aa3fef8a980075775b8c46f874823b5b4a15de327d2dbb3b6fd818480ba" +dependencies = [ + "bytemuck", +] + [[package]] name = "quick-error" version = "1.2.3" diff --git a/benches/Cargo.toml b/benches/Cargo.toml index 8124d337..c2cbf966 100644 --- a/benches/Cargo.toml +++ b/benches/Cargo.toml @@ -9,6 +9,10 @@ edition.workspace = true name = "perfenc" path = "src/perfenc.rs" +[features] +default = ["qoi"] +qoi = ["ironrdp/qoi"] + [dependencies] anyhow = "1.0.98" async-trait = "0.1.88" diff --git a/benches/src/perfenc.rs b/benches/src/perfenc.rs index 3f5a302f..6b57560c 100644 --- a/benches/src/perfenc.rs +++ b/benches/src/perfenc.rs @@ -28,7 +28,7 @@ async fn main() -> Result<(), anyhow::Error> { println!(" --width Width of the display (default: 3840)"); println!(" --height Height of the display (default: 2400)"); println!(" --codec Codec to use (default: remotefx)"); - println!(" Valid values: remotefx, bitmap, none"); + println!(" Valid values: qoi, remotefx, bitmap, none"); println!(" --fps Frames per second (default: none)"); std::process::exit(0); } @@ -52,6 +52,8 @@ async fn main() -> Result<(), anyhow::Error> { flags -= CmdFlags::SET_SURFACE_BITS; } OptCodec::None => {} + #[cfg(feature = "qoi")] + OptCodec::Qoi => update_codecs.set_qoi(Some(0)), }; let mut encoder = UpdateEncoder::new(DesktopSize { width, height }, flags, update_codecs); @@ -172,6 +174,8 @@ enum OptCodec { RemoteFX, Bitmap, None, + #[cfg(feature = "qoi")] + Qoi, } impl Default for OptCodec { @@ -188,6 +192,8 @@ impl core::str::FromStr for OptCodec { "remotefx" => Ok(Self::RemoteFX), "bitmap" => Ok(Self::Bitmap), "none" => Ok(Self::None), + #[cfg(feature = "qoi")] + "qoi" => Ok(Self::Qoi), _ => Err(anyhow::anyhow!("unknown codec: {}", s)), } } diff --git a/crates/ironrdp-client/Cargo.toml b/crates/ironrdp-client/Cargo.toml index a9cfc940..a09adfe3 100644 --- a/crates/ironrdp-client/Cargo.toml +++ b/crates/ironrdp-client/Cargo.toml @@ -27,6 +27,7 @@ test = false default = ["rustls"] rustls = ["ironrdp-tls/rustls", "tokio-tungstenite/rustls-tls-native-roots"] native-tls = ["ironrdp-tls/native-tls", "tokio-tungstenite/native-tls"] +qoi = ["ironrdp/qoi"] [dependencies] # Protocols diff --git a/crates/ironrdp-connector/Cargo.toml b/crates/ironrdp-connector/Cargo.toml index 906bb33e..3edb6990 100644 --- a/crates/ironrdp-connector/Cargo.toml +++ b/crates/ironrdp-connector/Cargo.toml @@ -16,7 +16,9 @@ doctest = false test = false [features] +default = [] arbitrary = ["dep:arbitrary"] +qoi = ["ironrdp-pdu/qoi"] [dependencies] ironrdp-svc = { path = "../ironrdp-svc", version = "0.4" } # public diff --git a/crates/ironrdp-pdu/Cargo.toml b/crates/ironrdp-pdu/Cargo.toml index 5946603c..a72336c7 100644 --- a/crates/ironrdp-pdu/Cargo.toml +++ b/crates/ironrdp-pdu/Cargo.toml @@ -19,6 +19,7 @@ doctest = false default = [] std = ["alloc", "ironrdp-error/std", "ironrdp-core/std"] alloc = ["ironrdp-core/alloc", "ironrdp-error/alloc"] +qoi = [] [dependencies] bitflags = "2.9" diff --git a/crates/ironrdp-pdu/src/rdp/capability_sets.rs b/crates/ironrdp-pdu/src/rdp/capability_sets.rs index 6dd189a0..cad2a83f 100644 --- a/crates/ironrdp-pdu/src/rdp/capability_sets.rs +++ b/crates/ironrdp-pdu/src/rdp/capability_sets.rs @@ -34,7 +34,7 @@ pub use self::bitmap_cache::{ pub use self::bitmap_codecs::{ client_codecs_capabilities, server_codecs_capabilities, BitmapCodecs, CaptureFlags, Codec, CodecId, CodecProperty, EntropyBits, Guid, NsCodec, RemoteFxContainer, RfxCaps, RfxCapset, RfxClientCapsContainer, RfxICap, RfxICapFlags, - CODEC_ID_NONE, CODEC_ID_REMOTEFX, + CODEC_ID_NONE, CODEC_ID_QOI, CODEC_ID_REMOTEFX, }; pub use self::brush::{Brush, SupportLevel}; pub use self::frame_acknowledge::FrameAcknowledge; diff --git a/crates/ironrdp-pdu/src/rdp/capability_sets/bitmap_codecs.rs b/crates/ironrdp-pdu/src/rdp/capability_sets/bitmap_codecs.rs index 01a7a0ee..807970ad 100644 --- a/crates/ironrdp-pdu/src/rdp/capability_sets/bitmap_codecs.rs +++ b/crates/ironrdp-pdu/src/rdp/capability_sets/bitmap_codecs.rs @@ -40,6 +40,9 @@ const GUID_REMOTEFX: Guid = Guid(0x7677_2f12, 0xbd72, 0x4463, 0xaf, 0xb3, 0xb7, const GUID_IMAGE_REMOTEFX: Guid = Guid(0x2744_ccd4, 0x9d8a, 0x4e74, 0x80, 0x3c, 0x0e, 0xcb, 0xee, 0xa1, 0x9c, 0x54); #[rustfmt::skip] const GUID_IGNORE: Guid = Guid(0x9c43_51a6, 0x3535, 0x42ae, 0x91, 0x0c, 0xcd, 0xfc, 0xe5, 0x76, 0x0b, 0x58); +#[rustfmt::skip] +#[cfg(feature="qoi")] +const GUID_QOI: Guid = Guid(0x4dae_9af8, 0xb399, 0x4df6, 0xb4, 0x3a, 0x66, 0x2f, 0xd9, 0xc0, 0xf5, 0xd6); #[derive(Debug, PartialEq, Eq)] pub struct Guid(u32, u16, u16, u8, u8, u8, u8, u8, u8, u8, u8); @@ -167,6 +170,8 @@ impl Encode for Codec { CodecProperty::RemoteFx(_) => GUID_REMOTEFX, CodecProperty::ImageRemoteFx(_) => GUID_IMAGE_REMOTEFX, CodecProperty::Ignore => GUID_IGNORE, + #[cfg(feature = "qoi")] + CodecProperty::Qoi => GUID_QOI, _ => return Err(other_err!("invalid codec")), }; guid.encode(dst)?; @@ -204,6 +209,8 @@ impl Encode for Codec { } }; } + #[cfg(feature = "qoi")] + CodecProperty::Qoi => dst.write_u16(0), CodecProperty::Ignore => dst.write_u16(0), CodecProperty::None => dst.write_u16(0), }; @@ -227,6 +234,8 @@ impl Encode for Codec { RemoteFxContainer::ClientContainer(container) => container.size(), RemoteFxContainer::ServerContainer(size) => *size, }, + #[cfg(feature = "qoi")] + CodecProperty::Qoi => 0, CodecProperty::Ignore => 0, CodecProperty::None => 0, } @@ -264,6 +273,13 @@ impl<'de> Decode<'de> for Codec { } } GUID_IGNORE => CodecProperty::Ignore, + #[cfg(feature = "qoi")] + GUID_QOI => { + if !property_buffer.is_empty() { + return Err(invalid_field_err!("qoi property", "must be empty")); + } + CodecProperty::Qoi + } _ => CodecProperty::None, }; @@ -283,6 +299,8 @@ pub enum CodecProperty { RemoteFx(RemoteFxContainer), ImageRemoteFx(RemoteFxContainer), Ignore, + #[cfg(feature = "qoi")] + Qoi, None, } @@ -620,12 +638,14 @@ pub struct CodecId(u8); pub const CODEC_ID_NONE: CodecId = CodecId(0); pub const CODEC_ID_REMOTEFX: CodecId = CodecId(3); +pub const CODEC_ID_QOI: CodecId = CodecId(0x0A); impl Debug for CodecId { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { let name = match self.0 { 0 => "None", 3 => "RemoteFx", + 0x0A => "QOI", _ => "unknown", }; write!(f, "CodecId({name})") @@ -637,6 +657,7 @@ impl CodecId { match value { 0 => Some(CODEC_ID_NONE), 3 => Some(CODEC_ID_REMOTEFX), + 0x0A => Some(CODEC_ID_QOI), _ => None, } } @@ -678,6 +699,7 @@ fn parse_codecs_config<'a>(codecs: &'a [&'a str]) -> Result Result>().join(", "); if !codec_names.is_empty() { return Err(format!("Unknown codecs: {codec_names}")); @@ -728,6 +759,7 @@ List of codecs: /// # List of codecs /// /// * `remotefx` (on by default) +/// * `qoi` (on by default, when feature "qoi") /// /// # Returns /// @@ -738,6 +770,7 @@ pub fn server_codecs_capabilities(config: &[&str]) -> Result>().join(", "); if !codec_names.is_empty() { return Err(format!("Unknown codecs: {codec_names}")); diff --git a/crates/ironrdp-server/Cargo.toml b/crates/ironrdp-server/Cargo.toml index 4910bc44..ec58b869 100644 --- a/crates/ironrdp-server/Cargo.toml +++ b/crates/ironrdp-server/Cargo.toml @@ -16,9 +16,10 @@ doctest = true test = false [features] -default = ["rayon"] +default = ["rayon", "qoi"] helper = ["dep:x509-cert", "dep:rustls-pemfile"] rayon = ["dep:rayon"] +qoi = ["dep:qoicoubeh", "ironrdp-pdu/qoi"] # Internal (PRIVATE!) features used to aid testing. # Don't rely on these whatsoever. They may disappear at any time. @@ -47,6 +48,7 @@ rustls-pemfile = { version = "2.2.0", optional = true } rayon = { version = "1.10.0", optional = true } bytes = "1" visibility = { version = "0.1", optional = true } +qoicoubeh = { version = "0.5", optional = true } [dev-dependencies] tokio = { version = "1", features = ["sync"] } diff --git a/crates/ironrdp-server/src/encoder/mod.rs b/crates/ironrdp-server/src/encoder/mod.rs index 1bd102ad..8388f5b4 100644 --- a/crates/ironrdp-server/src/encoder/mod.rs +++ b/crates/ironrdp-server/src/encoder/mod.rs @@ -33,18 +33,30 @@ enum CodecId { #[derive(Debug)] pub(crate) struct UpdateEncoderCodecs { remotefx: Option<(EntropyBits, u8)>, + #[cfg(feature = "qoi")] + qoi: Option, } impl UpdateEncoderCodecs { #[cfg_attr(feature = "__bench", visibility::make(pub))] pub(crate) fn new() -> Self { - Self { remotefx: None } + Self { + remotefx: None, + #[cfg(feature = "qoi")] + qoi: None, + } } #[cfg_attr(feature = "__bench", visibility::make(pub))] pub(crate) fn set_remotefx(&mut self, remotefx: Option<(EntropyBits, u8)>) { self.remotefx = remotefx } + + #[cfg(feature = "qoi")] + #[cfg_attr(feature = "__bench", visibility::make(pub))] + pub(crate) fn set_qoi(&mut self, qoi: Option) { + self.qoi = qoi + } } impl Default for UpdateEncoderCodecs { @@ -78,6 +90,11 @@ impl UpdateEncoder { bitmap = BitmapUpdater::RemoteFx(RemoteFxHandler::new(algo, id, desktop_size)); } + #[cfg(feature = "qoi")] + if let Some(id) = codecs.qoi { + bitmap = BitmapUpdater::Qoi(QoiHandler::new(id)); + } + bitmap } else { BitmapUpdater::Bitmap(BitmapHandler::new()) @@ -287,6 +304,8 @@ enum BitmapUpdater { None(NoneHandler), Bitmap(BitmapHandler), RemoteFx(RemoteFxHandler), + #[cfg(feature = "qoi")] + Qoi(QoiHandler), } impl BitmapUpdater { @@ -295,6 +314,8 @@ impl BitmapUpdater { Self::None(up) => up.handle(bitmap), Self::Bitmap(up) => up.handle(bitmap), Self::RemoteFx(up) => up.handle(bitmap), + #[cfg(feature = "qoi")] + Self::Qoi(up) => up.handle(bitmap), } } @@ -408,6 +429,44 @@ impl BitmapUpdateHandler for RemoteFxHandler { } } +#[cfg(feature = "qoi")] +#[derive(Clone, Debug)] +struct QoiHandler { + codec_id: u8, +} + +#[cfg(feature = "qoi")] +impl QoiHandler { + fn new(codec_id: u8) -> Self { + Self { codec_id } + } +} + +#[cfg(feature = "qoi")] +impl BitmapUpdateHandler for QoiHandler { + fn handle(&mut self, bitmap: &BitmapUpdate) -> Result { + use ironrdp_graphics::image_processing::PixelFormat::*; + + let raw_channels = match bitmap.format { + ARgb32 => qoi::RawChannels::Argb, + XRgb32 => qoi::RawChannels::Xrgb, + ABgr32 => qoi::RawChannels::Abgr, + XBgr32 => qoi::RawChannels::Xbgr, + BgrA32 => qoi::RawChannels::Bgra, + BgrX32 => qoi::RawChannels::Bgrx, + RgbA32 => qoi::RawChannels::Rgba, + RgbX32 => qoi::RawChannels::Rgbx, + }; + + let enc = qoi::EncoderBuilder::new(&bitmap.data, bitmap.width.get().into(), bitmap.height.get().into()) + .stride(bitmap.stride) + .raw_channels(raw_channels) + .build()?; + let data = enc.encode_to_vec()?; + set_surface(bitmap, self.codec_id, &data) + } +} + fn set_surface(bitmap: &BitmapUpdate, codec_id: u8, data: &[u8]) -> Result { let destination = ExclusiveRectangle { left: bitmap.x, diff --git a/crates/ironrdp-server/src/server.rs b/crates/ironrdp-server/src/server.rs index 0acb7603..9e43c537 100644 --- a/crates/ironrdp-server/src/server.rs +++ b/crates/ironrdp-server/src/server.rs @@ -55,6 +55,14 @@ impl RdpServerOptions { .iter() .any(|codec| matches!(codec.property, CodecProperty::RemoteFx(_))) } + + #[cfg(feature = "qoi")] + fn has_qoi(&self) -> bool { + self.codecs + .0 + .iter() + .any(|codec| matches!(codec.property, CodecProperty::Qoi)) + } } #[derive(Clone)] @@ -742,6 +750,10 @@ impl RdpServer { } } CodecProperty::NsCodec(_) => (), + #[cfg(feature = "qoi")] + CodecProperty::Qoi if self.opts.has_qoi() => { + update_codecs.set_qoi(Some(codec.id)); + } _ => (), } } diff --git a/crates/ironrdp-session/Cargo.toml b/crates/ironrdp-session/Cargo.toml index cd768204..93eee93d 100644 --- a/crates/ironrdp-session/Cargo.toml +++ b/crates/ironrdp-session/Cargo.toml @@ -15,6 +15,10 @@ categories.workspace = true doctest = false test = false +[features] +default = [] +qoi = ["dep:qoicoubeh", "ironrdp-pdu/qoi"] + [dependencies] ironrdp-core = { path = "../ironrdp-core", version = "0.1" } # public ironrdp-connector = { path = "../ironrdp-connector", version = "0.6" } # public # TODO: at some point, this dependency could be removed (good for compilation speed) @@ -25,7 +29,7 @@ ironrdp-graphics = { path = "../ironrdp-graphics", version = "0.4" } # public ironrdp-pdu = { path = "../ironrdp-pdu", version = "0.5", features = ["std"] } # public ironrdp-displaycontrol = { path = "../ironrdp-displaycontrol", version = "0.3" } tracing = { version = "0.1", features = ["log"] } +qoicoubeh = { version = "0.5", optional = true } [lints] workspace = true - diff --git a/crates/ironrdp-session/src/fast_path.rs b/crates/ironrdp-session/src/fast_path.rs index 59db1326..db34614e 100644 --- a/crates/ironrdp-session/src/fast_path.rs +++ b/crates/ironrdp-session/src/fast_path.rs @@ -361,6 +361,23 @@ impl Processor { .or(Some(rectangle)); } } + #[cfg(feature = "qoi")] + ironrdp_pdu::rdp::capability_sets::CODEC_ID_QOI => { + let (header, decoded) = qoi::decode_to_vec(bits.extended_bitmap_data.data) + .map_err(|e| reason_err!("QOI decode", "{}", e))?; + match header.channels { + qoi::Channels::Rgb => { + let rectangle = image.apply_rgb24(&decoded, &destination, false)?; + + update_rectangle = update_rectangle + .map(|rect: InclusiveRectangle| rect.union(&rectangle)) + .or(Some(rectangle)); + } + qoi::Channels::Rgba => { + warn!("Unsupported RGBA QOI data"); + } + } + } _ => { warn!("Unsupported codec ID: {}", bits.extended_bitmap_data.codec_id); } diff --git a/crates/ironrdp-testsuite-core/Cargo.toml b/crates/ironrdp-testsuite-core/Cargo.toml index 5c1f3fe9..96b03393 100644 --- a/crates/ironrdp-testsuite-core/Cargo.toml +++ b/crates/ironrdp-testsuite-core/Cargo.toml @@ -44,7 +44,7 @@ ironrdp-graphics.path = "../ironrdp-graphics" ironrdp-input.path = "../ironrdp-input" ironrdp-rdcleanpath.path = "../ironrdp-rdcleanpath" ironrdp-rdpsnd.path = "../ironrdp-rdpsnd" -ironrdp-session.path = "../ironrdp-session" +ironrdp-session = { path = "../ironrdp-session", features = ["qoi"] } ironrdp-propertyset.path = "../ironrdp-propertyset" ironrdp-rdpfile.path = "../ironrdp-rdpfile" png = "0.17" diff --git a/crates/ironrdp-testsuite-core/tests/session/mod.rs b/crates/ironrdp-testsuite-core/tests/session/mod.rs index fd03457b..dda2a99f 100644 --- a/crates/ironrdp-testsuite-core/tests/session/mod.rs +++ b/crates/ironrdp-testsuite-core/tests/session/mod.rs @@ -14,11 +14,23 @@ mod tests { let config = &["remotefx:on"]; let capabilities = client_codecs_capabilities(config).unwrap(); - assert_eq!(capabilities.0.len(), 1); - assert!(matches!(capabilities.0[0].property, CodecProperty::RemoteFx(_))); + assert!(capabilities + .0 + .iter() + .any(|cap| matches!(cap.property, CodecProperty::RemoteFx(_)))); let config = &["remotefx:off"]; let capabilities = client_codecs_capabilities(config).unwrap(); - assert_eq!(capabilities.0.len(), 0); + assert!(!capabilities + .0 + .iter() + .any(|cap| matches!(cap.property, CodecProperty::RemoteFx(_)))); + + let config = &["qoi:on"]; + let capabilities = client_codecs_capabilities(config).unwrap(); + assert!(capabilities + .0 + .iter() + .any(|cap| matches!(cap.property, CodecProperty::Qoi))); } } diff --git a/crates/ironrdp-web/Cargo.toml b/crates/ironrdp-web/Cargo.toml index 56ce9f8b..f9002f77 100644 --- a/crates/ironrdp-web/Cargo.toml +++ b/crates/ironrdp-web/Cargo.toml @@ -20,6 +20,7 @@ crate-type = ["cdylib", "rlib"] [features] default = ["panic_hook"] panic_hook = ["iron-remote-desktop/panic_hook"] +qoi = ["ironrdp/qoi"] [dependencies] # Protocols @@ -31,7 +32,7 @@ ironrdp = { path = "../ironrdp", features = [ "dvc", "cliprdr", "svc", - "displaycontrol" + "displaycontrol", ] } ironrdp-core.path = "../ironrdp-core" ironrdp-cliprdr-format.path = "../ironrdp-cliprdr-format" diff --git a/crates/ironrdp/Cargo.toml b/crates/ironrdp/Cargo.toml index e44d44e6..4ab878d6 100644 --- a/crates/ironrdp/Cargo.toml +++ b/crates/ironrdp/Cargo.toml @@ -31,6 +31,7 @@ dvc = ["dep:ironrdp-dvc"] rdpdr = ["dep:ironrdp-rdpdr"] rdpsnd = ["dep:ironrdp-rdpsnd"] displaycontrol = ["dep:ironrdp-displaycontrol"] +qoi = ["ironrdp-server?/qoi", "ironrdp-pdu?/qoi", "ironrdp-connector?/qoi", "ironrdp-session?/qoi"] # Internal (PRIVATE!) features used to aid testing. # Don't rely on these whatsoever. They may disappear at any time. __bench = ["ironrdp-server/__bench"]