Merge pull request #12 from Nitrokey/streaming

Implement large file streaming for certificates and other data objects

Cargo.toml

@@ -74,7 +74,7 @@ log-warn = []
 log-error = []
 
 [patch.crates-io]
-trussed = { git = "https://github.com/Nitrokey/trussed", tag = "v0.1.0-nitrokey.8"}
+trussed = { git = "https://github.com/Nitrokey/trussed", tag = "v0.1.0-nitrokey.9" }
 trussed-auth = { git = "https://github.com/trussed-dev/trussed-auth", tag = "v0.2.1"}
 littlefs2 = { git = "https://github.com/Nitrokey/littlefs2", tag = "v0.3.2-nitrokey-2" }
 

src/lib.rs

@@ -151,7 +151,7 @@ where
     ) -> Result {
         info!("PIV responding to {:02x?}", command);
         let parsed_command: Command = command.try_into()?;
-        info!("parsed: {:?}", &parsed_command);
+        info!("parsed: {:02x?}", &parsed_command);
         let reply = Reply(reply);
 
         match parsed_command {
@@ -907,15 +907,22 @@ impl<'a, T: trussed::Client + AuthClient + trussed::client::Ed255> LoadedAuthent
             _ => &[0x53],
         };
         reply.expand(tag)?;
-        let offset = reply.len();
         match container {
-            Container::KeyHistoryObject => self.get_key_history_object(reply.lend())?,
-            _ => match ContainerStorage(container).load(self.trussed, self.options.storage)? {
-                Some(data) => reply.expand(&data)?,
-                None => return Err(Status::NotFound),
-            },
+            Container::KeyHistoryObject => {
+                let offset = reply.len();
+                self.get_key_history_object(reply.lend())?;
+                reply.prepend_len(offset)?;
+            }
+            _ => {
+                if !ContainerStorage(container).load(
+                    self.trussed,
+                    self.options.storage,
+                    reply.lend(),
+                )? {
+                    return Err(Status::NotFound);
+                }
+            }
         }
-        reply.prepend_len(offset)?;
 
         Ok(())
     }

src/state.rs

@@ -8,15 +8,18 @@ use flexiber::EncodableHeapless;
 use heapless::Vec;
 use heapless_bytes::Bytes;
 use iso7816::Status;
+use trussed::types::OpenSeekFrom;
 use trussed::{
     api::reply::Metadata,
     config::MAX_MESSAGE_LENGTH,
     syscall, try_syscall,
     types::{KeyId, KeySerialization, Location, Mechanism, PathBuf, StorageAttributes},
+    utils,
 };
 use trussed_auth::AuthClient;
 
 use crate::piv_types::CardHolderUniqueIdentifier;
+use crate::reply::Reply;
 use crate::{constants::*, piv_types::AsymmetricAlgorithms};
 use crate::{
     container::{AsymmetricKeyReference, Container, ReadAccessRule, SecurityCondition},
@@ -617,6 +620,56 @@ fn load_if_exists(
     }
 }
 
+/// Returns false if the file does not exist
+fn load_if_exists_streaming<const R: usize>(
+    client: &mut impl trussed::Client,
+    location: Location,
+    path: &PathBuf,
+    mut buffer: Reply<'_, R>,
+) -> Result<bool, Status> {
+    let mut read_len = 0;
+    let file_len;
+    match try_syscall!(client.read_file_chunk(location, path.clone(), OpenSeekFrom::Start(0))) {
+        Ok(r) => {
+            read_len += r.data.len();
+            file_len = r.len;
+            buffer.append_len(file_len)?;
+            buffer.expand(&r.data)?;
+        }
+        Err(_) => match try_syscall!(client.entry_metadata(location, path.clone())) {
+            Ok(Metadata { metadata: None }) => return Ok(false),
+            Ok(Metadata {
+                metadata: Some(_metadata),
+            }) => {
+                error!("File {path} exists but couldn't be read: {_metadata:?}");
+                return Err(Status::UnspecifiedPersistentExecutionError);
+            }
+            Err(_err) => {
+                error!("File {path} couldn't be read: {_err:?}");
+                return Err(Status::UnspecifiedPersistentExecutionError);
+            }
+        },
+    }
+
+    while read_len < file_len {
+        match try_syscall!(client.read_file_chunk(
+            location,
+            path.clone(),
+            OpenSeekFrom::Start(read_len as u32)
+        )) {
+            Ok(r) => {
+                read_len += r.data.len();
+                buffer.expand(&r.data)?;
+            }
+            Err(_err) => {
+                error!("Failed to read chunk: {:?}", _err);
+            }
+        }
+    }
+
+    Ok(true)
+}
+
 #[derive(Clone, Copy, Debug)]
 pub struct ContainerStorage(pub Container);
 
@@ -703,13 +756,24 @@ impl ContainerStorage {
         }
     }
 
-    pub fn load(
+    // Write the length of the file and write
+    pub fn load<const R: usize>(
         self,
         client: &mut impl trussed::Client,
         storage: Location,
-    ) -> Result<Option<Bytes<MAX_MESSAGE_LENGTH>>, Status> {
-        load_if_exists(client, storage, &self.path())
-            .map(|data| data.or_else(|| self.default().map(Bytes::from)))
+        mut reply: Reply<'_, R>,
+    ) -> Result<bool, Status> {
+        if load_if_exists_streaming(client, storage, &self.path(), reply.lend())? {
+            return Ok(true);
+        }
+
+        if let Some(data) = self.default() {
+            reply.append_len(data.len())?;
+            reply.expand(&data)?;
+            Ok(true)
+        } else {
+            Ok(false)
+        }
     }
 
     pub fn save(
@@ -718,14 +782,9 @@ impl ContainerStorage {
         bytes: &[u8],
         storage: Location,
     ) -> Result<(), Status> {
-        let msg = Bytes::from(heapless::Vec::try_from(bytes).map_err(|_| {
-            error!("Buffer full");
-            Status::IncorrectDataParameter
-        })?);
-        try_syscall!(client.write_file(storage, self.path(), msg, None)).map_err(|_err| {
-            error!("Failed to store data: {_err:?}");
+        utils::write_all(client, storage, self.path(), bytes, None).map_err(|_err| {
+            error!("Failed to write data object: {:?}", _err);
             Status::UnspecifiedNonpersistentExecutionError
-        })?;
-        Ok(())
+        })
     }
 }

tests/pivy.rs

@@ -9,7 +9,7 @@ use card::with_vsc;
 
 use expectrl::{spawn, Eof, Regex, WaitStatus};
 
-use std::io::Write;
+use std::io::{Read, Write};
 use std::process::{Command, Stdio};
 
 #[test_log::test]
@@ -76,3 +76,74 @@ fn ecdh() {
         assert_eq!(p.wait().unwrap().code(), Some(0));
     });
 }
+
+const LARGE_CERT: &str = "-----BEGIN CERTIFICATE-----
+MIIHNTCCBh2gAwIBAgIUBeJLVUnOULY3fhLvjaWOZe/qWfYwDQYJKoZIhvcNAQEL
+BQAwggIoMQswCQYDVQQGEwJURTGBizCBiAYDVQQIDIGAVEVTVFRFU1RURVNUVEVT
+VFRFU1RURVNUVEVTVFRFU1RURVNUVEVTVFRFU1RURVNUVEVTVFRFU1RURVNUVEVT
+VFRFU1RURVNUVEVTVFRFU1RURVNUVEVTVFRFU1RURVNUVEVTVFRFU1RURVNUVEVT
+VFRFU1RURVNUVEVTVFRFU1QxgYswgYgGA1UEBwyBgFRFU1RURVNUVEVTVFRFU1RU
+RVNUVEVTVFRFU1RURVNUVEVTVFRFU1RURVNUVEVTVFRFU1RURVNUVEVTVFRFU1RU
+RVNUVEVTVFRFU1RURVNUVEVTVFRFU1RURVNUVEVTVFRFU1RURVNUVEVTVFRFU1RU
+RVNUVEVTVFRFU1RURVNUMUkwRwYDVQQKDEBURVNUVEVTVFRFU1RURVNUVEVTVFRF
+U1RURVNUVEVTVFRFU1RURVNUVEVTVFRFU1RURVNUVEVTVFRFU1RURVNUMUkwRwYD
+VQQLDEBURVNUVEVTVFRFU1RURVNUVEVTVFRFU1RURVNUVEVTVFRFU1RURVNUVEVT
+VFRFU1RURVNUVEVTVFRFU1RURVNUMUkwRwYDVQQDDEBURVNUVEVTVFRFU1RURVNU
+VEVTVFRFU1RURVNUVEVTVFRFU1RURVNUVEVTVFRFU1RURVNUVEVTVFRFU1RURVNU
+MRwwGgYJKoZIhvcNAQkBFg10ZXN0QHRlc3QuY29tMB4XDTIzMDMzMDA5NDg0NFoX
+DTI0MDMyOTA5NDg0NFowggIoMQswCQYDVQQGEwJURTGBizCBiAYDVQQIDIGAVEVT
+VFRFU1RURVNUVEVTVFRFU1RURVNUVEVTVFRFU1RURVNUVEVTVFRFU1RURVNUVEVT
+VFRFU1RURVNUVEVTVFRFU1RURVNUVEVTVFRFU1RURVNUVEVTVFRFU1RURVNUVEVT
+VFRFU1RURVNUVEVTVFRFU1RURVNUVEVTVFRFU1QxgYswgYgGA1UEBwyBgFRFU1RU
+RVNUVEVTVFRFU1RURVNUVEVTVFRFU1RURVNUVEVTVFRFU1RURVNUVEVTVFRFU1RU
+RVNUVEVTVFRFU1RURVNUVEVTVFRFU1RURVNUVEVTVFRFU1RURVNUVEVTVFRFU1RU
+RVNUVEVTVFRFU1RURVNUVEVTVFRFU1RURVNUMUkwRwYDVQQKDEBURVNUVEVTVFRF
+U1RURVNUVEVTVFRFU1RURVNUVEVTVFRFU1RURVNUVEVTVFRFU1RURVNUVEVTVFRF
+U1RURVNUMUkwRwYDVQQLDEBURVNUVEVTVFRFU1RURVNUVEVTVFRFU1RURVNUVEVT
+VFRFU1RURVNUVEVTVFRFU1RURVNUVEVTVFRFU1RURVNUMUkwRwYDVQQDDEBURVNU
+VEVTVFRFU1RURVNUVEVTVFRFU1RURVNUVEVTVFRFU1RURVNUVEVTVFRFU1RURVNU
+VEVTVFRFU1RURVNUMRwwGgYJKoZIhvcNAQkBFg10ZXN0QHRlc3QuY29tMIIBIjAN
+BgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAjEZtjVvccB3j/ZZWdor3YDWou0Ww
+JWc0A7bAaFKK2cWjY08atejeoeuOvqezAejhSgqA9R60B8LJSGFg6y3D3QJ3JOOx
+8ZodYIl0/QNfIHG1oaG9hp7zCaGlqyV6J+Bn1Sm3A6ElrNjb6Hkc8+bqqfH7gZbW
+w3vDgx6u3sgnB6QnP/Zg9+H/1Ws3rCEyU8eaJhQpi2JBzODLDGmVkoo07U4D/7TG
+nu5LgPBIRV0vmiCejMtpYhPCGAnTSdbhvKkNJAkZ8s225YlLFACgTVVmpcGb+cKu
+RVXxZXFI1sWeIz9RMflobkpemKxHSUtuQJxJMDbPyOPqwd5CRFHgs7tRBwIDAQAB
+o1MwUTAdBgNVHQ4EFgQUfRto8fDPPLA/ok5lgK7MypPSh54wHwYDVR0jBBgwFoAU
+fRto8fDPPLA/ok5lgK7MypPSh54wDwYDVR0TAQH/BAUwAwEB/zANBgkqhkiG9w0B
+AQsFAAOCAQEAWyy0drsTRfU8/J+rrX4trDb9o6iy7dSHyrpxo/TbaxBFTH69OJGb
+Q8YbutSq1a4m4UaSnGJYwarVCxmntjciz7byhfUwFEAdZ/rqwCeaqTdomGiYUisM
+Dmf/WiLYxRCpxr8tkkc332OlmHeBsDHKYY0G6dpdiTAGrjGNQZJJQc1wzy/+guZE
+UWr6jSVOel/u47jadbFK2/4a8ZnZEuEU0nn5h01lFY3fvrHr93Z3yzZ60LKeMszs
+SmDyoVI1XfNSJd8YbshGP91CVHFnDWDqo1JWV7hRev5g3XJfobIAAAqbL/H92BCT
+N4vF6RP8Ck9wj1OYq/w82MkgxOPleUju4Q==
+-----END CERTIFICATE-----
+";
+
+#[test_log::test]
+fn large_cert() {
+    with_vsc(|| {
+        let mut p = Command::new("pivy-tool")
+            .args(["write-cert", "9A"])
+            .stdin(Stdio::piped())
+            .stdout(Stdio::piped())
+            .spawn()
+            .unwrap();
+        let mut stdin = p.stdin.take().unwrap();
+        stdin.write_all(LARGE_CERT.as_bytes()).unwrap();
+        drop(stdin);
+        assert_eq!(p.wait().unwrap().code(), Some(0));
+
+        let mut p = Command::new("pivy-tool")
+            .args(["cert", "9A"])
+            .stdin(Stdio::piped())
+            .stdout(Stdio::piped())
+            .spawn()
+            .unwrap();
+        let mut stdout = p.stdout.take().unwrap();
+        let mut buf = String::new();
+        stdout.read_to_string(&mut buf).unwrap();
+        assert_eq!(&buf, LARGE_CERT);
+        assert_eq!(p.wait().unwrap().code(), Some(0));
+    });
+}