Merge branch 'master' into nfc-example-app

.github/workflows/cargo_fuzz.yml

@@ -29,3 +29,5 @@ jobs:
         run: cargo fuzz build
       - name: Cargo fuzz build (libraries/cbor)
         run: cd libraries/cbor && cargo fuzz build && cd ../..
+      - name: Cargo fuzz build (libraries/persistent_store)
+        run: cd libraries/persistent_store && cargo fuzz build && cd ../..

libraries/persistent_store/fuzz/.gitignore

@@ -0,0 +1,4 @@
+/Cargo.lock
+/artifacts/
+/corpus/
+/target/

libraries/persistent_store/fuzz/Cargo.toml

@@ -0,0 +1,21 @@
+[package]
+name = "fuzz-store"
+version = "0.0.0"
+authors = ["Julien Cretin <cretin@google.com>"]
+publish = false
+edition = "2018"
+
+[package.metadata]
+cargo-fuzz = true
+
+[dependencies]
+libfuzzer-sys = "0.3"
+persistent_store = { path = "..", features = ["std"] }
+
+# Prevent this from interfering with workspaces
+[workspace]
+members = ["."]
+
+[[bin]]
+name = "store"
+path = "fuzz_targets/store.rs"

libraries/persistent_store/fuzz/fuzz_targets/store.rs

@@ -0,0 +1,21 @@
+// Copyright 2019-2020 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#![no_main]
+
+use libfuzzer_sys::fuzz_target;
+
+fuzz_target!(|data: &[u8]| {
+    // TODO(ia0): Call fuzzing when implemented.
+});

libraries/persistent_store/fuzz/src/lib.rs

@@ -0,0 +1,191 @@
+// Copyright 2019-2020 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Fuzzing library for the persistent store.
+//!
+//! The overall design principles are (in order of precedence):
+//! - Determinism: fuzzing is a function from seeds (byte slices) to sequences of store
+//!   manipulations (things like creating a store, applying operations, interrupting operations,
+//!   interrupting reboots, checking invariant, etc). We can replay this function on the same input
+//!   to get the same sequence of manipulations (for the same fuzzing and store code).
+//! - Coverage: fuzzing tries to coverage as much different behaviors as possible for small seeds.
+//!   Ideally, each seed bit would control a branch decision in the tree of execution paths.
+//! - Surjectivity: all sequences of manipulations are reachable by fuzzing for some seed. The only
+//!   situation where coverage takes precedence over surjectivity is for the value of insert updates
+//!   where a pseudo-random generator is used to avoid wasting entropy.
+
+// TODO(ia0): Remove when used.
+#![allow(dead_code)]
+
+/// Bit-level entropy source based on a byte slice shared reference.
+///
+/// This is used to convert the byte slice provided by the fuzzer into the entropy used by the
+/// fuzzing code to generate a sequence of store manipulations, among other things. Entropy
+/// operations use the shortest necessary sequence of bits from the byte slice, such that fuzzer
+/// mutations of the byte slice have local impact or cascading effects towards future operations
+/// only.
+///
+/// The entropy has the following properties (in order of precedence):
+/// - It always returns a result.
+/// - It is deterministic: for a given slice and a given sequence of operations, the same results
+///   are returned. This permits to replay and debug fuzzing artifacts.
+/// - It uses the slice as a bit stream. In particular, it doesn't do big number arithmetic. This
+///   permits to have a simple implementation.
+/// - It doesn't waste information: for a given operation, the minimum integer number of bits is
+///   used to produce the result. As a consequence fractional bits can be wasted at each operation.
+/// - It uses the information uniformly: each bit is used exactly once, except when only a fraction
+///   of it is used. In particular, a bit is not used more than once. A consequence of each bit
+///   being used essentially once, is that the results are mostly uniformly distributed.
+///
+/// # Invariant
+///
+/// - The bit is a valid position in the slice, or one past: `bit <= 8 * data.len()`.
+struct Entropy<'a> {
+    /// The byte slice shared reference providing the entropy.
+    data: &'a [u8],
+
+    /// The bit position in the byte slice of the next entropy bit.
+    bit: usize,
+}
+
+impl Entropy<'_> {
+    /// Creates a bit-level entropy given a byte slice.
+    fn new(data: &[u8]) -> Entropy {
+        let bit = 0;
+        Entropy { data, bit }
+    }
+
+    /// Consumes the remaining entropy.
+    fn consume_all(&mut self) {
+        self.bit = 8 * self.data.len();
+    }
+
+    /// Returns whether there is entropy remaining.
+    fn is_empty(&self) -> bool {
+        assert!(self.bit <= 8 * self.data.len());
+        self.bit == 8 * self.data.len()
+    }
+
+    /// Reads a bit.
+    fn read_bit(&mut self) -> bool {
+        if self.is_empty() {
+            return false;
+        }
+        let b = self.bit;
+        self.bit += 1;
+        self.data[b / 8] & 1 << (b % 8) != 0
+    }
+
+    /// Reads a number with a given bit-width.
+    ///
+    /// # Preconditions
+    ///
+    /// - The number should fit in the return type: `n <= 8 * size_of::<usize>()`.
+    fn read_bits(&mut self, n: usize) -> usize {
+        assert!(n <= 8 * std::mem::size_of::<usize>());
+        let mut r = 0;
+        for i in 0..n {
+            r |= (self.read_bit() as usize) << i;
+        }
+        r
+    }
+
+    /// Reads a byte.
+    fn read_byte(&mut self) -> u8 {
+        self.read_bits(8) as u8
+    }
+
+    /// Reads a slice.
+    fn read_slice(&mut self, length: usize) -> Vec<u8> {
+        let mut result = Vec::with_capacity(length);
+        for _ in 0..length {
+            result.push(self.read_byte());
+        }
+        result
+    }
+
+    /// Reads a number between `min` and `max` (inclusive bounds).
+    ///
+    /// The distribution is uniform if the range width is a power of two. Otherwise, the minimum
+    /// amount of entropy is used (the next power of two) and the distribution is the closest to
+    /// uniform for that entropy.
+    ///
+    /// # Preconditions
+    ///
+    /// - The bounds should be correctly ordered: `min <= max`.
+    /// - The upper-bound should not be too large: `max < usize::max_value()`.
+    fn read_range(&mut self, min: usize, max: usize) -> usize {
+        assert!(min <= max && max < usize::max_value());
+        let count = max - min + 1;
+        let delta = self.read_bits(num_bits(count - 1)) % count;
+        min + delta
+    }
+}
+
+/// Returns the number of bits necessary to represent a number.
+fn num_bits(x: usize) -> usize {
+    8 * std::mem::size_of::<usize>() - x.leading_zeros() as usize
+}
+
+#[test]
+fn num_bits_ok() {
+    assert_eq!(num_bits(0), 0);
+    assert_eq!(num_bits(1), 1);
+    assert_eq!(num_bits(2), 2);
+    assert_eq!(num_bits(3), 2);
+    assert_eq!(num_bits(4), 3);
+    assert_eq!(num_bits(7), 3);
+    assert_eq!(num_bits(8), 4);
+    assert_eq!(num_bits(15), 4);
+    assert_eq!(num_bits(16), 5);
+    assert_eq!(
+        num_bits(usize::max_value()),
+        8 * std::mem::size_of::<usize>()
+    );
+}
+
+#[test]
+fn read_bit_ok() {
+    let mut entropy = Entropy::new(&[0b10110010]);
+    assert!(!entropy.read_bit());
+    assert!(entropy.read_bit());
+    assert!(!entropy.read_bit());
+    assert!(!entropy.read_bit());
+    assert!(entropy.read_bit());
+    assert!(entropy.read_bit());
+    assert!(!entropy.read_bit());
+    assert!(entropy.read_bit());
+}
+
+#[test]
+fn read_bits_ok() {
+    let mut entropy = Entropy::new(&[0x83, 0x92]);
+    assert_eq!(entropy.read_bits(4), 0x3);
+    assert_eq!(entropy.read_bits(8), 0x28);
+    assert_eq!(entropy.read_bits(2), 0b01);
+    assert_eq!(entropy.read_bits(2), 0b10);
+}
+
+#[test]
+fn read_range_ok() {
+    let mut entropy = Entropy::new(&[0b00101011]);
+    assert_eq!(entropy.read_range(0, 7), 0b011);
+    assert_eq!(entropy.read_range(1, 8), 1 + 0b101);
+    assert_eq!(entropy.read_range(4, 6), 4 + 0b00);
+    let mut entropy = Entropy::new(&[0b00101011]);
+    assert_eq!(entropy.read_range(0, 8), 0b1011 % 9);
+    assert_eq!(entropy.read_range(3, 15), 3 + 0b0010);
+    let mut entropy = Entropy::new(&[0x12, 0x34, 0x56, 0x78]);
+    assert_eq!(entropy.read_range(0, usize::max_value() - 1), 0x78563412);
+}

libraries/persistent_store/src/lib.rs

@@ -349,6 +349,8 @@
 extern crate alloc;
 
 mod buffer;
+#[cfg(feature = "std")]
+mod driver;
 mod format;
 #[cfg(feature = "std")]
 mod model;
@@ -357,6 +359,10 @@ mod store;
 
 pub use self::buffer::{BufferCorruptFunction, BufferOptions, BufferStorage};
 #[cfg(feature = "std")]
+pub use self::driver::{
+    StoreDriver, StoreDriverOff, StoreDriverOn, StoreInterruption, StoreInvariant,
+};
+#[cfg(feature = "std")]
 pub use self::model::{StoreModel, StoreOperation};
 pub use self::storage::{Storage, StorageError, StorageIndex, StorageResult};
 pub use self::store::{

libraries/persistent_store/src/store.rs

@@ -18,9 +18,11 @@ use crate::format::{
 };
 #[cfg(feature = "std")]
 pub use crate::model::{StoreModel, StoreOperation};
-#[cfg(feature = "std")]
-pub use crate::BufferStorage;
 use crate::{usize_to_nat, Nat, Storage, StorageError, StorageIndex};
+#[cfg(feature = "std")]
+pub use crate::{
+    BufferStorage, StoreDriver, StoreDriverOff, StoreDriverOn, StoreInterruption, StoreInvariant,
+};
 use alloc::vec::Vec;
 use core::cmp::{max, min, Ordering};
 #[cfg(feature = "std")]
@@ -1050,7 +1052,7 @@ impl Store<BufferStorage> {
     }
 
     /// Extracts the storage.
-    pub fn into_storage(self) -> BufferStorage {
+    pub fn extract_storage(self) -> BufferStorage {
         self.storage
     }
 
@@ -1233,3 +1235,207 @@ fn is_write_needed(source: &[u8], target: &[u8]) -> StoreResult<bool> {
     }
     Ok(false)
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::BufferOptions;
+
+    #[derive(Clone)]
+    struct Config {
+        word_size: usize,
+        page_size: usize,
+        num_pages: usize,
+        max_word_writes: usize,
+        max_page_erases: usize,
+    }
+
+    impl Config {
+        fn new_driver(&self) -> StoreDriverOff {
+            let options = BufferOptions {
+                word_size: self.word_size,
+                page_size: self.page_size,
+                max_word_writes: self.max_word_writes,
+                max_page_erases: self.max_page_erases,
+                strict_write: true,
+            };
+            StoreDriverOff::new(options, self.num_pages)
+        }
+    }
+
+    const MINIMAL: Config = Config {
+        word_size: 4,
+        page_size: 64,
+        num_pages: 5,
+        max_word_writes: 2,
+        max_page_erases: 9,
+    };
+
+    const NORDIC: Config = Config {
+        word_size: 4,
+        page_size: 0x1000,
+        num_pages: 20,
+        max_word_writes: 2,
+        max_page_erases: 10000,
+    };
+
+    const TITAN: Config = Config {
+        word_size: 4,
+        page_size: 0x800,
+        num_pages: 10,
+        max_word_writes: 2,
+        max_page_erases: 10000,
+    };
+
+    #[test]
+    fn nordic_capacity() {
+        let driver = NORDIC.new_driver().power_on().unwrap();
+        assert_eq!(driver.model().capacity().total, 19123);
+    }
+
+    #[test]
+    fn titan_capacity() {
+        let driver = TITAN.new_driver().power_on().unwrap();
+        assert_eq!(driver.model().capacity().total, 4315);
+    }
+
+    #[test]
+    fn minimal_virt_page_size() {
+        // Make sure a virtual page has 14 words. We use this property in the other tests below to
+        // know whether entries are spanning, starting, and ending pages.
+        assert_eq!(MINIMAL.new_driver().model().format().virt_page_size(), 14);
+    }
+
+    #[test]
+    fn init_ok() {
+        assert!(MINIMAL.new_driver().power_on().is_ok());
+    }
+
+    #[test]
+    fn insert_ok() {
+        let mut driver = MINIMAL.new_driver().power_on().unwrap();
+        // Empty entry.
+        driver.insert(0, &[]).unwrap();
+        driver.insert(1, &[]).unwrap();
+        driver.check().unwrap();
+        // Last word is erased but last bit is not user data.
+        driver.insert(0, &[0xff]).unwrap();
+        driver.insert(1, &[0xff]).unwrap();
+        driver.check().unwrap();
+        // Last word is erased and last bit is user data.
+        driver.insert(0, &[0xff, 0xff, 0xff, 0xff]).unwrap();
+        driver.insert(1, &[0xff, 0xff, 0xff, 0xff]).unwrap();
+        driver.insert(2, &[0x5c; 6]).unwrap();
+        driver.check().unwrap();
+        // Entry spans 2 pages.
+        assert_eq!(driver.store().tail().unwrap().get(), 13);
+        driver.insert(3, &[0x5c; 8]).unwrap();
+        driver.check().unwrap();
+        assert_eq!(driver.store().tail().unwrap().get(), 16);
+        // Entry ends a page.
+        driver.insert(2, &[0x93; (28 - 16 - 1) * 4]).unwrap();
+        driver.check().unwrap();
+        assert_eq!(driver.store().tail().unwrap().get(), 28);
+        // Entry starts a page.
+        driver.insert(3, &[0x81; 10]).unwrap();
+        driver.check().unwrap();
+    }
+
+    #[test]
+    fn remove_ok() {
+        let mut driver = MINIMAL.new_driver().power_on().unwrap();
+        // Remove absent entry.
+        driver.remove(0).unwrap();
+        driver.remove(1).unwrap();
+        driver.check().unwrap();
+        // Remove last inserted entry.
+        driver.insert(0, &[0x5c; 6]).unwrap();
+        driver.remove(0).unwrap();
+        driver.check().unwrap();
+        // Remove empty entries.
+        driver.insert(0, &[]).unwrap();
+        driver.insert(1, &[]).unwrap();
+        driver.remove(0).unwrap();
+        driver.remove(1).unwrap();
+        driver.check().unwrap();
+        // Remove entry with flipped bit.
+        driver.insert(0, &[0xff]).unwrap();
+        driver.insert(1, &[0xff; 4]).unwrap();
+        driver.remove(0).unwrap();
+        driver.remove(1).unwrap();
+        driver.check().unwrap();
+        // Write some entries with one spanning 2 pages.
+        driver.insert(2, &[0x93; 9]).unwrap();
+        assert_eq!(driver.store().tail().unwrap().get(), 13);
+        driver.insert(3, &[0x81; 10]).unwrap();
+        assert_eq!(driver.store().tail().unwrap().get(), 17);
+        driver.insert(4, &[0x76; 11]).unwrap();
+        driver.check().unwrap();
+        // Remove the entry spanning 2 pages.
+        driver.remove(3).unwrap();
+        driver.check().unwrap();
+        // Write some entries with one ending a page and one starting the next.
+        assert_eq!(driver.store().tail().unwrap().get(), 21);
+        driver.insert(2, &[0xd7; (28 - 21 - 1) * 4]).unwrap();
+        assert_eq!(driver.store().tail().unwrap().get(), 28);
+        driver.insert(4, &[0xe2; 21]).unwrap();
+        driver.check().unwrap();
+        // Remove them.
+        driver.remove(2).unwrap();
+        driver.remove(4).unwrap();
+        driver.check().unwrap();
+    }
+
+    #[test]
+    fn prepare_ok() {
+        let mut driver = MINIMAL.new_driver().power_on().unwrap();
+
+        // Don't compact if enough immediate capacity.
+        assert_eq!(driver.store().immediate_capacity().unwrap(), 39);
+        assert_eq!(driver.store().capacity().unwrap().remaining(), 34);
+        assert_eq!(driver.store().head().unwrap().get(), 0);
+        driver.store_mut().prepare(34).unwrap();
+        assert_eq!(driver.store().head().unwrap().get(), 0);
+
+        // Fill the store.
+        for key in 0..4 {
+            driver.insert(key, &[0x38; 28]).unwrap();
+        }
+        driver.check().unwrap();
+        assert_eq!(driver.store().immediate_capacity().unwrap(), 7);
+        assert_eq!(driver.store().capacity().unwrap().remaining(), 2);
+        // Removing entries increases available capacity but not immediate capacity.
+        driver.remove(0).unwrap();
+        driver.remove(2).unwrap();
+        driver.check().unwrap();
+        assert_eq!(driver.store().immediate_capacity().unwrap(), 7);
+        assert_eq!(driver.store().capacity().unwrap().remaining(), 18);
+
+        // Prepare for next write (7 words data + 1 word overhead).
+        assert_eq!(driver.store().head().unwrap().get(), 0);
+        driver.store_mut().prepare(8).unwrap();
+        driver.check().unwrap();
+        assert_eq!(driver.store().head().unwrap().get(), 16);
+        // The available capacity did not change, but the immediate capacity is above 8.
+        assert_eq!(driver.store().immediate_capacity().unwrap(), 14);
+        assert_eq!(driver.store().capacity().unwrap().remaining(), 18);
+    }
+
+    #[test]
+    fn reboot_ok() {
+        let mut driver = MINIMAL.new_driver().power_on().unwrap();
+
+        // Do some operations and reboot.
+        driver.insert(0, &[0x38; 24]).unwrap();
+        driver.insert(1, &[0x5c; 13]).unwrap();
+        driver = driver.power_off().power_on().unwrap();
+        driver.check().unwrap();
+
+        // Do more operations and reboot.
+        driver.insert(2, &[0x93; 1]).unwrap();
+        driver.remove(0).unwrap();
+        driver.insert(3, &[0xde; 9]).unwrap();
+        driver = driver.power_off().power_on().unwrap();
+        driver.check().unwrap();
+    }
+}

run_desktop_tests.sh

@@ -62,6 +62,9 @@ cargo fuzz build
 cd libraries/cbor
 cargo fuzz build
 cd ../..
+cd libraries/persistent_store
+cargo fuzz build
+cd ../..
 
 echo "Checking that CTAP2 builds and links properly (1 set of features)..."
 cargo build --release --target=thumbv7em-none-eabi --features with_ctap1