feat: add the library text core — unicode fold, filename parse, author keys (#2885)

lib/LibraryIndex/LibraryText.cpp

@@ -0,0 +1,346 @@
+#include "LibraryText.h"
+
+#include <Utf8.h>
+
+#include <algorithm>
+#include <array>
+#include <cstring>
+
+namespace library {
+namespace {
+
+// Letters with no canonical decomposition, plus the punctuation that would
+// otherwise become a space and break word shapes. Everything here is a distinct
+// letter in Unicode, so no amount of NFD gets us the ASCII form.
+struct CharMap {
+  uint32_t cp;
+  const char* replacement;
+};
+constexpr CharMap EXPLICIT_MAP[] = {
+    {0x00D8, "o"},   // Ø
+    {0x00F8, "o"},   // ø  — "Søren" folds to "soren", not "nesb"
+    {0x00C6, "ae"},  // Æ
+    {0x00E6, "ae"},  // æ
+    {0x0152, "oe"},  // Œ
+    {0x0153, "oe"},  // œ
+    {0x00DF, "ss"},  // ß
+    {0x0141, "l"},   // Ł
+    {0x0142, "l"},   // ł
+    {0x00D0, "d"},   // Ð
+    {0x00F0, "d"},   // ð
+    {0x0110, "d"},   // Đ
+    {0x0111, "d"},   // đ
+    {0x00DE, "th"},  // Þ
+    {0x00FE, "th"},  // þ
+    {0x0131, "i"},   // ı
+    {0x0027, "'"},   // ' — kept, not turned into a space, so "O'Brien" stays one word
+    {0x2019, "'"},   // ’ — and so the curly form folds to the same thing
+    {0x2018, "'"},   // ‘
+};
+
+const char* explicitMapping(const uint32_t cp) {
+  for (const auto& e : EXPLICIT_MAP) {
+    if (e.cp == cp) return e.replacement;
+  }
+  return nullptr;
+}
+
+// Reverse lookup in the NFC table: given a precomposed codepoint, return the
+// base letter it was composed from, or 0.
+//
+// The table is sorted by (base, mark) so this is a linear scan rather than a
+// binary search. It runs once per non-ASCII character at index-build time and
+// over a short query per keystroke, never over the whole index, so the scan is
+// not on any hot path. Reusing the generated table costs no extra flash — it is
+// already linked for utf8ComposeNfc().
+// Fully decompose: é -> e, and the rare doubly-accented forms (ế -> ê -> e).
+uint32_t stripDiacritics(uint32_t cp) {
+  for (int guard = 0; guard < 4; guard++) {
+    const uint32_t base = utf8DecomposedBase(cp);
+    if (base == 0) break;
+    cp = base;
+  }
+  return cp;
+}
+
+bool isAsciiAlnum(const uint32_t cp) {
+  return (cp >= '0' && cp <= '9') || (cp >= 'a' && cp <= 'z') || (cp >= 'A' && cp <= 'Z');
+}
+
+void appendLowerAscii(const uint32_t cp, std::string& out) {
+  out.push_back(static_cast<char>(cp >= 'A' && cp <= 'Z' ? cp - 'A' + 'a' : cp));
+}
+
+// Articles stripped from the head of sort and search keys. Display text never
+// goes through this.
+constexpr const char* ARTICLES[] = {"the ", "a ",   "an ", "le ",  "la ",  "les ", "l'",   "un ",
+                                    "une ", "de ",  "du ", "des ", "der ", "die ", "das ", "el ",
+                                    "los ", "las ", "il ", "lo ",  "gli ", "i ",   "o ",   "os "};
+
+// Views into `folded`, not copies: the caller keeps that string alive for as
+// long as the tokens, and a std::string per token costs an allocation each plus
+// 24 bytes of stack apiece -- 288 B for the twelve, over the 256 B this repo
+// asks callers to justify.
+void splitTokens(std::string_view folded, std::string_view* out, size_t maxTokens, size_t& count) {
+  count = 0;
+  size_t i = 0;
+  while (i < folded.size() && count < maxTokens) {
+    while (i < folded.size() && folded[i] == ' ') i++;
+    const size_t start = i;
+    while (i < folded.size() && folded[i] != ' ') i++;
+    if (i > start) out[count++] = folded.substr(start, i - start);
+  }
+}
+
+}  // namespace
+
+std::string fold(const std::string_view text, const bool stripArticle) {
+  std::string out;
+  out.reserve(text.size());
+
+  const auto* cursor = reinterpret_cast<const unsigned char*>(text.data());
+  const auto* end = cursor + text.size();
+  bool pendingSpace = false;
+
+  while (cursor < end) {
+    // The shared decoder stops on NUL and every std::string source is
+    // NUL-terminated, but fold's contract is string_view — and a view may end
+    // mid-sequence. Refuse to decode a lead byte whose continuation bytes lie
+    // past the end rather than trusting whatever sits there.
+    const unsigned char lead = *cursor;
+    const ptrdiff_t promised = lead < 0x80           ? 1
+                               : (lead >> 5) == 0x06 ? 2
+                               : (lead >> 4) == 0x0E ? 3
+                               : (lead >> 3) == 0x1E ? 4
+                                                     : 1;
+    if (promised > end - cursor) break;
+    const uint32_t cp = utf8NextCodepoint(&cursor);
+    if (cp == 0) break;
+
+    if (utf8IsCombiningMark(cp)) continue;  // already-decomposed input
+
+    const char* mapped = explicitMapping(cp);
+    if (mapped != nullptr) {
+      if (pendingSpace && !out.empty()) out.push_back(' ');
+      pendingSpace = false;
+      out.append(mapped);
+      continue;
+    }
+
+    const uint32_t base = stripDiacritics(cp);
+    if (isAsciiAlnum(base)) {
+      if (pendingSpace && !out.empty()) out.push_back(' ');
+      pendingSpace = false;
+      appendLowerAscii(base, out);
+      continue;
+    }
+
+    // Everything else — punctuation, symbols, unmapped scripts — separates
+    // words. Deferring the space keeps runs collapsed and drops trailing ones.
+    if (!out.empty()) pendingSpace = true;
+  }
+
+  if (stripArticle) {
+    for (const char* article : ARTICLES) {
+      const size_t len = strlen(article);
+      if (out.size() > len && out.compare(0, len, article) == 0) {
+        out.erase(0, len);
+        break;
+      }
+    }
+  }
+  return out;
+}
+
+std::string cleanPersonName(const std::string_view author) {
+  std::string out;
+  out.reserve(author.size());
+  int depth = 0;
+  for (size_t i = 0; i < author.size(); i++) {
+    const char c = author[i];
+    if (c == '[' || c == '(') {
+      depth++;
+      continue;
+    }
+    if (c == ']' || c == ')') {
+      if (depth > 0) depth--;
+      continue;
+    }
+    if (c == ';') break;  // secondary authors
+    if (depth > 0) continue;
+    if (c == '_') {
+      // "Herbert G_ Wells" — the underscore stands in for a full stop the
+      // filesystem would not take. Between letters it is an abbreviation dot;
+      // at the end of a word it is just noise.
+      const bool betweenLetters = i > 0 && i + 1 < author.size() &&
+                                  isalpha(static_cast<unsigned char>(author[i - 1])) &&
+                                  isalpha(static_cast<unsigned char>(author[i + 1]));
+      out.push_back(betweenLetters ? '.' : ' ');
+      continue;
+    }
+    out.push_back(c);
+  }
+
+  while (!out.empty() &&
+         (out.back() == ' ' || out.back() == ',' || out.back() == '-' || out.back() == '.' || out.back() == '_')) {
+    out.pop_back();
+  }
+  size_t start = 0;
+  while (start < out.size() && out[start] == ' ') start++;
+  out.erase(0, start);
+
+  // Collapse the space runs left behind by the removals.
+  std::string collapsed;
+  collapsed.reserve(out.size());
+  bool space = false;
+  for (const char c : out) {
+    if (c == ' ') {
+      space = true;
+      continue;
+    }
+    if (space && !collapsed.empty()) collapsed.push_back(' ');
+    space = false;
+    collapsed.push_back(c);
+  }
+
+  // "Austen, Jane" is the same person as "Jane Austen", and publishers use both.
+  // The spelling vote cannot settle it — with one book per author there is no
+  // majority — so the inverted form is turned round here instead. Only a single
+  // comma qualifies: "Smith, John, Jr." and lists of several authors are left
+  // exactly as they are rather than being scrambled.
+  const size_t comma = collapsed.find(',');
+  if (comma != std::string::npos && collapsed.find(',', comma + 1) == std::string::npos) {
+    std::string_view last(collapsed.data(), comma);
+    std::string_view first(collapsed.data() + comma + 1, collapsed.size() - comma - 1);
+    while (!first.empty() && first.front() == ' ') first.remove_prefix(1);
+    while (!last.empty() && last.back() == ' ') last.remove_suffix(1);
+    if (!first.empty() && !last.empty()) {
+      std::string swapped;
+      swapped.reserve(collapsed.size());
+      swapped.append(first);
+      swapped.push_back(' ');
+      swapped.append(last);
+      return swapped;
+    }
+  }
+  return collapsed;
+}
+
+std::string authorKey(const std::string_view author) {
+  // Drop bracketed spans ("George Sand [Sand, George]") and everything after
+  // a multi-author separator.
+  std::string cleaned;
+  cleaned.reserve(author.size());
+  int depth = 0;
+  for (const char c : author) {
+    if (c == '[' || c == '(') {
+      depth++;
+      continue;
+    }
+    if (c == ']' || c == ')') {
+      if (depth > 0) depth--;
+      continue;
+    }
+    if (c == ';') break;
+    if (depth == 0) cleaned.push_back(c);
+  }
+
+  const std::string folded = fold(cleaned);
+
+  constexpr size_t MAX_TOKENS = 12;
+  std::string_view tokens[MAX_TOKENS];
+  size_t count = 0;
+  splitTokens(folded, tokens, MAX_TOKENS, count);
+
+  // Initials carry no identity and appear inconsistently ("Herbert G Wells" vs
+  // "Herbert Wells"), so they must not change the key.
+  size_t kept = 0;
+  for (size_t i = 0; i < count; i++) {
+    if (tokens[i].size() > 1) tokens[kept++] = tokens[i];
+  }
+  std::sort(tokens, tokens + kept);
+
+  std::string key;
+  for (size_t i = 0; i < kept; i++) {
+    if (!key.empty()) key.push_back(' ');
+    key.append(tokens[i]);
+  }
+  // Truncate on bytes, not on a token boundary. Sorting puts a short forename
+  // first, so a whole-token cut would reduce "Wollstonecraft, Mary" to the key
+  // "alex" and merge every Alex in the library; the byte cut keeps
+  // "mary wollsto", which stays a prefix of the full key and discriminates.
+  if (key.size() > AUTHOR_KEY_MAX_BYTES) key.resize(AUTHOR_KEY_MAX_BYTES);
+  while (!key.empty() && key.back() == ' ') key.pop_back();
+  return key;
+}
+
+// fold() keeps the apostrophe, which is right for sorting — "L'Eneide" belongs
+// under L. For searching it is wrong: in French the word worth typing is the one
+// AFTER the apostrophe, so "eneide" must reach "L'Eneide" and "cote" must
+// reach "d'a cote". Treating it as a word boundary here leaves the sort untouched.
+bool isWordBreak(const char c) { return c == ' ' || c == '\''; }
+
+bool matchesQuery(const std::string_view haystack, const std::string_view needle) {
+  if (needle.empty()) return true;
+
+  // Walk the query one word at a time, and for each one scan the book's words for
+  // a prefix hit. Both strings are at most a couple of hundred bytes and this
+  // runs once per book per keypress, so a plain scan is cheaper than anything
+  // that would need building first.
+  size_t qs = 0;
+  while (qs < needle.size()) {
+    while (qs < needle.size() && isWordBreak(needle[qs])) qs++;
+    if (qs >= needle.size()) break;
+    size_t qe = qs;
+    while (qe < needle.size() && !isWordBreak(needle[qe])) qe++;
+    const std::string_view word = needle.substr(qs, qe - qs);
+
+    bool found = false;
+    size_t hs = 0;
+    while (hs < haystack.size() && !found) {
+      while (hs < haystack.size() && isWordBreak(haystack[hs])) hs++;
+      if (hs >= haystack.size()) break;
+      if (haystack.compare(hs, word.size(), word) == 0) {
+        found = true;
+        break;
+      }
+      while (hs < haystack.size() && !isWordBreak(haystack[hs])) hs++;
+    }
+    if (!found) return false;
+    qs = qe;
+  }
+  return true;
+}
+
+std::string surnameKey(const std::string_view displayAuthor) {
+  const std::string folded = fold(displayAuthor);
+  if (folded.empty()) return {};
+
+  size_t lastStart = std::string::npos;
+  size_t lastEnd = folded.size();
+  size_t i = folded.size();
+  while (i > 0) {
+    i--;
+    if (folded[i] != ' ') {
+      if (lastStart == std::string::npos) lastEnd = i + 1;
+      lastStart = i;
+    } else if (lastStart != std::string::npos) {
+      break;
+    }
+  }
+  if (lastStart == std::string::npos) return {};
+
+  std::string key;
+  key.reserve(folded.size() + 1);
+  key.append(folded, lastStart, lastEnd - lastStart);
+  // The given names follow, so two people sharing a surname stay in a stable,
+  // readable order rather than whichever the disk walk happened to produce.
+  if (lastStart > 0) {
+    key.push_back(' ');
+    key.append(folded, 0, lastStart);
+    while (!key.empty() && key.back() == ' ') key.pop_back();
+  }
+  return key;
+}
+
+}  // namespace library

lib/LibraryIndex/LibraryText.h

@@ -0,0 +1,111 @@
+#pragma once
+
+// Text normalisation for the Library index: the fold used for search and sort
+// keys, and the author key that merges spellings of one person.
+//
+// Nothing here reads a filename. Titles and authors come from the book's own
+// metadata; a book whose metadata carries no title falls back to its filename
+// stem, unparsed, and one with no author is grouped under Unknown.
+//
+// Pure functions over UTF-8, no hardware and no allocation beyond the returned
+// strings, so the whole unit is host-testable (test/library_text).
+//
+// Design notes that are easy to get wrong and were measured on a real card
+// (docs/superpowers/specs/2026-08-05-addendum-a-findability.md, A2.1-A2.8):
+//
+//   * The fold DECOMPOSES. `utf8ComposeNfc()` goes the other way, so a fold
+//     built on it passes on a card holding only decomposed text and then mangles
+//     the first precomposed file to arrive from Windows or Calibre. Both forms
+//     must produce the same output, and the host tests assert exactly that.
+//   * Some letters have no canonical decomposition at all — U+00F8 (ø) is a
+//     distinct letter, not o-with-stroke — so a decompose-only fold turns
+//     "Søren" into "Sren". Those need an explicit map.
+//   * The author KEY sorts its tokens, so for identity purposes name order
+//     stops mattering and no First/Last guess is needed there. Display-side
+//     helpers do use narrow rules — cleanPersonName inverts a single-comma
+//     "Last, First", surnameKey takes the last word — because showing
+//     "Austen, Jane" and "Jane Austen" as two people is worse than the guess.
+
+#include <cstdint>
+#include <string>
+#include <string_view>
+
+namespace library {
+
+// Longest author key written into an index record. Sized so a key fits the
+// record's fixed field; measured to collide 0 times over a 69-book library.
+inline constexpr size_t AUTHOR_KEY_MAX_BYTES = 12;
+
+// Casefold and strip diacritics for matching and sorting.
+//
+// Maps a handful of letters that have no canonical decomposition, decomposes the
+// rest and drops combining marks, lowercases ASCII alphanumerics, and turns
+// everything else into a single space. Space runs collapse and the result is
+// trimmed. Apostrophes survive as ASCII '\'' so "O'Malley" and "L'\xC3\x89n\xC3\xA9ideé"
+// keep their shape.
+//
+// `stripArticle` additionally removes one leading article ("the ", "le ", "la ",
+// ...) — correct for sort keys and search text, wrong for anything displayed.
+std::string fold(std::string_view text, bool stripArticle = false);
+
+// Tidy a person's name for DISPLAY, without reordering it.
+//
+// Drops bracketed spans ("George Sand [Sand, George]"), everything after a
+// multi-author separator, and the trailing underscores and punctuation that
+// exporters leave behind ("Lu Xun_", "Herbert G_ Wells"). An underscore
+// between letters becomes a full stop, since that is what it replaced in a name
+// a filesystem refused to hold.
+//
+// A single-comma "Austen, Jane" IS turned round into "Jane Austen" — with one
+// book per author the spelling vote has no majority to settle it, so the comma
+// is the one signal acted on here. Multi-comma names ("Smith, John, Jr.") and
+// author lists are left exactly as written. Harmonising the several spellings
+// of one person is done by picking the most common one that actually occurs,
+// which needs the whole library and so belongs to the index build.
+std::string cleanPersonName(std::string_view author);
+
+// Order-insensitive identity for one person, at most AUTHOR_KEY_MAX_BYTES.
+//
+// Drops bracketed spans and everything after ';' (multi-author separator), folds,
+// drops single-character tokens (initials), sorts the remaining tokens and joins
+// them. "Lu, Xun", "Xun, Lu" and "Lu Xun [Xun, Lu]" all
+// collapse to one key. Truncation is on BYTES, not a token boundary: the sort
+// puts a short forename first, so a whole-token cut would reduce
+// "Wollstonecraft, Mary" to "alex" and merge every Alex in the library; the byte
+// cut keeps "mary wollsto", still a prefix of the full key.
+std::string authorKey(std::string_view author);
+
+// Does a book match what has been typed so far?
+//
+// Both sides are already folded — accents stripped, case dropped, punctuation
+// turned to spaces — so "eneide" finds "L'Énéide" and "eluard" finds
+// "Éluard". `haystack` is the record's stored fold; `needle` is the query put
+// through the same fold.
+//
+// Every query word must PREFIX some word of the book. That is the rule that fits
+// the hardware: with no partial refresh, each keypress costs a full ~185 ms panel
+// repaint, so the reader wants to stop typing as early as possible. "dar mat"
+// — six keys — finds "Wuthering Heights", where a plain substring test would demand the
+// whole of one word and give nothing for the effort of a second.
+//
+// An empty query matches everything, so the list is the unfiltered shelf before
+// the first key is pressed.
+bool matchesQuery(std::string_view haystack, std::string_view needle);
+
+// Ordering key for a shelf sorted by author: surname first, then the rest.
+// "Herman Melville" becomes "melville herman", so the shelf reads C where a library
+// would put it.
+//
+// Deliberately NOT the same key as authorKey(). That one sorts a name's words so
+// that "Victor Hugo" and "Hugo Victor" hash alike and are recognised as one person;
+// it is a GROUPING key and would be wrong to order by. This is derived from the
+// DISPLAY name instead, which is safe because the spelling vote has already made
+// every book by one author show the same name — so a group cannot split across
+// two places on the shelf.
+//
+// The last word is taken as the surname. That is right for the western names on
+// this card and wrong for some others, which is a limit worth stating rather than
+// hiding: a single word name simply keys on itself.
+std::string surnameKey(std::string_view displayAuthor);
+
+}  // namespace library

lib/Utf8/Utf8.cpp

@@ -23,6 +23,14 @@ uint32_t utf8ComposePair(const uint32_t base, const uint32_t mark) {
 }
 }  // namespace
 
+uint32_t utf8DecomposedBase(const uint32_t cp) {
+  if (cp < 0x00C0) return 0;  // no precomposed Latin letter below this
+  for (const auto& e : kUtf8ComposeTable) {
+    if (e.composed == cp) return e.base;
+  }
+  return 0;
+}
+
 std::string utf8ComposeNfc(const std::string& in) {
   // Fast path: NFC composition can only change text that contains a combining
   // diacritical mark U+0300-036F (UTF-8 lead byte 0xCC or 0xCD). Plain ASCII and

lib/Utf8/Utf8.h

@@ -18,6 +18,18 @@ void utf8TruncateChars(std::string& str, size_t numChars);
 // stored in NFD (e.g. some EPUB chapter titles) otherwise renders broken.
 std::string utf8ComposeNfc(const std::string& in);
 
+// The base letter a precomposed codepoint decomposes to, or 0 when there is
+// none ("é" -> "e", but "ø" -> 0: it is a letter in its own right, not
+// o-with-stroke). Lives here rather than in a caller because the compose table
+// is a ~5 KB static array in the header: a second includer is a second copy in
+// flash.
+//
+// A linear scan, unlike utf8ComposePair's binary search above it: the table is
+// sorted by (base, mark), which this lookup searches against the grain. Adding
+// a second table sorted by composed would cost more flash than sharing this one
+// saves.
+uint32_t utf8DecomposedBase(uint32_t cp);
+
 // Truncate a raw char buffer to the last complete UTF-8 codepoint boundary.
 // Returns the new length (<= len). If the buffer ends mid-sequence, the
 // incomplete trailing bytes are excluded.

test/CMakeLists.txt

@@ -44,6 +44,7 @@ add_subdirectory(differential_rounding)
 add_subdirectory(hyphenation_eval)
 add_subdirectory(utf8_compose)
 add_subdirectory(opds_filename)
+add_subdirectory(library_text)
 add_subdirectory(minibidi_arabic)
 add_subdirectory(combining_marks)
 add_subdirectory(credential_integrity)

test/library_text/CMakeLists.txt

@@ -0,0 +1,17 @@
+add_executable(LibraryTextTest
+  LibraryTextTest.cpp
+  ${REPO_ROOT}/lib/LibraryIndex/LibraryText.cpp
+  ${REPO_ROOT}/lib/Utf8/Utf8.cpp
+)
+
+target_include_directories(LibraryTextTest PRIVATE
+  ${REPO_ROOT}/lib/LibraryIndex
+  ${REPO_ROOT}/lib/Utf8
+)
+
+target_link_libraries(LibraryTextTest PRIVATE
+  crosspoint_test_common
+  GTest::gtest_main
+)
+
+gtest_discover_tests(LibraryTextTest)

test/library_text/LibraryTextTest.cpp

@@ -0,0 +1,243 @@
+#include <gtest/gtest.h>
+
+#include <string>
+
+#include "LibraryText.h"
+
+using library::authorKey;
+using library::fold;
+
+namespace {
+
+// The same text in both Unicode normal forms. Every fold test runs both, because
+// a card holding only one form makes a one-sided test pass by accident — which
+// is exactly how a fold built on utf8ComposeNfc() survives until the first file
+// arrives from the other kind of machine.
+struct NormalisationPair {
+  const char* nfc;  // precomposed: e-acute is one codepoint
+  const char* nfd;  // decomposed: e followed by a combining acute
+  const char* expected;
+};
+
+constexpr NormalisationPair PAIRS[] = {
+    {"pand\xC3\xA9mie", "pande\xCC\x81mie", "pandemie"},
+    {"\xC3\x89"
+     "clipse totale",
+     "E\xCC\x81"
+     "clipse totale",
+     "eclipse totale"},
+    {"M\xC3\xA9moires", "Me\xCC\x81moires", "memoires"},
+    {"Ang\xC3\xA9lina", "Ange\xCC\x81lina", "angelina"},
+    {"R\xC3\xA9"
+     "camier",
+     "Re\xCC\x81"
+     "camier",
+     "recamier"},
+    {"Derri\xC3\xA8re les collines", "Derrie\xCC\x80re les collines", "derriere les collines"},
+};
+
+}  // namespace
+
+TEST(LibraryFold, BothNormalisationsAgree) {
+  for (const auto& p : PAIRS) {
+    EXPECT_EQ(fold(p.nfc), p.expected) << "NFC input: " << p.nfc;
+    EXPECT_EQ(fold(p.nfd), p.expected) << "NFD input: " << p.nfd;
+    EXPECT_EQ(fold(p.nfc), fold(p.nfd)) << "forms disagree for " << p.expected;
+  }
+}
+
+TEST(LibraryFold, LettersWithoutCanonicalDecomposition) {
+  // These have no NFD form at all, so a decompose-only fold silently deletes
+  // them. "Søren" losing its last letter is the case that motivated the map.
+  EXPECT_EQ(fold("S\xC3\xB8ren"), "soren");
+  EXPECT_EQ(fold("\xC3\x98rsted"), "orsted");
+  EXPECT_EQ(fold("\xC3\x86"
+                 "sop"),
+            "aesop");
+  EXPECT_EQ(fold("Stra\xC3\x9F"
+                 "e"),
+            "strasse");
+  EXPECT_EQ(fold("\xC5\x81odz"), "lodz");
+  EXPECT_EQ(fold("s\xC5\x93ur"), "soeur");
+}
+
+TEST(LibraryFold, ApostrophesSurviveInNamesAndElisions) {
+  // U+2019 is what exporters actually emit; folding it to a space would split
+  // "O'Malley" into two tokens and change both its sort place and its search.
+  // ("Malley", not "Brien": C++ hex escapes are greedy, so \x99B would parse
+  // as the single escape 0x99B.)
+  EXPECT_EQ(fold("O\xE2\x80\x99Malley"), "o'malley");
+  EXPECT_EQ(fold("O'Malley"), "o'malley");
+  EXPECT_EQ(fold("L\xE2\x80\x99\xC3\x89n\xC3\xA9ide"), "l'eneide");
+}
+
+TEST(LibraryFold, TypographicDashesAndQuotesFoldLikeTheirAsciiForms) {
+  // An em dash used to stay inside the fold word while an ASCII hyphen broke
+  // it, so the same title written both ways sorted and searched differently.
+  EXPECT_EQ(library::fold("a\u2014b"), library::fold("a-b"));
+  EXPECT_EQ(library::fold("a\u2013b"), library::fold("a-b"));
+  EXPECT_EQ(library::fold("\u201Cquoted\u201D"), library::fold("\"quoted\""));
+}
+
+TEST(LibraryFold, PunctuationSeparatesAndSpaceRunsCollapse) {
+  EXPECT_EQ(fold("Le juge Untel.T2.Le po\xC3\xA8me"), "le juge untel t2 le poeme");
+  EXPECT_EQ(fold("  spaced   out  "), "spaced out");
+  EXPECT_EQ(fold("a---b"), "a b");
+  EXPECT_EQ(fold("2085 _ Artificial"), "2085 artificial");
+  EXPECT_EQ(fold(""), "");
+  EXPECT_EQ(fold("!!!"), "");
+}
+
+TEST(LibraryFold, ArticleStrippingOnlyWhenAsked) {
+  EXPECT_EQ(fold("The Iliad"), "the iliad");
+  EXPECT_EQ(fold("The Iliad", true), "iliad");
+  EXPECT_EQ(fold("Les Mis\xC3\xA9rables", true), "miserables");
+  EXPECT_EQ(fold("L\xE2\x80\x99\xC3\x89n\xC3\xA9ide", true), "eneide");
+  // A title that IS an article-like word must not vanish.
+  EXPECT_EQ(fold("The", true), "the");
+}
+
+TEST(LibraryAuthorKey, OrderAndPunctuationDoNotMatter) {
+  const std::string expected = authorKey("Lu Xun");
+  EXPECT_FALSE(expected.empty());
+  for (const char* spelling : {"Lu, Xun", "Xun, Lu", "Lu Xun_", "Lu Xun [Xun, Lu]", "  lu   xun  "}) {
+    EXPECT_EQ(authorKey(spelling), expected) << spelling;
+  }
+}
+
+TEST(LibraryAuthorKey, InitialsAreIgnored) {
+  EXPECT_EQ(authorKey("Herbert G Wells"), authorKey("Herbert Wells"));
+  EXPECT_EQ(authorKey("Wells, Herbert G."), authorKey("Herbert Wells"));
+}
+
+TEST(LibraryAuthorKey, SecondaryAuthorsAndBracketsDropped) {
+  EXPECT_EQ(authorKey("Emile Erckmann; Alexandre Chatrian"), authorKey("Emile Erckmann"));
+  EXPECT_EQ(authorKey("George Sand [Sand, George]"), authorKey("George Sand"));
+}
+
+TEST(LibraryAuthorKey, DistinctPeopleDoNotCollide) {
+  EXPECT_NE(authorKey("Mary Wollstonecraft"), authorKey("Charlotte Bronte"));
+  EXPECT_NE(authorKey("Victor Hugo"), authorKey("Jules Verne"));
+}
+
+TEST(LibraryAuthorKey, FitsTheRecordFieldWithoutCollapsingToAForename) {
+  const std::string key = authorKey("Bartholomew Fitzgerald Wellington");
+  ASSERT_FALSE(key.empty());
+  EXPECT_LE(key.size(), library::AUTHOR_KEY_MAX_BYTES);
+  EXPECT_NE(key.back(), ' ');
+
+  // Sorting puts a short forename first, so cutting on a token boundary would
+  // reduce this to "mary" and merge every Alex in the library. The byte cut must
+  // keep enough of the surname to discriminate.
+  const std::string mary = authorKey("Wollstonecraft, Mary");
+  EXPECT_GT(mary.size(), 5u);
+  EXPECT_NE(mary, "mary");
+  EXPECT_NE(mary, authorKey("Mary Trevelyan"));
+
+  // A truncated key stays a prefix of the untruncated one, so grouping is stable
+  // however long the name is.
+  EXPECT_EQ(authorKey("Wollstonecraft, Maryse").rfind("mary", 0), 0u);
+
+  EXPECT_FALSE(authorKey("Nebuchadnezzarson").empty());
+  EXPECT_TRUE(authorKey("").empty());
+  EXPECT_TRUE(authorKey("Q. X. Z.").empty());  // initials only: no identity
+}
+
+// --- matchesQuery ------------------------------------------------------------
+//
+// Cases taken from the shape of the accented and
+// apostrophised titles real cards hold — what a naive matcher gets wrong.
+
+TEST(MatchesQuery, EmptyQueryMatchesEverything) {
+  EXPECT_TRUE(library::matchesQuery(library::fold("Wuthering Heights"), ""));
+}
+
+TEST(MatchesQuery, WholeWordMatches) {
+  EXPECT_TRUE(library::matchesQuery(library::fold("Wuthering Heights"), library::fold("heights")));
+}
+
+TEST(MatchesQuery, PrefixOfOneWordIsEnough) {
+  EXPECT_TRUE(library::matchesQuery(library::fold("Wuthering Heights"), library::fold("hei")));
+}
+
+// The point of the whole design: six keypresses instead of ten, on a panel where
+// each one costs a full repaint.
+TEST(MatchesQuery, EveryWordMayBeAbbreviated) {
+  EXPECT_TRUE(library::matchesQuery(library::fold("Wuthering Heights"), library::fold("wut hei")));
+}
+
+TEST(MatchesQuery, WordsNeedNotBeInOrder) {
+  EXPECT_TRUE(library::matchesQuery(library::fold("Wuthering Heights"), library::fold("heights wuthering")));
+}
+
+TEST(MatchesQuery, EveryWordMustHit) {
+  EXPECT_FALSE(library::matchesQuery(library::fold("Wuthering Heights"), library::fold("wuthering blue")));
+}
+
+// A prefix, not a substring: "eights" is inside "heights" but starts no word.
+TEST(MatchesQuery, MidWordDoesNotMatch) {
+  EXPECT_FALSE(library::matchesQuery(library::fold("Wuthering Heights"), library::fold("eights")));
+}
+
+TEST(MatchesQuery, AccentsAreIgnoredOnBothSides) {
+  EXPECT_TRUE(library::matchesQuery(library::fold("L'Énéide"), library::fold("eneide")));
+  EXPECT_TRUE(library::matchesQuery(library::fold("L'Eneide"), library::fold("énéide")));
+  EXPECT_TRUE(library::matchesQuery(library::fold("Éluard"), library::fold("eluard")));
+}
+
+TEST(MatchesQuery, ApostropheSplitsWords) {
+  EXPECT_TRUE(library::matchesQuery(library::fold("Le bureau d'à côté"), library::fold("cote")));
+}
+
+TEST(MatchesQuery, CaseIsIgnored) {
+  EXPECT_TRUE(library::matchesQuery(library::fold("Wuthering Heights"), library::fold("HEIGHTS")));
+}
+
+// The stored fold is capped at 96 bytes, so a query word beyond that cannot be
+// found. Asserted rather than left implicit: it is the one place a search can
+// honestly fail to find a book that is really there.
+TEST(MatchesQuery, LongTitlesAreOnlySearchableWithinTheStoredFold) {
+  const std::string longTitle(120, 'a');
+  const std::string folded = library::fold(longTitle + " needle").substr(0, 96);
+  EXPECT_FALSE(library::matchesQuery(folded, library::fold("needle")));
+}
+
+// --- inverted author names ---------------------------------------------------
+
+TEST(CleanPersonName, InvertedNameIsTurnedRound) {
+  EXPECT_EQ(library::cleanPersonName("Austen, Jane"), "Jane Austen");
+  EXPECT_EQ(library::cleanPersonName("Wollstonecraft, Mary"), "Mary Wollstonecraft");
+}
+
+TEST(CleanPersonName, PlainNameIsUntouched) { EXPECT_EQ(library::cleanPersonName("Emily Bronte"), "Emily Bronte"); }
+
+// Two commas mean a suffix or a list, not an inversion — leave it alone rather
+// than scramble it.
+TEST(CleanPersonName, MultipleCommasAreLeftAlone) {
+  // The trailing full stop is stripped by the existing noise rules.
+  EXPECT_EQ(library::cleanPersonName("Smith, John, Jr."), "Smith, John, Jr");
+}
+
+TEST(CleanPersonName, DanglingCommaIsNotAnInversion) { EXPECT_EQ(library::cleanPersonName("Austen,"), "Austen"); }
+
+// --- surnameKey --------------------------------------------------------------
+
+TEST(SurnameKey, SurnameLeadsThenGivenNames) {
+  EXPECT_EQ(library::surnameKey("Herman Melville"), "melville herman");
+  EXPECT_EQ(library::surnameKey("Mary Wollstonecraft"), "wollstonecraft mary");
+}
+
+TEST(SurnameKey, SingleWordKeysOnItself) { EXPECT_EQ(library::surnameKey("Voltaire"), "voltaire"); }
+
+TEST(SurnameKey, AccentsAreFolded) { EXPECT_EQ(library::surnameKey("Paul Éluard"), "eluard paul"); }
+
+TEST(SurnameKey, ThreeWordNamesTakeTheLast) { EXPECT_EQ(library::surnameKey("Herbert G. Wells"), "wells herbert g"); }
+
+TEST(SurnameKey, EmptyStaysEmpty) { EXPECT_EQ(library::surnameKey(""), ""); }
+
+// The whole point of keying off the DISPLAY name: the spelling vote has already
+// made every book by one author show one name, so a group cannot land in two
+// places even though "Victor Hugo" and "Hugo Victor" both exist in the wild.
+TEST(SurnameKey, HarmonisedDisplayNameKeepsAGroupTogether) {
+  EXPECT_NE(library::surnameKey("Victor Hugo"), library::surnameKey("Hugo Victor"));
+}