diff --git a/client/CMakeLists.txt b/client/CMakeLists.txt index d8cbc7cb3..db358960d 100644 --- a/client/CMakeLists.txt +++ b/client/CMakeLists.txt @@ -343,6 +343,11 @@ set (TARGET_SOURCES ${PM3_ROOT}/client/src/cipurse/cipursecore.c ${PM3_ROOT}/client/src/cipurse/cipursetest.c ${PM3_ROOT}/client/src/loclass/cipher.c + ${PM3_ROOT}/client/src/loclass/cipher_bs.c + ${PM3_ROOT}/client/src/loclass/cipher_bs_avx2.c + ${PM3_ROOT}/client/src/loclass/cipher_bs_avx512.c + ${PM3_ROOT}/client/src/loclass/cipher_bs_dispatch.c + ${PM3_ROOT}/client/src/loclass/cipher_bs_neon.c ${PM3_ROOT}/client/src/loclass/cipherutils.c ${PM3_ROOT}/client/src/loclass/elite_crack.c ${PM3_ROOT}/client/src/loclass/hash1_brute.c diff --git a/client/Makefile b/client/Makefile index 308fe2de9..6854ebcd6 100644 --- a/client/Makefile +++ b/client/Makefile @@ -806,6 +806,11 @@ SRCS = mifare/aiddesfire.c \ iso7816/apduinfo.c \ iso7816/iso7816core.c \ loclass/cipher.c \ + loclass/cipher_bs.c \ + loclass/cipher_bs_avx2.c \ + loclass/cipher_bs_avx512.c \ + loclass/cipher_bs_dispatch.c \ + loclass/cipher_bs_neon.c \ loclass/cipherutils.c \ loclass/elite_crack.c \ loclass/ikeys.c \ diff --git a/client/experimental_lib/CMakeLists.txt b/client/experimental_lib/CMakeLists.txt index ff08543a4..27ceb0108 100644 --- a/client/experimental_lib/CMakeLists.txt +++ b/client/experimental_lib/CMakeLists.txt @@ -263,6 +263,11 @@ set (TARGET_SOURCES ${PM3_ROOT}/client/src/cipurse/cipursecore.c ${PM3_ROOT}/client/src/cipurse/cipursetest.c ${PM3_ROOT}/client/src/loclass/cipher.c + ${PM3_ROOT}/client/src/loclass/cipher_bs.c + ${PM3_ROOT}/client/src/loclass/cipher_bs_avx2.c + ${PM3_ROOT}/client/src/loclass/cipher_bs_avx512.c + ${PM3_ROOT}/client/src/loclass/cipher_bs_dispatch.c + ${PM3_ROOT}/client/src/loclass/cipher_bs_neon.c ${PM3_ROOT}/client/src/loclass/cipherutils.c ${PM3_ROOT}/client/src/loclass/elite_crack.c ${PM3_ROOT}/client/src/loclass/hash1_brute.c diff --git a/client/src/cmdhficlass.c b/client/src/cmdhficlass.c index 0cfb13c54..f68ab6eb2 100644 --- a/client/src/cmdhficlass.c +++ b/client/src/cmdhficlass.c @@ -33,6 +33,8 @@ #include "des.h" #include "loclass/cipherutils.h" #include "loclass/cipher.h" +#include "loclass/cipher_bs.h" +#include "loclass/cipher_bs_dispatch.h" #include "loclass/ikeys.h" #include "loclass/elite_crack.h" #include "fileutils.h" @@ -6108,15 +6110,46 @@ typedef struct { pthread_mutex_t *log_lock; } thread_args_t; +// Lock-guarded "found" announcement; factored out because the bitslice and +// scalar paths both reach it. +static void legbrute_announce(thread_args_t *args, const uint8_t div_key[8]) { + pthread_mutex_lock(args->log_lock); + if (!*(args->found)) { + *args->found = true; + PrintAndLogEx(NORMAL, "\n"); + PrintAndLogEx(SUCCESS, "Found valid raw key " _GREEN_("%s"), sprint_hex_inrow(div_key, 8)); + PrintAndLogEx(HINT, "Hint: Run `"_YELLOW_("hf iclass unhash -k %s")"` to find the needed pre-images", sprint_hex_inrow(div_key, 8)); + PrintAndLogEx(INFO, "Done!"); + PrintAndLogEx(NORMAL, ""); + } + pthread_mutex_unlock(args->log_lock); +} + // HF iClass legbrute - Brute-force worker thread static void *brute_thread(void *args_void) { thread_args_t *args = (thread_args_t *)args_void; uint8_t div_key[8]; - uint8_t mac[4]; - uint8_t verification_mac[4]; uint64_t index = args->index_start; + // Scalar per-candidate hot-loop expansions (used in the alignment + // prefix/tail and for MAC2 verification after a bitslice hit). + uint8_t y_bits1[96]; + uint8_t y_bits2[96]; + prepare_ccnr_bits(args->CCNR1, y_bits1); + prepare_ccnr_bits(args->CCNR2, y_bits2); + + // Bitslice expansions for the fast path. The backend is picked once at + // startup (widest SIMD width the CPU supports: AVX-512 > AVX2 > NEON > u64). + // MAC2 stays scalar — it only runs on the vanishingly rare MAC1 collisions, + // so there is no reason to pre-expand it. + const bs_backend_t *bs = bs_best_backend(); + const uint64_t bs_align = (uint64_t)(bs->width - 1); + uint64_t y_bits1_bs[96 * BS_MAX_WORDS]; + uint64_t target_mac1_bs[32 * BS_MAX_WORDS]; + bs->prepare_ccnr(args->CCNR1, y_bits1_bs); + bs->prepare_mac(args->MAC_TAG1, target_mac1_bs); + if (args->debug) { pthread_mutex_lock(args->log_lock); @@ -6144,28 +6177,55 @@ static void *brute_thread(void *args_void) { uint32_t progress_countdown = 1000000; while (index < args->index_end && !*(args->found) && !*(args->aborted)) { - generate_key_block_inverted(args->startingKey, index, div_key); - doMAC_brute(args->CCNR1, div_key, mac); + uint64_t step; + const uint64_t remaining = args->index_end - index; - if (memcmp(mac, args->MAC_TAG1, 4) == 0) { - doMAC_brute(args->CCNR2, div_key, verification_mac); - if (memcmp(verification_mac, args->MAC_TAG2, 4) == 0) { - pthread_mutex_lock(args->log_lock); - if (!*(args->found)) { - *args->found = true; - PrintAndLogEx(NORMAL, "\n"); - PrintAndLogEx(SUCCESS, "Found valid raw key " _GREEN_("%s"), sprint_hex_inrow(div_key, 8)); - PrintAndLogEx(HINT, "Hint: Run `"_YELLOW_("hf iclass unhash -k %s")"` to find the needed pre-images", sprint_hex_inrow(div_key, 8)); - PrintAndLogEx(INFO, "Done!"); - PrintAndLogEx(NORMAL, ""); + if ((index & bs_align) == 0 && remaining >= (uint64_t)bs->width) { + // Fast path: bs->width-wide bitslice MAC1 sweep. Build the key + // schedule for W consecutive candidates and test them in parallel. + uint64_t kb[64 * BS_MAX_WORDS]; + bs->build_key(args->startingKey, index, kb); + + uint64_t match[BS_MAX_WORDS]; + bs->match(y_bits1_bs, kb, target_mac1_bs, match); + + // MAC1 collisions are ~W / 2^32 per batch on average (i.e., zero + // until the true key's batch is reached). For each surviving lane, + // reconstruct its div_key and verify against MAC2 with the scalar + // prebit matcher. + bool done = false; + for (int w = 0; w < bs->words && !done; w++) { + uint64_t m = match[w]; + while (m != 0) { + const int L = __builtin_ctzll(m); + m &= m - 1; + const uint64_t cand = index + (uint64_t)(w * 64 + L); + generate_key_block_inverted(args->startingKey, cand, div_key); + if (doMAC_brute_match_prebit(y_bits2, div_key, args->MAC_TAG2)) { + legbrute_announce(args, div_key); + done = true; + break; + } } - pthread_mutex_unlock(args->log_lock); - break; } + + step = (uint64_t)bs->width; + } else { + // Scalar fallback: non-64-aligned prefix, the sub-64-candidate + // tail, or any thread whose slice is not a multiple of 64. + generate_key_block_inverted(args->startingKey, index, div_key); + + if (doMAC_brute_match_prebit(y_bits1, div_key, args->MAC_TAG1)) { + if (doMAC_brute_match_prebit(y_bits2, div_key, args->MAC_TAG2)) { + legbrute_announce(args, div_key); + } + } + + step = 1; } uint64_t thread_progress = index - args->index_start; - if (--progress_countdown == 0 && !*(args->found)) { + if (progress_countdown <= step && !*(args->found)) { progress_countdown = 1000000; if (args->thread_id == 0) { @@ -6207,8 +6267,10 @@ static void *brute_thread(void *args_void) { pthread_mutex_unlock(args->log_lock); } + } else { + progress_countdown -= (uint32_t)step; } - index++; + index += step; } return NULL; } @@ -6225,7 +6287,9 @@ static int CmdHFiClassLegBrute_MT(uint8_t epurse[8], uint8_t macs[8], uint8_t ma PrintAndLogEx(INFO, "Capping threads at available CPU count (%d)", max_threads); thread_count = max_threads; } - PrintAndLogEx(INFO, "Bruteforcing using " _YELLOW_("%u") " threads", thread_count); + const bs_backend_t *bs = bs_best_backend(); + PrintAndLogEx(INFO, "Bruteforcing using " _YELLOW_("%u") " threads, " _YELLOW_("%s") " bitslice (%d lanes)", + thread_count, bs->name, bs->width); PrintAndLogEx(NORMAL, ""); uint8_t CCNR[12], CCNR2[12], MAC_TAG[4], MAC_TAG2[4]; diff --git a/client/src/loclass/cipher.c b/client/src/loclass/cipher.c index 62d997244..0f3b067cb 100644 --- a/client/src/loclass/cipher.c +++ b/client/src/loclass/cipher.c @@ -233,6 +233,62 @@ void doMAC_brute(const uint8_t *cc_nr, const uint8_t *div_key, uint8_t mac[4]) { output_bytes(div_key, &s, mac, 4); } +// Expand 12 cc_nr bytes into 96 LSB-first cipher input bits. Done once per thread +// so the per-candidate hot loop never re-shifts the same bytes. +void prepare_ccnr_bits(const uint8_t *cc_nr, uint8_t y_bits[96]) { + for (int i = 0; i < 12; i++) { + uint8_t b = cc_nr[i]; + y_bits[i * 8 + 0] = b & 1; + y_bits[i * 8 + 1] = (b >> 1) & 1; + y_bits[i * 8 + 2] = (b >> 2) & 1; + y_bits[i * 8 + 3] = (b >> 3) & 1; + y_bits[i * 8 + 4] = (b >> 4) & 1; + y_bits[i * 8 + 5] = (b >> 5) & 1; + y_bits[i * 8 + 6] = (b >> 6) & 1; + y_bits[i * 8 + 7] = (b >> 7) & 1; + } +} + +// Pre-expanded-input variant of doMAC_brute with per-byte early-out comparison. +// Returns true iff the 32-bit MAC matches target_mac. On miss we bail after the +// first non-matching byte, skipping the remaining output ticks. +bool doMAC_brute_match_prebit(const uint8_t *y_bits, const uint8_t *div_key, const uint8_t target_mac[4]) { + State_t s = init(div_key); + + for (int i = 0; i < 96; i++) { + successor(div_key, &s, y_bits[i]); + } + + for (int byte_idx = 0; byte_idx < 4; byte_idx++) { + uint8_t bout = 0; + bout |= (s.r & 0x4) >> 2; + successor(div_key, &s, 0); + bout |= (s.r & 0x4) >> 1; + successor(div_key, &s, 0); + bout |= (s.r & 0x4); + successor(div_key, &s, 0); + bout |= (s.r & 0x4) << 1; + successor(div_key, &s, 0); + bout |= (s.r & 0x4) << 2; + successor(div_key, &s, 0); + bout |= (s.r & 0x4) << 3; + successor(div_key, &s, 0); + bout |= (s.r & 0x4) << 4; + successor(div_key, &s, 0); + bout |= (s.r & 0x4) << 5; + + if (bout != target_mac[byte_idx]) { + return false; + } + + if (byte_idx < 3) { + successor(div_key, &s, 0); + } + } + + return true; +} + void doMAC_N(uint8_t *address_data_p, uint8_t address_data_size, uint8_t *div_key_p, uint8_t mac[4]) { uint8_t *address_data; uint8_t div_key[8]; diff --git a/client/src/loclass/cipher.h b/client/src/loclass/cipher.h index 923e09809..edbbbd030 100644 --- a/client/src/loclass/cipher.h +++ b/client/src/loclass/cipher.h @@ -35,12 +35,22 @@ #ifndef CIPHER_H #define CIPHER_H #include +#include #include "pm3_cmd.h" void doMAC(uint8_t *cc_nr_p, uint8_t *div_key_p, uint8_t mac[4]); void doMAC_brute(const uint8_t *cc_nr, const uint8_t *div_key, uint8_t mac[4]); void doMAC_N(uint8_t *address_data_p, uint8_t address_data_size, uint8_t *div_key_p, uint8_t mac[4]); +// Expand 12 cc_nr bytes into 96 LSB-first input bits for the cipher. Call once per thread +// and pass the result to doMAC_brute_match_prebit to skip the per-candidate shift work. +void prepare_ccnr_bits(const uint8_t *cc_nr, uint8_t y_bits[96]); + +// doMAC_brute variant with pre-expanded input bits and early-out MAC comparison. +// Returns true iff the 32-bit MAC for (cc_nr, div_key) equals target_mac. +// Aborts after the first non-matching output byte, saving ~75% of output ticks on misses. +bool doMAC_brute_match_prebit(const uint8_t *y_bits, const uint8_t *div_key, const uint8_t target_mac[4]); + #ifndef ON_DEVICE int testMAC(void); #endif diff --git a/client/src/loclass/cipher_bs.c b/client/src/loclass/cipher_bs.c new file mode 100644 index 000000000..dc13e7ad7 --- /dev/null +++ b/client/src/loclass/cipher_bs.c @@ -0,0 +1,235 @@ +//----------------------------------------------------------------------------- +// Copyright (C) Proxmark3 contributors. See AUTHORS.md for details. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// See LICENSE.txt for the text of the license. +//----------------------------------------------------------------------------- +// 64-wide bitsliced iClass cipher MAC. Mirrors m64000_a3-pure.cl but targets +// portable uint64_t so every CPU gets 64 parallel candidates per bs_tick, +// regardless of SSE/AVX availability. +//----------------------------------------------------------------------------- + +#include "cipher_bs.h" + +#define BS_ALL_ONES (~(uint64_t)0) + +// Bitsliced 3-bit select over 8 key bytes. Each z/nz is the bit across all +// 64 lanes. Returns the mux result per lane. +static inline uint64_t bs_mux8(uint64_t z0, uint64_t z1, uint64_t z2, + uint64_t nz0, uint64_t nz1, uint64_t nz2, + uint64_t v0, uint64_t v1, uint64_t v2, uint64_t v3, + uint64_t v4, uint64_t v5, uint64_t v6, uint64_t v7) { + const uint64_t a0 = (z2 & v1) | (nz2 & v0); + const uint64_t a1 = (z2 & v3) | (nz2 & v2); + const uint64_t a2 = (z2 & v5) | (nz2 & v4); + const uint64_t a3 = (z2 & v7) | (nz2 & v6); + const uint64_t b0 = (z1 & a1) | (nz1 & a0); + const uint64_t b1 = (z1 & a3) | (nz1 & a2); + return (z0 & b1) | (nz0 & b0); +} + +// Bitsliced 8-bit add (ripple-carry). out = (a + b) mod 256 per lane. +static inline void bs_add8(const uint64_t *a, const uint64_t *b, uint64_t *out) { + uint64_t carry = 0; + for (int i = 0; i < 8; i++) { + const uint64_t x = a[i] ^ b[i]; + out[i] = x ^ carry; + carry = (a[i] & b[i]) | (carry & x); + } +} + +// One cipher tick, bitsliced. t[16], b[8], l[8], r[8] are the cipher state +// where array index is bit position (LSB first) and each slot is a uint64_t +// holding that bit for 64 lanes. kb[64] is the bitsliced key schedule. +// y_bs is the input bit broadcast to all 64 lanes (0 or all-ones). +static inline void bs_tick(uint64_t *t, uint64_t *b, uint64_t *l, uint64_t *r, + const uint64_t *kb, uint64_t y_bs) { + + const uint64_t Tt = t[15] ^ t[14] ^ t[10] ^ t[8] ^ t[5] ^ t[4] ^ t[1] ^ t[0]; + const uint64_t Bt = b[6] ^ b[5] ^ b[4] ^ b[0]; + + const uint64_t cr0 = r[7], cr1 = r[6], cr2 = r[5], cr3 = r[4]; + const uint64_t cr4 = r[3], cr5 = r[2], cr6 = r[1], cr7 = r[0]; + + const uint64_t new_t = Tt ^ cr0 ^ cr4; + const uint64_t new_b = Bt ^ cr7; + + // Shift t (bit 0 drops out, new_t goes in at bit 15) and b similarly. + t[0] = t[1]; t[1] = t[2]; t[2] = t[3]; t[3] = t[4]; + t[4] = t[5]; t[5] = t[6]; t[6] = t[7]; t[7] = t[8]; + t[8] = t[9]; t[9] = t[10]; t[10] = t[11]; t[11] = t[12]; + t[12] = t[13]; t[13] = t[14]; t[14] = t[15]; t[15] = new_t; + + b[0] = b[1]; b[1] = b[2]; b[2] = b[3]; b[3] = b[4]; + b[4] = b[5]; b[5] = b[6]; b[6] = b[7]; b[7] = new_b; + + const uint64_t ncr3 = ~cr3; + const uint64_t ncr5 = ~cr5; + + const uint64_t z0 = (cr0 & cr2) ^ (cr1 & ncr3) ^ (cr2 | cr4); + const uint64_t z1 = (cr0 | cr2) ^ (cr5 | cr7) ^ cr1 ^ cr6 ^ Tt ^ y_bs; + const uint64_t z2 = (cr3 & ncr5) ^ (cr4 & cr6) ^ cr7 ^ Tt; + + const uint64_t nz0 = ~z0, nz1 = ~z1, nz2 = ~z2; + + uint64_t val[8]; + for (int bit = 0; bit < 8; bit++) { + val[bit] = bs_mux8(z0, z1, z2, nz0, nz1, nz2, + kb[0 * 8 + bit], kb[1 * 8 + bit], + kb[2 * 8 + bit], kb[3 * 8 + bit], + kb[4 * 8 + bit], kb[5 * 8 + bit], + kb[6 * 8 + bit], kb[7 * 8 + bit]); + } + + val[0] ^= b[0]; val[1] ^= b[1]; val[2] ^= b[2]; val[3] ^= b[3]; + val[4] ^= b[4]; val[5] ^= b[5]; val[6] ^= b[6]; val[7] ^= b[7]; + + uint64_t old_r[8]; + for (int i = 0; i < 8; i++) old_r[i] = r[i]; + + // r = val + l ; l = r + old_r + bs_add8(val, l, r); + bs_add8(r, old_r, l); +} + +void prepare_ccnr_bits_bs(const uint8_t *cc_nr, uint64_t y_bits_bs[96]) { + for (int i = 0; i < 12; i++) { + const uint8_t byte = cc_nr[i]; + y_bits_bs[i * 8 + 0] = (byte & 1) ? BS_ALL_ONES : 0; + y_bits_bs[i * 8 + 1] = ((byte >> 1) & 1) ? BS_ALL_ONES : 0; + y_bits_bs[i * 8 + 2] = ((byte >> 2) & 1) ? BS_ALL_ONES : 0; + y_bits_bs[i * 8 + 3] = ((byte >> 3) & 1) ? BS_ALL_ONES : 0; + y_bits_bs[i * 8 + 4] = ((byte >> 4) & 1) ? BS_ALL_ONES : 0; + y_bits_bs[i * 8 + 5] = ((byte >> 5) & 1) ? BS_ALL_ONES : 0; + y_bits_bs[i * 8 + 6] = ((byte >> 6) & 1) ? BS_ALL_ONES : 0; + y_bits_bs[i * 8 + 7] = ((byte >> 7) & 1) ? BS_ALL_ONES : 0; + } +} + +void prepare_target_mac_bs(const uint8_t target_mac[4], uint64_t target_mac_bs[32]) { + for (int i = 0; i < 4; i++) { + const uint8_t byte = target_mac[i]; + for (int bit = 0; bit < 8; bit++) { + target_mac_bs[i * 8 + bit] = ((byte >> bit) & 1) ? BS_ALL_ONES : 0; + } + } +} + +// Lane patterns for L bits 0..5 (L = 0..63). Each pattern has bit L set iff +// bit k of L is 1. These are constant for 64-aligned batches — bits 0..5 of +// (index_start + L) collapse to the pure lane index L. +static const uint64_t LANE_BITS[6] = { + 0xAAAAAAAAAAAAAAAAULL, // L bit 0 + 0xCCCCCCCCCCCCCCCCULL, // L bit 1 + 0xF0F0F0F0F0F0F0F0ULL, // L bit 2 + 0xFF00FF00FF00FF00ULL, // L bit 3 + 0xFFFF0000FFFF0000ULL, // L bit 4 + 0xFFFFFFFF00000000ULL, // L bit 5 +}; + +void build_bitslice_key_64(const uint8_t partial_key[8], uint64_t index_start, uint64_t kb[64]) { + + // Low 3 bits of each key byte are fixed by the partial key and broadcast. + for (int j = 0; j < 8; j++) { + kb[j * 8 + 0] = (partial_key[j] & 0x01) ? BS_ALL_ONES : 0; + kb[j * 8 + 1] = (partial_key[j] & 0x02) ? BS_ALL_ONES : 0; + kb[j * 8 + 2] = (partial_key[j] & 0x04) ? BS_ALL_ONES : 0; + } + + // High 5 bits per byte come from the 40-bit index. Byte j consumes index + // bits [5*(7-j) .. 5*(7-j)+4]. For a 64-aligned index_start, bits 0..5 of + // (index_start + L) equal the lane index L, so they pick up LANE_BITS; + // bits 6..39 are broadcast from index_start. + for (int j = 0; j < 8; j++) { + const int base_bit = 5 * (7 - j); + for (int k = 0; k < 5; k++) { + const int idx_bit = base_bit + k; + uint64_t pattern; + if (idx_bit < 6) { + pattern = LANE_BITS[idx_bit]; + } else { + pattern = ((index_start >> idx_bit) & 1) ? BS_ALL_ONES : 0; + } + kb[j * 8 + 3 + k] = pattern; + } + } +} + +// init(k) for the iClass cipher with k[0] variable and b, t constants: +// l = ((k[0] ^ 0x4c) + 0xEC) & 0xff +// r = ((k[0] ^ 0x4c) + 0x21) & 0xff +// b = 0x4C, t = 0xE012 +// Bitsliced: flip kb[0..7] bits that differ under XOR with 0x4C, then bs_add8 +// against the broadcast 0xEC / 0x21 bit patterns. +static inline void bs_init_state(const uint64_t kb[64], + uint64_t l[8], uint64_t r[8], + uint64_t b[8], uint64_t t[16]) { + + // 0x4C = 0b01001100 → bits set at positions 2, 3, 6. + uint64_t k0xor[8]; + k0xor[0] = kb[0]; + k0xor[1] = kb[1]; + k0xor[2] = kb[2] ^ BS_ALL_ONES; + k0xor[3] = kb[3] ^ BS_ALL_ONES; + k0xor[4] = kb[4]; + k0xor[5] = kb[5]; + k0xor[6] = kb[6] ^ BS_ALL_ONES; + k0xor[7] = kb[7]; + + // 0xEC = 0b11101100 → LSB-first bit pattern: 0,0,1,1,0,1,1,1 + const uint64_t ec[8] = {0, 0, BS_ALL_ONES, BS_ALL_ONES, 0, BS_ALL_ONES, BS_ALL_ONES, BS_ALL_ONES}; + // 0x21 = 0b00100001 → LSB-first: 1,0,0,0,0,1,0,0 + const uint64_t x21[8] = {BS_ALL_ONES, 0, 0, 0, 0, BS_ALL_ONES, 0, 0}; + + bs_add8(k0xor, ec, l); + bs_add8(k0xor, x21, r); + + // b = 0x4C → 0,0,1,1,0,0,1,0 (LSB first) + b[0] = 0; b[1] = 0; + b[2] = BS_ALL_ONES; b[3] = BS_ALL_ONES; + b[4] = 0; b[5] = 0; + b[6] = BS_ALL_ONES; b[7] = 0; + + // t = 0xE012 → LSB-first across 16 bits: + // 0xE012 = 0b1110_0000_0001_0010 + // bit0=0 bit1=1 bit2=0 bit3=0 bit4=1 bit5=0 bit6=0 bit7=0 + // bit8=0 bit9=0 bit10=0 bit11=0 bit12=0 bit13=1 bit14=1 bit15=1 + t[ 0] = 0; t[ 1] = BS_ALL_ONES; t[ 2] = 0; t[ 3] = 0; + t[ 4] = BS_ALL_ONES; t[ 5] = 0; t[ 6] = 0; t[ 7] = 0; + t[ 8] = 0; t[ 9] = 0; t[10] = 0; t[11] = 0; + t[12] = 0; t[13] = BS_ALL_ONES; t[14] = BS_ALL_ONES; t[15] = BS_ALL_ONES; +} + +uint64_t doMAC_brute_match64(const uint64_t y_bits_bs[96], const uint64_t kb[64], const uint64_t target_mac_bs[32]) { + + uint64_t l[8], r[8], b[8], t[16]; + bs_init_state(kb, l, r, b, t); + + // 96-tick input phase: consume the 96 cc_nr bits. + for (int i = 0; i < 96; i++) { + bs_tick(t, b, l, r, kb, y_bits_bs[i]); + } + + // 32-tick output phase with per-byte early-exit. r[2] at tick t yields + // bit (t mod 8) of MAC byte (t / 8), so we AND away lanes whose output + // disagrees with target_mac_bs[t]. Bail once every lane is eliminated. + uint64_t mac_match = BS_ALL_ONES; + + for (int tick = 0; tick < 32; tick++) { + mac_match &= ~(r[2] ^ target_mac_bs[tick]); + + if ((tick == 7 || tick == 15 || tick == 23) && mac_match == 0) { + return 0; + } + + if (tick < 31) { + bs_tick(t, b, l, r, kb, 0); + } + } + + return mac_match; +} diff --git a/client/src/loclass/cipher_bs.h b/client/src/loclass/cipher_bs.h new file mode 100644 index 000000000..8b36e85ec --- /dev/null +++ b/client/src/loclass/cipher_bs.h @@ -0,0 +1,43 @@ +//----------------------------------------------------------------------------- +// Copyright (C) Proxmark3 contributors. See AUTHORS.md for details. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// See LICENSE.txt for the text of the license. +//----------------------------------------------------------------------------- +// 64-wide bitsliced iClass cipher MAC for the legbrute hot loop. Ports the +// approach used in the hashcat m64000 kernel to portable C (uint64_t lanes). +// Each "bit" of the cipher state is stored as a uint64_t holding that bit for +// 64 parallel candidates; all arithmetic is expressed as bitwise ops plus a +// ripple-carry adder, so one bs_tick advances 64 MACs at once. +//----------------------------------------------------------------------------- + +#ifndef CIPHER_BS_H +#define CIPHER_BS_H + +#include + +// Expand 12 cc_nr bytes into 96 LSB-first bit masks (0 or all-ones) shared +// across the 64 lanes. Call once per thread. +void prepare_ccnr_bits_bs(const uint8_t *cc_nr, uint64_t y_bits_bs[96]); + +// Expand a 4-byte target MAC into 32 bit masks for per-tick comparison. +// target_mac_bs[t] == ~0 iff the target bit at tick t is 1. +// Tick t corresponds to bit (t mod 8) of target_mac[t / 8] (matches the +// scalar output_bytes packing order). +void prepare_target_mac_bs(const uint8_t target_mac[4], uint64_t target_mac_bs[32]); + +// Build the 64-wide bitsliced key for 64 consecutive candidates starting at +// index_start. index_start MUST be a multiple of 64 so the low 6 index bits +// are the pure lane index. +void build_bitslice_key_64(const uint8_t partial_key[8], uint64_t index_start, uint64_t kb[64]); + +// Run the bitsliced MAC against a pre-expanded target and return a 64-bit +// lane mask: bit L is set iff candidate (index_start + L) produces target_mac. +// Early-exits after any 8-bit MAC byte that rules out every lane. +uint64_t doMAC_brute_match64(const uint64_t y_bits_bs[96], const uint64_t kb[64], const uint64_t target_mac_bs[32]); + +#endif // CIPHER_BS_H diff --git a/client/src/loclass/cipher_bs_avx2.c b/client/src/loclass/cipher_bs_avx2.c new file mode 100644 index 000000000..e7d564851 --- /dev/null +++ b/client/src/loclass/cipher_bs_avx2.c @@ -0,0 +1,275 @@ +//----------------------------------------------------------------------------- +// Copyright (C) Proxmark3 contributors. See AUTHORS.md for details. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// See LICENSE.txt for the text of the license. +//----------------------------------------------------------------------------- +// AVX2 256-wide bitsliced iClass cipher MAC. Mirrors cipher_bs.c exactly; +// the only substantive change is the lane type (__m256i in place of uint64_t). +//----------------------------------------------------------------------------- + +#include "cipher_bs_avx2.h" + +#if defined(__x86_64__) || defined(__i386__) || defined(_M_X64) || defined(_M_IX86) + +#include + +#if defined(__GNUC__) || defined(__clang__) +#pragma GCC push_options +#pragma GCC target("avx2") +#endif + +#define BS_ZERO _mm256_setzero_si256() +#define BS_ONES _mm256_set1_epi64x(-1) + +static inline __m256i bs_not(__m256i v) { return _mm256_xor_si256(v, BS_ONES); } + +static inline __m256i bs_mux8(__m256i z0, __m256i z1, __m256i z2, + __m256i nz0, __m256i nz1, __m256i nz2, + __m256i v0, __m256i v1, __m256i v2, __m256i v3, + __m256i v4, __m256i v5, __m256i v6, __m256i v7) { + const __m256i a0 = _mm256_or_si256(_mm256_and_si256(z2, v1), _mm256_and_si256(nz2, v0)); + const __m256i a1 = _mm256_or_si256(_mm256_and_si256(z2, v3), _mm256_and_si256(nz2, v2)); + const __m256i a2 = _mm256_or_si256(_mm256_and_si256(z2, v5), _mm256_and_si256(nz2, v4)); + const __m256i a3 = _mm256_or_si256(_mm256_and_si256(z2, v7), _mm256_and_si256(nz2, v6)); + const __m256i b0 = _mm256_or_si256(_mm256_and_si256(z1, a1), _mm256_and_si256(nz1, a0)); + const __m256i b1 = _mm256_or_si256(_mm256_and_si256(z1, a3), _mm256_and_si256(nz1, a2)); + return _mm256_or_si256(_mm256_and_si256(z0, b1), _mm256_and_si256(nz0, b0)); +} + +static inline void bs_add8(const __m256i *a, const __m256i *b, __m256i *out) { + __m256i carry = BS_ZERO; + for (int i = 0; i < 8; i++) { + const __m256i x = _mm256_xor_si256(a[i], b[i]); + out[i] = _mm256_xor_si256(x, carry); + carry = _mm256_or_si256(_mm256_and_si256(a[i], b[i]), _mm256_and_si256(carry, x)); + } +} + +static inline void bs_tick(__m256i *t, __m256i *b, __m256i *l, __m256i *r, + const __m256i *kb, __m256i y_bs) { + + const __m256i Tt = _mm256_xor_si256(_mm256_xor_si256(_mm256_xor_si256(_mm256_xor_si256( + _mm256_xor_si256(_mm256_xor_si256(_mm256_xor_si256(t[15], t[14]), t[10]), t[8]), + t[5]), t[4]), t[1]), t[0]); + const __m256i Bt = _mm256_xor_si256(_mm256_xor_si256(_mm256_xor_si256(b[6], b[5]), b[4]), b[0]); + + const __m256i cr0 = r[7], cr1 = r[6], cr2 = r[5], cr3 = r[4]; + const __m256i cr4 = r[3], cr5 = r[2], cr6 = r[1], cr7 = r[0]; + + const __m256i new_t = _mm256_xor_si256(_mm256_xor_si256(Tt, cr0), cr4); + const __m256i new_b = _mm256_xor_si256(Bt, cr7); + + for (int i = 0; i < 15; i++) t[i] = t[i + 1]; + t[15] = new_t; + for (int i = 0; i < 7; i++) b[i] = b[i + 1]; + b[7] = new_b; + + const __m256i ncr3 = bs_not(cr3); + const __m256i ncr5 = bs_not(cr5); + + const __m256i z0 = _mm256_xor_si256(_mm256_xor_si256(_mm256_and_si256(cr0, cr2), _mm256_and_si256(cr1, ncr3)), + _mm256_or_si256(cr2, cr4)); + const __m256i z1 = _mm256_xor_si256(_mm256_xor_si256(_mm256_xor_si256(_mm256_xor_si256(_mm256_xor_si256( + _mm256_or_si256(cr0, cr2), _mm256_or_si256(cr5, cr7)), cr1), cr6), Tt), y_bs); + const __m256i z2 = _mm256_xor_si256(_mm256_xor_si256(_mm256_xor_si256( + _mm256_and_si256(cr3, ncr5), _mm256_and_si256(cr4, cr6)), cr7), Tt); + + const __m256i nz0 = bs_not(z0); + const __m256i nz1 = bs_not(z1); + const __m256i nz2 = bs_not(z2); + + __m256i val[8]; + for (int bit = 0; bit < 8; bit++) { + val[bit] = bs_mux8(z0, z1, z2, nz0, nz1, nz2, + kb[0 * 8 + bit], kb[1 * 8 + bit], + kb[2 * 8 + bit], kb[3 * 8 + bit], + kb[4 * 8 + bit], kb[5 * 8 + bit], + kb[6 * 8 + bit], kb[7 * 8 + bit]); + } + + for (int i = 0; i < 8; i++) val[i] = _mm256_xor_si256(val[i], b[i]); + + __m256i old_r[8]; + for (int i = 0; i < 8; i++) old_r[i] = r[i]; + bs_add8(val, l, r); + bs_add8(r, old_r, l); +} + +// Lane patterns for L bit k (L = 0..255). Pattern bit L set iff bit k of L +// equals 1. Stored as 4 × uint64_t (low word = lanes 0..63, etc.) so they +// match the __m256i memory layout on little-endian x86. +static const uint64_t LANE_BITS_256_RAW[8][BS256_WORDS] = { + {0xAAAAAAAAAAAAAAAAULL, 0xAAAAAAAAAAAAAAAAULL, 0xAAAAAAAAAAAAAAAAULL, 0xAAAAAAAAAAAAAAAAULL}, // k=0 + {0xCCCCCCCCCCCCCCCCULL, 0xCCCCCCCCCCCCCCCCULL, 0xCCCCCCCCCCCCCCCCULL, 0xCCCCCCCCCCCCCCCCULL}, // k=1 + {0xF0F0F0F0F0F0F0F0ULL, 0xF0F0F0F0F0F0F0F0ULL, 0xF0F0F0F0F0F0F0F0ULL, 0xF0F0F0F0F0F0F0F0ULL}, // k=2 + {0xFF00FF00FF00FF00ULL, 0xFF00FF00FF00FF00ULL, 0xFF00FF00FF00FF00ULL, 0xFF00FF00FF00FF00ULL}, // k=3 + {0xFFFF0000FFFF0000ULL, 0xFFFF0000FFFF0000ULL, 0xFFFF0000FFFF0000ULL, 0xFFFF0000FFFF0000ULL}, // k=4 + {0xFFFFFFFF00000000ULL, 0xFFFFFFFF00000000ULL, 0xFFFFFFFF00000000ULL, 0xFFFFFFFF00000000ULL}, // k=5 + {0x0000000000000000ULL, 0xFFFFFFFFFFFFFFFFULL, 0x0000000000000000ULL, 0xFFFFFFFFFFFFFFFFULL}, // k=6 + {0x0000000000000000ULL, 0x0000000000000000ULL, 0xFFFFFFFFFFFFFFFFULL, 0xFFFFFFFFFFFFFFFFULL}, // k=7 +}; + +static inline __m256i lane_bits_256(int k) { + return _mm256_loadu_si256((const __m256i *)LANE_BITS_256_RAW[k]); +} + +static inline void bs_init_state(const __m256i kb[64], + __m256i l[8], __m256i r[8], + __m256i b[8], __m256i t[16]) { + + __m256i k0xor[8]; + k0xor[0] = kb[0]; + k0xor[1] = kb[1]; + k0xor[2] = _mm256_xor_si256(kb[2], BS_ONES); + k0xor[3] = _mm256_xor_si256(kb[3], BS_ONES); + k0xor[4] = kb[4]; + k0xor[5] = kb[5]; + k0xor[6] = _mm256_xor_si256(kb[6], BS_ONES); + k0xor[7] = kb[7]; + + // 0xEC LSB-first: 0,0,1,1,0,1,1,1 + const __m256i ec[8] = {BS_ZERO, BS_ZERO, BS_ONES, BS_ONES, BS_ZERO, BS_ONES, BS_ONES, BS_ONES}; + // 0x21 LSB-first: 1,0,0,0,0,1,0,0 + const __m256i x21[8] = {BS_ONES, BS_ZERO, BS_ZERO, BS_ZERO, BS_ZERO, BS_ONES, BS_ZERO, BS_ZERO}; + + bs_add8(k0xor, ec, l); + bs_add8(k0xor, x21, r); + + // b = 0x4C LSB-first: 0,0,1,1,0,0,1,0 + b[0] = BS_ZERO; b[1] = BS_ZERO; b[2] = BS_ONES; b[3] = BS_ONES; + b[4] = BS_ZERO; b[5] = BS_ZERO; b[6] = BS_ONES; b[7] = BS_ZERO; + + // t = 0xE012 LSB-first: 0,1,0,0,1,0,0,0, 0,0,0,0,0,1,1,1 + t[ 0] = BS_ZERO; t[ 1] = BS_ONES; t[ 2] = BS_ZERO; t[ 3] = BS_ZERO; + t[ 4] = BS_ONES; t[ 5] = BS_ZERO; t[ 6] = BS_ZERO; t[ 7] = BS_ZERO; + t[ 8] = BS_ZERO; t[ 9] = BS_ZERO; t[10] = BS_ZERO; t[11] = BS_ZERO; + t[12] = BS_ZERO; t[13] = BS_ONES; t[14] = BS_ONES; t[15] = BS_ONES; +} + +void prepare_ccnr_bits_bs256(const uint8_t *cc_nr, uint64_t y_bits_bs[96 * BS256_WORDS]) { + __m256i *y = (__m256i *)y_bits_bs; + for (int i = 0; i < 12; i++) { + const uint8_t byte = cc_nr[i]; + for (int bit = 0; bit < 8; bit++) { + _mm256_storeu_si256(&y[i * 8 + bit], ((byte >> bit) & 1) ? BS_ONES : BS_ZERO); + } + } +} + +void prepare_target_mac_bs256(const uint8_t target_mac[4], uint64_t target_mac_bs[32 * BS256_WORDS]) { + __m256i *tm = (__m256i *)target_mac_bs; + for (int i = 0; i < 4; i++) { + const uint8_t byte = target_mac[i]; + for (int bit = 0; bit < 8; bit++) { + _mm256_storeu_si256(&tm[i * 8 + bit], ((byte >> bit) & 1) ? BS_ONES : BS_ZERO); + } + } +} + +void build_bitslice_key_256(const uint8_t partial_key[8], uint64_t index_start, uint64_t kb[64 * BS256_WORDS]) { + __m256i *k = (__m256i *)kb; + + for (int j = 0; j < 8; j++) { + _mm256_storeu_si256(&k[j * 8 + 0], (partial_key[j] & 0x01) ? BS_ONES : BS_ZERO); + _mm256_storeu_si256(&k[j * 8 + 1], (partial_key[j] & 0x02) ? BS_ONES : BS_ZERO); + _mm256_storeu_si256(&k[j * 8 + 2], (partial_key[j] & 0x04) ? BS_ONES : BS_ZERO); + } + + for (int j = 0; j < 8; j++) { + const int base_bit = 5 * (7 - j); + for (int kk = 0; kk < 5; kk++) { + const int idx_bit = base_bit + kk; + __m256i pattern; + if (idx_bit < 8) { + pattern = lane_bits_256(idx_bit); + } else { + pattern = ((index_start >> idx_bit) & 1) ? BS_ONES : BS_ZERO; + } + _mm256_storeu_si256(&k[j * 8 + 3 + kk], pattern); + } + } +} + +void doMAC_brute_match256(const uint64_t y_bits_bs[96 * BS256_WORDS], + const uint64_t kb[64 * BS256_WORDS], + const uint64_t target_mac_bs[32 * BS256_WORDS], + uint64_t match_out[BS256_WORDS]) { + + // Load key into local __m256i array for fast access. + __m256i k[64]; + const __m256i *kb_m = (const __m256i *)kb; + for (int i = 0; i < 64; i++) k[i] = _mm256_loadu_si256(&kb_m[i]); + + __m256i l[8], r[8], b[8], t[16]; + bs_init_state(k, l, r, b, t); + + const __m256i *y = (const __m256i *)y_bits_bs; + const __m256i *tm = (const __m256i *)target_mac_bs; + + for (int i = 0; i < 96; i++) { + bs_tick(t, b, l, r, k, _mm256_loadu_si256(&y[i])); + } + + __m256i mac_match = BS_ONES; + for (int tick = 0; tick < 32; tick++) { + const __m256i diff = _mm256_xor_si256(r[2], _mm256_loadu_si256(&tm[tick])); + // mac_match &= ~diff → andnot(diff, mac_match) = ~diff & mac_match + mac_match = _mm256_andnot_si256(diff, mac_match); + + if ((tick == 7 || tick == 15 || tick == 23) && _mm256_testz_si256(mac_match, mac_match)) { + for (int i = 0; i < BS256_WORDS; i++) match_out[i] = 0; + return; + } + + if (tick < 31) { + bs_tick(t, b, l, r, k, BS_ZERO); + } + } + + _mm256_storeu_si256((__m256i *)match_out, mac_match); +} + +#if defined(__GNUC__) || defined(__clang__) +#pragma GCC pop_options +#endif + +bool bs_avx2_supported(void) { + static int cached = -1; + if (cached < 0) { +#if defined(__GNUC__) || defined(__clang__) + __builtin_cpu_init(); + cached = __builtin_cpu_supports("avx2") ? 1 : 0; +#else + cached = 0; +#endif + } + return cached != 0; +} + +#else // non-x86 build: everything is a no-op, bs_avx2_supported returns false. + +bool bs_avx2_supported(void) { return false; } + +void prepare_ccnr_bits_bs256(const uint8_t *cc_nr, uint64_t y_bits_bs[96 * BS256_WORDS]) { + (void)cc_nr; (void)y_bits_bs; +} +void prepare_target_mac_bs256(const uint8_t target_mac[4], uint64_t target_mac_bs[32 * BS256_WORDS]) { + (void)target_mac; (void)target_mac_bs; +} +void build_bitslice_key_256(const uint8_t partial_key[8], uint64_t index_start, uint64_t kb[64 * BS256_WORDS]) { + (void)partial_key; (void)index_start; (void)kb; +} +void doMAC_brute_match256(const uint64_t y_bits_bs[96 * BS256_WORDS], + const uint64_t kb[64 * BS256_WORDS], + const uint64_t target_mac_bs[32 * BS256_WORDS], + uint64_t match_out[BS256_WORDS]) { + (void)y_bits_bs; (void)kb; (void)target_mac_bs; + for (int i = 0; i < BS256_WORDS; i++) match_out[i] = 0; +} + +#endif diff --git a/client/src/loclass/cipher_bs_avx2.h b/client/src/loclass/cipher_bs_avx2.h new file mode 100644 index 000000000..a4690d2c1 --- /dev/null +++ b/client/src/loclass/cipher_bs_avx2.h @@ -0,0 +1,58 @@ +//----------------------------------------------------------------------------- +// Copyright (C) Proxmark3 contributors. See AUTHORS.md for details. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// See LICENSE.txt for the text of the license. +//----------------------------------------------------------------------------- +// 256-wide bitsliced iClass cipher MAC (AVX2). Same algorithm as cipher_bs.c +// but swaps uint64_t lanes for __m256i, yielding 256 candidates per bs_tick +// (4x throughput over the portable u64 path on AVX2-capable x86). +// +// The interface hides __m256i behind plain uint64_t arrays so callers do not +// need the AVX2 headers. Each __m256i occupies BS256_WORDS (= 4) uint64_t; +// the implementation uses unaligned loads/stores, so no special alignment +// is required from the caller. +// +// On non-x86 builds (or when AVX2 is absent at runtime) every function in +// this header is a safe no-op; callers should gate use with the return value +// of bs_avx2_supported(). +//----------------------------------------------------------------------------- + +#ifndef CIPHER_BS_AVX2_H +#define CIPHER_BS_AVX2_H + +#include +#include + +#define BS256_WIDTH 256 +#define BS256_WORDS 4 + +// Returns true iff the running CPU supports AVX2 and this translation unit +// was built for an x86 target. Cached after first call. +bool bs_avx2_supported(void); + +// Expand 12 cc_nr bytes into 96 bit masks broadcast across 256 lanes. +void prepare_ccnr_bits_bs256(const uint8_t *cc_nr, uint64_t y_bits_bs[96 * BS256_WORDS]); + +// Expand a 4-byte target MAC into 32 bit masks for per-tick comparison. +void prepare_target_mac_bs256(const uint8_t target_mac[4], uint64_t target_mac_bs[32 * BS256_WORDS]); + +// Build the 256-wide bitsliced key for 256 consecutive candidates starting +// at index_start. index_start MUST be a multiple of 256 so the low 8 index +// bits reduce to the lane index. +void build_bitslice_key_256(const uint8_t partial_key[8], uint64_t index_start, uint64_t kb[64 * BS256_WORDS]); + +// Run the bitsliced MAC and write a 256-bit lane match mask into match_out +// (match_out[0] = lanes 0..63, match_out[1] = lanes 64..127, ..., match_out[3] +// = lanes 192..255). All zeros on complete miss or on an early-out eliminating +// every lane. +void doMAC_brute_match256(const uint64_t y_bits_bs[96 * BS256_WORDS], + const uint64_t kb[64 * BS256_WORDS], + const uint64_t target_mac_bs[32 * BS256_WORDS], + uint64_t match_out[BS256_WORDS]); + +#endif // CIPHER_BS_AVX2_H diff --git a/client/src/loclass/cipher_bs_avx512.c b/client/src/loclass/cipher_bs_avx512.c new file mode 100644 index 000000000..6656fb7b8 --- /dev/null +++ b/client/src/loclass/cipher_bs_avx512.c @@ -0,0 +1,279 @@ +//----------------------------------------------------------------------------- +// Copyright (C) Proxmark3 contributors. See AUTHORS.md for details. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// See LICENSE.txt for the text of the license. +//----------------------------------------------------------------------------- +// AVX-512F 512-wide bitsliced iClass cipher MAC. Same algorithm as cipher_bs.c +// with __m512i lanes. Nine LANE_BITS patterns cover the log2(512) = 9 lane- +// varying index bits. +//----------------------------------------------------------------------------- + +#include "cipher_bs_avx512.h" + +#if defined(__x86_64__) || defined(_M_X64) + +#include + +#if defined(__GNUC__) || defined(__clang__) +#pragma GCC push_options +#pragma GCC target("avx512f") +#endif + +#define BS_ZERO _mm512_setzero_si512() +#define BS_ONES _mm512_set1_epi64(-1) + +static inline __m512i bs_not(__m512i v) { return _mm512_xor_si512(v, BS_ONES); } + +static inline __m512i bs_mux8(__m512i z0, __m512i z1, __m512i z2, + __m512i nz0, __m512i nz1, __m512i nz2, + __m512i v0, __m512i v1, __m512i v2, __m512i v3, + __m512i v4, __m512i v5, __m512i v6, __m512i v7) { + const __m512i a0 = _mm512_or_si512(_mm512_and_si512(z2, v1), _mm512_and_si512(nz2, v0)); + const __m512i a1 = _mm512_or_si512(_mm512_and_si512(z2, v3), _mm512_and_si512(nz2, v2)); + const __m512i a2 = _mm512_or_si512(_mm512_and_si512(z2, v5), _mm512_and_si512(nz2, v4)); + const __m512i a3 = _mm512_or_si512(_mm512_and_si512(z2, v7), _mm512_and_si512(nz2, v6)); + const __m512i b0 = _mm512_or_si512(_mm512_and_si512(z1, a1), _mm512_and_si512(nz1, a0)); + const __m512i b1 = _mm512_or_si512(_mm512_and_si512(z1, a3), _mm512_and_si512(nz1, a2)); + return _mm512_or_si512(_mm512_and_si512(z0, b1), _mm512_and_si512(nz0, b0)); +} + +static inline void bs_add8(const __m512i *a, const __m512i *b, __m512i *out) { + __m512i carry = BS_ZERO; + for (int i = 0; i < 8; i++) { + const __m512i x = _mm512_xor_si512(a[i], b[i]); + out[i] = _mm512_xor_si512(x, carry); + carry = _mm512_or_si512(_mm512_and_si512(a[i], b[i]), _mm512_and_si512(carry, x)); + } +} + +static inline void bs_tick(__m512i *t, __m512i *b, __m512i *l, __m512i *r, + const __m512i *kb, __m512i y_bs) { + + const __m512i Tt = _mm512_xor_si512(_mm512_xor_si512(_mm512_xor_si512(_mm512_xor_si512( + _mm512_xor_si512(_mm512_xor_si512(_mm512_xor_si512(t[15], t[14]), t[10]), t[8]), + t[5]), t[4]), t[1]), t[0]); + const __m512i Bt = _mm512_xor_si512(_mm512_xor_si512(_mm512_xor_si512(b[6], b[5]), b[4]), b[0]); + + const __m512i cr0 = r[7], cr1 = r[6], cr2 = r[5], cr3 = r[4]; + const __m512i cr4 = r[3], cr5 = r[2], cr6 = r[1], cr7 = r[0]; + + const __m512i new_t = _mm512_xor_si512(_mm512_xor_si512(Tt, cr0), cr4); + const __m512i new_b = _mm512_xor_si512(Bt, cr7); + + for (int i = 0; i < 15; i++) t[i] = t[i + 1]; + t[15] = new_t; + for (int i = 0; i < 7; i++) b[i] = b[i + 1]; + b[7] = new_b; + + const __m512i ncr3 = bs_not(cr3); + const __m512i ncr5 = bs_not(cr5); + + const __m512i z0 = _mm512_xor_si512(_mm512_xor_si512(_mm512_and_si512(cr0, cr2), _mm512_and_si512(cr1, ncr3)), + _mm512_or_si512(cr2, cr4)); + const __m512i z1 = _mm512_xor_si512(_mm512_xor_si512(_mm512_xor_si512(_mm512_xor_si512(_mm512_xor_si512( + _mm512_or_si512(cr0, cr2), _mm512_or_si512(cr5, cr7)), cr1), cr6), Tt), y_bs); + const __m512i z2 = _mm512_xor_si512(_mm512_xor_si512(_mm512_xor_si512( + _mm512_and_si512(cr3, ncr5), _mm512_and_si512(cr4, cr6)), cr7), Tt); + + const __m512i nz0 = bs_not(z0); + const __m512i nz1 = bs_not(z1); + const __m512i nz2 = bs_not(z2); + + __m512i val[8]; + for (int bit = 0; bit < 8; bit++) { + val[bit] = bs_mux8(z0, z1, z2, nz0, nz1, nz2, + kb[0 * 8 + bit], kb[1 * 8 + bit], + kb[2 * 8 + bit], kb[3 * 8 + bit], + kb[4 * 8 + bit], kb[5 * 8 + bit], + kb[6 * 8 + bit], kb[7 * 8 + bit]); + } + + for (int i = 0; i < 8; i++) val[i] = _mm512_xor_si512(val[i], b[i]); + + __m512i old_r[8]; + for (int i = 0; i < 8; i++) old_r[i] = r[i]; + bs_add8(val, l, r); + bs_add8(r, old_r, l); +} + +// Lane patterns for L = 0..511, stored as 8 × uint64_t (word w covers lanes +// w*64 .. w*64+63, bit b of word w = lane w*64+b). +static const uint64_t LANE_BITS_512_RAW[9][BS512_WORDS] = { + // k=0..5 are the same in-word pattern repeated across all 8 words. + {0xAAAAAAAAAAAAAAAAULL, 0xAAAAAAAAAAAAAAAAULL, 0xAAAAAAAAAAAAAAAAULL, 0xAAAAAAAAAAAAAAAAULL, + 0xAAAAAAAAAAAAAAAAULL, 0xAAAAAAAAAAAAAAAAULL, 0xAAAAAAAAAAAAAAAAULL, 0xAAAAAAAAAAAAAAAAULL}, // k=0 + {0xCCCCCCCCCCCCCCCCULL, 0xCCCCCCCCCCCCCCCCULL, 0xCCCCCCCCCCCCCCCCULL, 0xCCCCCCCCCCCCCCCCULL, + 0xCCCCCCCCCCCCCCCCULL, 0xCCCCCCCCCCCCCCCCULL, 0xCCCCCCCCCCCCCCCCULL, 0xCCCCCCCCCCCCCCCCULL}, // k=1 + {0xF0F0F0F0F0F0F0F0ULL, 0xF0F0F0F0F0F0F0F0ULL, 0xF0F0F0F0F0F0F0F0ULL, 0xF0F0F0F0F0F0F0F0ULL, + 0xF0F0F0F0F0F0F0F0ULL, 0xF0F0F0F0F0F0F0F0ULL, 0xF0F0F0F0F0F0F0F0ULL, 0xF0F0F0F0F0F0F0F0ULL}, // k=2 + {0xFF00FF00FF00FF00ULL, 0xFF00FF00FF00FF00ULL, 0xFF00FF00FF00FF00ULL, 0xFF00FF00FF00FF00ULL, + 0xFF00FF00FF00FF00ULL, 0xFF00FF00FF00FF00ULL, 0xFF00FF00FF00FF00ULL, 0xFF00FF00FF00FF00ULL}, // k=3 + {0xFFFF0000FFFF0000ULL, 0xFFFF0000FFFF0000ULL, 0xFFFF0000FFFF0000ULL, 0xFFFF0000FFFF0000ULL, + 0xFFFF0000FFFF0000ULL, 0xFFFF0000FFFF0000ULL, 0xFFFF0000FFFF0000ULL, 0xFFFF0000FFFF0000ULL}, // k=4 + {0xFFFFFFFF00000000ULL, 0xFFFFFFFF00000000ULL, 0xFFFFFFFF00000000ULL, 0xFFFFFFFF00000000ULL, + 0xFFFFFFFF00000000ULL, 0xFFFFFFFF00000000ULL, 0xFFFFFFFF00000000ULL, 0xFFFFFFFF00000000ULL}, // k=5 + // k=6: alternate zero/all-ones per word. + {0x0000000000000000ULL, 0xFFFFFFFFFFFFFFFFULL, 0x0000000000000000ULL, 0xFFFFFFFFFFFFFFFFULL, + 0x0000000000000000ULL, 0xFFFFFFFFFFFFFFFFULL, 0x0000000000000000ULL, 0xFFFFFFFFFFFFFFFFULL}, // k=6 + // k=7: 00110011 across words. + {0x0000000000000000ULL, 0x0000000000000000ULL, 0xFFFFFFFFFFFFFFFFULL, 0xFFFFFFFFFFFFFFFFULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, 0xFFFFFFFFFFFFFFFFULL, 0xFFFFFFFFFFFFFFFFULL}, // k=7 + // k=8: low half zero, high half all-ones. + {0x0000000000000000ULL, 0x0000000000000000ULL, 0x0000000000000000ULL, 0x0000000000000000ULL, + 0xFFFFFFFFFFFFFFFFULL, 0xFFFFFFFFFFFFFFFFULL, 0xFFFFFFFFFFFFFFFFULL, 0xFFFFFFFFFFFFFFFFULL}, // k=8 +}; + +static inline __m512i lane_bits_512(int k) { + return _mm512_loadu_si512((const void *)LANE_BITS_512_RAW[k]); +} + +static inline void bs_init_state(const __m512i kb[64], + __m512i l[8], __m512i r[8], + __m512i b[8], __m512i t[16]) { + + __m512i k0xor[8]; + k0xor[0] = kb[0]; + k0xor[1] = kb[1]; + k0xor[2] = _mm512_xor_si512(kb[2], BS_ONES); + k0xor[3] = _mm512_xor_si512(kb[3], BS_ONES); + k0xor[4] = kb[4]; + k0xor[5] = kb[5]; + k0xor[6] = _mm512_xor_si512(kb[6], BS_ONES); + k0xor[7] = kb[7]; + + const __m512i ec[8] = {BS_ZERO, BS_ZERO, BS_ONES, BS_ONES, BS_ZERO, BS_ONES, BS_ONES, BS_ONES}; + const __m512i x21[8] = {BS_ONES, BS_ZERO, BS_ZERO, BS_ZERO, BS_ZERO, BS_ONES, BS_ZERO, BS_ZERO}; + + bs_add8(k0xor, ec, l); + bs_add8(k0xor, x21, r); + + b[0] = BS_ZERO; b[1] = BS_ZERO; b[2] = BS_ONES; b[3] = BS_ONES; + b[4] = BS_ZERO; b[5] = BS_ZERO; b[6] = BS_ONES; b[7] = BS_ZERO; + + t[ 0] = BS_ZERO; t[ 1] = BS_ONES; t[ 2] = BS_ZERO; t[ 3] = BS_ZERO; + t[ 4] = BS_ONES; t[ 5] = BS_ZERO; t[ 6] = BS_ZERO; t[ 7] = BS_ZERO; + t[ 8] = BS_ZERO; t[ 9] = BS_ZERO; t[10] = BS_ZERO; t[11] = BS_ZERO; + t[12] = BS_ZERO; t[13] = BS_ONES; t[14] = BS_ONES; t[15] = BS_ONES; +} + +void prepare_ccnr_bits_bs512(const uint8_t *cc_nr, uint64_t y_bits_bs[96 * BS512_WORDS]) { + for (int i = 0; i < 12; i++) { + const uint8_t byte = cc_nr[i]; + for (int bit = 0; bit < 8; bit++) { + _mm512_storeu_si512((void *)&y_bits_bs[(i * 8 + bit) * BS512_WORDS], + ((byte >> bit) & 1) ? BS_ONES : BS_ZERO); + } + } +} + +void prepare_target_mac_bs512(const uint8_t target_mac[4], uint64_t target_mac_bs[32 * BS512_WORDS]) { + for (int i = 0; i < 4; i++) { + const uint8_t byte = target_mac[i]; + for (int bit = 0; bit < 8; bit++) { + _mm512_storeu_si512((void *)&target_mac_bs[(i * 8 + bit) * BS512_WORDS], + ((byte >> bit) & 1) ? BS_ONES : BS_ZERO); + } + } +} + +void build_bitslice_key_512(const uint8_t partial_key[8], uint64_t index_start, uint64_t kb[64 * BS512_WORDS]) { + + for (int j = 0; j < 8; j++) { + _mm512_storeu_si512((void *)&kb[(j * 8 + 0) * BS512_WORDS], (partial_key[j] & 0x01) ? BS_ONES : BS_ZERO); + _mm512_storeu_si512((void *)&kb[(j * 8 + 1) * BS512_WORDS], (partial_key[j] & 0x02) ? BS_ONES : BS_ZERO); + _mm512_storeu_si512((void *)&kb[(j * 8 + 2) * BS512_WORDS], (partial_key[j] & 0x04) ? BS_ONES : BS_ZERO); + } + + for (int j = 0; j < 8; j++) { + const int base_bit = 5 * (7 - j); + for (int kk = 0; kk < 5; kk++) { + const int idx_bit = base_bit + kk; + __m512i pattern; + if (idx_bit < 9) { + pattern = lane_bits_512(idx_bit); + } else { + pattern = ((index_start >> idx_bit) & 1) ? BS_ONES : BS_ZERO; + } + _mm512_storeu_si512((void *)&kb[(j * 8 + 3 + kk) * BS512_WORDS], pattern); + } + } +} + +void doMAC_brute_match512(const uint64_t y_bits_bs[96 * BS512_WORDS], + const uint64_t kb[64 * BS512_WORDS], + const uint64_t target_mac_bs[32 * BS512_WORDS], + uint64_t match_out[BS512_WORDS]) { + + __m512i k[64]; + for (int i = 0; i < 64; i++) k[i] = _mm512_loadu_si512((const void *)&kb[i * BS512_WORDS]); + + __m512i l[8], r[8], b[8], t[16]; + bs_init_state(k, l, r, b, t); + + for (int i = 0; i < 96; i++) { + bs_tick(t, b, l, r, k, _mm512_loadu_si512((const void *)&y_bits_bs[i * BS512_WORDS])); + } + + __m512i mac_match = BS_ONES; + for (int tick = 0; tick < 32; tick++) { + const __m512i diff = _mm512_xor_si512(r[2], _mm512_loadu_si512((const void *)&target_mac_bs[tick * BS512_WORDS])); + mac_match = _mm512_andnot_si512(diff, mac_match); + + if ((tick == 7 || tick == 15 || tick == 23) && + _mm512_test_epi64_mask(mac_match, mac_match) == 0) { + for (int i = 0; i < BS512_WORDS; i++) match_out[i] = 0; + return; + } + + if (tick < 31) { + bs_tick(t, b, l, r, k, BS_ZERO); + } + } + + _mm512_storeu_si512((void *)match_out, mac_match); +} + +#if defined(__GNUC__) || defined(__clang__) +#pragma GCC pop_options +#endif + +bool bs_avx512_supported(void) { + static int cached = -1; + if (cached < 0) { +#if defined(__GNUC__) || defined(__clang__) + __builtin_cpu_init(); + cached = __builtin_cpu_supports("avx512f") ? 1 : 0; +#else + cached = 0; +#endif + } + return cached != 0; +} + +#else // non-x86 build + +bool bs_avx512_supported(void) { return false; } + +void prepare_ccnr_bits_bs512(const uint8_t *cc_nr, uint64_t y_bits_bs[96 * BS512_WORDS]) { + (void)cc_nr; (void)y_bits_bs; +} +void prepare_target_mac_bs512(const uint8_t target_mac[4], uint64_t target_mac_bs[32 * BS512_WORDS]) { + (void)target_mac; (void)target_mac_bs; +} +void build_bitslice_key_512(const uint8_t partial_key[8], uint64_t index_start, uint64_t kb[64 * BS512_WORDS]) { + (void)partial_key; (void)index_start; (void)kb; +} +void doMAC_brute_match512(const uint64_t y_bits_bs[96 * BS512_WORDS], + const uint64_t kb[64 * BS512_WORDS], + const uint64_t target_mac_bs[32 * BS512_WORDS], + uint64_t match_out[BS512_WORDS]) { + (void)y_bits_bs; (void)kb; (void)target_mac_bs; + for (int i = 0; i < BS512_WORDS; i++) match_out[i] = 0; +} + +#endif diff --git a/client/src/loclass/cipher_bs_avx512.h b/client/src/loclass/cipher_bs_avx512.h new file mode 100644 index 000000000..ad20e5092 --- /dev/null +++ b/client/src/loclass/cipher_bs_avx512.h @@ -0,0 +1,38 @@ +//----------------------------------------------------------------------------- +// Copyright (C) Proxmark3 contributors. See AUTHORS.md for details. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// See LICENSE.txt for the text of the license. +//----------------------------------------------------------------------------- +// 512-wide bitsliced iClass cipher MAC (AVX-512F). 8x throughput over the +// portable u64 bitslice on hosts with AVX-512F. +// +// Buffers are exposed as plain uint64_t arrays to avoid leaking __m512i into +// callers that may not be AVX-512-built. Each __m512i occupies BS512_WORDS +// (= 8) uint64_t. +//----------------------------------------------------------------------------- + +#ifndef CIPHER_BS_AVX512_H +#define CIPHER_BS_AVX512_H + +#include +#include + +#define BS512_WIDTH 512 +#define BS512_WORDS 8 + +bool bs_avx512_supported(void); + +void prepare_ccnr_bits_bs512(const uint8_t *cc_nr, uint64_t y_bits_bs[96 * BS512_WORDS]); +void prepare_target_mac_bs512(const uint8_t target_mac[4], uint64_t target_mac_bs[32 * BS512_WORDS]); +void build_bitslice_key_512(const uint8_t partial_key[8], uint64_t index_start, uint64_t kb[64 * BS512_WORDS]); +void doMAC_brute_match512(const uint64_t y_bits_bs[96 * BS512_WORDS], + const uint64_t kb[64 * BS512_WORDS], + const uint64_t target_mac_bs[32 * BS512_WORDS], + uint64_t match_out[BS512_WORDS]); + +#endif // CIPHER_BS_AVX512_H diff --git a/client/src/loclass/cipher_bs_dispatch.c b/client/src/loclass/cipher_bs_dispatch.c new file mode 100644 index 000000000..f04539901 --- /dev/null +++ b/client/src/loclass/cipher_bs_dispatch.c @@ -0,0 +1,79 @@ +//----------------------------------------------------------------------------- +// Copyright (C) Proxmark3 contributors. See AUTHORS.md for details. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// See LICENSE.txt for the text of the license. +//----------------------------------------------------------------------------- + +#include "cipher_bs_dispatch.h" +#include "cipher_bs.h" +#include "cipher_bs_avx2.h" +#include "cipher_bs_avx512.h" +#include "cipher_bs_neon.h" + +// The u64 match function returns its mask instead of writing through a +// pointer; wrap it to match the uniform backend signature. +static void u64_match_adapter(const uint64_t *y, const uint64_t *kb, + const uint64_t *tgt, uint64_t *out) { + out[0] = doMAC_brute_match64(y, kb, tgt); +} + +static const bs_backend_t backend_u64 = { + .width = 64, + .words = 1, + .name = "u64", + .prepare_ccnr = prepare_ccnr_bits_bs, + .prepare_mac = prepare_target_mac_bs, + .build_key = build_bitslice_key_64, + .match = u64_match_adapter, +}; + +static const bs_backend_t backend_neon = { + .width = 128, + .words = 2, + .name = "NEON", + .prepare_ccnr = prepare_ccnr_bits_bs128, + .prepare_mac = prepare_target_mac_bs128, + .build_key = build_bitslice_key_128, + .match = doMAC_brute_match128, +}; + +static const bs_backend_t backend_avx2 = { + .width = 256, + .words = 4, + .name = "AVX2", + .prepare_ccnr = prepare_ccnr_bits_bs256, + .prepare_mac = prepare_target_mac_bs256, + .build_key = build_bitslice_key_256, + .match = doMAC_brute_match256, +}; + +static const bs_backend_t backend_avx512 = { + .width = 512, + .words = 8, + .name = "AVX-512", + .prepare_ccnr = prepare_ccnr_bits_bs512, + .prepare_mac = prepare_target_mac_bs512, + .build_key = build_bitslice_key_512, + .match = doMAC_brute_match512, +}; + +const bs_backend_t *bs_best_backend(void) { + static const bs_backend_t *cached = NULL; + if (cached != NULL) return cached; + + if (bs_avx512_supported()) { + cached = &backend_avx512; + } else if (bs_avx2_supported()) { + cached = &backend_avx2; + } else if (bs_neon_supported()) { + cached = &backend_neon; + } else { + cached = &backend_u64; + } + return cached; +} diff --git a/client/src/loclass/cipher_bs_dispatch.h b/client/src/loclass/cipher_bs_dispatch.h new file mode 100644 index 000000000..2b535c90c --- /dev/null +++ b/client/src/loclass/cipher_bs_dispatch.h @@ -0,0 +1,41 @@ +//----------------------------------------------------------------------------- +// Copyright (C) Proxmark3 contributors. See AUTHORS.md for details. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// See LICENSE.txt for the text of the license. +//----------------------------------------------------------------------------- +// Runtime dispatcher for the legbrute bitslice backends. Returns the widest +// bitslice implementation the CPU supports: AVX-512 (512 lanes) > AVX2 (256) +// > NEON (128) > portable u64 (64). +// +// Buffers in the returned backend must be sized for backend->words uint64_t +// per logical bit; callers can stack-allocate using BS_MAX_WORDS to cover +// every backend. +//----------------------------------------------------------------------------- + +#ifndef CIPHER_BS_DISPATCH_H +#define CIPHER_BS_DISPATCH_H + +#include + +#define BS_MAX_WORDS 8 // AVX-512 + +typedef struct bs_backend_s { + int width; // 64, 128, 256, or 512 + int words; // width / 64 + const char *name; + void (*prepare_ccnr)(const uint8_t *cc_nr, uint64_t *y_bits_bs); + void (*prepare_mac)(const uint8_t target_mac[4], uint64_t *target_mac_bs); + void (*build_key)(const uint8_t partial_key[8], uint64_t index_start, uint64_t *kb); + void (*match)(const uint64_t *y_bits_bs, const uint64_t *kb, const uint64_t *target_mac_bs, uint64_t *match_out); +} bs_backend_t; + +// Returns the widest backend the current CPU supports. Never NULL (u64 is +// the universal fallback). Cached after first call; safe to call repeatedly. +const bs_backend_t *bs_best_backend(void); + +#endif // CIPHER_BS_DISPATCH_H diff --git a/client/src/loclass/cipher_bs_neon.c b/client/src/loclass/cipher_bs_neon.c new file mode 100644 index 000000000..d3d6bf1a5 --- /dev/null +++ b/client/src/loclass/cipher_bs_neon.c @@ -0,0 +1,248 @@ +//----------------------------------------------------------------------------- +// Copyright (C) Proxmark3 contributors. See AUTHORS.md for details. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// See LICENSE.txt for the text of the license. +//----------------------------------------------------------------------------- +// ARM NEON 128-wide bitsliced iClass cipher MAC. Same algorithm as cipher_bs.c +// with uint64x2_t lanes. Seven LANE_BITS patterns cover log2(128) = 7 bits. +//----------------------------------------------------------------------------- + +#include "cipher_bs_neon.h" + +#if defined(__ARM_NEON) || defined(__ARM_NEON__) + +#include + +#define BS_ZERO vdupq_n_u64(0) +#define BS_ONES vdupq_n_u64(~(uint64_t)0) + +static inline uint64x2_t bs_not(uint64x2_t v) { return veorq_u64(v, BS_ONES); } + +static inline uint64x2_t bs_mux8(uint64x2_t z0, uint64x2_t z1, uint64x2_t z2, + uint64x2_t nz0, uint64x2_t nz1, uint64x2_t nz2, + uint64x2_t v0, uint64x2_t v1, uint64x2_t v2, uint64x2_t v3, + uint64x2_t v4, uint64x2_t v5, uint64x2_t v6, uint64x2_t v7) { + const uint64x2_t a0 = vorrq_u64(vandq_u64(z2, v1), vandq_u64(nz2, v0)); + const uint64x2_t a1 = vorrq_u64(vandq_u64(z2, v3), vandq_u64(nz2, v2)); + const uint64x2_t a2 = vorrq_u64(vandq_u64(z2, v5), vandq_u64(nz2, v4)); + const uint64x2_t a3 = vorrq_u64(vandq_u64(z2, v7), vandq_u64(nz2, v6)); + const uint64x2_t b0 = vorrq_u64(vandq_u64(z1, a1), vandq_u64(nz1, a0)); + const uint64x2_t b1 = vorrq_u64(vandq_u64(z1, a3), vandq_u64(nz1, a2)); + return vorrq_u64(vandq_u64(z0, b1), vandq_u64(nz0, b0)); +} + +static inline void bs_add8(const uint64x2_t *a, const uint64x2_t *b, uint64x2_t *out) { + uint64x2_t carry = BS_ZERO; + for (int i = 0; i < 8; i++) { + const uint64x2_t x = veorq_u64(a[i], b[i]); + out[i] = veorq_u64(x, carry); + carry = vorrq_u64(vandq_u64(a[i], b[i]), vandq_u64(carry, x)); + } +} + +static inline void bs_tick(uint64x2_t *t, uint64x2_t *b, uint64x2_t *l, uint64x2_t *r, + const uint64x2_t *kb, uint64x2_t y_bs) { + + const uint64x2_t Tt = veorq_u64(veorq_u64(veorq_u64(veorq_u64( + veorq_u64(veorq_u64(veorq_u64(t[15], t[14]), t[10]), t[8]), + t[5]), t[4]), t[1]), t[0]); + const uint64x2_t Bt = veorq_u64(veorq_u64(veorq_u64(b[6], b[5]), b[4]), b[0]); + + const uint64x2_t cr0 = r[7], cr1 = r[6], cr2 = r[5], cr3 = r[4]; + const uint64x2_t cr4 = r[3], cr5 = r[2], cr6 = r[1], cr7 = r[0]; + + const uint64x2_t new_t = veorq_u64(veorq_u64(Tt, cr0), cr4); + const uint64x2_t new_b = veorq_u64(Bt, cr7); + + for (int i = 0; i < 15; i++) t[i] = t[i + 1]; + t[15] = new_t; + for (int i = 0; i < 7; i++) b[i] = b[i + 1]; + b[7] = new_b; + + const uint64x2_t ncr3 = bs_not(cr3); + const uint64x2_t ncr5 = bs_not(cr5); + + const uint64x2_t z0 = veorq_u64(veorq_u64(vandq_u64(cr0, cr2), vandq_u64(cr1, ncr3)), + vorrq_u64(cr2, cr4)); + const uint64x2_t z1 = veorq_u64(veorq_u64(veorq_u64(veorq_u64(veorq_u64( + vorrq_u64(cr0, cr2), vorrq_u64(cr5, cr7)), cr1), cr6), Tt), y_bs); + const uint64x2_t z2 = veorq_u64(veorq_u64(veorq_u64( + vandq_u64(cr3, ncr5), vandq_u64(cr4, cr6)), cr7), Tt); + + const uint64x2_t nz0 = bs_not(z0); + const uint64x2_t nz1 = bs_not(z1); + const uint64x2_t nz2 = bs_not(z2); + + uint64x2_t val[8]; + for (int bit = 0; bit < 8; bit++) { + val[bit] = bs_mux8(z0, z1, z2, nz0, nz1, nz2, + kb[0 * 8 + bit], kb[1 * 8 + bit], + kb[2 * 8 + bit], kb[3 * 8 + bit], + kb[4 * 8 + bit], kb[5 * 8 + bit], + kb[6 * 8 + bit], kb[7 * 8 + bit]); + } + + for (int i = 0; i < 8; i++) val[i] = veorq_u64(val[i], b[i]); + + uint64x2_t old_r[8]; + for (int i = 0; i < 8; i++) old_r[i] = r[i]; + bs_add8(val, l, r); + bs_add8(r, old_r, l); +} + +// Lane patterns for L = 0..127, stored as 2 × uint64_t (low word = lanes 0..63, +// high word = lanes 64..127). +static const uint64_t LANE_BITS_128_RAW[7][BS128_WORDS] = { + {0xAAAAAAAAAAAAAAAAULL, 0xAAAAAAAAAAAAAAAAULL}, // k=0 + {0xCCCCCCCCCCCCCCCCULL, 0xCCCCCCCCCCCCCCCCULL}, // k=1 + {0xF0F0F0F0F0F0F0F0ULL, 0xF0F0F0F0F0F0F0F0ULL}, // k=2 + {0xFF00FF00FF00FF00ULL, 0xFF00FF00FF00FF00ULL}, // k=3 + {0xFFFF0000FFFF0000ULL, 0xFFFF0000FFFF0000ULL}, // k=4 + {0xFFFFFFFF00000000ULL, 0xFFFFFFFF00000000ULL}, // k=5 + {0x0000000000000000ULL, 0xFFFFFFFFFFFFFFFFULL}, // k=6 +}; + +static inline uint64x2_t lane_bits_128(int k) { + return vld1q_u64(LANE_BITS_128_RAW[k]); +} + +static inline void bs_init_state(const uint64x2_t kb[64], + uint64x2_t l[8], uint64x2_t r[8], + uint64x2_t b[8], uint64x2_t t[16]) { + + uint64x2_t k0xor[8]; + k0xor[0] = kb[0]; + k0xor[1] = kb[1]; + k0xor[2] = veorq_u64(kb[2], BS_ONES); + k0xor[3] = veorq_u64(kb[3], BS_ONES); + k0xor[4] = kb[4]; + k0xor[5] = kb[5]; + k0xor[6] = veorq_u64(kb[6], BS_ONES); + k0xor[7] = kb[7]; + + const uint64x2_t ec[8] = {BS_ZERO, BS_ZERO, BS_ONES, BS_ONES, BS_ZERO, BS_ONES, BS_ONES, BS_ONES}; + const uint64x2_t x21[8] = {BS_ONES, BS_ZERO, BS_ZERO, BS_ZERO, BS_ZERO, BS_ONES, BS_ZERO, BS_ZERO}; + + bs_add8(k0xor, ec, l); + bs_add8(k0xor, x21, r); + + b[0] = BS_ZERO; b[1] = BS_ZERO; b[2] = BS_ONES; b[3] = BS_ONES; + b[4] = BS_ZERO; b[5] = BS_ZERO; b[6] = BS_ONES; b[7] = BS_ZERO; + + t[ 0] = BS_ZERO; t[ 1] = BS_ONES; t[ 2] = BS_ZERO; t[ 3] = BS_ZERO; + t[ 4] = BS_ONES; t[ 5] = BS_ZERO; t[ 6] = BS_ZERO; t[ 7] = BS_ZERO; + t[ 8] = BS_ZERO; t[ 9] = BS_ZERO; t[10] = BS_ZERO; t[11] = BS_ZERO; + t[12] = BS_ZERO; t[13] = BS_ONES; t[14] = BS_ONES; t[15] = BS_ONES; +} + +void prepare_ccnr_bits_bs128(const uint8_t *cc_nr, uint64_t y_bits_bs[96 * BS128_WORDS]) { + for (int i = 0; i < 12; i++) { + const uint8_t byte = cc_nr[i]; + for (int bit = 0; bit < 8; bit++) { + vst1q_u64(&y_bits_bs[(i * 8 + bit) * BS128_WORDS], + ((byte >> bit) & 1) ? BS_ONES : BS_ZERO); + } + } +} + +void prepare_target_mac_bs128(const uint8_t target_mac[4], uint64_t target_mac_bs[32 * BS128_WORDS]) { + for (int i = 0; i < 4; i++) { + const uint8_t byte = target_mac[i]; + for (int bit = 0; bit < 8; bit++) { + vst1q_u64(&target_mac_bs[(i * 8 + bit) * BS128_WORDS], + ((byte >> bit) & 1) ? BS_ONES : BS_ZERO); + } + } +} + +void build_bitslice_key_128(const uint8_t partial_key[8], uint64_t index_start, uint64_t kb[64 * BS128_WORDS]) { + + for (int j = 0; j < 8; j++) { + vst1q_u64(&kb[(j * 8 + 0) * BS128_WORDS], (partial_key[j] & 0x01) ? BS_ONES : BS_ZERO); + vst1q_u64(&kb[(j * 8 + 1) * BS128_WORDS], (partial_key[j] & 0x02) ? BS_ONES : BS_ZERO); + vst1q_u64(&kb[(j * 8 + 2) * BS128_WORDS], (partial_key[j] & 0x04) ? BS_ONES : BS_ZERO); + } + + for (int j = 0; j < 8; j++) { + const int base_bit = 5 * (7 - j); + for (int kk = 0; kk < 5; kk++) { + const int idx_bit = base_bit + kk; + uint64x2_t pattern; + if (idx_bit < 7) { + pattern = lane_bits_128(idx_bit); + } else { + pattern = ((index_start >> idx_bit) & 1) ? BS_ONES : BS_ZERO; + } + vst1q_u64(&kb[(j * 8 + 3 + kk) * BS128_WORDS], pattern); + } + } +} + +// True iff every lane of v is zero. Portable NEON (works on AArch32 + AArch64). +static inline bool bs128_all_zero(uint64x2_t v) { + return (vgetq_lane_u64(v, 0) | vgetq_lane_u64(v, 1)) == 0; +} + +void doMAC_brute_match128(const uint64_t y_bits_bs[96 * BS128_WORDS], + const uint64_t kb[64 * BS128_WORDS], + const uint64_t target_mac_bs[32 * BS128_WORDS], + uint64_t match_out[BS128_WORDS]) { + + uint64x2_t k[64]; + for (int i = 0; i < 64; i++) k[i] = vld1q_u64(&kb[i * BS128_WORDS]); + + uint64x2_t l[8], r[8], b[8], t[16]; + bs_init_state(k, l, r, b, t); + + for (int i = 0; i < 96; i++) { + bs_tick(t, b, l, r, k, vld1q_u64(&y_bits_bs[i * BS128_WORDS])); + } + + uint64x2_t mac_match = BS_ONES; + for (int tick = 0; tick < 32; tick++) { + const uint64x2_t diff = veorq_u64(r[2], vld1q_u64(&target_mac_bs[tick * BS128_WORDS])); + // mac_match &= ~diff → vbicq_u64(mac_match, diff) = mac_match AND NOT diff + mac_match = vbicq_u64(mac_match, diff); + + if ((tick == 7 || tick == 15 || tick == 23) && bs128_all_zero(mac_match)) { + match_out[0] = 0; match_out[1] = 0; + return; + } + + if (tick < 31) { + bs_tick(t, b, l, r, k, BS_ZERO); + } + } + + vst1q_u64(match_out, mac_match); +} + +bool bs_neon_supported(void) { return true; } + +#else // no NEON + +bool bs_neon_supported(void) { return false; } + +void prepare_ccnr_bits_bs128(const uint8_t *cc_nr, uint64_t y_bits_bs[96 * BS128_WORDS]) { + (void)cc_nr; (void)y_bits_bs; +} +void prepare_target_mac_bs128(const uint8_t target_mac[4], uint64_t target_mac_bs[32 * BS128_WORDS]) { + (void)target_mac; (void)target_mac_bs; +} +void build_bitslice_key_128(const uint8_t partial_key[8], uint64_t index_start, uint64_t kb[64 * BS128_WORDS]) { + (void)partial_key; (void)index_start; (void)kb; +} +void doMAC_brute_match128(const uint64_t y_bits_bs[96 * BS128_WORDS], + const uint64_t kb[64 * BS128_WORDS], + const uint64_t target_mac_bs[32 * BS128_WORDS], + uint64_t match_out[BS128_WORDS]) { + (void)y_bits_bs; (void)kb; (void)target_mac_bs; + for (int i = 0; i < BS128_WORDS; i++) match_out[i] = 0; +} + +#endif diff --git a/client/src/loclass/cipher_bs_neon.h b/client/src/loclass/cipher_bs_neon.h new file mode 100644 index 000000000..bafb217ab --- /dev/null +++ b/client/src/loclass/cipher_bs_neon.h @@ -0,0 +1,36 @@ +//----------------------------------------------------------------------------- +// Copyright (C) Proxmark3 contributors. See AUTHORS.md for details. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// See LICENSE.txt for the text of the license. +//----------------------------------------------------------------------------- +// 128-wide bitsliced iClass cipher MAC (ARM NEON). 2x throughput over the +// portable u64 bitslice on AArch64 (NEON is mandatory there) and on 32-bit +// ARM builds that enable NEON. Buffers are plain uint64_t arrays so the +// header stays toolchain-agnostic. +//----------------------------------------------------------------------------- + +#ifndef CIPHER_BS_NEON_H +#define CIPHER_BS_NEON_H + +#include +#include + +#define BS128_WIDTH 128 +#define BS128_WORDS 2 + +bool bs_neon_supported(void); + +void prepare_ccnr_bits_bs128(const uint8_t *cc_nr, uint64_t y_bits_bs[96 * BS128_WORDS]); +void prepare_target_mac_bs128(const uint8_t target_mac[4], uint64_t target_mac_bs[32 * BS128_WORDS]); +void build_bitslice_key_128(const uint8_t partial_key[8], uint64_t index_start, uint64_t kb[64 * BS128_WORDS]); +void doMAC_brute_match128(const uint64_t y_bits_bs[96 * BS128_WORDS], + const uint64_t kb[64 * BS128_WORDS], + const uint64_t target_mac_bs[32 * BS128_WORDS], + uint64_t match_out[BS128_WORDS]); + +#endif // CIPHER_BS_NEON_H