Verified Commit 88cb7851 authored by Tom Almeida's avatar Tom Almeida
Browse files

Initial commit to new repository

parents
/target/
Cargo.lock
[package]
name = "seahash"
version = "3.0.5"
authors = ["ticki <ticki@users.noreply.github.com>"]
description = "A blazingly fast, portable hash function with proven statistical guarantees."
repository = "https://github.com/ticki/tfs"
documentation = "https://docs.rs/seahash"
license = "MIT"
keywords = ["hash", "hashing", "checksum", "checsumming", "portable"]
exclude = ["target", "Cargo.lock"]
<div align="center"><img alt="Logo" width="200" src="./logo.png" /></div>
===================
SeaHash: A bizarrely fast hash function.
SeaHash is a hash function with performance better than (around 3-20% improvement) xxHash and
MetroHash. Furthermore, SeaHash has mathematically provable statistical guarantees.
In action:
[![The hash function in action.](http://ticki.github.io/img/seahash_construction_diagram.svg)](http://ticki.github.io/img/seahash_construction_diagram.svg)
#![feature(test)]
extern crate test;
extern crate seahash;
#[bench]
fn gigabyte(b: &mut test::Bencher) {
b.iter(|| {
let mut x = 0;
let mut buf = [15; 4096];
for _ in 0..250000 {
x ^= seahash::hash(&buf);
buf[0] += buf[0].wrapping_add(1);
}
x
})
}
logo.png

16.1 KB

//! A highly optimized version of SeaHash.
use core::slice;
use helper;
/// A SeaHash state.
#[derive(Clone)]
pub struct State {
/// `a`
a: u64,
/// `b`
b: u64,
/// `c`
c: u64,
/// `d`
d: u64,
/// The number of written bytes.
written: u64,
}
impl State {
/// Create a new state vector with some initial values.
pub fn new(a: u64, b: u64, c: u64, d: u64) -> State {
State {
a: a,
b: b,
c: c,
d: d,
written: 0,
}
}
/// Hash a buffer with some seed.
pub fn hash(buf: &[u8], (mut a, mut b, mut c, mut d): (u64, u64, u64, u64)) -> State {
unsafe {
// We use 4 different registers to store seperate hash states, because this allows us
// to update them seperately, and consequently exploiting ILP to update the states in
// parallel.
// The pointer to the current bytes.
let mut ptr = buf.as_ptr();
/// The end of the "main segment", i.e. the biggest buffer s.t. the length is divisible
/// by 32.
let end_ptr = buf.as_ptr().offset(buf.len() as isize & !0x1F);
while end_ptr > ptr {
// Modern CPUs allow the pointer arithmetic to be done in place, hence not
// introducing tmpvars.
a ^= helper::read_u64(ptr);
b ^= helper::read_u64(ptr.offset(8));
c ^= helper::read_u64(ptr.offset(16));
d ^= helper::read_u64(ptr.offset(24));
// Increment the pointer.
ptr = ptr.offset(32);
// Diffuse the updated registers. We hope that each of these are executed in
// parallel.
a = helper::diffuse(a);
b = helper::diffuse(b);
c = helper::diffuse(c);
d = helper::diffuse(d);
}
// Calculate the number of excessive bytes. These are bytes that could not be handled
// in the loop above.
let mut excessive = buf.len() as usize + buf.as_ptr() as usize - end_ptr as usize;
// Handle the excessive bytes.
match excessive {
0 => {},
1...7 => {
// 1 or more excessive.
// Write the last excessive bytes (<8 bytes).
a ^= helper::read_int(slice::from_raw_parts(ptr as *const u8, excessive));
// Diffuse.
a = helper::diffuse(a);
},
8 => {
// 8 bytes excessive.
// Mix in the partial block.
a ^= helper::read_u64(ptr);
// Diffuse.
a = helper::diffuse(a);
},
9...15 => {
// More than 8 bytes excessive.
// Mix in the partial block.
a ^= helper::read_u64(ptr);
// Write the last excessive bytes (<8 bytes).
excessive = excessive - 8;
b ^= helper::read_int(slice::from_raw_parts(ptr.offset(8), excessive));
// Diffuse.
a = helper::diffuse(a);
b = helper::diffuse(b);
},
16 => {
// 16 bytes excessive.
// Mix in the partial block.
a ^= helper::read_u64(ptr);
b ^= helper::read_u64(ptr.offset(8));
// Diffuse.
a = helper::diffuse(a);
b = helper::diffuse(b);
},
17...23 => {
// 16 bytes or more excessive.
// Mix in the partial block.
a ^= helper::read_u64(ptr);
b ^= helper::read_u64(ptr.offset(8));
// Write the last excessive bytes (<8 bytes).
excessive = excessive - 16;
c ^= helper::read_int(slice::from_raw_parts(ptr.offset(16), excessive));
// Diffuse.
a = helper::diffuse(a);
b = helper::diffuse(b);
c = helper::diffuse(c);
},
24 => {
// 24 bytes excessive.
// Mix in the partial block.
a ^= helper::read_u64(ptr);
b ^= helper::read_u64(ptr.offset(8));
c ^= helper::read_u64(ptr.offset(16));
// Diffuse.
a = helper::diffuse(a);
b = helper::diffuse(b);
c = helper::diffuse(c);
},
_ => {
// More than 24 bytes excessive.
// Mix in the partial block.
a ^= helper::read_u64(ptr);
b ^= helper::read_u64(ptr.offset(8));
c ^= helper::read_u64(ptr.offset(16));
// Write the last excessive bytes (<8 bytes).
excessive = excessive - 24;
d ^= helper::read_int(slice::from_raw_parts(ptr.offset(24), excessive));
// Diffuse.
a = helper::diffuse(a);
b = helper::diffuse(b);
c = helper::diffuse(c);
d = helper::diffuse(d);
}
}
}
State {
a: a,
b: b,
c: c,
d: d,
written: buf.len() as u64,
}
}
/// Write another 64-bit integer into the state.
pub fn push(&mut self, x: u64) {
let mut a = self.a;
// Mix `x` into `a`.
a = helper::diffuse(a ^ x);
// Rotate around.
// _______________________
// | v
// a <---- b <---- c <---- d
self.a = self.b;
self.b = self.c;
self.c = self.d;
self.d = a;
// Increase the written bytes counter.
self.written += 8;
}
/// Remove the most recently written 64-bit integer from the state.
///
/// Given the value of the most recently written u64 `last`, remove it from the state.
pub fn pop(&mut self, last: u64) {
// Decrese the written bytes counter.
self.written -= 8;
// Remove the recently written data.
self.d = helper::undiffuse(self.d) ^ last;
let mut a = self.a;
// Rotate back.
// _______________________
// v |
// a ----> b ----> c ----> d
self.a = self.d;
self.b = a;
self.c = self.b;
self.d = self.c;
}
/// Finalize the state.
#[inline]
pub fn finalize(self) -> u64 {
let State { written, mut a, b, mut c, d } = self;
// XOR the states together. Even though XOR is commutative, it doesn't matter, because the
// state vector's initial components are mutually distinct, and thus swapping even and odd
// chunks will affect the result, because it is sensitive to the initial condition.
a ^= b;
c ^= d;
a ^= c;
// XOR the number of written bytes in order to make the excessive bytes zero-sensitive
// (without this, two excessive zeros would be equivalent to three excessive zeros). This
// is know as length padding.
a ^= written;
// We diffuse to make the excessive bytes discrete (i.e. small changes shouldn't give small
// changes in the output).
helper::diffuse(a)
}
}
/// Hash some buffer.
///
/// This is a highly optimized implementation of SeaHash. It implements numerous techniques to
/// improve performance:
///
/// - Register allocation: This makes a great deal out of making sure everything fits into
/// registers such that minimal memory accesses are needed. This works quite successfully on most
/// CPUs, and the only time it reads from memory is when it fetches the data of the buffer.
/// - Bulk reads: Like most other good hash functions, we read 8 bytes a time. This obviously
/// improves performance a lot
/// - Independent updates: We make sure very few statements next to each other depends on the
/// other. This means that almost always the CPU will be able to run the instructions in parallel.
/// - Loop unrolling: The hot loop is unrolled such that very little branches (one every 32 bytes)
/// are needed.
///
/// and more.
///
/// The seed of this hash function is prechosen.
pub fn hash(buf: &[u8]) -> u64 {
hash_seeded(buf, 0x16f11fe89b0d677c, 0xb480a793d8e6c86c, 0x6fe2e5aaf078ebc9, 0x14f994a4c5259381)
}
/// Hash some buffer according to a chosen seed.
///
/// The keys are expected to be chosen from a uniform distribution. The keys should be mutually
/// distinct to avoid issues with collisions if the lanes are permuted.
///
/// This is not secure, as [the key can be extracted with a bit of computational
/// work](https://github.com/ticki/tfs/issues/5), as such, it is recommended to have a fallback
/// hash function (adaptive hashing) in the case of hash flooding. It can be considered unbroken if
/// the output is not known (i.e. no malicious party has access to the raw values of the keys, only
/// a permutation thereof).), however I absolutely do not recommend using it for this. If you want
/// to be strict, this should only be used as a layer of obfuscation, such that the fallback (e.g.
/// SipHash) is harder to trigger.
///
/// In the future, I might strengthen the security if possible while having backward compatibility
/// with the default initialization vector.
pub fn hash_seeded(buf: &[u8], a: u64, b: u64, c: u64, d: u64) -> u64 {
State::hash(buf, (a, b, c, d)).finalize()
}
#[cfg(test)]
mod tests {
use super::*;
use reference;
fn hash_match(a: &[u8]) {
assert_eq!(hash(a), reference::hash(a));
assert_eq!(hash_seeded(a, 1, 1, 1, 1), reference::hash_seeded(a, 1, 1, 1, 1));
assert_eq!(hash_seeded(a, 500, 2873, 2389, 9283), reference::hash_seeded(a, 500, 2873, 2389, 9283));
assert_eq!(hash_seeded(a, 238945723984, 872894734, 239478243, 28937498234), reference::hash_seeded(a, 238945723984, 872894734, 239478243, 28937498234));
assert_eq!(hash_seeded(a, !0, !0, !0, !0), reference::hash_seeded(a, !0, !0, !0, !0));
assert_eq!(hash_seeded(a, 0, 0, 0, 0), reference::hash_seeded(a, 0, 0, 0, 0));
}
#[test]
fn zero() {
let arr = [0; 4096];
for n in 0..4096 {
hash_match(&arr[0..n]);
}
}
#[test]
fn seq() {
let mut buf = [0; 4096];
for i in 0..4096 {
buf[i] = i as u8;
}
hash_match(&buf);
}
#[test]
fn position_depedent() {
let mut buf1 = [0; 4098];
for i in 0..4098 {
buf1[i] = i as u8;
}
let mut buf2 = [0; 4098];
for i in 0..4098 {
buf2[i] = i as u8 ^ 1;
}
assert!(hash(&buf1) != hash(&buf2));
}
#[test]
fn shakespear() {
hash_match(b"to be or not to be");
hash_match(b"love is a wonderful terrible thing");
}
#[test]
fn zero_senitive() {
assert_ne!(hash(&[1, 2, 3, 4]), hash(&[1, 0, 2, 3, 4]));
assert_ne!(hash(&[1, 2, 3, 4]), hash(&[1, 0, 0, 2, 3, 4]));
assert_ne!(hash(&[1, 2, 3, 4]), hash(&[1, 2, 3, 4, 0]));
assert_ne!(hash(&[1, 2, 3, 4]), hash(&[0, 1, 2, 3, 4]));
assert_ne!(hash(&[0, 0, 0]), hash(&[0, 0, 0, 0, 0]));
}
#[test]
fn not_equal() {
assert_ne!(hash(b"to be or not to be "), hash(b"to be or not to be"));
assert_ne!(hash(b"jkjke"), hash(b"jkjk"));
assert_ne!(hash(b"ijkjke"), hash(b"ijkjk"));
assert_ne!(hash(b"iijkjke"), hash(b"iijkjk"));
assert_ne!(hash(b"iiijkjke"), hash(b"iiijkjk"));
assert_ne!(hash(b"iiiijkjke"), hash(b"iiiijkjk"));
assert_ne!(hash(b"iiiiijkjke"), hash(b"iiiiijkjk"));
assert_ne!(hash(b"iiiiiijkjke"), hash(b"iiiiiijkjk"));
assert_ne!(hash(b"iiiiiiijkjke"), hash(b"iiiiiiijkjk"));
assert_ne!(hash(b"iiiiiiiijkjke"), hash(b"iiiiiiiijkjk"));
assert_ne!(hash(b"ab"), hash(b"bb"));
}
#[test]
fn push() {
let mut state = State::new(1, 2, 3, 4);
state.push(!0);
state.push(0);
assert_eq!(hash_seeded(&[0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0, 0, 0, 0, 0, 0, 0, 0], 1, 2, 3, 4), state.finalize());
}
#[test]
fn pop() {
let mut state = State::new(1, 2, 3, 4);
state.push(!0);
state.push(0);
state.pop(0);
assert_eq!(hash_seeded(&[0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF], 1, 2, 3, 4), state.finalize());
}
}
//! Helper functions.
/// Read a buffer smaller than 8 bytes into an integer in little-endian.
///
/// This assumes that `buf.len() < 8`. If this is not satisfied, the behavior is unspecified.
#[inline(always)]
pub fn read_int(buf: &[u8]) -> u64 {
// Because we want to make sure that it is register allocated, we fetch this into a variable.
// It will likely make no difference anyway, though.
let ptr = buf.as_ptr();
unsafe {
// Break it down to reads of integers with widths in total spanning the buffer. This minimizes
// the number of reads
match buf.len() {
// u8.
1 => *ptr as u64,
// u16.
2 => (*(ptr as *const u16)).to_le() as u64,
// u16 + u8.
3 => {
let a = (*(ptr as *const u16)).to_le() as u64;
let b = *ptr.offset(2) as u64;
a | (b << 16)
},
// u32.
4 => (*(ptr as *const u32)).to_le() as u64,
// u32 + u8.
5 => {
let a = (*(ptr as *const u32)).to_le() as u64;
let b = *ptr.offset(4) as u64;
a | (b << 32)
},
// u32 + u16.
6 => {
let a = (*(ptr as *const u32)).to_le() as u64;
let b = (*(ptr.offset(4) as *const u16)).to_le() as u64;
a | (b << 32)
},
// u32 + u16 + u8.
7 => {
let a = (*(ptr as *const u32)).to_le() as u64;
let b = (*(ptr.offset(4) as *const u16)).to_le() as u64;
let c = *ptr.offset(6) as u64;
a | (b << 32) | (c << 48)
},
_ => 0,
}
}
}
/// Read a little-endian 64-bit integer from some buffer.
#[inline(always)]
pub unsafe fn read_u64(ptr: *const u8) -> u64 {
#[cfg(target_pointer_width = "32")]
{
// We cannot be sure about the memory layout of a potentially emulated 64-bit integer, so
// we read it manually. If possible, the compiler should emit proper instructions.
(*(ptr as *const u32)).to_le() as u64 | ((*(ptr as *const u32)).to_le() as u64) << 32
}
#[cfg(target_pointer_width = "64")]
{
(*(ptr as *const u64)).to_le()
}
}
/// The diffusion function.
///
/// This is a bijective function emitting chaotic behavior. Such functions are used as building
/// blocks for hash functions.
pub fn diffuse(mut x: u64) -> u64 {
// These are derived from the PCG RNG's round. Thanks to @Veedrac for proposing this. The basic
// idea is that we use dynamic shifts, which are determined by the input itself. The shift is
// chosen by the higher bits, which means that changing those flips the lower bits, which
// scatters upwards because of the multiplication.
x = x.wrapping_mul(0x6eed0e9da4d94a4f);
let a = x >> 32;
let b = x >> 60;
x ^= a >> b;
x = x.wrapping_mul(0x6eed0e9da4d94a4f);
x
}
/// Reverse the `diffuse` function.
pub fn undiffuse(mut x: u64) -> u64 {
// 0x2f72b4215a3d8caf is the modular multiplicative inverse of the constant used in `diffuse`.
x = x.wrapping_mul(0x2f72b4215a3d8caf);
let a = x >> 32;
let b = x >> 60;
x ^= a >> b;
x = x.wrapping_mul(0x2f72b4215a3d8caf);
x
}
#[cfg(test)]
mod tests {
use super::*;
fn diffuse_test(x: u64, y: u64) {
assert_eq!(diffuse(x), y);
assert_eq!(x, undiffuse(y));
assert_eq!(undiffuse(diffuse(x)), x);
}
#[test]
fn read_int_() {
assert_eq!(read_int(&[2, 3]), 770);
assert_eq!(read_int(&[3, 2]), 515);
assert_eq!(read_int(&[3, 2, 5]), 328195);
}
#[test]
fn read_u64_() {
unsafe {
assert_eq!(read_u64([1, 0, 0, 0, 0, 0, 0, 0].as_ptr()), 1);
assert_eq!(read_u64([2, 1, 0, 0, 0, 0, 0, 0].as_ptr()), 258);
}
}
#[test]
fn diffuse_test_vectors() {
diffuse_test(94203824938, 17289265692384716055);
diffuse_test(0xDEADBEEF, 12110756357096144265);
diffuse_test(0, 0);
diffuse_test(1, 15197155197312260123);
diffuse_test(2, 1571904453004118546);
diffuse_test(3, 16467633989910088880);
}
}
//! SeaHash: A blazingly fast, portable hash function with proven statistical guarantees.
//!
//! SeaHash is a hash function with performance better than (around 3-20% improvement) xxHash and
//! MetroHash. Furthermore, SeaHash has mathematically provable statistical guarantees.
//!
//! SeaHash is a portable hash function, meaning that the output is not dependent on the hosting
//! architecture, and makes no assumptions on endianness or the alike. This stable layout allows it
//! to be used for on-disk/permanent storage (e.g. checksums).
//!
//! # Design, advantages, and features
//!
//! - **High quality**: It beats most other general purpose hash functions because it provides full
//! avalanche inbetween state updates.
//! - **Performance**: SeaHash beats every high-quality (grading 10/10 in smhasher) hash function
//! that I know of.
//! - **Provable quality guarantees**: Contrary to most other non-cryptographic hash function,
//! SeaHash can be proved to satisfy the avalanche criterion as well as BIC.
//! - **Parallelizable**: Consists of multiple, independent states to take advantage of ILP and/or
//! software threads.
//! - **Bulk reads**: Reads 8 or 4 bytes a time.
//! - **Stable and portable**: Does not depend on the target architecture, and produces a stable
//! value, which is only changed in major version bumps.
//! - **Keyed**: Designed to not leak the seed/key. Note that it has not gone through
//! cryptoanalysis yet, so the keyed version shouldn't be relied on when security is needed.
//! - **Hardware accelerateable**: SeaHash is designed such that ASICs can implement it with really
//! high performance.
//!
//! # A word of warning!
//!
//! This is **not** a cryptographic function, and it certainly should not be used as one. If you
//! want a good cryptographic hash function, you should use SHA-3 (Keccak) or BLAKE2.
//!
//! It is not secure, nor does it aim to be. It aims to have high quality pseudorandom output and
//! few collisions, as well as being fast.
//!
//! # Benchmark
//!
//! On normal hardware, it is expected to run with a rate around 5.9-6.7 GB/S on a 2.5 GHz CPU.
//! Further improvement can be seen when hashing very big buffers in parallel.
//!
//! | Function | Quality | Cycles per byte (lower is better) | Author
//! |-------------|---------------|-----------------------------------|-------------------
//! | **SeaHash** | **Excellent** | **0.24** | **Ticki**
//! | xxHash | Excellent | 0.31 | Collet
//! | MetroHash | Excellent | 0.35 | Rogers
//! | Murmur | Excellent | 0.64 | Appleby
//! | Rabin | Medium | 1.51 | Rabin
//! | CityHash | Excellent | 1.62 | Pike, Alakuijala
//! | LoseLose | Terrible | 2.01 | Kernighan, Ritchie
//! | FNV | Poor | 3.12 | Fowler, Noll, Vo
//! | SipHash | Pseudorandom | 3.21 | Aumasson, Bernstein
//! | CRC | Good | 3.91 | Peterson
//! | DJB2 | Poor | 4.13 | Bernstein
//!
//! ## Ideal architecture
//!
//! SeaHash is designed and optimized for the most common architecture in use:
//!