Initial commit to new repository

.gitignore

+/target/
+Cargo.lock

Cargo.toml

+[package]
+name = "seahash"
+version = "3.0.5"
+authors = ["ticki <ticki@users.noreply.github.com>"]
+description = "A blazingly fast, portable hash function with proven statistical guarantees."
+repository = "https://github.com/ticki/tfs"
+documentation = "https://docs.rs/seahash"
+license = "MIT"
+keywords = ["hash", "hashing", "checksum", "checsumming", "portable"]
+exclude = ["target", "Cargo.lock"]

README.md

+<div align="center"><img alt="Logo" width="200" src="./logo.png" /></div>
+===================
+
+SeaHash: A bizarrely fast hash function.
+
+SeaHash is a hash function with performance better than (around 3-20% improvement) xxHash and
+MetroHash. Furthermore, SeaHash has mathematically provable statistical guarantees.
+
+In action:
+
+[![The hash function in action.](http://ticki.github.io/img/seahash_construction_diagram.svg)](http://ticki.github.io/img/seahash_construction_diagram.svg)

benches/gigabyte.rs

+#![feature(test)]
+
+extern crate test;
+extern crate seahash;
+
+#[bench]
+fn gigabyte(b: &mut test::Bencher) {
+    b.iter(|| {
+        let mut x = 0;
+        let mut buf = [15; 4096];
+
+        for _ in 0..250000 {
+            x ^= seahash::hash(&buf);
+            buf[0] += buf[0].wrapping_add(1);
+        }
+
+        x
+    })
+}

src/buffer.rs

+//! A highly optimized version of SeaHash.
+
+use core::slice;
+
+use helper;
+
+/// A SeaHash state.
+#[derive(Clone)]
+pub struct State {
+    /// `a`
+    a: u64,
+    /// `b`
+    b: u64,
+    /// `c`
+    c: u64,
+    /// `d`
+    d: u64,
+    /// The number of written bytes.
+    written: u64,
+}
+
+impl State {
+    /// Create a new state vector with some initial values.
+    pub fn new(a: u64, b: u64, c: u64, d: u64) -> State {
+        State {
+            a: a,
+            b: b,
+            c: c,
+            d: d,
+            written: 0,
+        }
+    }
+
+    /// Hash a buffer with some seed.
+    pub fn hash(buf: &[u8], (mut a, mut b, mut c, mut d): (u64, u64, u64, u64)) -> State {
+        unsafe {
+            // We use 4 different registers to store seperate hash states, because this allows us
+            // to update them seperately, and consequently exploiting ILP to update the states in
+            // parallel.
+
+            // The pointer to the current bytes.
+            let mut ptr = buf.as_ptr();
+            /// The end of the "main segment", i.e. the biggest buffer s.t. the length is divisible
+            /// by 32.
+            let end_ptr = buf.as_ptr().offset(buf.len() as isize & !0x1F);
+
+            while end_ptr > ptr {
+                // Modern CPUs allow the pointer arithmetic to be done in place, hence not
+                // introducing tmpvars.
+                a ^= helper::read_u64(ptr);
+                b ^= helper::read_u64(ptr.offset(8));
+                c ^= helper::read_u64(ptr.offset(16));
+                d ^= helper::read_u64(ptr.offset(24));
+
+                // Increment the pointer.
+                ptr = ptr.offset(32);
+
+                // Diffuse the updated registers. We hope that each of these are executed in
+                // parallel.
+                a = helper::diffuse(a);
+                b = helper::diffuse(b);
+                c = helper::diffuse(c);
+                d = helper::diffuse(d);
+            }
+
+            // Calculate the number of excessive bytes. These are bytes that could not be handled
+            // in the loop above.
+            let mut excessive = buf.len() as usize + buf.as_ptr() as usize - end_ptr as usize;
+            // Handle the excessive bytes.
+            match excessive {
+                0 => {},
+                1...7 => {
+                    // 1 or more excessive.
+
+                    // Write the last excessive bytes (<8 bytes).
+                    a ^= helper::read_int(slice::from_raw_parts(ptr as *const u8, excessive));
+
+                    // Diffuse.
+                    a = helper::diffuse(a);
+                },
+                8 => {
+                    // 8 bytes excessive.
+
+                    // Mix in the partial block.
+                    a ^= helper::read_u64(ptr);
+
+                    // Diffuse.
+                    a = helper::diffuse(a);
+                },
+                9...15 => {
+                    // More than 8 bytes excessive.
+
+                    // Mix in the partial block.
+                    a ^= helper::read_u64(ptr);
+
+                    // Write the last excessive bytes (<8 bytes).
+                    excessive = excessive - 8;
+                    b ^= helper::read_int(slice::from_raw_parts(ptr.offset(8), excessive));
+
+                    // Diffuse.
+                    a = helper::diffuse(a);
+                    b = helper::diffuse(b);
+
+                },
+                16 => {
+                    // 16 bytes excessive.
+
+                    // Mix in the partial block.
+                    a ^= helper::read_u64(ptr);
+                    b ^= helper::read_u64(ptr.offset(8));
+
+                    // Diffuse.
+                    a = helper::diffuse(a);
+                    b = helper::diffuse(b);
+                },
+                17...23 => {
+                    // 16 bytes or more excessive.
+
+                    // Mix in the partial block.
+                    a ^= helper::read_u64(ptr);
+                    b ^= helper::read_u64(ptr.offset(8));
+
+                    // Write the last excessive bytes (<8 bytes).
+                    excessive = excessive - 16;
+                    c ^= helper::read_int(slice::from_raw_parts(ptr.offset(16), excessive));
+
+                    // Diffuse.
+                    a = helper::diffuse(a);
+                    b = helper::diffuse(b);
+                    c = helper::diffuse(c);
+                },
+                24 => {
+                    // 24 bytes excessive.
+
+                    // Mix in the partial block.
+                    a ^= helper::read_u64(ptr);
+                    b ^= helper::read_u64(ptr.offset(8));
+                    c ^= helper::read_u64(ptr.offset(16));
+
+                    // Diffuse.
+                    a = helper::diffuse(a);
+                    b = helper::diffuse(b);
+                    c = helper::diffuse(c);
+                },
+                _ => {
+                    // More than 24 bytes excessive.
+
+                    // Mix in the partial block.
+                    a ^= helper::read_u64(ptr);
+                    b ^= helper::read_u64(ptr.offset(8));
+                    c ^= helper::read_u64(ptr.offset(16));
+
+                    // Write the last excessive bytes (<8 bytes).
+                    excessive = excessive - 24;
+                    d ^= helper::read_int(slice::from_raw_parts(ptr.offset(24), excessive));
+
+                    // Diffuse.
+                    a = helper::diffuse(a);
+                    b = helper::diffuse(b);
+                    c = helper::diffuse(c);
+                    d = helper::diffuse(d);
+                }
+            }
+        }
+
+        State {
+            a: a,
+            b: b,
+            c: c,
+            d: d,
+            written: buf.len() as u64,
+        }
+    }
+
+    /// Write another 64-bit integer into the state.
+    pub fn push(&mut self, x: u64) {
+        let mut a = self.a;
+
+        // Mix `x` into `a`.
+        a = helper::diffuse(a ^ x);
+
+        //  Rotate around.
+        //  _______________________
+        // |                       v
+        // a <---- b <---- c <---- d
+        self.a = self.b;
+        self.b = self.c;
+        self.c = self.d;
+        self.d = a;
+
+        // Increase the written bytes counter.
+        self.written += 8;
+    }
+
+    /// Remove the most recently written 64-bit integer from the state.
+    ///
+    /// Given the value of the most recently written u64 `last`, remove it from the state.
+    pub fn pop(&mut self, last: u64) {
+        // Decrese the written bytes counter.
+        self.written -= 8;
+
+        // Remove the recently written data.
+        self.d = helper::undiffuse(self.d) ^ last;
+
+        let mut a = self.a;
+
+        //  Rotate back.
+        //  _______________________
+        // v                       |
+        // a ----> b ----> c ----> d
+        self.a = self.d;
+        self.b = a;
+        self.c = self.b;
+        self.d = self.c;
+    }
+
+    /// Finalize the state.
+    #[inline]
+    pub fn finalize(self) -> u64 {
+        let State { written, mut a, b, mut c, d } = self;
+
+        // XOR the states together. Even though XOR is commutative, it doesn't matter, because the
+        // state vector's initial components are mutually distinct, and thus swapping even and odd
+        // chunks will affect the result, because it is sensitive to the initial condition.
+        a ^= b;
+        c ^= d;
+        a ^= c;
+        // XOR the number of written bytes in order to make the excessive bytes zero-sensitive
+        // (without this, two excessive zeros would be equivalent to three excessive zeros). This
+        // is know as length padding.
+        a ^= written;
+
+        // We diffuse to make the excessive bytes discrete (i.e. small changes shouldn't give small
+        // changes in the output).
+        helper::diffuse(a)
+    }
+}
+
+/// Hash some buffer.
+///
+/// This is a highly optimized implementation of SeaHash. It implements numerous techniques to
+/// improve performance:
+///
+/// - Register allocation: This makes a great deal out of making sure everything fits into
+///   registers such that minimal memory accesses are needed. This works quite successfully on most
+///   CPUs, and the only time it reads from memory is when it fetches the data of the buffer.
+/// - Bulk reads: Like most other good hash functions, we read 8 bytes a time. This obviously
+///   improves performance a lot
+/// - Independent updates: We make sure very few statements next to each other depends on the
+///   other. This means that almost always the CPU will be able to run the instructions in parallel.
+/// - Loop unrolling: The hot loop is unrolled such that very little branches (one every 32 bytes)
+///   are needed.
+///
+/// and more.
+///
+/// The seed of this hash function is prechosen.
+pub fn hash(buf: &[u8]) -> u64 {
+    hash_seeded(buf, 0x16f11fe89b0d677c, 0xb480a793d8e6c86c, 0x6fe2e5aaf078ebc9, 0x14f994a4c5259381)
+}
+
+/// Hash some buffer according to a chosen seed.
+///
+/// The keys are expected to be chosen from a uniform distribution. The keys should be mutually
+/// distinct to avoid issues with collisions if the lanes are permuted.
+///
+/// This is not secure, as [the key can be extracted with a bit of computational
+/// work](https://github.com/ticki/tfs/issues/5), as such, it is recommended to have a fallback
+/// hash function (adaptive hashing) in the case of hash flooding. It can be considered unbroken if
+/// the output is not known (i.e. no malicious party has access to the raw values of the keys, only
+/// a permutation thereof).), however I absolutely do not recommend using it for this. If you want
+/// to be strict, this should only be used as a layer of obfuscation, such that the fallback (e.g.
+/// SipHash) is harder to trigger.
+///
+/// In the future, I might strengthen the security if possible while having backward compatibility
+/// with the default initialization vector.
+pub fn hash_seeded(buf: &[u8], a: u64, b: u64, c: u64, d: u64) -> u64 {
+    State::hash(buf, (a, b, c, d)).finalize()
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    use reference;
+
+    fn hash_match(a: &[u8]) {
+        assert_eq!(hash(a), reference::hash(a));
+        assert_eq!(hash_seeded(a, 1, 1, 1, 1), reference::hash_seeded(a, 1, 1, 1, 1));
+        assert_eq!(hash_seeded(a, 500, 2873, 2389, 9283), reference::hash_seeded(a, 500, 2873, 2389, 9283));
+        assert_eq!(hash_seeded(a, 238945723984, 872894734, 239478243, 28937498234), reference::hash_seeded(a, 238945723984, 872894734, 239478243, 28937498234));
+        assert_eq!(hash_seeded(a, !0, !0, !0, !0), reference::hash_seeded(a, !0, !0, !0, !0));
+        assert_eq!(hash_seeded(a, 0, 0, 0, 0), reference::hash_seeded(a, 0, 0, 0, 0));
+    }
+
+    #[test]
+    fn zero() {
+        let arr = [0; 4096];
+        for n in 0..4096 {
+            hash_match(&arr[0..n]);
+        }
+    }
+
+    #[test]
+    fn seq() {
+        let mut buf = [0; 4096];
+        for i in 0..4096 {
+            buf[i] = i as u8;
+        }
+        hash_match(&buf);
+    }
+
+
+    #[test]
+    fn position_depedent() {
+        let mut buf1 = [0; 4098];
+        for i in 0..4098 {
+            buf1[i] = i as u8;
+        }
+        let mut buf2 = [0; 4098];
+        for i in 0..4098 {
+            buf2[i] = i as u8 ^ 1;
+        }
+
+        assert!(hash(&buf1) != hash(&buf2));
+    }
+
+    #[test]
+    fn shakespear() {
+        hash_match(b"to be or not to be");
+        hash_match(b"love is a wonderful terrible thing");
+    }
+
+    #[test]
+    fn zero_senitive() {
+        assert_ne!(hash(&[1, 2, 3, 4]), hash(&[1, 0, 2, 3, 4]));
+        assert_ne!(hash(&[1, 2, 3, 4]), hash(&[1, 0, 0, 2, 3, 4]));
+        assert_ne!(hash(&[1, 2, 3, 4]), hash(&[1, 2, 3, 4, 0]));
+        assert_ne!(hash(&[1, 2, 3, 4]), hash(&[0, 1, 2, 3, 4]));
+        assert_ne!(hash(&[0, 0, 0]), hash(&[0, 0, 0, 0, 0]));
+    }
+
+    #[test]
+    fn not_equal() {
+        assert_ne!(hash(b"to be or not to be "), hash(b"to be or not to be"));
+        assert_ne!(hash(b"jkjke"), hash(b"jkjk"));
+        assert_ne!(hash(b"ijkjke"), hash(b"ijkjk"));
+        assert_ne!(hash(b"iijkjke"), hash(b"iijkjk"));
+        assert_ne!(hash(b"iiijkjke"), hash(b"iiijkjk"));
+        assert_ne!(hash(b"iiiijkjke"), hash(b"iiiijkjk"));
+        assert_ne!(hash(b"iiiiijkjke"), hash(b"iiiiijkjk"));
+        assert_ne!(hash(b"iiiiiijkjke"), hash(b"iiiiiijkjk"));
+        assert_ne!(hash(b"iiiiiiijkjke"), hash(b"iiiiiiijkjk"));
+        assert_ne!(hash(b"iiiiiiiijkjke"), hash(b"iiiiiiiijkjk"));
+        assert_ne!(hash(b"ab"), hash(b"bb"));
+    }
+
+    #[test]
+    fn push() {
+        let mut state = State::new(1, 2, 3, 4);
+        state.push(!0);
+        state.push(0);
+
+        assert_eq!(hash_seeded(&[0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0, 0, 0, 0, 0, 0, 0, 0], 1, 2, 3, 4), state.finalize());
+    }
+
+    #[test]
+    fn pop() {
+        let mut state = State::new(1, 2, 3, 4);
+        state.push(!0);
+        state.push(0);
+        state.pop(0);
+
+        assert_eq!(hash_seeded(&[0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF], 1, 2, 3, 4), state.finalize());
+    }
+}

src/helper.rs

+//! Helper functions.
+
+/// Read a buffer smaller than 8 bytes into an integer in little-endian.
+///
+/// This assumes that `buf.len() < 8`. If this is not satisfied, the behavior is unspecified.
+#[inline(always)]
+pub fn read_int(buf: &[u8]) -> u64 {
+    // Because we want to make sure that it is register allocated, we fetch this into a variable.
+    // It will likely make no difference anyway, though.
+    let ptr = buf.as_ptr();
+
+    unsafe {
+        // Break it down to reads of integers with widths in total spanning the buffer. This minimizes
+        // the number of reads
+        match buf.len() {
+            // u8.
+            1 => *ptr as u64,
+            // u16.
+            2 => (*(ptr as *const u16)).to_le() as u64,
+            // u16 + u8.
+            3 => {
+                let a = (*(ptr as *const u16)).to_le() as u64;
+                let b = *ptr.offset(2) as u64;
+
+                a | (b << 16)
+            },
+            // u32.
+            4 => (*(ptr as *const u32)).to_le() as u64,
+            // u32 + u8.
+            5 => {
+                let a = (*(ptr as *const u32)).to_le() as u64;
+                let b = *ptr.offset(4) as u64;
+
+                a | (b << 32)
+            },
+            // u32 + u16.
+            6 => {
+                let a = (*(ptr as *const u32)).to_le() as u64;
+                let b = (*(ptr.offset(4) as *const u16)).to_le() as u64;
+
+                a | (b << 32)
+            },
+            // u32 + u16 + u8.
+            7 => {
+                let a = (*(ptr as *const u32)).to_le() as u64;
+                let b = (*(ptr.offset(4) as *const u16)).to_le() as u64;
+                let c = *ptr.offset(6) as u64;
+
+                a | (b << 32) | (c << 48)
+            },
+            _ => 0,
+        }
+    }
+}
+
+/// Read a little-endian 64-bit integer from some buffer.
+#[inline(always)]
+pub unsafe fn read_u64(ptr: *const u8) -> u64 {
+    #[cfg(target_pointer_width = "32")]
+    {
+        // We cannot be sure about the memory layout of a potentially emulated 64-bit integer, so
+        // we read it manually. If possible, the compiler should emit proper instructions.
+        (*(ptr as *const u32)).to_le() as u64 | ((*(ptr as *const u32)).to_le() as u64) << 32
+    }
+
+    #[cfg(target_pointer_width = "64")]
+    {
+        (*(ptr as *const u64)).to_le()
+    }
+}
+
+/// The diffusion function.
+///
+/// This is a bijective function emitting chaotic behavior. Such functions are used as building
+/// blocks for hash functions.
+pub fn diffuse(mut x: u64) -> u64 {
+    // These are derived from the PCG RNG's round. Thanks to @Veedrac for proposing this. The basic
+    // idea is that we use dynamic shifts, which are determined by the input itself. The shift is
+    // chosen by the higher bits, which means that changing those flips the lower bits, which
+    // scatters upwards because of the multiplication.
+
+    x = x.wrapping_mul(0x6eed0e9da4d94a4f);
+    let a = x >> 32;
+    let b = x >> 60;
+    x ^= a >> b;
+    x = x.wrapping_mul(0x6eed0e9da4d94a4f);
+
+    x
+}
+
+/// Reverse the `diffuse` function.
+pub fn undiffuse(mut x: u64) -> u64 {
+    // 0x2f72b4215a3d8caf is the modular multiplicative inverse of the constant used in `diffuse`.
+
+    x = x.wrapping_mul(0x2f72b4215a3d8caf);
+    let a = x >> 32;
+    let b = x >> 60;
+    x ^= a >> b;
+    x = x.wrapping_mul(0x2f72b4215a3d8caf);
+
+    x
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn diffuse_test(x: u64, y: u64) {
+        assert_eq!(diffuse(x), y);
+        assert_eq!(x, undiffuse(y));
+        assert_eq!(undiffuse(diffuse(x)), x);
+    }
+
+    #[test]
+    fn read_int_() {
+        assert_eq!(read_int(&[2, 3]), 770);
+        assert_eq!(read_int(&[3, 2]), 515);
+        assert_eq!(read_int(&[3, 2, 5]), 328195);
+    }
+
+    #[test]
+    fn read_u64_() {
+        unsafe {
+            assert_eq!(read_u64([1, 0, 0, 0, 0, 0, 0, 0].as_ptr()), 1);
+            assert_eq!(read_u64([2, 1, 0, 0, 0, 0, 0, 0].as_ptr()), 258);
+        }
+    }
+
+    #[test]
+    fn diffuse_test_vectors() {
+        diffuse_test(94203824938, 17289265692384716055);
+        diffuse_test(0xDEADBEEF, 12110756357096144265);
+        diffuse_test(0, 0);
+        diffuse_test(1, 15197155197312260123);
+        diffuse_test(2, 1571904453004118546);
+        diffuse_test(3, 16467633989910088880);
+    }
+}

src/lib.rs

+//! SeaHash: A blazingly fast, portable hash function with proven statistical guarantees.
+//!
+//! SeaHash is a hash function with performance better than (around 3-20% improvement) xxHash and
+//! MetroHash. Furthermore, SeaHash has mathematically provable statistical guarantees.
+//!
+//! SeaHash is a portable hash function, meaning that the output is not dependent on the hosting
+//! architecture, and makes no assumptions on endianness or the alike. This stable layout allows it
+//! to be used for on-disk/permanent storage (e.g. checksums).
+//!
+//! # Design, advantages, and features
+//!
+//! - **High quality**: It beats most other general purpose hash functions because it provides full
+//!   avalanche inbetween state updates.
+//! - **Performance**: SeaHash beats every high-quality (grading 10/10 in smhasher) hash function
+//!    that I know of.
+//! - **Provable quality guarantees**: Contrary to most other non-cryptographic hash function,
+//!   SeaHash can be proved to satisfy the avalanche criterion as well as BIC.
+//! - **Parallelizable**: Consists of multiple, independent states to take advantage of ILP and/or
+//!   software threads.
+//! - **Bulk reads**: Reads 8 or 4 bytes a time.
+//! - **Stable and portable**: Does not depend on the target architecture, and produces a stable
+//!   value, which is only changed in major version bumps.
+//! - **Keyed**: Designed to not leak the seed/key. Note that it has not gone through
+//!   cryptoanalysis yet, so the keyed version shouldn't be relied on when security is needed.
+//! - **Hardware accelerateable**: SeaHash is designed such that ASICs can implement it with really
+//!   high performance.
+//!
+//! # A word of warning!
+//!
+//! This is **not** a cryptographic function, and it certainly should not be used as one. If you
+//! want a good cryptographic hash function, you should use SHA-3 (Keccak) or BLAKE2.
+//!
+//! It is not secure, nor does it aim to be. It aims to have high quality pseudorandom output and
+//! few collisions, as well as being fast.
+//!
+//! # Benchmark
+//!
+//! On normal hardware, it is expected to run with a rate around 5.9-6.7 GB/S on a 2.5 GHz CPU.
+//! Further improvement can be seen when hashing very big buffers in parallel.
+//!
+//! | Function    | Quality       | Cycles per byte (lower is better) | Author
+//! |-------------|---------------|-----------------------------------|-------------------
+//! | **SeaHash** | **Excellent** | **0.24**                          | **Ticki**
+//! | xxHash      | Excellent     | 0.31                              | Collet
+//! | MetroHash   | Excellent     | 0.35                              | Rogers
+//! | Murmur      | Excellent     | 0.64                              | Appleby
+//! | Rabin       | Medium        | 1.51                              | Rabin
+//! | CityHash    | Excellent     | 1.62                              | Pike, Alakuijala
+//! | LoseLose    | Terrible      | 2.01                              | Kernighan, Ritchie
+//! | FNV         | Poor          | 3.12                              | Fowler, Noll, Vo
+//! | SipHash     | Pseudorandom  | 3.21                              | Aumasson, Bernstein
+//! | CRC         | Good          | 3.91                              | Peterson
+//! | DJB2        | Poor          | 4.13                              | Bernstein
+//!
+//! ## Ideal architecture
+//!