openethereum/util/bloom/src/lib.rs

// Copyright 2015-2019 Parity Technologies (UK) Ltd.
// This file is part of Parity Ethereum.

// Parity Ethereum is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.

// Parity Ethereum is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU General Public License for more details.

// You should have received a copy of the GNU General Public License
// along with Parity Ethereum.  If not, see <http://www.gnu.org/licenses/>.

extern crate siphasher;

use siphasher::sip::SipHasher;
use std::{
    cmp,
    collections::HashSet,
    f64,
    hash::{Hash, Hasher},
    mem,
};

/// BitVec structure with journalling
/// Every time any of the blocks is getting set it's index is tracked
/// and can be then drained by `drain` method
struct BitVecJournal {
    elems: Vec<u64>,
    journal: HashSet<usize>,
}

impl BitVecJournal {
    pub fn new(size: usize) -> BitVecJournal {
        let extra = if size % 64 > 0 { 1 } else { 0 };
        BitVecJournal {
            elems: vec![0u64; size / 64 + extra],
            journal: HashSet::new(),
        }
    }

    pub fn from_parts(parts: &[u64]) -> BitVecJournal {
        BitVecJournal {
            elems: parts.to_vec(),
            journal: HashSet::new(),
        }
    }

    pub fn set(&mut self, index: usize) {
        let e_index = index / 64;
        let bit_index = index % 64;
        let val = self.elems.get_mut(e_index).unwrap();
        *val |= 1u64 << bit_index;
        self.journal.insert(e_index);
    }

    pub fn get(&self, index: usize) -> bool {
        let e_index = index / 64;
        let bit_index = index % 64;
        self.elems[e_index] & (1 << bit_index) != 0
    }

    pub fn drain(&mut self) -> Vec<(usize, u64)> {
        let journal = mem::replace(&mut self.journal, HashSet::new()).into_iter();
        journal
            .map(|idx| (idx, self.elems[idx]))
            .collect::<Vec<(usize, u64)>>()
    }

    pub fn saturation(&self) -> f64 {
        self.elems
            .iter()
            .fold(0u64, |acc, e| acc + e.count_ones() as u64) as f64
            / (self.elems.len() * 64) as f64
    }
}

/// Bloom filter structure
pub struct Bloom {
    bitmap: BitVecJournal,
    bitmap_bits: u64,
    k_num: u32,
}

impl Bloom {
    /// Create a new bloom filter structure.
    /// bitmap_size is the size in bytes (not bits) that will be allocated in memory
    /// items_count is an estimation of the maximum number of items to store.
    pub fn new(bitmap_size: usize, items_count: usize) -> Bloom {
        assert!(bitmap_size > 0 && items_count > 0);
        let bitmap_bits = (bitmap_size as u64) * 8u64;
        let k_num = Bloom::optimal_k_num(bitmap_bits, items_count);
        let bitmap = BitVecJournal::new(bitmap_bits as usize);
        Bloom {
            bitmap: bitmap,
            bitmap_bits: bitmap_bits,
            k_num: k_num,
        }
    }

    /// Initializes bloom filter from saved state
    pub fn from_parts(parts: &[u64], k_num: u32) -> Bloom {
        let bitmap_size = parts.len() * 8;
        let bitmap_bits = (bitmap_size as u64) * 8u64;
        let bitmap = BitVecJournal::from_parts(parts);
        Bloom {
            bitmap: bitmap,
            bitmap_bits: bitmap_bits,
            k_num: k_num,
        }
    }

    /// Create a new bloom filter structure.
    /// items_count is an estimation of the maximum number of items to store.
    /// fp_p is the wanted rate of false positives, in ]0.0, 1.0[
    pub fn new_for_fp_rate(items_count: usize, fp_p: f64) -> Bloom {
        let bitmap_size = Bloom::compute_bitmap_size(items_count, fp_p);
        Bloom::new(bitmap_size, items_count)
    }

    /// Compute a recommended bitmap size for items_count items
    /// and a fp_p rate of false positives.
    /// fp_p obviously has to be within the ]0.0, 1.0[ range.
    pub fn compute_bitmap_size(items_count: usize, fp_p: f64) -> usize {
        assert!(items_count > 0);
        assert!(fp_p > 0.0 && fp_p < 1.0);
        let log2 = f64::consts::LN_2;
        let log2_2 = log2 * log2;
        ((items_count as f64) * f64::ln(fp_p) / (-8.0 * log2_2)).ceil() as usize
    }

    /// Records the presence of an item.
    pub fn set<T>(&mut self, item: T)
    where
        T: Hash,
    {
        let base_hash = Bloom::sip_hash(&item);
        for k_i in 0..self.k_num {
            let bit_offset = (Bloom::bloom_hash(base_hash, k_i) % self.bitmap_bits) as usize;
            self.bitmap.set(bit_offset);
        }
    }

    /// Check if an item is present in the set.
    /// There can be false positives, but no false negatives.
    pub fn check<T>(&self, item: T) -> bool
    where
        T: Hash,
    {
        let base_hash = Bloom::sip_hash(&item);
        for k_i in 0..self.k_num {
            let bit_offset = (Bloom::bloom_hash(base_hash, k_i) % self.bitmap_bits) as usize;
            if !self.bitmap.get(bit_offset) {
                return false;
            }
        }
        true
    }

    /// Return the number of bits in the filter
    pub fn number_of_bits(&self) -> u64 {
        self.bitmap_bits
    }

    /// Return the number of hash functions used for `check` and `set`
    pub fn number_of_hash_functions(&self) -> u32 {
        self.k_num
    }

    fn optimal_k_num(bitmap_bits: u64, items_count: usize) -> u32 {
        let m = bitmap_bits as f64;
        let n = items_count as f64;
        let k_num = (m / n * f64::ln(2.0f64)).ceil() as u32;
        cmp::max(k_num, 1)
    }

    fn sip_hash<T>(item: &T) -> u64
    where
        T: Hash,
    {
        let mut sip = SipHasher::new();
        item.hash(&mut sip);
        let hash = sip.finish();
        hash
    }

    fn bloom_hash(base_hash: u64, k_i: u32) -> u64 {
        if k_i < 2 {
            base_hash
        } else {
            base_hash.wrapping_add((k_i as u64).wrapping_mul(base_hash) % 0xffffffffffffffc5)
        }
    }

    /// Drains the bloom journal returning the updated bloom part
    pub fn drain_journal(&mut self) -> BloomJournal {
        BloomJournal {
            entries: self.bitmap.drain(),
            hash_functions: self.k_num,
        }
    }

    /// Returns the ratio of set bits in the bloom filter to the total bits
    pub fn saturation(&self) -> f64 {
        self.bitmap.saturation()
    }
}

/// Bloom journal
/// Returns the tuple of (bloom part index, bloom part value) where each one is representing
/// an index of bloom parts that was updated since the last drain
pub struct BloomJournal {
    pub hash_functions: u32,
    pub entries: Vec<(usize, u64)>,
}

#[cfg(test)]
mod tests {
    use super::Bloom;
    use std::collections::HashSet;

    #[test]
    fn get_set() {
        let mut bloom = Bloom::new(10, 80);
        let key = vec![115u8, 99];
        assert!(!bloom.check(&key));
        bloom.set(&key);
        assert!(bloom.check(&key));
    }

    #[test]
    fn journalling() {
        let initial = vec![0u64; 8];
        let mut bloom = Bloom::from_parts(&initial, 3);
        bloom.set(&vec![5u8, 4]);
        let drain = bloom.drain_journal();

        assert_eq!(2, drain.entries.len())
    }

    #[test]
    fn saturation() {
        let initial = vec![0u64; 8];
        let mut bloom = Bloom::from_parts(&initial, 3);
        bloom.set(&vec![5u8, 4]);

        let full = bloom.saturation();
        // 2/8/64 = 0.00390625
        assert!(full >= 0.0039f64 && full <= 0.004f64);
    }

    #[test]
    fn hash_backward_compatibility_for_new() {
        let ss = vec![
            "you",
            "should",
            "not",
            "break",
            "hash",
            "backward",
            "compatibility",
        ];
        let mut bloom = Bloom::new(16, 8);
        for s in ss.iter() {
            bloom.set(&s);
        }

        let drained_elems: HashSet<u64> = bloom
            .drain_journal()
            .entries
            .into_iter()
            .map(|t| t.1)
            .collect();
        let expected: HashSet<u64> = [2094615114573771027u64, 244675582389208413u64]
            .iter()
            .cloned()
            .collect();
        assert_eq!(drained_elems, expected);
        assert_eq!(bloom.k_num, 12);
    }

    #[test]
    fn hash_backward_compatibility_for_from_parts() {
        let stored_state = vec![2094615114573771027u64, 244675582389208413u64];
        let k_num = 12;
        let bloom = Bloom::from_parts(&stored_state, k_num);

        let ss = vec![
            "you",
            "should",
            "not",
            "break",
            "hash",
            "backward",
            "compatibility",
        ];
        let tt = vec!["this", "doesnot", "exist"];
        for s in ss.iter() {
            assert!(bloom.check(&s));
        }
        for s in tt.iter() {
            assert!(!bloom.check(&s));
        }
    }
}