FARGOS/VISTA Object Management Environment Core  ..
FARGOS/VISTA Object Management Environment Core Table of Contents
bloom_filter.hpp
Go to the documentation of this file.
1 #ifndef _BLOOM_FILTER_HPP_
2 #define _BLOOM_FILTER_HPP_ "$Id: bloom_filter.hpp 454 2020-07-23 20:22:23Z geoff $"
4 // Copyright (C) 2012 - 2019 FARGOS Development, LLC
6 
12 
46 template<typename HASHTYPE, typename KEYTYPE=uint32_t> class BloomFilterForKey {
47 protected:
48  enum {
49  BIT_WIDTH = sizeof(HASHTYPE) * 8, // assuming 8 bits per byte
50  TBL_SIZE = (1 << BIT_WIDTH),
51  HASH_MASK = (1 << BIT_WIDTH) - 1
52  };
53  uint64_t presenceTable[TBL_SIZE / 64];
54 
61 #define SHIFT_N_BITS_TO_POSITION(v,n,f,t) (((f >= t) ? ((v) >> (f - t)) : ((v) << (t - f))) & (((1 << n) - 1) << t))
62 
65  HASHTYPE hash1(KEYTYPE key) const OME_CONST_FUNCTION {
66  KEYTYPE k1 = key >> BIT_WIDTH;
67  KEYTYPE k2 = key + k1;
68  HASHTYPE result = (key + k2) & ((1 << (BIT_WIDTH - 2)) - 1);
69  result |= 3 << (BIT_WIDTH - 2);
70  return (result);
71  }
72 
73  HASHTYPE hash2(KEYTYPE key) const OME_CONST_FUNCTION {
74  /* We expect 0's in the high-order bits for normal integer keys.
75  * Assuming little-endian order, we'll also expect 0's in the high-order
76  * bits if the source was a string; the least significant bits will
77  * correspond to the first character in string.
78  */
79  // bits 0-4 to bits 0-4
80  KEYTYPE k1 = SHIFT_N_BITS_TO_POSITION(key, 5, 0, 0);
81  // bits 12-8 to bits 9-5
82  KEYTYPE k2 = SHIFT_N_BITS_TO_POSITION(key, 3, 8, 5);
83  // bits 19-16 to bits 13-10
84  KEYTYPE k3 = SHIFT_N_BITS_TO_POSITION(key, 4, 16, 8);
85  // bits 25,24 to bits 15,14
86  KEYTYPE k4 = SHIFT_N_BITS_TO_POSITION(key, 4, 24, 12);
87  HASHTYPE result = ((k1 ^ k2 ^ k3 ^ k4) + key) & HASH_MASK;
88  // NOTE: constant expression, so evaluated at compile-time
89  if (sizeof(KEYTYPE) == 8) { // 64-bit value
90  KEYTYPE k5 = SHIFT_N_BITS_TO_POSITION(key, 5, 32, 16);
91  KEYTYPE k6 = SHIFT_N_BITS_TO_POSITION(key, 3, 40, 21);
92  KEYTYPE k7 = SHIFT_N_BITS_TO_POSITION(key, 4, 48, 24);
93  KEYTYPE k8 = SHIFT_N_BITS_TO_POSITION(key, 4, 52, 28);
94  result = ((k5 ^ k6 ^ k7 ^ k8) + key) & HASH_MASK;
95  }
96  return (result);
97  }
98 
99  HASHTYPE hash3(KEYTYPE key) const OME_CONST_FUNCTION {
100  KEYTYPE k1 = key >> (BIT_WIDTH - 1);
101  HASHTYPE result = (key - k1) & ((1 << (BIT_WIDTH - 2)) - 1);
102  result |= 1 << (BIT_WIDTH - 1);
103  return (result);
104  }
105 
106  void setPresenceBit(HASHTYPE hashedVal) {
107  uint32_t sub = hashedVal >> 6; // 64 = 2^6
108  OME_PREFETCH(presenceTable + sub, 1, 3); // read-write, retain if at all possible
109  uint32_t bit = hashedVal & 63; // get bit
110  presenceTable[sub] |= (static_cast<uint64_t>(1) << bit);
111  }
112 
113  bool isPresenceBitSet(HASHTYPE hashedVal) const {
114  uint32_t sub = hashedVal >> 6; // 64 = 2^6
115  OME_PREFETCH(presenceTable + sub, 0, 3); // read-only, retain if at all possible
116  uint32_t bit = hashedVal & 63; // get bit
117  uint64_t bitVal = presenceTable[sub] & (static_cast<uint64_t>(1) << bit);
118  return ((bitVal != 0));
119  }
120 
121 public:
124  memset(presenceTable, 0, sizeof(presenceTable));
125  }
126 
128  }
129 
131  void clear() {
132  memset(presenceTable, 0, sizeof(presenceTable));
133  }
134 
139  void addKey(KEYTYPE key) OME_ALWAYS_INLINE {
140  HASHTYPE h1 = hash1(key);
141  HASHTYPE h2 = hash2(key);
142  HASHTYPE h3 = hash3(key);
143  setPresenceBit(h1);
144  setPresenceBit(h2);
145  setPresenceBit(h3);
146  }
147 
157  bool couldBeSelected(KEYTYPE key) const OME_ALWAYS_INLINE {
158  HASHTYPE h1 = hash1(key);
159  if (isPresenceBitSet(h1) == false) {
160  return (false);
161  }
162  HASHTYPE h2 = hash2(key);
163  if (isPresenceBitSet(h2) == false) {
164  return (false);
165  }
166  HASHTYPE h3 = hash3(key);
167  if (isPresenceBitSet(h3) == false) {
168  return (false);
169  }
170  // all hashes indicate a collision
171  return (true);
172  }
173 }; // end class BloomFilterForKey
174 
178 #endif
179 /* vim: set expandtab shiftwidth=4 tabstop=4: */
BloomFilterForKey::BIT_WIDTH
@ BIT_WIDTH
Definition: bloom_filter.hpp:49
BloomFilterForKey::presenceTable
uint64_t presenceTable[TBL_SIZE/64]
Definition: bloom_filter.hpp:53
BloomFilterForKey::setPresenceBit
void setPresenceBit(HASHTYPE hashedVal)
Definition: bloom_filter.hpp:106
BloomFilterForKey
Templated Bloom filter.
Definition: bloom_filter.hpp:46
BloomFilterForKey::HASH_MASK
@ HASH_MASK
Definition: bloom_filter.hpp:51
BloomFilterForKey::couldBeSelected
bool couldBeSelected(KEYTYPE key) const OME_ALWAYS_INLINE
Test to see if key could be of interest.
Definition: bloom_filter.hpp:157
BloomFilterForKey::clear
void clear()
Clear all presence bits from the Bloom filter.
Definition: bloom_filter.hpp:131
OME_PREFETCH
#define OME_PREFETCH(addr, rw, locality)
Macro to request prefetch.
Definition: compiler_hints.h:362
BloomFilterForKey::hash3
HASHTYPE hash3(KEYTYPE key) const OME_CONST_FUNCTION
Definition: bloom_filter.hpp:99
BloomFilterForKey::hash1
HASHTYPE hash1(KEYTYPE key) const OME_CONST_FUNCTION
Hash 1 for Bloom filter.
Definition: bloom_filter.hpp:65
BloomFilterForKey::TBL_SIZE
@ TBL_SIZE
Definition: bloom_filter.hpp:50
BloomFilterForKey::hash2
HASHTYPE hash2(KEYTYPE key) const OME_CONST_FUNCTION
Definition: bloom_filter.hpp:73
BloomFilterForKey::addKey
void addKey(KEYTYPE key) OME_ALWAYS_INLINE
Add a key to the Bloom filter.
Definition: bloom_filter.hpp:139
OME_CONST_FUNCTION
#define OME_CONST_FUNCTION
Mark as an idempotent function that only accesses arguments – no global data.
Definition: compiler_hints.h:390
BloomFilterForKey::isPresenceBitSet
bool isPresenceBitSet(HASHTYPE hashedVal) const
Definition: bloom_filter.hpp:113
SHIFT_N_BITS_TO_POSITION
#define SHIFT_N_BITS_TO_POSITION(v, n, f, t)
Definition: bloom_filter.hpp:61
compiler_hints.h
Compiler-specific macros to provide performance-related hints.
OME_ALWAYS_INLINE
#define OME_ALWAYS_INLINE
Tell the compiler to alway inline a function, regardless of optimization level.
Definition: compiler_hints.h:364
BloomFilterForKey::BloomFilterForKey
BloomFilterForKey()
Construct a Bloom filter.
Definition: bloom_filter.hpp:123
BloomFilterForKey::~BloomFilterForKey
~BloomFilterForKey()
Definition: bloom_filter.hpp:127
Generated: Fri Jul 31 2020 18:19:13
Support Information