Add per-HashTable stats
[WebKit-https.git] / Source / WTF / wtf / HashTable.h
1 /*
2  * Copyright (C) 2005, 2006, 2007, 2008, 2011, 2012 Apple Inc. All rights reserved.
3  * Copyright (C) 2008 David Levin <levin@chromium.org>
4  *
5  * This library is free software; you can redistribute it and/or
6  * modify it under the terms of the GNU Library General Public
7  * License as published by the Free Software Foundation; either
8  * version 2 of the License, or (at your option) any later version.
9  *
10  * This library is distributed in the hope that it will be useful,
11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
13  * Library General Public License for more details.
14  *
15  * You should have received a copy of the GNU Library General Public License
16  * along with this library; see the file COPYING.LIB.  If not, write to
17  * the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
18  * Boston, MA 02110-1301, USA.
19  *
20  */
21
22 #ifndef WTF_HashTable_h
23 #define WTF_HashTable_h
24
25 #include <wtf/Alignment.h>
26 #include <wtf/Assertions.h>
27 #include <wtf/DataLog.h>
28 #include <wtf/FastMalloc.h>
29 #include <wtf/HashTraits.h>
30 #include <wtf/StdLibExtras.h>
31 #include <wtf/Threading.h>
32 #include <wtf/ValueCheck.h>
33
34 #ifndef NDEBUG
35 // Required for CHECK_HASHTABLE_ITERATORS.
36 #include <wtf/OwnPtr.h>
37 #include <wtf/PassOwnPtr.h>
38 #endif
39
40 namespace WTF {
41
42 #define DUMP_HASHTABLE_STATS 0
43 #define DUMP_HASHTABLE_STATS_PER_TABLE 0
44
45 // Enables internal WTF consistency checks that are invoked automatically. Non-WTF callers can call checkTableConsistency() even if internal checks are disabled.
46 #define CHECK_HASHTABLE_CONSISTENCY 0
47
48 #ifdef NDEBUG
49 #define CHECK_HASHTABLE_ITERATORS 0
50 #define CHECK_HASHTABLE_USE_AFTER_DESTRUCTION 0
51 #else
52 #define CHECK_HASHTABLE_ITERATORS 1
53 #define CHECK_HASHTABLE_USE_AFTER_DESTRUCTION 1
54 #endif
55
56 #if DUMP_HASHTABLE_STATS
57
58     struct HashTableStats {
59         // The following variables are all atomically incremented when modified.
60         WTF_EXPORTDATA static int numAccesses;
61         WTF_EXPORTDATA static int numRehashes;
62         WTF_EXPORTDATA static int numRemoves;
63         WTF_EXPORTDATA static int numReinserts;
64
65         // The following variables are only modified in the recordCollisionAtCount method within a mutex.
66         WTF_EXPORTDATA static int maxCollisions;
67         WTF_EXPORTDATA static int numCollisions;
68         WTF_EXPORTDATA static int collisionGraph[4096];
69
70         WTF_EXPORT_PRIVATE static void recordCollisionAtCount(int count);
71         WTF_EXPORT_PRIVATE static void dumpStats();
72     };
73
74 #endif
75
76     template<typename Key, typename Value, typename Extractor, typename HashFunctions, typename Traits, typename KeyTraits>
77     class HashTable;
78     template<typename Key, typename Value, typename Extractor, typename HashFunctions, typename Traits, typename KeyTraits>
79     class HashTableIterator;
80     template<typename Key, typename Value, typename Extractor, typename HashFunctions, typename Traits, typename KeyTraits>
81     class HashTableConstIterator;
82
83     template<typename Key, typename Value, typename Extractor, typename HashFunctions, typename Traits, typename KeyTraits>
84     void addIterator(const HashTable<Key, Value, Extractor, HashFunctions, Traits, KeyTraits>*,
85         HashTableConstIterator<Key, Value, Extractor, HashFunctions, Traits, KeyTraits>*);
86
87     template<typename Key, typename Value, typename Extractor, typename HashFunctions, typename Traits, typename KeyTraits>
88     void removeIterator(HashTableConstIterator<Key, Value, Extractor, HashFunctions, Traits, KeyTraits>*);
89
90 #if !CHECK_HASHTABLE_ITERATORS
91
92     template<typename Key, typename Value, typename Extractor, typename HashFunctions, typename Traits, typename KeyTraits>
93     inline void addIterator(const HashTable<Key, Value, Extractor, HashFunctions, Traits, KeyTraits>*,
94         HashTableConstIterator<Key, Value, Extractor, HashFunctions, Traits, KeyTraits>*) { }
95
96     template<typename Key, typename Value, typename Extractor, typename HashFunctions, typename Traits, typename KeyTraits>
97     inline void removeIterator(HashTableConstIterator<Key, Value, Extractor, HashFunctions, Traits, KeyTraits>*) { }
98
99 #endif
100
101     typedef enum { HashItemKnownGood } HashItemKnownGoodTag;
102
103     template<typename Key, typename Value, typename Extractor, typename HashFunctions, typename Traits, typename KeyTraits>
104     class HashTableConstIterator {
105     private:
106         typedef HashTable<Key, Value, Extractor, HashFunctions, Traits, KeyTraits> HashTableType;
107         typedef HashTableIterator<Key, Value, Extractor, HashFunctions, Traits, KeyTraits> iterator;
108         typedef HashTableConstIterator<Key, Value, Extractor, HashFunctions, Traits, KeyTraits> const_iterator;
109         typedef Value ValueType;
110         typedef const ValueType& ReferenceType;
111         typedef const ValueType* PointerType;
112
113         friend class HashTable<Key, Value, Extractor, HashFunctions, Traits, KeyTraits>;
114         friend class HashTableIterator<Key, Value, Extractor, HashFunctions, Traits, KeyTraits>;
115
116         void skipEmptyBuckets()
117         {
118             while (m_position != m_endPosition && HashTableType::isEmptyOrDeletedBucket(*m_position))
119                 ++m_position;
120         }
121
122         HashTableConstIterator(const HashTableType* table, PointerType position, PointerType endPosition)
123             : m_position(position), m_endPosition(endPosition)
124         {
125             addIterator(table, this);
126             skipEmptyBuckets();
127         }
128
129         HashTableConstIterator(const HashTableType* table, PointerType position, PointerType endPosition, HashItemKnownGoodTag)
130             : m_position(position), m_endPosition(endPosition)
131         {
132             addIterator(table, this);
133         }
134
135     public:
136         HashTableConstIterator()
137         {
138             addIterator(static_cast<const HashTableType*>(0), this);
139         }
140
141         // default copy, assignment and destructor are OK if CHECK_HASHTABLE_ITERATORS is 0
142
143 #if CHECK_HASHTABLE_ITERATORS
144         ~HashTableConstIterator()
145         {
146             removeIterator(this);
147         }
148
149         HashTableConstIterator(const const_iterator& other)
150             : m_position(other.m_position), m_endPosition(other.m_endPosition)
151         {
152             addIterator(other.m_table, this);
153         }
154
155         const_iterator& operator=(const const_iterator& other)
156         {
157             m_position = other.m_position;
158             m_endPosition = other.m_endPosition;
159
160             removeIterator(this);
161             addIterator(other.m_table, this);
162
163             return *this;
164         }
165 #endif
166
167         PointerType get() const
168         {
169             checkValidity();
170             return m_position;
171         }
172         ReferenceType operator*() const { return *get(); }
173         PointerType operator->() const { return get(); }
174
175         const_iterator& operator++()
176         {
177             checkValidity();
178             ASSERT(m_position != m_endPosition);
179             ++m_position;
180             skipEmptyBuckets();
181             return *this;
182         }
183
184         // postfix ++ intentionally omitted
185
186         // Comparison.
187         bool operator==(const const_iterator& other) const
188         {
189             checkValidity(other);
190             return m_position == other.m_position;
191         }
192         bool operator!=(const const_iterator& other) const
193         {
194             checkValidity(other);
195             return m_position != other.m_position;
196         }
197         bool operator==(const iterator& other) const
198         {
199             return *this == static_cast<const_iterator>(other);
200         }
201         bool operator!=(const iterator& other) const
202         {
203             return *this != static_cast<const_iterator>(other);
204         }
205
206     private:
207         void checkValidity() const
208         {
209 #if CHECK_HASHTABLE_ITERATORS
210             ASSERT(m_table);
211 #endif
212         }
213
214
215 #if CHECK_HASHTABLE_ITERATORS
216         void checkValidity(const const_iterator& other) const
217         {
218             ASSERT(m_table);
219             ASSERT_UNUSED(other, other.m_table);
220             ASSERT(m_table == other.m_table);
221         }
222 #else
223         void checkValidity(const const_iterator&) const { }
224 #endif
225
226         PointerType m_position;
227         PointerType m_endPosition;
228
229 #if CHECK_HASHTABLE_ITERATORS
230     public:
231         // Any modifications of the m_next or m_previous of an iterator that is in a linked list of a HashTable::m_iterator,
232         // should be guarded with m_table->m_mutex.
233         mutable const HashTableType* m_table;
234         mutable const_iterator* m_next;
235         mutable const_iterator* m_previous;
236 #endif
237     };
238
239     template<typename Key, typename Value, typename Extractor, typename HashFunctions, typename Traits, typename KeyTraits>
240     class HashTableIterator {
241     private:
242         typedef HashTable<Key, Value, Extractor, HashFunctions, Traits, KeyTraits> HashTableType;
243         typedef HashTableIterator<Key, Value, Extractor, HashFunctions, Traits, KeyTraits> iterator;
244         typedef HashTableConstIterator<Key, Value, Extractor, HashFunctions, Traits, KeyTraits> const_iterator;
245         typedef Value ValueType;
246         typedef ValueType& ReferenceType;
247         typedef ValueType* PointerType;
248
249         friend class HashTable<Key, Value, Extractor, HashFunctions, Traits, KeyTraits>;
250
251         HashTableIterator(HashTableType* table, PointerType pos, PointerType end) : m_iterator(table, pos, end) { }
252         HashTableIterator(HashTableType* table, PointerType pos, PointerType end, HashItemKnownGoodTag tag) : m_iterator(table, pos, end, tag) { }
253
254     public:
255         HashTableIterator() { }
256
257         // default copy, assignment and destructor are OK
258
259         PointerType get() const { return const_cast<PointerType>(m_iterator.get()); }
260         ReferenceType operator*() const { return *get(); }
261         PointerType operator->() const { return get(); }
262
263         iterator& operator++() { ++m_iterator; return *this; }
264
265         // postfix ++ intentionally omitted
266
267         // Comparison.
268         bool operator==(const iterator& other) const { return m_iterator == other.m_iterator; }
269         bool operator!=(const iterator& other) const { return m_iterator != other.m_iterator; }
270         bool operator==(const const_iterator& other) const { return m_iterator == other; }
271         bool operator!=(const const_iterator& other) const { return m_iterator != other; }
272
273         operator const_iterator() const { return m_iterator; }
274
275     private:
276         const_iterator m_iterator;
277     };
278
279     using std::swap;
280
281     // Work around MSVC's standard library, whose swap for pairs does not swap by component.
282     template<typename T> inline void hashTableSwap(T& a, T& b)
283     {
284         swap(a, b);
285     }
286
287     // Swap pairs by component, in case of pair members that specialize swap.
288     template<typename T, typename U> inline void hashTableSwap(std::pair<T, U>& a, std::pair<T, U>& b)
289     {
290         swap(a.first, b.first);
291         swap(a.second, b.second);
292     }
293
294     template<typename T, bool useSwap> struct Mover;
295     template<typename T> struct Mover<T, true> { static void move(T& from, T& to) { hashTableSwap(from, to); } };
296     template<typename T> struct Mover<T, false> { static void move(T& from, T& to) { to = from; } };
297
298     template<typename HashFunctions> class IdentityHashTranslator {
299     public:
300         template<typename T> static unsigned hash(const T& key) { return HashFunctions::hash(key); }
301         template<typename T> static bool equal(const T& a, const T& b) { return HashFunctions::equal(a, b); }
302         template<typename T, typename U> static void translate(T& location, const U&, const T& value) { location = value; }
303     };
304
305     template<typename IteratorType> struct HashTableAddResult {
306         HashTableAddResult(IteratorType iter, bool isNewEntry) : iterator(iter), isNewEntry(isNewEntry) { }
307         IteratorType iterator;
308         bool isNewEntry;
309     };
310
311     template<typename Key, typename Value, typename Extractor, typename HashFunctions, typename Traits, typename KeyTraits>
312     class HashTable {
313     public:
314         typedef HashTableIterator<Key, Value, Extractor, HashFunctions, Traits, KeyTraits> iterator;
315         typedef HashTableConstIterator<Key, Value, Extractor, HashFunctions, Traits, KeyTraits> const_iterator;
316         typedef Traits ValueTraits;
317         typedef Key KeyType;
318         typedef Value ValueType;
319         typedef IdentityHashTranslator<HashFunctions> IdentityTranslatorType;
320         typedef HashTableAddResult<iterator> AddResult;
321
322 #if DUMP_HASHTABLE_STATS_PER_TABLE
323         struct Stats {
324             Stats()
325                 : numAccesses(0)
326                 , numRehashes(0)
327                 , numRemoves(0)
328                 , numReinserts(0)
329                 , maxCollisions(0)
330                 , numCollisions(0)
331                 , collisionGraph()
332             {
333             }
334
335             int numAccesses;
336             int numRehashes;
337             int numRemoves;
338             int numReinserts;
339
340             int maxCollisions;
341             int numCollisions;
342             int collisionGraph[4096];
343
344             void recordCollisionAtCount(int count)
345             {
346                 if (count > maxCollisions)
347                     maxCollisions = count;
348                 numCollisions++;
349                 collisionGraph[count]++;
350             }
351
352             void dumpStats()
353             {
354                 dataLog("\nWTF::HashTable::Stats dump\n\n");
355                 dataLog("%d accesses\n", numAccesses);
356                 dataLog("%d total collisions, average %.2f probes per access\n", numCollisions, 1.0 * (numAccesses + numCollisions) / numAccesses);
357                 dataLog("longest collision chain: %d\n", maxCollisions);
358                 for (int i = 1; i <= maxCollisions; i++) {
359                     dataLog("  %d lookups with exactly %d collisions (%.2f%% , %.2f%% with this many or more)\n", collisionGraph[i], i, 100.0 * (collisionGraph[i] - collisionGraph[i+1]) / numAccesses, 100.0 * collisionGraph[i] / numAccesses);
360                 }
361                 dataLog("%d rehashes\n", numRehashes);
362                 dataLog("%d reinserts\n", numReinserts);
363             }
364         };
365 #endif
366
367         HashTable();
368         ~HashTable() 
369         {
370             invalidateIterators(); 
371             if (m_table)
372                 deallocateTable(m_table, m_tableSize);
373 #if CHECK_HASHTABLE_USE_AFTER_DESTRUCTION
374             m_table = (ValueType*)(uintptr_t)0xbbadbeef;
375 #endif
376         }
377
378         HashTable(const HashTable&);
379         void swap(HashTable&);
380         HashTable& operator=(const HashTable&);
381
382         // When the hash table is empty, just return the same iterator for end as for begin.
383         // This is more efficient because we don't have to skip all the empty and deleted
384         // buckets, and iterating an empty table is a common case that's worth optimizing.
385         iterator begin() { return isEmpty() ? end() : makeIterator(m_table); }
386         iterator end() { return makeKnownGoodIterator(m_table + m_tableSize); }
387         const_iterator begin() const { return isEmpty() ? end() : makeConstIterator(m_table); }
388         const_iterator end() const { return makeKnownGoodConstIterator(m_table + m_tableSize); }
389
390         int size() const { return m_keyCount; }
391         int capacity() const { return m_tableSize; }
392         bool isEmpty() const { return !m_keyCount; }
393
394         AddResult add(const ValueType& value) { return add<IdentityTranslatorType>(Extractor::extract(value), value); }
395
396         // A special version of add() that finds the object by hashing and comparing
397         // with some other type, to avoid the cost of type conversion if the object is already
398         // in the table.
399         template<typename HashTranslator, typename T, typename Extra> AddResult add(const T& key, const Extra&);
400         template<typename HashTranslator, typename T, typename Extra> AddResult addPassingHashCode(const T& key, const Extra&);
401
402         iterator find(const KeyType& key) { return find<IdentityTranslatorType>(key); }
403         const_iterator find(const KeyType& key) const { return find<IdentityTranslatorType>(key); }
404         bool contains(const KeyType& key) const { return contains<IdentityTranslatorType>(key); }
405
406         template<typename HashTranslator, typename T> iterator find(const T&);
407         template<typename HashTranslator, typename T> const_iterator find(const T&) const;
408         template<typename HashTranslator, typename T> bool contains(const T&) const;
409
410         void remove(const KeyType&);
411         void remove(iterator);
412         void removeWithoutEntryConsistencyCheck(iterator);
413         void removeWithoutEntryConsistencyCheck(const_iterator);
414         void clear();
415
416         static bool isEmptyBucket(const ValueType& value) { return isHashTraitsEmptyValue<KeyTraits>(Extractor::extract(value)); }
417         static bool isDeletedBucket(const ValueType& value) { return KeyTraits::isDeletedValue(Extractor::extract(value)); }
418         static bool isEmptyOrDeletedBucket(const ValueType& value) { return isEmptyBucket(value) || isDeletedBucket(value); }
419
420         ValueType* lookup(const Key& key) { return lookup<IdentityTranslatorType>(key); }
421         template<typename HashTranslator, typename T> ValueType* lookup(const T&);
422
423 #if !ASSERT_DISABLED
424         void checkTableConsistency() const;
425 #else
426         static void checkTableConsistency() { }
427 #endif
428 #if CHECK_HASHTABLE_CONSISTENCY
429         void internalCheckTableConsistency() const { checkTableConsistency(); }
430         void internalCheckTableConsistencyExceptSize() const { checkTableConsistencyExceptSize(); }
431 #else
432         static void internalCheckTableConsistencyExceptSize() { }
433         static void internalCheckTableConsistency() { }
434 #endif
435
436     private:
437         static ValueType* allocateTable(int size);
438         static void deallocateTable(ValueType* table, int size);
439
440         typedef std::pair<ValueType*, bool> LookupType;
441         typedef std::pair<LookupType, unsigned> FullLookupType;
442
443         LookupType lookupForWriting(const Key& key) { return lookupForWriting<IdentityTranslatorType>(key); };
444         template<typename HashTranslator, typename T> FullLookupType fullLookupForWriting(const T&);
445         template<typename HashTranslator, typename T> LookupType lookupForWriting(const T&);
446
447         template<typename HashTranslator, typename T> void checkKey(const T&);
448
449         void removeAndInvalidateWithoutEntryConsistencyCheck(ValueType*);
450         void removeAndInvalidate(ValueType*);
451         void remove(ValueType*);
452
453         bool shouldExpand() const { return (m_keyCount + m_deletedCount) * m_maxLoad >= m_tableSize; }
454         bool mustRehashInPlace() const { return m_keyCount * m_minLoad < m_tableSize * 2; }
455         bool shouldShrink() const { return m_keyCount * m_minLoad < m_tableSize && m_tableSize > KeyTraits::minimumTableSize; }
456         void expand();
457         void shrink() { rehash(m_tableSize / 2); }
458
459         void rehash(int newTableSize);
460         void reinsert(ValueType&);
461
462         static void initializeBucket(ValueType& bucket);
463         static void deleteBucket(ValueType& bucket) { bucket.~ValueType(); Traits::constructDeletedValue(bucket); }
464
465         FullLookupType makeLookupResult(ValueType* position, bool found, unsigned hash)
466             { return FullLookupType(LookupType(position, found), hash); }
467
468         iterator makeIterator(ValueType* pos) { return iterator(this, pos, m_table + m_tableSize); }
469         const_iterator makeConstIterator(ValueType* pos) const { return const_iterator(this, pos, m_table + m_tableSize); }
470         iterator makeKnownGoodIterator(ValueType* pos) { return iterator(this, pos, m_table + m_tableSize, HashItemKnownGood); }
471         const_iterator makeKnownGoodConstIterator(ValueType* pos) const { return const_iterator(this, pos, m_table + m_tableSize, HashItemKnownGood); }
472
473 #if !ASSERT_DISABLED
474         void checkTableConsistencyExceptSize() const;
475 #else
476         static void checkTableConsistencyExceptSize() { }
477 #endif
478
479 #if CHECK_HASHTABLE_ITERATORS
480         void invalidateIterators();
481 #else
482         static void invalidateIterators() { }
483 #endif
484
485         static const int m_maxLoad = 2;
486         static const int m_minLoad = 6;
487
488         ValueType* m_table;
489         int m_tableSize;
490         int m_tableSizeMask;
491         int m_keyCount;
492         int m_deletedCount;
493
494 #if CHECK_HASHTABLE_ITERATORS
495     public:
496         // All access to m_iterators should be guarded with m_mutex.
497         mutable const_iterator* m_iterators;
498         // Use OwnPtr so HashTable can still be memmove'd or memcpy'ed.
499         mutable OwnPtr<Mutex> m_mutex;
500 #endif
501
502 #if DUMP_HASHTABLE_STATS_PER_TABLE
503     public:
504         mutable OwnPtr<Stats> m_stats;
505 #endif
506     };
507
508     template<typename Key, typename Value, typename Extractor, typename HashFunctions, typename Traits, typename KeyTraits>
509     inline HashTable<Key, Value, Extractor, HashFunctions, Traits, KeyTraits>::HashTable()
510         : m_table(0)
511         , m_tableSize(0)
512         , m_tableSizeMask(0)
513         , m_keyCount(0)
514         , m_deletedCount(0)
515 #if CHECK_HASHTABLE_ITERATORS
516         , m_iterators(0)
517         , m_mutex(adoptPtr(new Mutex))
518 #endif
519 #if DUMP_HASHTABLE_STATS_PER_TABLE
520         , m_stats(adoptPtr(new Stats))
521 #endif
522     {
523     }
524
525     inline unsigned doubleHash(unsigned key)
526     {
527         key = ~key + (key >> 23);
528         key ^= (key << 12);
529         key ^= (key >> 7);
530         key ^= (key << 2);
531         key ^= (key >> 20);
532         return key;
533     }
534
535 #if ASSERT_DISABLED
536
537     template<typename Key, typename Value, typename Extractor, typename HashFunctions, typename Traits, typename KeyTraits>
538     template<typename HashTranslator, typename T>
539     inline void HashTable<Key, Value, Extractor, HashFunctions, Traits, KeyTraits>::checkKey(const T&)
540     {
541     }
542
543 #else
544
545     template<typename Key, typename Value, typename Extractor, typename HashFunctions, typename Traits, typename KeyTraits>
546     template<typename HashTranslator, typename T>
547     void HashTable<Key, Value, Extractor, HashFunctions, Traits, KeyTraits>::checkKey(const T& key)
548     {
549         if (!HashFunctions::safeToCompareToEmptyOrDeleted)
550             return;
551         ASSERT(!HashTranslator::equal(KeyTraits::emptyValue(), key));
552         AlignedBuffer<sizeof(ValueType), WTF_ALIGN_OF(ValueType)> deletedValueBuffer;
553         ValueType* deletedValuePtr = reinterpret_cast_ptr<ValueType*>(deletedValueBuffer.buffer);
554         ValueType& deletedValue = *deletedValuePtr;
555         Traits::constructDeletedValue(deletedValue);
556         ASSERT(!HashTranslator::equal(Extractor::extract(deletedValue), key));
557     }
558
559 #endif
560
561     template<typename Key, typename Value, typename Extractor, typename HashFunctions, typename Traits, typename KeyTraits>
562     template<typename HashTranslator, typename T>
563     inline Value* HashTable<Key, Value, Extractor, HashFunctions, Traits, KeyTraits>::lookup(const T& key)
564     {
565         checkKey<HashTranslator>(key);
566
567         int k = 0;
568         int sizeMask = m_tableSizeMask;
569         ValueType* table = m_table;
570         unsigned h = HashTranslator::hash(key);
571         int i = h & sizeMask;
572
573         if (!table)
574             return 0;
575
576 #if DUMP_HASHTABLE_STATS
577         atomicIncrement(&HashTableStats::numAccesses);
578         int probeCount = 0;
579 #endif
580
581 #if DUMP_HASHTABLE_STATS_PER_TABLE
582         ++m_stats->numAccesses;
583         int perTableProbeCount = 0;
584 #endif
585
586         while (1) {
587             ValueType* entry = table + i;
588                 
589             // we count on the compiler to optimize out this branch
590             if (HashFunctions::safeToCompareToEmptyOrDeleted) {
591                 if (HashTranslator::equal(Extractor::extract(*entry), key))
592                     return entry;
593                 
594                 if (isEmptyBucket(*entry))
595                     return 0;
596             } else {
597                 if (isEmptyBucket(*entry))
598                     return 0;
599                 
600                 if (!isDeletedBucket(*entry) && HashTranslator::equal(Extractor::extract(*entry), key))
601                     return entry;
602             }
603 #if DUMP_HASHTABLE_STATS
604             ++probeCount;
605             HashTableStats::recordCollisionAtCount(probeCount);
606 #endif
607
608 #if DUMP_HASHTABLE_STATS_PER_TABLE
609             ++perTableProbeCount;
610             m_stats->recordCollisionAtCount(perTableProbeCount);
611 #endif
612
613             if (k == 0)
614                 k = 1 | doubleHash(h);
615             i = (i + k) & sizeMask;
616         }
617     }
618
619     template<typename Key, typename Value, typename Extractor, typename HashFunctions, typename Traits, typename KeyTraits>
620     template<typename HashTranslator, typename T>
621     inline typename HashTable<Key, Value, Extractor, HashFunctions, Traits, KeyTraits>::LookupType HashTable<Key, Value, Extractor, HashFunctions, Traits, KeyTraits>::lookupForWriting(const T& key)
622     {
623         ASSERT(m_table);
624         checkKey<HashTranslator>(key);
625
626         int k = 0;
627         ValueType* table = m_table;
628         int sizeMask = m_tableSizeMask;
629         unsigned h = HashTranslator::hash(key);
630         int i = h & sizeMask;
631
632 #if DUMP_HASHTABLE_STATS
633         atomicIncrement(&HashTableStats::numAccesses);
634         int probeCount = 0;
635 #endif
636
637 #if DUMP_HASHTABLE_STATS_PER_TABLE
638         ++m_stats->numAccesses;
639         int perTableProbeCount = 0;
640 #endif
641
642         ValueType* deletedEntry = 0;
643
644         while (1) {
645             ValueType* entry = table + i;
646             
647             // we count on the compiler to optimize out this branch
648             if (HashFunctions::safeToCompareToEmptyOrDeleted) {
649                 if (isEmptyBucket(*entry))
650                     return LookupType(deletedEntry ? deletedEntry : entry, false);
651                 
652                 if (HashTranslator::equal(Extractor::extract(*entry), key))
653                     return LookupType(entry, true);
654                 
655                 if (isDeletedBucket(*entry))
656                     deletedEntry = entry;
657             } else {
658                 if (isEmptyBucket(*entry))
659                     return LookupType(deletedEntry ? deletedEntry : entry, false);
660             
661                 if (isDeletedBucket(*entry))
662                     deletedEntry = entry;
663                 else if (HashTranslator::equal(Extractor::extract(*entry), key))
664                     return LookupType(entry, true);
665             }
666 #if DUMP_HASHTABLE_STATS
667             ++probeCount;
668             HashTableStats::recordCollisionAtCount(probeCount);
669 #endif
670
671 #if DUMP_HASHTABLE_STATS_PER_TABLE
672             ++perTableProbeCount;
673             m_stats->recordCollisionAtCount(perTableProbeCount);
674 #endif
675
676             if (k == 0)
677                 k = 1 | doubleHash(h);
678             i = (i + k) & sizeMask;
679         }
680     }
681
682     template<typename Key, typename Value, typename Extractor, typename HashFunctions, typename Traits, typename KeyTraits>
683     template<typename HashTranslator, typename T>
684     inline typename HashTable<Key, Value, Extractor, HashFunctions, Traits, KeyTraits>::FullLookupType HashTable<Key, Value, Extractor, HashFunctions, Traits, KeyTraits>::fullLookupForWriting(const T& key)
685     {
686         ASSERT(m_table);
687         checkKey<HashTranslator>(key);
688
689         int k = 0;
690         ValueType* table = m_table;
691         int sizeMask = m_tableSizeMask;
692         unsigned h = HashTranslator::hash(key);
693         int i = h & sizeMask;
694
695 #if DUMP_HASHTABLE_STATS
696         atomicIncrement(&HashTableStats::numAccesses);
697         int probeCount = 0;
698 #endif
699
700 #if DUMP_HASHTABLE_STATS_PER_TABLE
701         ++m_stats->numAccesses;
702         int perTableProbeCount = 0;
703 #endif
704
705         ValueType* deletedEntry = 0;
706
707         while (1) {
708             ValueType* entry = table + i;
709             
710             // we count on the compiler to optimize out this branch
711             if (HashFunctions::safeToCompareToEmptyOrDeleted) {
712                 if (isEmptyBucket(*entry))
713                     return makeLookupResult(deletedEntry ? deletedEntry : entry, false, h);
714                 
715                 if (HashTranslator::equal(Extractor::extract(*entry), key))
716                     return makeLookupResult(entry, true, h);
717                 
718                 if (isDeletedBucket(*entry))
719                     deletedEntry = entry;
720             } else {
721                 if (isEmptyBucket(*entry))
722                     return makeLookupResult(deletedEntry ? deletedEntry : entry, false, h);
723             
724                 if (isDeletedBucket(*entry))
725                     deletedEntry = entry;
726                 else if (HashTranslator::equal(Extractor::extract(*entry), key))
727                     return makeLookupResult(entry, true, h);
728             }
729 #if DUMP_HASHTABLE_STATS
730             ++probeCount;
731             HashTableStats::recordCollisionAtCount(probeCount);
732 #endif
733
734 #if DUMP_HASHTABLE_STATS_PER_TABLE
735             ++perTableProbeCount;
736             m_stats->recordCollisionAtCount(perTableProbeCount);
737 #endif
738
739             if (k == 0)
740                 k = 1 | doubleHash(h);
741             i = (i + k) & sizeMask;
742         }
743     }
744
745     template<bool emptyValueIsZero> struct HashTableBucketInitializer;
746
747     template<> struct HashTableBucketInitializer<false> {
748         template<typename Traits, typename Value> static void initialize(Value& bucket)
749         {
750             new (NotNull, &bucket) Value(Traits::emptyValue());
751         }
752     };
753
754     template<> struct HashTableBucketInitializer<true> {
755         template<typename Traits, typename Value> static void initialize(Value& bucket)
756         {
757             // This initializes the bucket without copying the empty value.
758             // That makes it possible to use this with types that don't support copying.
759             // The memset to 0 looks like a slow operation but is optimized by the compilers.
760             memset(&bucket, 0, sizeof(bucket));
761         }
762     };
763     
764     template<typename Key, typename Value, typename Extractor, typename HashFunctions, typename Traits, typename KeyTraits>
765     inline void HashTable<Key, Value, Extractor, HashFunctions, Traits, KeyTraits>::initializeBucket(ValueType& bucket)
766     {
767         HashTableBucketInitializer<Traits::emptyValueIsZero>::template initialize<Traits>(bucket);
768     }
769
770     template<typename Key, typename Value, typename Extractor, typename HashFunctions, typename Traits, typename KeyTraits>
771     template<typename HashTranslator, typename T, typename Extra>
772     inline typename HashTable<Key, Value, Extractor, HashFunctions, Traits, KeyTraits>::AddResult HashTable<Key, Value, Extractor, HashFunctions, Traits, KeyTraits>::add(const T& key, const Extra& extra)
773     {
774         checkKey<HashTranslator>(key);
775
776         invalidateIterators();
777
778         if (!m_table)
779             expand();
780
781         internalCheckTableConsistency();
782
783         ASSERT(m_table);
784
785         int k = 0;
786         ValueType* table = m_table;
787         int sizeMask = m_tableSizeMask;
788         unsigned h = HashTranslator::hash(key);
789         int i = h & sizeMask;
790
791 #if DUMP_HASHTABLE_STATS
792         atomicIncrement(&HashTableStats::numAccesses);
793         int probeCount = 0;
794 #endif
795
796 #if DUMP_HASHTABLE_STATS_PER_TABLE
797         ++m_stats->numAccesses;
798         int perTableProbeCount = 0;
799 #endif
800
801         ValueType* deletedEntry = 0;
802         ValueType* entry;
803         while (1) {
804             entry = table + i;
805             
806             // we count on the compiler to optimize out this branch
807             if (HashFunctions::safeToCompareToEmptyOrDeleted) {
808                 if (isEmptyBucket(*entry))
809                     break;
810                 
811                 if (HashTranslator::equal(Extractor::extract(*entry), key))
812                     return AddResult(makeKnownGoodIterator(entry), false);
813                 
814                 if (isDeletedBucket(*entry))
815                     deletedEntry = entry;
816             } else {
817                 if (isEmptyBucket(*entry))
818                     break;
819             
820                 if (isDeletedBucket(*entry))
821                     deletedEntry = entry;
822                 else if (HashTranslator::equal(Extractor::extract(*entry), key))
823                     return AddResult(makeKnownGoodIterator(entry), false);
824             }
825 #if DUMP_HASHTABLE_STATS
826             ++probeCount;
827             HashTableStats::recordCollisionAtCount(probeCount);
828 #endif
829
830 #if DUMP_HASHTABLE_STATS_PER_TABLE
831             ++perTableProbeCount;
832             m_stats->recordCollisionAtCount(perTableProbeCount);
833 #endif
834
835             if (k == 0)
836                 k = 1 | doubleHash(h);
837             i = (i + k) & sizeMask;
838         }
839
840         if (deletedEntry) {
841             initializeBucket(*deletedEntry);
842             entry = deletedEntry;
843             --m_deletedCount; 
844         }
845
846         HashTranslator::translate(*entry, key, extra);
847
848         ++m_keyCount;
849         
850         if (shouldExpand()) {
851             // FIXME: This makes an extra copy on expand. Probably not that bad since
852             // expand is rare, but would be better to have a version of expand that can
853             // follow a pivot entry and return the new position.
854             KeyType enteredKey = Extractor::extract(*entry);
855             expand();
856             AddResult result(find(enteredKey), true);
857             ASSERT(result.iterator != end());
858             return result;
859         }
860         
861         internalCheckTableConsistency();
862         
863         return AddResult(makeKnownGoodIterator(entry), true);
864     }
865
866     template<typename Key, typename Value, typename Extractor, typename HashFunctions, typename Traits, typename KeyTraits>
867     template<typename HashTranslator, typename T, typename Extra>
868     inline typename HashTable<Key, Value, Extractor, HashFunctions, Traits, KeyTraits>::AddResult HashTable<Key, Value, Extractor, HashFunctions, Traits, KeyTraits>::addPassingHashCode(const T& key, const Extra& extra)
869     {
870         checkKey<HashTranslator>(key);
871
872         invalidateIterators();
873
874         if (!m_table)
875             expand();
876
877         internalCheckTableConsistency();
878
879         FullLookupType lookupResult = fullLookupForWriting<HashTranslator>(key);
880
881         ValueType* entry = lookupResult.first.first;
882         bool found = lookupResult.first.second;
883         unsigned h = lookupResult.second;
884         
885         if (found)
886             return AddResult(makeKnownGoodIterator(entry), false);
887         
888         if (isDeletedBucket(*entry)) {
889             initializeBucket(*entry);
890             --m_deletedCount;
891         }
892         
893         HashTranslator::translate(*entry, key, extra, h);
894         ++m_keyCount;
895         if (shouldExpand()) {
896             // FIXME: This makes an extra copy on expand. Probably not that bad since
897             // expand is rare, but would be better to have a version of expand that can
898             // follow a pivot entry and return the new position.
899             KeyType enteredKey = Extractor::extract(*entry);
900             expand();
901             AddResult result(find(enteredKey), true);
902             ASSERT(result.iterator != end());
903             return result;
904         }
905
906         internalCheckTableConsistency();
907
908         return AddResult(makeKnownGoodIterator(entry), true);
909     }
910
911     template<typename Key, typename Value, typename Extractor, typename HashFunctions, typename Traits, typename KeyTraits>
912     inline void HashTable<Key, Value, Extractor, HashFunctions, Traits, KeyTraits>::reinsert(ValueType& entry)
913     {
914         ASSERT(m_table);
915         ASSERT(!lookupForWriting(Extractor::extract(entry)).second);
916         ASSERT(!isDeletedBucket(*(lookupForWriting(Extractor::extract(entry)).first)));
917 #if DUMP_HASHTABLE_STATS
918         atomicIncrement(&HashTableStats::numReinserts);
919 #endif
920 #if DUMP_HASHTABLE_STATS_PER_TABLE
921         ++m_stats->numReinserts;
922 #endif
923
924         Mover<ValueType, Traits::needsDestruction>::move(entry, *lookupForWriting(Extractor::extract(entry)).first);
925     }
926
927     template<typename Key, typename Value, typename Extractor, typename HashFunctions, typename Traits, typename KeyTraits>
928     template <typename HashTranslator, typename T> 
929     typename HashTable<Key, Value, Extractor, HashFunctions, Traits, KeyTraits>::iterator HashTable<Key, Value, Extractor, HashFunctions, Traits, KeyTraits>::find(const T& key)
930     {
931         if (!m_table)
932             return end();
933
934         ValueType* entry = lookup<HashTranslator>(key);
935         if (!entry)
936             return end();
937
938         return makeKnownGoodIterator(entry);
939     }
940
941     template<typename Key, typename Value, typename Extractor, typename HashFunctions, typename Traits, typename KeyTraits>
942     template <typename HashTranslator, typename T> 
943     typename HashTable<Key, Value, Extractor, HashFunctions, Traits, KeyTraits>::const_iterator HashTable<Key, Value, Extractor, HashFunctions, Traits, KeyTraits>::find(const T& key) const
944     {
945         if (!m_table)
946             return end();
947
948         ValueType* entry = const_cast<HashTable*>(this)->lookup<HashTranslator>(key);
949         if (!entry)
950             return end();
951
952         return makeKnownGoodConstIterator(entry);
953     }
954
955     template<typename Key, typename Value, typename Extractor, typename HashFunctions, typename Traits, typename KeyTraits>
956     template <typename HashTranslator, typename T> 
957     bool HashTable<Key, Value, Extractor, HashFunctions, Traits, KeyTraits>::contains(const T& key) const
958     {
959         if (!m_table)
960             return false;
961
962         return const_cast<HashTable*>(this)->lookup<HashTranslator>(key);
963     }
964
965     template<typename Key, typename Value, typename Extractor, typename HashFunctions, typename Traits, typename KeyTraits>
966     void HashTable<Key, Value, Extractor, HashFunctions, Traits, KeyTraits>::removeAndInvalidateWithoutEntryConsistencyCheck(ValueType* pos)
967     {
968         invalidateIterators();
969         remove(pos);
970     }
971
972     template<typename Key, typename Value, typename Extractor, typename HashFunctions, typename Traits, typename KeyTraits>
973     void HashTable<Key, Value, Extractor, HashFunctions, Traits, KeyTraits>::removeAndInvalidate(ValueType* pos)
974     {
975         invalidateIterators();
976         internalCheckTableConsistency();
977         remove(pos);
978     }
979
980     template<typename Key, typename Value, typename Extractor, typename HashFunctions, typename Traits, typename KeyTraits>
981     void HashTable<Key, Value, Extractor, HashFunctions, Traits, KeyTraits>::remove(ValueType* pos)
982     {
983 #if DUMP_HASHTABLE_STATS
984         atomicIncrement(&HashTableStats::numRemoves);
985 #endif
986 #if DUMP_HASHTABLE_STATS_PER_TABLE
987         ++m_stats->numRemoves;
988 #endif
989
990         deleteBucket(*pos);
991         ++m_deletedCount;
992         --m_keyCount;
993
994         if (shouldShrink())
995             shrink();
996
997         internalCheckTableConsistency();
998     }
999
1000     template<typename Key, typename Value, typename Extractor, typename HashFunctions, typename Traits, typename KeyTraits>
1001     inline void HashTable<Key, Value, Extractor, HashFunctions, Traits, KeyTraits>::remove(iterator it)
1002     {
1003         if (it == end())
1004             return;
1005
1006         removeAndInvalidate(const_cast<ValueType*>(it.m_iterator.m_position));
1007     }
1008
1009     template<typename Key, typename Value, typename Extractor, typename HashFunctions, typename Traits, typename KeyTraits>
1010     inline void HashTable<Key, Value, Extractor, HashFunctions, Traits, KeyTraits>::removeWithoutEntryConsistencyCheck(iterator it)
1011     {
1012         if (it == end())
1013             return;
1014
1015         removeAndInvalidateWithoutEntryConsistencyCheck(const_cast<ValueType*>(it.m_iterator.m_position));
1016     }
1017
1018     template<typename Key, typename Value, typename Extractor, typename HashFunctions, typename Traits, typename KeyTraits>
1019     inline void HashTable<Key, Value, Extractor, HashFunctions, Traits, KeyTraits>::removeWithoutEntryConsistencyCheck(const_iterator it)
1020     {
1021         if (it == end())
1022             return;
1023
1024         removeAndInvalidateWithoutEntryConsistencyCheck(const_cast<ValueType*>(it.m_position));
1025     }
1026
1027     template<typename Key, typename Value, typename Extractor, typename HashFunctions, typename Traits, typename KeyTraits>
1028     inline void HashTable<Key, Value, Extractor, HashFunctions, Traits, KeyTraits>::remove(const KeyType& key)
1029     {
1030         remove(find(key));
1031     }
1032
1033     template<typename Key, typename Value, typename Extractor, typename HashFunctions, typename Traits, typename KeyTraits>
1034     Value* HashTable<Key, Value, Extractor, HashFunctions, Traits, KeyTraits>::allocateTable(int size)
1035     {
1036         // would use a template member function with explicit specializations here, but
1037         // gcc doesn't appear to support that
1038         if (Traits::emptyValueIsZero)
1039             return static_cast<ValueType*>(fastZeroedMalloc(size * sizeof(ValueType)));
1040         ValueType* result = static_cast<ValueType*>(fastMalloc(size * sizeof(ValueType)));
1041         for (int i = 0; i < size; i++)
1042             initializeBucket(result[i]);
1043         return result;
1044     }
1045
1046     template<typename Key, typename Value, typename Extractor, typename HashFunctions, typename Traits, typename KeyTraits>
1047     void HashTable<Key, Value, Extractor, HashFunctions, Traits, KeyTraits>::deallocateTable(ValueType* table, int size)
1048     {
1049         if (Traits::needsDestruction) {
1050             for (int i = 0; i < size; ++i) {
1051                 if (!isDeletedBucket(table[i]))
1052                     table[i].~ValueType();
1053             }
1054         }
1055         fastFree(table);
1056     }
1057
1058     template<typename Key, typename Value, typename Extractor, typename HashFunctions, typename Traits, typename KeyTraits>
1059     void HashTable<Key, Value, Extractor, HashFunctions, Traits, KeyTraits>::expand()
1060     {
1061         int newSize;
1062         if (m_tableSize == 0)
1063             newSize = KeyTraits::minimumTableSize;
1064         else if (mustRehashInPlace())
1065             newSize = m_tableSize;
1066         else
1067             newSize = m_tableSize * 2;
1068
1069         rehash(newSize);
1070     }
1071
1072     template<typename Key, typename Value, typename Extractor, typename HashFunctions, typename Traits, typename KeyTraits>
1073     void HashTable<Key, Value, Extractor, HashFunctions, Traits, KeyTraits>::rehash(int newTableSize)
1074     {
1075         internalCheckTableConsistencyExceptSize();
1076
1077         int oldTableSize = m_tableSize;
1078         ValueType* oldTable = m_table;
1079
1080 #if DUMP_HASHTABLE_STATS
1081         if (oldTableSize != 0)
1082             atomicIncrement(&HashTableStats::numRehashes);
1083 #endif
1084
1085 #if DUMP_HASHTABLE_STATS_PER_TABLE
1086         if (oldTableSize != 0)
1087             ++m_stats->numRehashes;
1088 #endif
1089
1090         m_tableSize = newTableSize;
1091         m_tableSizeMask = newTableSize - 1;
1092         m_table = allocateTable(newTableSize);
1093
1094         for (int i = 0; i != oldTableSize; ++i)
1095             if (!isEmptyOrDeletedBucket(oldTable[i]))
1096                 reinsert(oldTable[i]);
1097
1098         m_deletedCount = 0;
1099
1100         deallocateTable(oldTable, oldTableSize);
1101
1102         internalCheckTableConsistency();
1103     }
1104
1105     template<typename Key, typename Value, typename Extractor, typename HashFunctions, typename Traits, typename KeyTraits>
1106     void HashTable<Key, Value, Extractor, HashFunctions, Traits, KeyTraits>::clear()
1107     {
1108         invalidateIterators();
1109         if (!m_table)
1110             return;
1111
1112         deallocateTable(m_table, m_tableSize);
1113         m_table = 0;
1114         m_tableSize = 0;
1115         m_tableSizeMask = 0;
1116         m_keyCount = 0;
1117     }
1118
1119     template<typename Key, typename Value, typename Extractor, typename HashFunctions, typename Traits, typename KeyTraits>
1120     HashTable<Key, Value, Extractor, HashFunctions, Traits, KeyTraits>::HashTable(const HashTable& other)
1121         : m_table(0)
1122         , m_tableSize(0)
1123         , m_tableSizeMask(0)
1124         , m_keyCount(0)
1125         , m_deletedCount(0)
1126 #if CHECK_HASHTABLE_ITERATORS
1127         , m_iterators(0)
1128         , m_mutex(adoptPtr(new Mutex))
1129 #endif
1130 #if DUMP_HASHTABLE_STATS_PER_TABLE
1131         , m_stats(adoptPtr(new Stats(*other.m_stats)))
1132 #endif
1133     {
1134         // Copy the hash table the dumb way, by adding each element to the new table.
1135         // It might be more efficient to copy the table slots, but it's not clear that efficiency is needed.
1136         const_iterator end = other.end();
1137         for (const_iterator it = other.begin(); it != end; ++it)
1138             add(*it);
1139     }
1140
1141     template<typename Key, typename Value, typename Extractor, typename HashFunctions, typename Traits, typename KeyTraits>
1142     void HashTable<Key, Value, Extractor, HashFunctions, Traits, KeyTraits>::swap(HashTable& other)
1143     {
1144         invalidateIterators();
1145         other.invalidateIterators();
1146
1147         ValueType* tmp_table = m_table;
1148         m_table = other.m_table;
1149         other.m_table = tmp_table;
1150
1151         int tmp_tableSize = m_tableSize;
1152         m_tableSize = other.m_tableSize;
1153         other.m_tableSize = tmp_tableSize;
1154
1155         int tmp_tableSizeMask = m_tableSizeMask;
1156         m_tableSizeMask = other.m_tableSizeMask;
1157         other.m_tableSizeMask = tmp_tableSizeMask;
1158
1159         int tmp_keyCount = m_keyCount;
1160         m_keyCount = other.m_keyCount;
1161         other.m_keyCount = tmp_keyCount;
1162
1163         int tmp_deletedCount = m_deletedCount;
1164         m_deletedCount = other.m_deletedCount;
1165         other.m_deletedCount = tmp_deletedCount;
1166
1167 #if DUMP_HASHTABLE_STATS_PER_TABLE
1168         m_stats.swap(other.m_stats);
1169 #endif
1170     }
1171
1172     template<typename Key, typename Value, typename Extractor, typename HashFunctions, typename Traits, typename KeyTraits>
1173     HashTable<Key, Value, Extractor, HashFunctions, Traits, KeyTraits>& HashTable<Key, Value, Extractor, HashFunctions, Traits, KeyTraits>::operator=(const HashTable& other)
1174     {
1175         HashTable tmp(other);
1176         swap(tmp);
1177         return *this;
1178     }
1179
1180 #if !ASSERT_DISABLED
1181
1182     template<typename Key, typename Value, typename Extractor, typename HashFunctions, typename Traits, typename KeyTraits>
1183     void HashTable<Key, Value, Extractor, HashFunctions, Traits, KeyTraits>::checkTableConsistency() const
1184     {
1185         checkTableConsistencyExceptSize();
1186         ASSERT(!m_table || !shouldExpand());
1187         ASSERT(!shouldShrink());
1188     }
1189
1190     template<typename Key, typename Value, typename Extractor, typename HashFunctions, typename Traits, typename KeyTraits>
1191     void HashTable<Key, Value, Extractor, HashFunctions, Traits, KeyTraits>::checkTableConsistencyExceptSize() const
1192     {
1193         if (!m_table)
1194             return;
1195
1196         int count = 0;
1197         int deletedCount = 0;
1198         for (int j = 0; j < m_tableSize; ++j) {
1199             ValueType* entry = m_table + j;
1200             if (isEmptyBucket(*entry))
1201                 continue;
1202
1203             if (isDeletedBucket(*entry)) {
1204                 ++deletedCount;
1205                 continue;
1206             }
1207
1208             const_iterator it = find(Extractor::extract(*entry));
1209             ASSERT(entry == it.m_position);
1210             ++count;
1211
1212             ValueCheck<Key>::checkConsistency(it->first);
1213         }
1214
1215         ASSERT(count == m_keyCount);
1216         ASSERT(deletedCount == m_deletedCount);
1217         ASSERT(m_tableSize >= KeyTraits::minimumTableSize);
1218         ASSERT(m_tableSizeMask);
1219         ASSERT(m_tableSize == m_tableSizeMask + 1);
1220     }
1221
1222 #endif // ASSERT_DISABLED
1223
1224 #if CHECK_HASHTABLE_ITERATORS
1225
1226     template<typename Key, typename Value, typename Extractor, typename HashFunctions, typename Traits, typename KeyTraits>
1227     void HashTable<Key, Value, Extractor, HashFunctions, Traits, KeyTraits>::invalidateIterators()
1228     {
1229         MutexLocker lock(*m_mutex);
1230         const_iterator* next;
1231         for (const_iterator* p = m_iterators; p; p = next) {
1232             next = p->m_next;
1233             p->m_table = 0;
1234             p->m_next = 0;
1235             p->m_previous = 0;
1236         }
1237         m_iterators = 0;
1238     }
1239
1240     template<typename Key, typename Value, typename Extractor, typename HashFunctions, typename Traits, typename KeyTraits>
1241     void addIterator(const HashTable<Key, Value, Extractor, HashFunctions, Traits, KeyTraits>* table,
1242         HashTableConstIterator<Key, Value, Extractor, HashFunctions, Traits, KeyTraits>* it)
1243     {
1244         it->m_table = table;
1245         it->m_previous = 0;
1246
1247         // Insert iterator at head of doubly-linked list of iterators.
1248         if (!table) {
1249             it->m_next = 0;
1250         } else {
1251             MutexLocker lock(*table->m_mutex);
1252             ASSERT(table->m_iterators != it);
1253             it->m_next = table->m_iterators;
1254             table->m_iterators = it;
1255             if (it->m_next) {
1256                 ASSERT(!it->m_next->m_previous);
1257                 it->m_next->m_previous = it;
1258             }
1259         }
1260     }
1261
1262     template<typename Key, typename Value, typename Extractor, typename HashFunctions, typename Traits, typename KeyTraits>
1263     void removeIterator(HashTableConstIterator<Key, Value, Extractor, HashFunctions, Traits, KeyTraits>* it)
1264     {
1265         typedef HashTable<Key, Value, Extractor, HashFunctions, Traits, KeyTraits> HashTableType;
1266         typedef HashTableConstIterator<Key, Value, Extractor, HashFunctions, Traits, KeyTraits> const_iterator;
1267
1268         // Delete iterator from doubly-linked list of iterators.
1269         if (!it->m_table) {
1270             ASSERT(!it->m_next);
1271             ASSERT(!it->m_previous);
1272         } else {
1273             MutexLocker lock(*it->m_table->m_mutex);
1274             if (it->m_next) {
1275                 ASSERT(it->m_next->m_previous == it);
1276                 it->m_next->m_previous = it->m_previous;
1277             }
1278             if (it->m_previous) {
1279                 ASSERT(it->m_table->m_iterators != it);
1280                 ASSERT(it->m_previous->m_next == it);
1281                 it->m_previous->m_next = it->m_next;
1282             } else {
1283                 ASSERT(it->m_table->m_iterators == it);
1284                 it->m_table->m_iterators = it->m_next;
1285             }
1286         }
1287
1288         it->m_table = 0;
1289         it->m_next = 0;
1290         it->m_previous = 0;
1291     }
1292
1293 #endif // CHECK_HASHTABLE_ITERATORS
1294
1295     // iterator adapters
1296
1297     template<typename HashTableType, typename ValueType> struct HashTableConstIteratorAdapter {
1298         HashTableConstIteratorAdapter() {}
1299         HashTableConstIteratorAdapter(const typename HashTableType::const_iterator& impl) : m_impl(impl) {}
1300
1301         const ValueType* get() const { return (const ValueType*)m_impl.get(); }
1302         const ValueType& operator*() const { return *get(); }
1303         const ValueType* operator->() const { return get(); }
1304
1305         HashTableConstIteratorAdapter& operator++() { ++m_impl; return *this; }
1306         // postfix ++ intentionally omitted
1307
1308         typename HashTableType::const_iterator m_impl;
1309     };
1310
1311     template<typename HashTableType, typename ValueType> struct HashTableIteratorAdapter {
1312         HashTableIteratorAdapter() {}
1313         HashTableIteratorAdapter(const typename HashTableType::iterator& impl) : m_impl(impl) {}
1314
1315         ValueType* get() const { return (ValueType*)m_impl.get(); }
1316         ValueType& operator*() const { return *get(); }
1317         ValueType* operator->() const { return get(); }
1318
1319         HashTableIteratorAdapter& operator++() { ++m_impl; return *this; }
1320         // postfix ++ intentionally omitted
1321
1322         operator HashTableConstIteratorAdapter<HashTableType, ValueType>() {
1323             typename HashTableType::const_iterator i = m_impl;
1324             return i;
1325         }
1326
1327         typename HashTableType::iterator m_impl;
1328     };
1329
1330     template<typename T, typename U>
1331     inline bool operator==(const HashTableConstIteratorAdapter<T, U>& a, const HashTableConstIteratorAdapter<T, U>& b)
1332     {
1333         return a.m_impl == b.m_impl;
1334     }
1335
1336     template<typename T, typename U>
1337     inline bool operator!=(const HashTableConstIteratorAdapter<T, U>& a, const HashTableConstIteratorAdapter<T, U>& b)
1338     {
1339         return a.m_impl != b.m_impl;
1340     }
1341
1342     template<typename T, typename U>
1343     inline bool operator==(const HashTableIteratorAdapter<T, U>& a, const HashTableIteratorAdapter<T, U>& b)
1344     {
1345         return a.m_impl == b.m_impl;
1346     }
1347
1348     template<typename T, typename U>
1349     inline bool operator!=(const HashTableIteratorAdapter<T, U>& a, const HashTableIteratorAdapter<T, U>& b)
1350     {
1351         return a.m_impl != b.m_impl;
1352     }
1353
1354     // All 4 combinations of ==, != and Const,non const.
1355     template<typename T, typename U>
1356     inline bool operator==(const HashTableConstIteratorAdapter<T, U>& a, const HashTableIteratorAdapter<T, U>& b)
1357     {
1358         return a.m_impl == b.m_impl;
1359     }
1360
1361     template<typename T, typename U>
1362     inline bool operator!=(const HashTableConstIteratorAdapter<T, U>& a, const HashTableIteratorAdapter<T, U>& b)
1363     {
1364         return a.m_impl != b.m_impl;
1365     }
1366
1367     template<typename T, typename U>
1368     inline bool operator==(const HashTableIteratorAdapter<T, U>& a, const HashTableConstIteratorAdapter<T, U>& b)
1369     {
1370         return a.m_impl == b.m_impl;
1371     }
1372
1373     template<typename T, typename U>
1374     inline bool operator!=(const HashTableIteratorAdapter<T, U>& a, const HashTableConstIteratorAdapter<T, U>& b)
1375     {
1376         return a.m_impl != b.m_impl;
1377     }
1378
1379 } // namespace WTF
1380
1381 #include <wtf/HashIterators.h>
1382
1383 #endif // WTF_HashTable_h