Refactor the code cache to handle large number of properties on the global object.

A separate object type for the code cache have been added. This object has two different code caches. The first one (default_cache) is a fixed array organized in the same way as the as the code cache was before. The second cache (global_access_cache) is for code stubs to access the global object. This cache is organized as a hash table taking the property name and code flags as the key.

The reason for separating the global access stubs into a hash table representation is that the number of these is not bounded in the same was as the other types.

BUG=613
Review URL: http://codereview.chromium.org/652119

git-svn-id: http://v8.googlecode.com/svn/branches/bleeding_edge@3952 ce2b1a6d-e550-0410-aec6-3dcde31c8c00

src/heap.cc

@@ -1206,7 +1206,7 @@ Object* Heap::AllocateMap(InstanceType instance_type, int instance_size) {
   map->set_inobject_properties(0);
   map->set_pre_allocated_property_fields(0);
   map->set_instance_descriptors(empty_descriptor_array());
-  map->set_code_cache(empty_fixed_array());
+  map->set_code_cache(undefined_value());
   map->set_unused_property_fields(0);
   map->set_bit_field(0);
   map->set_bit_field2(1 << Map::kIsExtensible);
@@ -1221,6 +1221,16 @@ Object* Heap::AllocateMap(InstanceType instance_type, int instance_size) {
 }
 
 
+Object* Heap::AllocateCodeCache() {
+  Object* result = AllocateStruct(CODE_CACHE_TYPE);
+  if (result->IsFailure()) return result;
+  CodeCache* code_cache = CodeCache::cast(result);
+  code_cache->set_default_cache(empty_fixed_array());
+  code_cache->set_normal_type_cache(undefined_value());
+  return code_cache;
+}
+
+
 const Heap::StringTypeTable Heap::string_type_table[] = {
 #define STRING_TYPE_ELEMENT(type, size, name, camel_name)                      \
   {type, size, k##camel_name##MapRootIndex},
@@ -1277,13 +1287,13 @@ bool Heap::CreateInitialMaps() {
 
   // Fix the instance_descriptors for the existing maps.
   meta_map()->set_instance_descriptors(empty_descriptor_array());
-  meta_map()->set_code_cache(empty_fixed_array());
+  meta_map()->set_code_cache(undefined_value());
 
   fixed_array_map()->set_instance_descriptors(empty_descriptor_array());
-  fixed_array_map()->set_code_cache(empty_fixed_array());
+  fixed_array_map()->set_code_cache(undefined_value());
 
   oddball_map()->set_instance_descriptors(empty_descriptor_array());
-  oddball_map()->set_code_cache(empty_fixed_array());
+  oddball_map()->set_code_cache(undefined_value());
 
   // Fix prototype object for existing maps.
   meta_map()->set_prototype(null_value());

src/heap.h

@@ -345,6 +345,9 @@ class Heap : public AllStatic {
   // Allocate a map for the specified function
   static Object* AllocateInitialMap(JSFunction* fun);
 
+  // Allocates an empty code cache.
+  static Object* AllocateCodeCache();
+
   // Allocates and fully initializes a String.  There are two String
   // encodings: ASCII and two byte. One should choose between the three string
   // allocation functions based on the encoding of the string buffer used to

src/objects-debug.cc

@@ -644,6 +644,24 @@ void Map::MapVerify() {
 }
 
 
+void CodeCache::CodeCachePrint() {
+  HeapObject::PrintHeader("CodeCache");
+  PrintF("\n - default_cache: ");
+  default_cache()->ShortPrint();
+  PrintF("\n - normal_type_cache: ");
+  normal_type_cache()->ShortPrint();
+}
+
+
+void CodeCache::CodeCacheVerify() {
+  VerifyHeapPointer(default_cache());
+  VerifyHeapPointer(normal_type_cache());
+  ASSERT(default_cache()->IsFixedArray());
+  ASSERT(normal_type_cache()->IsUndefined()
+         || normal_type_cache()->IsCodeCacheHashTable());
+}
+
+
 void FixedArray::FixedArrayPrint() {
   HeapObject::PrintHeader("FixedArray");
   PrintF(" - length: %d", length());

src/objects-inl.h

@@ -564,6 +564,11 @@ bool Object::IsCompilationCacheTable() {
 }
 
 
+bool Object::IsCodeCacheHashTable() {
+  return IsHashTable();
+}
+
+
 bool Object::IsMapCache() {
   return IsHashTable();
 }
@@ -1563,6 +1568,7 @@ CAST_ACCESSOR(FixedArray)
 CAST_ACCESSOR(DescriptorArray)
 CAST_ACCESSOR(SymbolTable)
 CAST_ACCESSOR(CompilationCacheTable)
+CAST_ACCESSOR(CodeCacheHashTable)
 CAST_ACCESSOR(MapCache)
 CAST_ACCESSOR(String)
 CAST_ACCESSOR(SeqString)
@@ -2255,7 +2261,7 @@ void Map::set_prototype(Object* value, WriteBarrierMode mode) {
 
 ACCESSORS(Map, instance_descriptors, DescriptorArray,
           kInstanceDescriptorsOffset)
-ACCESSORS(Map, code_cache, FixedArray, kCodeCacheOffset)
+ACCESSORS(Map, code_cache, Object, kCodeCacheOffset)
 ACCESSORS(Map, constructor, Object, kConstructorOffset)
 
 ACCESSORS(JSFunction, shared, SharedFunctionInfo, kSharedFunctionInfoOffset)
@@ -2393,6 +2399,9 @@ INT_ACCESSORS(SharedFunctionInfo, this_property_assignments_count,
               kThisPropertyAssignmentsCountOffset)
 
 
+ACCESSORS(CodeCache, default_cache, FixedArray, kDefaultCacheOffset)
+ACCESSORS(CodeCache, normal_type_cache, Object, kNormalTypeCacheOffset)
+
 bool Script::HasValidSource() {
   Object* src = this->source();
   if (!src->IsString()) return true;
@@ -2993,8 +3002,8 @@ void Map::ClearCodeCache() {
   // No write barrier is needed since empty_fixed_array is not in new space.
   // Please note this function is used during marking:
   //  - MarkCompactCollector::MarkUnmarkedObject
-  ASSERT(!Heap::InNewSpace(Heap::raw_unchecked_empty_fixed_array()));
-  WRITE_FIELD(this, kCodeCacheOffset, Heap::raw_unchecked_empty_fixed_array());
+  ASSERT(!Heap::InNewSpace(Heap::undefined_value()));
+  WRITE_FIELD(this, kCodeCacheOffset, Heap::undefined_value());
 }
 
 

src/objects.cc

@@ -3033,19 +3033,79 @@ Object* Map::CopyDropTransitions() {
 
 
 Object* Map::UpdateCodeCache(String* name, Code* code) {
+  // Allocate the code cache if not present.
+  if (code_cache()->IsUndefined()) {
+    Object* result = Heap::AllocateCodeCache();
+    if (result->IsFailure()) return result;
+    set_code_cache(result);
+  }
+
+  // Update the code cache.
+  return CodeCache::cast(code_cache())->Update(name, code);
+}
+
+
+Object* Map::FindInCodeCache(String* name, Code::Flags flags) {
+  // Do a lookup if a code cache exists.
+  if (!code_cache()->IsUndefined()) {
+    return CodeCache::cast(code_cache())->Lookup(name, flags);
+  } else {
+    return Heap::undefined_value();
+  }
+}
+
+
+int Map::IndexInCodeCache(Code* code) {
+  // Get the internal index if a code cache exists.
+  if (!code_cache()->IsUndefined()) {
+    return CodeCache::cast(code_cache())->GetIndex(code);
+  }
+  return -1;
+}
+
+
+void Map::RemoveFromCodeCache(int index) {
+  // No GC is supposed to happen between a call to IndexInCodeCache and
+  // RemoveFromCodeCache so the code cache must be there.
+  ASSERT(!code_cache()->IsUndefined());
+  return CodeCache::cast(code_cache())->RemoveByIndex(index);
+}
+
+
+Object* CodeCache::Update(String* name, Code* code) {
   ASSERT(code->ic_state() == MONOMORPHIC);
-  FixedArray* cache = code_cache();
 
-  // When updating the code cache we disregard the type encoded in the
+  // The number of monomorphic stubs for normal load/store/call IC's can grow to
+  // a large number and therefore they need to go into a hash table. They are
+  // used to load global properties from cells.
+  if (code->type() == NORMAL) {
+    // Make sure that a hash table is allocated for the normal load code cache.
+    if (normal_type_cache()->IsUndefined()) {
+      Object* result =
+          CodeCacheHashTable::Allocate(CodeCacheHashTable::kInitialSize);
+      if (result->IsFailure()) return result;
+      set_normal_type_cache(result);
+    }
+    return UpdateNormalTypeCache(name, code);
+  } else {
+    ASSERT(default_cache()->IsFixedArray());
+    return UpdateDefaultCache(name, code);
+  }
+}
+
+
+Object* CodeCache::UpdateDefaultCache(String* name, Code* code) {
+  // When updating the default code cache we disregard the type encoded in the
   // flags. This allows call constant stubs to overwrite call field
   // stubs, etc.
   Code::Flags flags = Code::RemoveTypeFromFlags(code->flags());
 
   // First check whether we can update existing code cache without
   // extending it.
+  FixedArray* cache = default_cache();
   int length = cache->length();
   int deleted_index = -1;
-  for (int i = 0; i < length; i += 2) {
+  for (int i = 0; i < length; i += kCodeCacheEntrySize) {
     Object* key = cache->get(i);
     if (key->IsNull()) {
       if (deleted_index < 0) deleted_index = i;
@@ -3053,14 +3113,15 @@ Object* Map::UpdateCodeCache(String* name, Code* code) {
     }
     if (key->IsUndefined()) {
       if (deleted_index >= 0) i = deleted_index;
-      cache->set(i + 0, name);
-      cache->set(i + 1, code);
+      cache->set(i + kCodeCacheEntryNameOffset, name);
+      cache->set(i + kCodeCacheEntryCodeOffset, code);
       return this;
     }
     if (name->Equals(String::cast(key))) {
-      Code::Flags found = Code::cast(cache->get(i + 1))->flags();
+      Code::Flags found =
+          Code::cast(cache->get(i + kCodeCacheEntryCodeOffset))->flags();
       if (Code::RemoveTypeFromFlags(found) == flags) {
-        cache->set(i + 1, code);
+        cache->set(i + kCodeCacheEntryCodeOffset, code);
         return this;
       }
     }
@@ -3069,61 +3130,180 @@ Object* Map::UpdateCodeCache(String* name, Code* code) {
   // Reached the end of the code cache.  If there were deleted
   // elements, reuse the space for the first of them.
   if (deleted_index >= 0) {
-    cache->set(deleted_index + 0, name);
-    cache->set(deleted_index + 1, code);
+    cache->set(deleted_index + kCodeCacheEntryNameOffset, name);
+    cache->set(deleted_index + kCodeCacheEntryCodeOffset, code);
     return this;
   }
 
-  // Extend the code cache with some new entries (at least one).
-  int new_length = length + ((length >> 1) & ~1) + 2;
-  ASSERT((new_length & 1) == 0);  // must be a multiple of two
+  // Extend the code cache with some new entries (at least one). Must be a
+  // multiple of the entry size.
+  int new_length = length + ((length >> 1)) + kCodeCacheEntrySize;
+  new_length = new_length - new_length % kCodeCacheEntrySize;
+  ASSERT((new_length % kCodeCacheEntrySize) == 0);
   Object* result = cache->CopySize(new_length);
   if (result->IsFailure()) return result;
 
   // Add the (name, code) pair to the new cache.
   cache = FixedArray::cast(result);
-  cache->set(length + 0, name);
-  cache->set(length + 1, code);
-  set_code_cache(cache);
+  cache->set(length + kCodeCacheEntryNameOffset, name);
+  cache->set(length + kCodeCacheEntryCodeOffset, code);
+  set_default_cache(cache);
   return this;
 }
 
 
-Object* Map::FindInCodeCache(String* name, Code::Flags flags) {
-  FixedArray* cache = code_cache();
+Object* CodeCache::UpdateNormalTypeCache(String* name, Code* code) {
+  // Adding a new entry can cause a new cache to be allocated.
+  CodeCacheHashTable* cache = CodeCacheHashTable::cast(normal_type_cache());
+  Object* new_cache = cache->Put(name, code);
+  if (new_cache->IsFailure()) return new_cache;
+  set_normal_type_cache(new_cache);
+  return this;
+}
+
+
+Object* CodeCache::Lookup(String* name, Code::Flags flags) {
+  if (Code::ExtractTypeFromFlags(flags) == NORMAL) {
+    return LookupNormalTypeCache(name, flags);
+  } else {
+    return LookupDefaultCache(name, flags);
+  }
+}
+
+
+Object* CodeCache::LookupDefaultCache(String* name, Code::Flags flags) {
+  FixedArray* cache = default_cache();
   int length = cache->length();
-  for (int i = 0; i < length; i += 2) {
-    Object* key = cache->get(i);
+  for (int i = 0; i < length; i += kCodeCacheEntrySize) {
+    Object* key = cache->get(i + kCodeCacheEntryNameOffset);
     // Skip deleted elements.
     if (key->IsNull()) continue;
     if (key->IsUndefined()) return key;
     if (name->Equals(String::cast(key))) {
-      Code* code = Code::cast(cache->get(i + 1));
-      if (code->flags() == flags) return code;
+      Code* code = Code::cast(cache->get(i + kCodeCacheEntryCodeOffset));
+      if (code->flags() == flags) {
+        return code;
+      }
     }
   }
   return Heap::undefined_value();
 }
 
 
-int Map::IndexInCodeCache(Code* code) {
-  FixedArray* array = code_cache();
+Object* CodeCache::LookupNormalTypeCache(String* name, Code::Flags flags) {
+  if (!normal_type_cache()->IsUndefined()) {
+    CodeCacheHashTable* cache = CodeCacheHashTable::cast(normal_type_cache());
+    return cache->Lookup(name, flags);
+  } else {
+    return Heap::undefined_value();
+  }
+}
+
+
+int CodeCache::GetIndex(Code* code) {
+  // This is not used for normal load/store/call IC's.
+  ASSERT(code->type() != NORMAL);
+
+  FixedArray* array = default_cache();
   int len = array->length();
-  for (int i = 0; i < len; i += 2) {
-    if (array->get(i + 1) == code) return i + 1;
+  for (int i = 0; i < len; i += kCodeCacheEntrySize) {
+    if (array->get(i + kCodeCacheEntryCodeOffset) == code) return i + 1;
   }
   return -1;
 }
 
 
-void Map::RemoveFromCodeCache(int index) {
-  FixedArray* array = code_cache();
+void CodeCache::RemoveByIndex(int index) {
+  FixedArray* array = default_cache();
   ASSERT(array->length() >= index && array->get(index)->IsCode());
   // Use null instead of undefined for deleted elements to distinguish
   // deleted elements from unused elements.  This distinction is used
   // when looking up in the cache and when updating the cache.
-  array->set_null(index - 1);  // key
-  array->set_null(index);  // code
+  ASSERT_EQ(1, kCodeCacheEntryCodeOffset - kCodeCacheEntryNameOffset);
+  array->set_null(index - 1);  // Name.
+  array->set_null(index);  // Code.
+}
+
+
+// The key in the code cache hash table consists of the property name and the
+// code object. The actual match is on the name and the code flags. If a key
+// is created using the flags and not a code object it can only be used for
+// lookup not to create a new entry.
+class CodeCacheHashTableKey : public HashTableKey {
+ public:
+  CodeCacheHashTableKey(String* name, Code::Flags flags)
+      : name_(name), flags_(flags), code_(NULL) { }
+
+  CodeCacheHashTableKey(String* name, Code* code)
+      : name_(name),
+        flags_(code->flags()),
+        code_(code) { }
+
+
+  bool IsMatch(Object* other) {
+    if (!other->IsFixedArray()) return false;
+    FixedArray* pair = FixedArray::cast(other);
+    String* name = String::cast(pair->get(0));
+    Code::Flags flags = Code::cast(pair->get(1))->flags();
+    if (flags != flags_) {
+      return false;
+    }
+    return name_->Equals(name);
+  }
+
+  static uint32_t NameFlagsHashHelper(String* name, Code::Flags flags) {
+    return name->Hash() ^ flags;
+  }
+
+  uint32_t Hash() { return NameFlagsHashHelper(name_, flags_); }
+
+  uint32_t HashForObject(Object* obj) {
+    FixedArray* pair = FixedArray::cast(obj);
+    String* name = String::cast(pair->get(0));
+    Code* code = Code::cast(pair->get(1));
+    return NameFlagsHashHelper(name, code->flags());
+  }
+
+  Object* AsObject() {
+    ASSERT(code_ != NULL);
+    Object* obj = Heap::AllocateFixedArray(2);
+    if (obj->IsFailure()) return obj;
+    FixedArray* pair = FixedArray::cast(obj);
+    pair->set(0, name_);
+    pair->set(1, code_);
+    return pair;
+  }
+
+ private:
+  String* name_;
+  Code::Flags flags_;
+  Code* code_;
+};
+
+
+Object* CodeCacheHashTable::Lookup(String* name, Code::Flags flags) {
+  CodeCacheHashTableKey key(name, flags);
+  int entry = FindEntry(&key);
+  if (entry == kNotFound) return Heap::undefined_value();
+  return get(EntryToIndex(entry) + 1);
+}
+
+
+Object* CodeCacheHashTable::Put(String* name, Code* code) {
+  CodeCacheHashTableKey key(name, code);
+  Object* obj = EnsureCapacity(1, &key);
+
+  // Don't use this, as the table might have grown.
+  CodeCacheHashTable* cache = reinterpret_cast<CodeCacheHashTable*>(obj);
+
+  int entry = cache->FindInsertionEntry(key.Hash());
+  Object* k = key.AsObject();
+  if (k->IsFailure()) return k;
+
+  cache->set(EntryToIndex(entry), k);
+  cache->set(EntryToIndex(entry) + 1, code);
+  cache->ElementAdded();
+  return cache;
 }
 
 
@@ -6981,7 +7161,6 @@ Object* HashTable<Shape, Key>::EnsureCapacity(int n, Key key) {
 }
 
 
-
 template<typename Shape, typename Key>
 uint32_t HashTable<Shape, Key>::FindInsertionEntry(uint32_t hash) {
   uint32_t capacity = Capacity();

src/objects.h

@@ -72,6 +72,7 @@
 //             - Dictionary
 //             - SymbolTable
 //             - CompilationCacheTable
+//             - CodeCacheHashTable
 //             - MapCache
 //           - Context
 //           - GlobalContext
@@ -102,6 +103,7 @@
 //         - TypeSwitchInfo
 //         - DebugInfo
 //         - BreakPointInfo
+//         - CodeCache
 //
 // Formats of Object*:
 //  Smi:        [31 bit signed int] 0
@@ -269,6 +271,7 @@ enum PropertyNormalizationMode {
   V(SIGNATURE_INFO_TYPE)                                                       \
   V(TYPE_SWITCH_INFO_TYPE)                                                     \
   V(SCRIPT_TYPE)                                                               \
+  V(CODE_CACHE_TYPE)                                                           \
                                                                                \
   V(JS_VALUE_TYPE)                                                             \
   V(JS_OBJECT_TYPE)                                                            \
@@ -364,7 +367,8 @@ enum PropertyNormalizationMode {
   V(OBJECT_TEMPLATE_INFO, ObjectTemplateInfo, object_template_info)            \
   V(SIGNATURE_INFO, SignatureInfo, signature_info)                             \
   V(TYPE_SWITCH_INFO, TypeSwitchInfo, type_switch_info)                        \
-  V(SCRIPT, Script, script)
+  V(SCRIPT, Script, script)                                                    \
+  V(CODE_CACHE, CodeCache, code_cache)
 
 #ifdef ENABLE_DEBUGGER_SUPPORT
 #define STRUCT_LIST_DEBUGGER(V)                                                \
@@ -468,6 +472,7 @@ enum InstanceType {
   SIGNATURE_INFO_TYPE,
   TYPE_SWITCH_INFO_TYPE,
   SCRIPT_TYPE,
+  CODE_CACHE_TYPE,
 #ifdef ENABLE_DEBUGGER_SUPPORT
   DEBUG_INFO_TYPE,
   BREAK_POINT_INFO_TYPE,
@@ -601,6 +606,7 @@ class Object BASE_EMBEDDED {
   inline bool IsDictionary();
   inline bool IsSymbolTable();
   inline bool IsCompilationCacheTable();
+  inline bool IsCodeCacheHashTable();
   inline bool IsMapCache();
   inline bool IsPrimitive();
   inline bool IsGlobalObject();
@@ -2922,7 +2928,7 @@ class Map: public HeapObject {
   DECL_ACCESSORS(instance_descriptors, DescriptorArray)
 
   // [stub cache]: contains stubs compiled for this map.
-  DECL_ACCESSORS(code_cache, FixedArray)
+  DECL_ACCESSORS(code_cache, Object)
 
   Object* CopyDropDescriptors();
 
@@ -3026,6 +3032,11 @@ class Map: public HeapObject {
   static const int kNeedsLoading = 0;
   static const int kIsExtensible = 1;
 
+  // Layout of the default cache. It holds alternating name and code objects.
+  static const int kCodeCacheEntrySize = 2;
+  static const int kCodeCacheEntryNameOffset = 0;
+  static const int kCodeCacheEntryCodeOffset = 1;
+
  private:
   DISALLOW_IMPLICIT_CONSTRUCTORS(Map);
 };
@@ -3711,6 +3722,93 @@ class CompilationCacheTable: public HashTable<CompilationCacheShape,
 };
 
 
+class CodeCache: public Struct {
+ public:
+  DECL_ACCESSORS(default_cache, FixedArray)
+  DECL_ACCESSORS(normal_type_cache, Object)
+
+  // Add the code object to the cache.
+  Object* Update(String* name, Code* code);
+
+  // Lookup code object in the cache. Returns code object if found.
+  Object* Lookup(String* name, Code::Flags flags);
+
+  // Get the internal index of a code object in the cache. Returns -1 if the
+  // code object is not in that cache. This index can be used to later call
+  // RemoveByIndex. The cache cannot be modified between a call to GetIndex and
+  // RemoveByIndex.
+  int GetIndex(Code* code);
+
+  // Remove an object from the cache with the provided internal index.
+  void RemoveByIndex(int index);
+
+  static inline CodeCache* cast(Object* obj);
+
+#ifdef DEBUG
+  void CodeCachePrint();
+  void CodeCacheVerify();
+#endif
+
+  static const int kDefaultCacheOffset = HeapObject::kHeaderSize;
+  static const int kNormalTypeCacheOffset =
+      kDefaultCacheOffset + kPointerSize;
+  static const int kSize = kNormalTypeCacheOffset + kPointerSize;
+
+ private:
+  Object* UpdateDefaultCache(String* name, Code* code);
+  Object* UpdateNormalTypeCache(String* name, Code* code);
+  Object* LookupDefaultCache(String* name, Code::Flags flags);
+  Object* LookupNormalTypeCache(String* name, Code::Flags flags);
+
+  // Code cache layout of the default cache. Elements are alternating name and
+  // code objects for non normal load/store/call IC's.
+  static const int kCodeCacheEntrySize = 2;
+  static const int kCodeCacheEntryNameOffset = 0;
+  static const int kCodeCacheEntryCodeOffset = 1;
+
+  DISALLOW_IMPLICIT_CONSTRUCTORS(CodeCache);
+};
+
+
+class CodeCacheHashTableShape {
+ public:
+  static inline bool IsMatch(HashTableKey* key, Object* value) {
+    return key->IsMatch(value);
+  }
+
+  static inline uint32_t Hash(HashTableKey* key) {
+    return key->Hash();
+  }
+
+  static inline uint32_t HashForObject(HashTableKey* key, Object* object) {
+    return key->HashForObject(object);
+  }
+
+  static Object* AsObject(HashTableKey* key) {
+    return key->AsObject();
+  }
+
+  static const int kPrefixSize = 0;
+  static const int kEntrySize = 2;
+};
+
+
+class CodeCacheHashTable: public HashTable<CodeCacheHashTableShape,
+                                           HashTableKey*> {
+ public:
+  Object* Lookup(String* name, Code::Flags flags);
+  Object* Put(String* name, Code* code);
+
+  static inline CodeCacheHashTable* cast(Object* obj);
+
+  // Initial size of the fixed array backing the hash table.
+  static const int kInitialSize = 64;
+
+ private:
+  DISALLOW_IMPLICIT_CONSTRUCTORS(CodeCacheHashTable);
+};
+
+
 enum AllowNullsFlag {ALLOW_NULLS, DISALLOW_NULLS};
 enum RobustnessFlag {ROBUST_STRING_TRAVERSAL, FAST_STRING_TRAVERSAL};