Changeset 577

Timestamp:

11/06/07 13:05:26

Author:

fumanchu

Message:

Crazycache: Test and fix for some indexing errors.

Files:

• branches/crazycache/dejavu/storage/caching.py (modified) (1 diff)
• branches/crazycache/dejavu/storage/storememcached.py (modified) (12 diffs)
• branches/crazycache/dejavu/test/zoo_fixture.py (modified) (6 diffs)

branches/crazycache/dejavu/storage/caching.py

@@ -94 +94 @@
-        # Try to retrieve units using a cached index.
-        if cls.identifiers and cls in self.cache.classes:
-
-
-
-
-
-
+
+
+
+
+
+
+
+
-        
-        # Query storage.

branches/crazycache/dejavu/storage/storememcached.py

@@ -386 +386 @@
-                unit = self.client.get(key)
-                if unit is not None:
+                    unit.cleanse()
-                    units.append(unit)
-        return units
@@ -513 +514 @@
-        keyattrs = self.primary_keys[cls]
-        
-Get a cached list of identifier-tuple
+Find the best index for the given filter
-        for index in indexset:
-            if set(filters.keys()) >= set(index):
-                indexcriteria = dict([(k, filters[k]) for k in index])
-                break
-        else:
@@ -522 +522 @@
-            return None
-        
-
+
+
-        if ids is None:
-            # Not in the cache. Grab the list of id-tuples from nextstore.
-
+
+
+
+
-            # Then cache the list result for next time. Note that index
-            # contents are unordered.
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
+
+
+
-        
-        # Now query the nextstore for any units that the cache missed...
+        misses = [k for k in ids if k not in units]
-        if self.index_stride:
-            # ...in chunks of length: self.index_stride.
@@ -580 +565 @@
-                    pass
-        
+        indexset.filter(index, filters, units)
+        
-        return units.values()
-
@@ -614 +601 @@
-        return iter(self.indices)
-    
-
-
-
-
+
+
+
+
+
+
+
-        """
-        criteria = ["%s=%s" % (k, str(v).replace(" ", "+"))
-filters.iteritems()
+criteria
-        return self._key_template % ",".join(criteria)
-    
-
-
+
+
+
+
+
+
+
-        
-        The ids returned will be a list of tuples of the form:
-            tuple([getattr(unit, name) for name in primary_keys[cls]])
-        """
-filters
+criteria
-        ids = self.store.client.get(cache_key)
-        if self.store.logflags & logflags.IO:
@@ -640 +635 @@
-        return ids
-    
-
-
+
+
+
+
+
+
+
-        
-        The ids provided MUST be a list of tuples of the form:
-            tuple([getattr(unit, name) for name in primary_keys[cls]])
-        """
-filters
+criteria
-        if self.store.logflags & logflags.IO:
-            self.store.log(logflags.IO.message("INDEX PUT (%s) len %r: %r" %
@@ -658 +658 @@
-            tuple([getattr(unit, name) for name in primary_keys[cls]])
-        
-
+
+
+
-        """
-        clsname = self.cls.__name__
@@ -671 +673 @@
-                unit = data.get(k, None)
-                if unit is not None:
+                    unit.cleanse()
-                    units[i] = unit
-        else:
@@ -694 +697 @@
-            # to perform single gets against memcached, rather than the
-            # get_multi calls that self.xrecall performs.
-indexcriteria = dict([(k, filters[k]) for k in index])
-index
+criteria = [(k, filters[k]) for k in index]
+
-            if ids:
-                for id in ids:
@@ -722 +725 @@
-        index, although it may and often should contain additional entries.
-        """
-indexcriteria = dict([(k, filters[k]) for k in index])
-index
+criteria = [(k, filters[k]) for k in index]
+
-        if ids:
-            units = self.scan(ids)
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-    
-    def add(self, unit):
@@ -751 +765 @@
-                       for name in self.store.primary_keys[self.cls]])
-        for index in self.indices:
-indexcriteria = dict([(k, getattr(unit, k)) for k in index])
-index
+criteria = [(k, getattr(unit, k)) for k in index]
+
-            if ident not in indexnode:
-                indexnode.append(ident)
-index
+
-    
-    def discard(self, unit):
@@ -762 +776 @@
-                       for name in self.store.primary_keys[self.cls]])
-        for index in self.indices:
-indexcriteria = dict([(k, getattr(unit, k)) for k in index])
-index
+criteria = [(k, getattr(unit, k)) for k in index]
+
-            if ident in indexnode:
-                indexnode.remove(ident)
-index
-
+
+

branches/crazycache/dejavu/test/zoo_fixture.py

@@ -62 +62 @@
-
-class Animal(Unit):
+    Class = UnitProperty(index=True)
-    Species = UnitProperty(hints={'bytes': 100})
-    ZooID = UnitProperty(int, index=True)
@@ -256 +257 @@
-            
-            # Animals
-
+Class='Mammalia', 
-            self.assertEqual(leopard.PreviousZoos, None)
-            box.memorize(leopard)
@@ -264 +265 @@
-            leopard.LastEscape = datetime.datetime(2004, 12, 21, 8, 15, 0, 999907)
-            
-
+Class='Mammalia', 
-                          LastEscape = datetime.datetime(2007, 9, 24,
-                                                         16, 18, 42))
-            box.memorize(lion)
-            
-
+
+
-                                # Test our 8000-byte limit
-                                PreviousZoos=["f" * (8000 - 14)]))
-            
-
+
+
-            box.memorize(tiger)
-            
-            # Override Legs.default with itself just to make sure it works.
-
+Class='Mammalia', 
-            # Notice that ostrich.PreviousZoos is [], whereas leopard is None.
-
+
+
-                                Lifespan=103.2))
-
-
-
+Class='Chilopoda', 
+
+Class='Aves', 
-            box.memorize(emp)
-
+Class='Aves', 
-                            LastEscape = datetime.datetime(2007, 9, 20,
-                                                           19, 10, 14))
@@ -292 +296 @@
-            seaworld.add(emp, adelie)
-            
-
+Class='Diplopoda', 
-            millipede.PreviousZoos = [WAP.Name]
-            box.memorize(millipede)
@@ -299 +303 @@
-            
-            # Add a mother and child to test relationships
-
+Class='Mammalia', 
-            box.memorize(bai_yun)   # ID = 11
-            self.assertEqual(bai_yun.ID, 11)
-
+
+
-            box.memorize(hua_mei)   # ID = 12
-            self.assertEqual(hua_mei.ID, 12)
@@ -539 +544 @@
-        finally:
-            box.flush_all()
+    
+    def test_4a_Indexing(self):
+        # Different subsets of an index leaf node must return correct data.
+        for legs, species in [(2, set(['Ape', 'Ape'])),
+                              (4, set(['Leopard', 'Lion', 'Tiger', 'Bear']))]:
+            animals = root.recall(Animal, dict(Class='Mammalia', Legs=legs))
+            self.assertEqual(set([a.Species for a in animals]), species)
-    
-    def test_5_Aggregates(self):