Changeset 119

Timestamp:

12/07/05 02:13:22

Author:

fumanchu

Message:

Fix for #27 (OUTER JOIN capability).

1. New UnitJoin? class, which is automatically returned from Unit class & Unit class (for INNER), and from << and >> (LEFT and RIGHT joins).
   
2. All warnings changed from UserWarning? to new dejavu.StorageWarning?.
   
3. New warning for MSAccess if MEMO is used for join.
   

Files:

• trunk/arenas.py (modified) (2 diffs)
• trunk/errors.py (modified) (2 diffs)
• trunk/storage/db.py (modified) (6 diffs)
• trunk/storage/storeado.py (modified) (4 diffs)
• trunk/storage/storemysql.py (modified) (1 diff)
• trunk/storage/storeodbc.py (modified) (3 diffs)
• trunk/storage/storeshelve.py (modified) (1 diff)
• trunk/storage/storesqlite.py (modified) (2 diffs)
• trunk/test/test_dejavu.py (modified) (9 diffs)
• trunk/test/zoo_fixture.py (modified) (5 diffs)
• trunk/units.py (modified) (5 diffs)

trunk/arenas.py

@@ -301 +301 @@
-                       LOGRECALL)
-        
-
-
-
+
+
+
+
-                raise ValueError(u"multirecall() does not support multiple"
-                                 u" classes in disparate stores.")
@@ -312 +313 @@
-        # in read-only scripts, it should be OK for now. But if you mutate
-        # Units and then call multirecall, expect inconsistent results.
-
+first
-            confirmed = True
-            for index in xrange(len(unitset)):

trunk/errors.py

@@ -1 +1 @@
-
-
+
+
-
-class DejavuError(Exception):
@@ -19 +20 @@
-    pass
-
+class StorageWarning(UserWarning):
+    """Warning about functionality which is not supported by all SM's."""
+    pass

trunk/storage/db.py

@@ -153 +153 @@
-                          "using %s. Values may be stored incorrectly."
-                          % (precision, self.numeric_max_precision,
-
+
+
-            precision = self.numeric_max_precision
-        # Assume most people use decimal for money; default scale = 2.
@@ -185 +186 @@
-                          "using %s. Values may be stored incorrectly."
-                          % (bytes, self.numeric_max_precision,
-
+
+
-            bytes = self.numeric_max_precision
-        return "NUMERIC(%s, 0)" % bytes
@@ -761 +763 @@
-        idlen = self.identifier_length
-        if idlen and len(ident) > idlen:
-
+
+
-            ident = ident[:idlen]
-        return '"' + ident + '"'
@@ -1107 +1110 @@
-                          "It may take an absurd amount of time to run, "
-                          "since each unit must be fully-formed. %s"
-
+
+
-            for unit in self.recall(cls, expr):
-                # Use tuples for hashability
@@ -1136 +1140 @@
-                          "It may take an absurd amount of time to run, "
-                          "since each unit must be fully-formed. %s"
-
+
+
-            vals = {}
-            for unit in self.recall(cls, expr):
@@ -1154 +1159 @@
-                     for row in data]
-    
-multiselect(self, classes, expr
+join(self, unitjoin
-        t = self.tablename
-        i = self.identifier
-        
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-        if expr is None:
-            expr = logic.Expression(lambda *args: True)
-        w, imp = self.where(tablenames, expr)
-        
-
-
+
+
+
+
+
+
+
-        columns = []
-        joins = []
-        basecls = firstcls = classes[0]
-        for cls in classes:
-            # Place the identifier properties first
-it
+them
-            idnames = [prop.key for prop in cls.identifiers]
+            
-            keys = idnames + [k for k in cls.properties() if k not in idnames]
-            columns.extend([(cls, k) for k in keys])
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
+
+
-        statement = ("SELECT %s FROM %s WHERE %s" %
-u', '.join(tablenames)
+joins
-        return statement, imp, columns
-    

trunk/storage/storeado.py

@@ -456 +456 @@
-                          "not allow comparisons on string fields larger "
-                          "than 8000 characters. Some of your data may be "
-
+, dejavu.StorageWarning
-            bytes = 8000
-        # 8000 *bytes* is the absolute upper limit, based on T_SQL docs for
@@ -531 +531 @@
-                          "using %s. Values may be stored incorrectly."
-                          % (precision, self.numeric_max_precision,
-
+
+
-            precision = self.numeric_max_precision
-        if scale > 4:
@@ -537 +538 @@
-                          "using %s. Values may be stored incorrectly."
-                          % (scale, cls.__name__, key,
-
+
+
-        
-        # MS Access doesn't let us control precision and scale directly.
@@ -599 +601 @@
-            # MEMO is 1 GB max when set programatically (only 64K when set
-            # in Access UI). But then, 1 GB is the limit for the whole DB.
+            for assoc in cls._associations.itervalues():
+                if assoc.nearKey == key:
+                    warnings.warn("Memo fields cannot be used as join keys. "
+                                  "You should set %s.%s(hints={'bytes': 255})"
+                                  % (cls.__name__, key),
+                                  dejavu.StorageWarning)
-            return u"MEMO"
-

trunk/storage/storemysql.py

@@ -158 +158 @@
-        idlen = self.identifier_length
-        if idlen and len(ident) > idlen:
-
+
+
-            ident = ident[:idlen]
-        return '`' + ident + '`'

trunk/storage/storeodbc.py

@@ -19 +19 @@
-        warnings.warn("The precision of %s.%s cannot be determined for "
-                      "ODBC stores. Values may be stored incorrectly."
-
+, dejavu.StorageWarning
-        return u"NUMERIC"
-    
@@ -25 +25 @@
-        warnings.warn("The precision of %s.%s cannot be determined for "
-                      "ODBC stores. Values may be stored incorrectly."
-
+, dejavu.StorageWarning
-        return u"NUMERIC"
-    
@@ -39 +39 @@
-        warnings.warn("The precision of %s.%s cannot be determined for "
-                      "ODBC stores. Values may be stored incorrectly."
-
+, dejavu.StorageWarning
-        return u"NUMERIC"
-

trunk/storage/storeshelve.py

@@ -168 +168 @@
-    def multirecall(self, classes, expr):
-        """multirecall(classes, expr) -> Full inner join units."""
-
-firstcls = classes[0]
-tables = {}
-joins = dict([(cls, None) for cls in classes])
+if expr is None:
+    expr = logic.Expression(lambda *args: True)
+
+firstcls = list(classes)[0]
-        # TODO: deconstruct expr into a set of subexpr's, one for
-        # each class in classes.
-        filters = dict([(cls, None) for cls in classes])
-        
-
-
-
-
-
-
-
-
+
+
+
+
+
+
-            else:
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-            if expr(*unitrow):
-                yield unitrow

trunk/storage/storesqlite.py

@@ -15 +15 @@
-                       "wildcard literals." % _version)
-    import warnings
-
+, dejavu.StorageWarning
-
-
@@ -204 +204 @@
-            self.execute(u'CREATE INDEX %s ON %s (%s);' %
-                         (i, tablename, self.identifier(index)))
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+

trunk/test/test_dejavu.py

@@ -1 +1 @@
-import datetime
-import unittest
+import warnings
-
-import dejavu
-from dejavu import storage
- import zoo_fixture
+.zoo_fixture import *
-
-
+
+
-
-
@@ -13 +15 @@
-    def setUp(self):
-        # CleanUP The Database!
-zoo_fixture.
-zoo_fixture.
+
+
-            animal.forget()
-zoo_fixture.
+
-            zoo_thing.forget()
-    
@@ -22 +24 @@
-        # Instance creation and population
-        f = datetime.date(1916, 10, 2)
-zoo_fixture.
+
-        self.assertEqual(z.dirty(), True)
-zoo_fixture.
+
-        self.assertEqual(z.ID, None)
-        self.assertEqual(z.Name, 'San Diego Zoo')
@@ -31 +33 @@
-        self.assertEqual(z.__class__.ID.index, True)
-        
-zoo_fixture.
+
-        self.assertEqual(a.dirty(), True)
-        self.assertEqual(a.ID, None)
@@ -39 +41 @@
-        
-        # Sandboxing
-zoo_fixture.
+
-        s.memorize(z)
-        self.assertEqual(z.ID, 1)
@@ -59 +61 @@
-        
-        # Create some animals in a sandbox
-zoo_fixture.
-zoo_fixture.
-zoo_fixture.
+
+
+
-        
-        animals = []
@@ -67 +69 @@
-        
-        # Start a new sandbox with no cache
-zoo_fixture.
+
-        
-        # get animals alternating from two different xrecalls
-        animals_is_stopped = False
-        animals2_is_stopped = False
-zoo_fixture.
-zoo_fixture.
+
+
-        while not (animals_is_stopped and animals2_is_stopped):
-            try:
@@ -99 +101 @@
-        
-        # Create an animal in a sandbox, but retain a reference to it
-zoo_fixture.
-zoo_fixture.
+
+
-        box.memorize(bat)
-        
@@ -107 +109 @@
-        
-        # Retrieve the Unit from the same sandbox again.
-zoo_fixture.
+
-        
-        # Retrieve the Unit from a new sandbox.
-        # Units should be different, and their
-        # UnitProperties should be different.
-zoo_fixture.arena.new_sandbox().unit(zoo_fixture.
+arena.new_sandbox().unit(
-        self.assert_(bat3 is not bat)
-        self.assertEqual(bat3.Legs, 4)
+    
+    def test_UnitJoin(self):
+        box = arena.new_sandbox()
+        tree = Animal & Zoo
+        self.assertEqual(str(tree), "(Animal & Zoo)")
+        tree = Animal << Zoo
+        self.assertEqual(str(tree), "(Animal << Zoo)")
+        tree = Animal >> Zoo
+        self.assertEqual(str(tree), "(Animal >> Zoo)")
+        
+        trees = []
+        def make_tree():
+            trees.append( (Animal & Zoo) >> Exhibit )
+        
+        warnings.filterwarnings("error", category=dejavu.StorageWarning)
+        try:
+            self.assertRaises(dejavu.StorageWarning, make_tree)
+        finally:
+            warnings.filters.pop(0)
+        
+        # Since we raised the warning, our first make_tree failed.
+        warnings.filterwarnings("ignore", category=dejavu.StorageWarning)
+        try:
+            make_tree()
+        finally:
+            warnings.filters.pop(0)
+        
+        self.assertEqual(str(trees[0]), "((Animal & Zoo) >> Exhibit)")
+        tree = trees[0] & (Visit << Vet) & Exhibit
+        self.assertEqual(str(tree), "((((Animal & Zoo) >> Exhibit) & "
+                                    "(Visit << Vet)) & Exhibit)")
+        self.assertEqual(list(tree), [Animal, Zoo, Exhibit, Visit, Vet, Exhibit])
-
-

trunk/test/zoo_fixture.py

@@ -3 +3 @@
-import datetime
-import os
+try:
+    import pythoncom
+except ImportError:
+    pythoncom = None
+
+try:
+    set
+except NameError:
+    from sets import Set as set
+
-import threading
-import unittest
+import warnings
-
-try:
@@ -431 +442 @@
-        f = logic.Expression(lambda z, a: z.Name == 'San Diego Zoo')
-        zooed_animals = [(z, a) for z, a in
-[Zoo, Animal]
+Zoo & Animal
-        self.assertEqual(len(zooed_animals), 2)
-        
@@ -448 +459 @@
-        leo = logic.Expression(lambda z, a: a.Species == 'Leopard')
-        zooed_animals = [(z, a) for z, a in
-[Zoo, Animal]
+Zoo & Animal
-        self.assertEqual(len(zooed_animals), 0)
+        
+        # Now try the same expr with INNER, LEFT, and RIGHT JOINs.
+        zooed_animals = list(box.multirecall(Zoo & Animal))
+        self.assertEqual(len(zooed_animals), 6)
+        self.assertEqual(set([(z.Name, a.Species) for z, a in zooed_animals]),
+                         set([("Wild Animal Park", "Leopard"),
+                              ("Wild Animal Park", "Lion"),
+                              ("San Diego Zoo", "Tiger"),
+                              ("San Diego Zoo", "Millipede"),
+                              ("Sea_World", "Emperor Penguin"),
+                              ("Sea_World", "Adelie Penguin")]))
+        
+        zooed_animals = list(box.multirecall(Zoo >> Animal))
+        self.assertEqual(len(zooed_animals), 12)
+        self.assertEqual(set([(z.Name, a.Species) for z, a in zooed_animals]),
+                         set([("Wild Animal Park", "Leopard"),
+                              ("Wild Animal Park", "Lion"),
+                              ("San Diego Zoo", "Tiger"),
+                              ("San Diego Zoo", "Millipede"),
+                              ("Sea_World", "Emperor Penguin"),
+                              ("Sea_World", "Adelie Penguin"),
+                              (None, "Slug"),
+                              (None, "Bear"),
+                              (None, "Ostrich"),
+                              (None, "Centipede"),
+                              (None, "Ape"),
+                              (None, "Ape"),
+                              ]))
+        
+        zooed_animals = list(box.multirecall(Zoo << Animal))
+        self.assertEqual(len(zooed_animals), 7)
+        self.assertEqual(set([(z.Name, a.Species) for z, a in zooed_animals]),
+                         set([("Wild Animal Park", "Leopard"),
+                              ("Wild Animal Park", "Lion"),
+                              ("San Diego Zoo", "Tiger"),
+                              ("San Diego Zoo", "Millipede"),
+                              ("Sea_World", "Emperor Penguin"),
+                              ("Sea_World", "Adelie Penguin"),
+                              (u'Montr\xe9al Biod\xf4me', None),
+                              ]))
-        
-        # Try a multiple-arg expression
-        f = logic.Expression(lambda a, z: a.Legs >= 4 and z.Admission < 10)
-[Animal, Zoo]
+Animal & Zoo
-        self.assertEqual(len(animal_zoos), 4)
-        names = [a.Species for a, z in animal_zoos]
@@ -460 +511 @@
-        
-        # Let's try three joined classes just for the sadistic fun of it.
+        tree = (Animal >> Zoo) >> Vet
-        f = logic.Expression(lambda a, z, v: z.Name == 'Sea_World')
-
-
+
-        self.assertEqual(len(azv), 2)
+        
+        # MSAccess can't handle an INNER JOIN nested in an OUTER JOIN.
+        # Test that this fails for MSAccess, but works for other SM's.
+        trees = []
+        def make_tree():
+            trees.append( (Animal & Zoo) >> Vet )
+        warnings.filterwarnings("ignore", category=dejavu.StorageWarning)
+        try:
+            make_tree()
+        finally:
+            warnings.filters.pop(0)
+        
+        azv = []
+        def set_azv():
+            f = logic.Expression(lambda a, z, v: z.Name == 'Sea_World')
+            azv.append([(a, z, v) for a, z, v in
+                        box.multirecall(trees[0], f)])
+        
+        smname = arena.stores['testSM'].__class__.__name__
+        if smname in ("StorageManagerADO_MSAccess",):
+            self.assertRaises(pythoncom.com_error, set_azv)
+        else:
+            set_azv()
+            self.assertEqual(len(azv[0]), 2)
-    
-    def test_Editing(self):
@@ -589 +664 @@
-        self.assertEqual(len(snaps[1]), 2)
-        self.assertEqual(eng.last_snapshot(), snaps[1])
-
+##
-##    def test_Transactions(self):
-##        box = arena.new_sandbox()

trunk/units.py

@@ -1 @@
-import sha
-import types
-
-try:
-    import cPickle as pickle
@@ -7 +4 @@
-    import pickle
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
-           'Unit', 'UnitProperty', 'MetaUnit',
-           'UnitSequencerInteger', 'UnitSequencerNull',
@@ -109 +110 @@
-            f += expr
-        return unit.sandbox.recall(self.farClass, f)
+
+
+class UnitJoin(object):
+    
+    def __init__(self, class1, class2, leftbiased=None):
+        self.class1 = class1
+        self.class2 = class2
+        self.leftbiased = leftbiased
+        
+        # From http://msdn.microsoft.com/library/en-us/
+        #           dnacc2k/html/acintsql.asp#acintsql_joins
+        # "OUTER JOINs can be nested inside INNER JOINs in a multi-table
+        # join, but INNER JOINs cannot be nested inside OUTER JOINs."
+        if leftbiased is not None:
+            if ((isinstance(class1, UnitJoin) and class1.leftbiased is None)
+                or (isinstance(class1, UnitJoin) and class1.leftbiased is None)):
+                warnings.warn("Some StorageManagers cannot nest an INNER "
+                              "JOIN within an OUTER JOIN. Consider rewriting "
+                              "your join tree.", errors.StorageWarning)
+    
+    def __str__(self):
+        if self.leftbiased is None:
+            op = "&"
+        elif self.leftbiased is True:
+            op = "<<"
+        else:
+            op = ">>"
+        if isinstance(self.class1, UnitJoin):
+            name1 = str(self.class1)
+        else:
+            name1 = self.class1.__name__
+        if isinstance(self.class2, UnitJoin):
+            name2 = str(self.class2)
+        else:
+            name2 = self.class2.__name__
+        return "(%s %s %s)" % (name1, op, name2)
+    __repr__ = __str__
+    
+    def __iter__(self):
+        def genclasses():
+            if isinstance(self.class1, MetaUnit):
+                yield self.class1
+            else:
+                for cls in iter(self.class1):
+                    yield cls
+            if isinstance(self.class2, MetaUnit):
+                yield self.class2
+            else:
+                for cls in iter(self.class2):
+                    yield cls
+        return genclasses()
+    
+    def __lshift__(self, other):
+        if isinstance(other, (MetaUnit, UnitJoin)):
+            return UnitJoin(self, other, leftbiased=True)
+        else:
+            raise TypeError("Joined classes must be UnitJoin or Unit subclasses.")
+    __rrshift__ = __lshift__
+    
+    def __rshift__(self, other):
+        if isinstance(other, (MetaUnit, UnitJoin)):
+            return UnitJoin(self, other, leftbiased=False)
+        else:
+            raise TypeError("Joined classes must be UnitJoin or Unit subclasses.")
+    __rlshift__ = __rshift__
+    
+    def __add__(self, other):
+        if isinstance(other, (MetaUnit, UnitJoin)):
+            return UnitJoin(self, other)
+        else:
+            raise TypeError("Joined classes must be UnitJoin or Unit subclasses.")
+    __and__ = __add__
+    
+    def __radd__(self, other):
+        if isinstance(other, (MetaUnit, UnitJoin)):
+            return UnitJoin(other, self)
+        else:
+            raise TypeError("Joined classes must be UnitJoin or Unit subclasses.")
+    __rand__ = __radd__
-
-
@@ -310 +390 @@
-        cls._properties = props
-        cls._associations = assocs
+    
+    def __lshift__(self, other):
+        if isinstance(other, (MetaUnit, UnitJoin)):
+            return UnitJoin(self, other, leftbiased=True)
+        else:
+            raise TypeError("Joined classes must be UnitJoin or Unit subclasses.")
+    __rrshift__ = __lshift__
+    
+    def __rshift__(self, other):
+        if isinstance(other, (MetaUnit, UnitJoin)):
+            return UnitJoin(self, other, leftbiased=False)
+        else:
+            raise TypeError("Joined classes must be UnitJoin or Unit subclasses.")
+    __rlshift__ = __rshift__
+    
+    def __add__(self, other):
+        if isinstance(other, (MetaUnit, UnitJoin)):
+            return UnitJoin(self, other)
+        else:
+            raise TypeError("Joined classes must be UnitJoin or Unit subclasses.")
+    __and__ = __add__
+    
+    def __radd__(self, other):
+        if isinstance(other, (MetaUnit, UnitJoin)):
+            return UnitJoin(other, self)
+        else:
+            raise TypeError("Joined classes must be UnitJoin or Unit subclasses.")
+    __rand__ = __radd__
-
-
@@ -519 +627 @@
-                # If far key is already set, it will simply be overwritten.
-                setattr(unit, ua.farKey, nearval)
-