Contact: fumanchu@aminus.org

Log in as guest/dejavu to create tickets

root/trunk/dejavu/units.py

Revision 563 (checked in by fumanchu, 1 year ago)

Optimization to ToOne?.related: call unit() instead of xrecall() (if no expr) in order to hit key-value stores.

  • Property svn:eol-style set to native
Line 
1 try:
2     from decimal import Decimal as decimal
3 except ImportError:
4     decimal = None
5
6 try:
7     import cPickle as pickle
8 except ImportError:
9     import pickle
10
11 import sha
12 import types
13 import warnings
14
15 from dejavu import errors, logic
16
17
18 __all__ = ['UnitAssociation', 'ToMany', 'ToOne', 'UnitJoin',
19            'Unit', 'UnitProperty', 'TriggerProperty', 'MetaUnit',
20            'UnitSequencerInteger', 'UnitSequencer',
21            'UnitSequencerUnicode',
22 ##           '_define_fixedpoint_states', '_fix_fixedpoint_cmp',
23            ]
24
25 def _fix_fixedpoint_cmp():
26     """Add methods to fixedpoint to support pickling."""
27     import fixedpoint
28     def __cmp__(self, other):
29         if other is None:
30             return 1
31         xn, yn, p = fixedpoint._norm(self, other, FixedPoint=type(self))
32         return cmp(xn, yn)
33     fixedpoint.FixedPoint.__cmp__ = __cmp__
34
35 def _define_fixedpoint_states():
36     """Add methods to fixedpoint to support pickling."""
37     import fixedpoint
38    
39     if not hasattr(fixedpoint.FixedPoint, "__getstate__"):
40         def __getstate__(self):
41             return (self.n, self.p)
42         fixedpoint.FixedPoint.__getstate__ = __getstate__
43        
44         def __setstate__(self, v):
45             self.n, self.p = v
46         fixedpoint.FixedPoint.__setstate__ = __setstate__
47
48
49 ###########################################################################
50 ##                                                                       ##
51 ##                             Associations                              ##
52 ##                                                                       ##
53 ###########################################################################
54
55
56 class UnitAssociation(object):
57     """Non-data descriptor method to retrieve related Units via attributes."""
58    
59     to_many = None
60    
61     def __init__(self, nearKey, farClass, farKey):
62         # Since the keys will be used as kwarg keys, they must be strings.
63         self.nearKey = str(nearKey)
64         self.farKey = str(farKey)
65        
66         self.nearClass = None
67         self.farClass = farClass
68    
69     def __get__(self, unit, unitclass=None):
70         if unit is None:
71             # When calling on the class instead of an instance...
72             return self
73         else:
74             m = types.MethodType(self.related, unit, unitclass)
75             return m
76    
77     def __delete__(self, unit):
78         raise AttributeError("Unit Associations may not be deleted.")
79    
80     def related(self, unit, expr=None, order=None, limit=None, offset=None):
81         """Return unit(s) on the far side of this relation."""
82         raise NotImplementedError
83
84
85 class ToOne(UnitAssociation):
86    
87     # If True, a single near value maps to multiple far values; False otherwise.
88     to_many = False
89    
90     def related(self, unit, expr=None, order=None, limit=None, offset=None):
91         """Return the single unit on the far side of this relation."""
92         value = getattr(unit, self.nearKey)
93         if value is None:
94             return None
95        
96         if expr is None:
97             # Optimize with a unit() call so key-value stores can be hit.
98             return unit.sandbox.unit(self.farClass, **{self.farKey: value})
99         else:
100             expr = logic.combine(expr, {self.farKey: value})
101             units = unit.sandbox.xrecall(self.farClass, expr, order=order,
102                                          limit=1, offset=offset)
103             try:
104                 return units.next()
105             except StopIteration:
106                 return None
107
108
109 class ToMany(UnitAssociation):
110    
111     # If True, a single near value maps to multiple far values; False otherwise.
112     to_many = True
113    
114     def related(self, unit, expr=None, order=None, limit=None, offset=None):
115         """Return all units on the far side of this relation."""
116         value = getattr(unit, self.nearKey)
117         if value is None:
118             return []
119        
120         expr = logic.combine(expr, {self.farKey: value})
121         return unit.sandbox.recall(self.farClass, expr, order=order,
122                                    limit=limit, offset=offset)
123
124
125 class UnitJoin(object):
126     """A join between two Unit classes."""
127    
128     def __init__(self, class1, class2, leftbiased=None):
129         self.class1 = class1
130         self.class2 = class2
131         self.leftbiased = leftbiased
132         self.path = None
133        
134         # From http://msdn.microsoft.com/library/en-us/
135         #           dnacc2k/html/acintsql.asp#acintsql_joins
136         # "OUTER JOINs can be nested inside INNER JOINs in a multi-table
137         # join, but INNER JOINs cannot be nested inside OUTER JOINs."
138         if leftbiased is not None:
139             if ((isinstance(class1, UnitJoin) and class1.leftbiased is None)
140                 or (isinstance(class2, UnitJoin) and class2.leftbiased is None)):
141                 warnings.warn("Some StorageManagers cannot nest an INNER "
142                               "JOIN within an OUTER JOIN. Consider rewriting "
143                               "your join tree.", errors.StorageWarning)
144    
145     def __str__(self):
146         if self.leftbiased is None:
147             op = "&"
148         elif self.leftbiased is True:
149             op = "<<"
150         else:
151             op = ">>"
152         if isinstance(self.class1, UnitJoin):
153             name1 = str(self.class1)
154         elif isinstance(self.class1, type):
155             name1 = self.class1.__name__
156         else:
157             name1 = repr(self.class1)
158        
159         if isinstance(self.class2, UnitJoin):
160             name2 = str(self.class2)
161         elif isinstance(self.class2, type):
162             name2 = self.class2.__name__
163         else:
164             name2 = repr(self.class2)
165        
166         return "(%s %s %s)" % (name1, op, name2)
167     __repr__ = __str__
168    
169     def __iter__(self):
170         def genclasses():
171             if isinstance(self.class1, UnitJoin):
172                 for cls in iter(self.class1):
173                     yield cls
174             else:
175                 yield self.class1
176             if isinstance(self.class2, UnitJoin):
177                 for cls in iter(self.class2):
178                     yield cls
179             else:
180                 yield self.class2
181         return genclasses()
182    
183     def __lshift__(self, other):
184         if isinstance(other, (MetaUnit, UnitJoin)):
185             return UnitJoin(self, other, leftbiased=True)
186         else:
187             raise TypeError("Joined classes must be UnitJoin or Unit subclasses.")
188     __rrshift__ = __lshift__
189    
190     def __rshift__(self, other):
191         if isinstance(other, (MetaUnit, UnitJoin)):
192             return UnitJoin(self, other, leftbiased=False)
193         else:
194             raise TypeError("Joined classes must be UnitJoin or Unit subclasses.")
195     __rlshift__ = __rshift__
196    
197     def __add__(self, other):
198         if isinstance(other, (MetaUnit, UnitJoin)):
199             return UnitJoin(self, other)
200         else:
201             raise TypeError("Joined classes must be UnitJoin or Unit subclasses.")
202     __and__ = __add__
203    
204     def __radd__(self, other):
205         if isinstance(other, (MetaUnit, UnitJoin)):
206             return UnitJoin(other, self)
207         else:
208             raise TypeError("Joined classes must be UnitJoin or Unit subclasses.")
209     __rand__ = __radd__
210    
211     def __eq__(self, other):
212         return (self.class1 == other.class1 and
213                 self.class2 == other.class2 and
214                 self.leftbiased == other.leftbiased and
215                 self.path == other.path)
216
217
218 ###########################################################################
219 ##                                                                       ##
220 ##                            Unit Sequencers                            ##
221 ##                                                                       ##
222 ###########################################################################
223
224
225 # All Units must possess at least one UnitProperty which is an identifier.
226 # The sequencing of identifiers depends upon their type and the particular
227 # needs of the class. Pick one of these UnitSequencers to fit your subclass.
228 # When creating new sequencers, you should aim to generate identifiers that
229 # obey the builtin max() and min() functions.
230
231 class UnitSequencer(object):
232     """A base class for Unit identifier Sequencers. Sequencing will error.
233     
234     In many cases, identifier values simply have no algorithmic sequence;
235     for example, a set of Employee Units might use Social Security
236     Numbers for identifiers (which you should never, ever do ;).
237     
238     In other cases, sequencing will be best handled by custom algorithms
239     within application code; that is, the job of abstracting the sequence
240     logic would not be worth the effort.
241     """
242    
243     def __init__(self, type=unicode):
244         self.type = type
245    
246     def valid_id(self, identity):
247         """If the given identity tuple is syntactically valid, return True.
248         
249         Note that this method makes no other assertions about the given
250         identity; in particular, it does not check for duplicated values.
251         """
252         for val in identity:
253             if val is None:
254                 return False
255         return True
256    
257     def assign(self, unit, sequence):
258         """Set a valid identifier on the given unit.
259         
260         The given sequence may be used to determine the 'next' valid id.
261         If provided, it should be the entire set of existing identifiers.
262         """
263         raise StopIteration("No sequence defined.")
264
265
266 class UnitSequencerInteger(UnitSequencer):
267     """A sequencer for Unit identifiers, where id[i+1] == id[i] + 1."""
268    
269     def __init__(self, type=int, initial=1):
270         self.type = type
271         self.initial = initial
272    
273     def valid_id(self, identity):
274         """If the given identity tuple is syntactically valid, return True."""
275         return identity != (None,)
276    
277     def assign(self, unit, sequence):
278         """Set a valid identifier on the given unit.
279         
280         The given sequence may be used to determine the 'next' valid id.
281         If provided, it should be the entire set of existing identifiers.
282         """
283         newvalue = self.initial
284         if sequence:
285             m = max(sequence)
286             if m != (None,):
287                 newvalue = m[0] + 1
288         setattr(unit, unit.identifiers[0], newvalue)
289
290
291 class UnitSequencerUnicode(UnitSequencer):
292     """A sequencer for Unit identifiers, where next('abc') == 'abd'."""
293    
294     def __init__(self, type=unicode, width=6,
295                  range="abcdefghijklmnopqrstuvwxyz"):
296         self.type = type
297         self.width = width
298         self.range = range
299    
300     def valid_id(self, identity):
301         """If the given identity tuple is syntactically valid, return True."""
302         return identity != (None,)
303    
304     def assign(self, unit, sequence):
305         """Set a valid identifier on the given unit.
306         
307         The given sequence may be used to determine the 'next' valid id.
308         If provided, it should be the entire set of existing identifiers.
309         """
310         r = self.range
311         newvalue = r[0] * self.width
312         if sequence:
313             maxid = max(sequence)[0]
314             if len(maxid) != self.width:
315                 raise OverflowError("'%s' is not of width %s." %
316                                     (maxid, self.width))
317             for i in range(self.width - 1, -1, -1):
318                 pos = r.index(maxid[i]) + 1
319                 if pos >= len(r) or pos < 0:
320                     maxid = maxid[:i] + r[0] + maxid[i+1:]
321                 else:
322                     maxid = maxid[:i] + r[pos] + maxid[i+1:]
323                     break
324             else:
325                 raise OverflowError("Next identifier exceeds width %s."
326                                     % self.width)
327             newvalue = maxid
328         setattr(unit, unit.identifiers[0], newvalue)
329
330
331 ###########################################################################
332 ##                                                                       ##
333 ##                                 Units                                 ##
334 ##                                                                       ##
335 ###########################################################################
336
337
338 class UnitProperty(object):
339     """UnitProperty(type=unicode, index=False, hints={}, key=None, default=None)
340     Data descriptor for Unit data which will persist in storage.
341     
342     hints: A dictionary which provides named hints to Storage Managers
343         concerning the nature of the data. A common use, for example,
344         is to inform Managers that would usually store unicode strings
345         as strings of length 255, that a particular value should be
346         a larger object; this is done with a 'bytes' mapping, such as:
347         hints = {u'bytes': 0}, where 0 implies no limit. Canonical storage
348         hint names and implementation details may be found in /storage
349         documentation.
350     """
351    
352     def __init__(self, type=unicode, index=False, hints=None, key=None, default=None):
353         if type.__name__ == 'FixedPoint':
354             # fixedpoint can't handle "FixedPoint() != None" in Python 2.4
355             _fix_fixedpoint_cmp()
356            
357             # fixedpoint.Fixedpoint can't be pickled because it
358             # defines __slots__ but not __getstate__. Provide it.
359             _define_fixedpoint_states()
360        
361         self.type = type
362         self.index = index
363         if hints is None: hints = {}
364         self.hints = hints
365         self.key = key
366         self.default = default
367    
368     def _get_default(self):
369         return self._default
370     def _set_default(self, value):
371         if self.coerce:
372             value = self.coerce(None, value)
373         self._default = value
374     default = property(_get_default, _set_default,
375                        doc="""Default value of this property for new units.""")
376    
377     def __get__(self, unit, unitclass=None):
378         if unit is None:
379             # When calling on the class instead of an instance...
380             return self
381         else:
382             return unit._properties[self.key]
383    
384     def __set__(self, unit, value):
385         if self.coerce:
386             value = self.coerce(unit, value)
387         oldvalue = unit._properties[self.key]
388         if oldvalue != value:
389             unit._properties[self.key] = value
390    
391     def coerce(self, unit, value):
392         """Coerce the given value to the proper type for this property.
393         
394         In the base class, the 'unit' arg is not used. When overriding
395         this class, you should allow for meaningful results even if
396         the supplied 'unit' arg is None.
397         """
398         if value is not None:
399             selftype = self.type
400            
401             if not isinstance(value, selftype):
402                 # Try to cast the value to self.type.
403                 try:
404                     value = selftype(value)
405                 except Exception, x:
406                     msg = ("%r is type %r (expected %r)" %
407                            (value, type(value), selftype))
408                     x.args += (msg,)
409                     raise
410            
411             # The final indignity ;)
412             if decimal and (selftype is decimal):
413                 scale = self.hints.get('scale', None)
414                 if scale:
415                     value = value.quantize(decimal("." + ("0" * scale)))
416         return value
417    
418     def __delete__(self, unit):
419         raise AttributeError("Unit Properties may not be deleted.")
420    
421     def __str__(self):
422         cls = self.__class__
423         return ("%s.%s(type=%s, index=%s, hints=%r, key=%r, default=%r)"
424                 % (cls.__module__, cls.__name__, self.type.__name__,
425                    self.index, self.hints, self.key, self.default))
426     __repr__ = __str__
427
428
429 class TriggerProperty(UnitProperty):
430     """UnitProperty subclass for managing immediate triggers on set.
431     
432     The __set__ method will call the on_set method, which should then
433     deal with the new value.
434     """
435    
436     def __set__(self, unit, value):
437         if self.coerce:
438             value = self.coerce(unit, value)
439         oldvalue = unit._properties[self.key]
440         if oldvalue != value:
441             unit._properties[self.key] = value
442             if unit.sandbox:
443                 self.on_set(unit, oldvalue)
444    
445     def on_set(self, unit, oldvalue):
446         """Overridable hook for when this property is __set__."""
447         pass
448
449
450 class MetaUnit(type):
451    
452     def __init__(cls, name, bases, dct):
453         # Make a copy of the parent class' _associations, and store
454         # it in the _associations attribute of this subclass. In this
455         # manner, Unit Associations should propagate down to subclasses,
456         # but not back up to superclasses.
457         if hasattr(cls, "_associations"):
458             assocs = cls._associations.copy()
459         else:
460             assocs = {}
461        
462         # Make a copy of the parent class' properties, and store
463         # it in the properties attribute of this subclass. In this
464         # manner, Unit Property keys should propagate down to subclasses,
465         # but not back up to superclasses.
466         if hasattr(cls, "properties"):
467             props = list(cls.properties)
468         else:
469             props = []
470        
471         for name, val in dct.iteritems():
472             # Now grab any new UnitProperties defined in this class.
473             # Overwrite any properties defined in superclasses.
474             if isinstance(val, UnitProperty):
475                 # If the UnitProperty.key is None,
476                 # supply it from the attribute name (name).
477                 if val.key is None:
478                     val.key = name
479                 if name not in props:
480                     props.append(name)
481            
482             # Remove any properties from the parent class if requested
483             # (request by binding the parent's UnitProperty.key to None).
484             if val is None and name in props:
485                 props.remove(name)
486            
487             # Now grab any new UnitAssociations defined in this class.
488             if isinstance(val, UnitAssociation):
489                 val.nearClass = cls
490                 assocs[name] = val
491        
492         cls.properties = props
493         cls._associations = assocs
494        
495         # Keep backward compatibility from 1.4 to 1.5. See ticket #48.
496         ident = dct.get('identifiers', ())
497         if ident:
498             newident = []
499             for val in ident:
500                 if isinstance(val, UnitProperty):
501                 &nbs