root/trunk/__init__.py

import ConfigParser
import containers
import logic
import xray

from dejavu.analysis import *
from dejavu.readme import *


###########################################################################
##                                                                       ##
##                                 Units                                 ##
##                                                                       ##
###########################################################################


# All Units currently must possess an 'ID' UnitProperty. The sequencing of
# IDs depends upon their type and the particular needs of the class. Pick
# one of these UnitSequencers to fit your subclass.

# At the moment, no ID sequences are allowed None as a value, since this
# signals a Unit which needs to be sequenced when memorized. In addition,
# you should aim to create new sequencers which generate IDs that obey
# the builtin max() and min() functions.

class UnitSequencerNull(object):
    """A null sequencer for Unit IDs. Sequencing will error.
    
    In many cases, ID values simply have no algorithmic sequence;
    for example, a set of Employee Units might use Social Security
    Numbers for IDs (which you should never, ever do ;).
    
    In other cases, sequencing will be best handled by custom algorithms
    within application code; that is, the job of abstracting the sequence
    logic would not be worth the effort.
    """
    
    def __init__(self, type=unicode):
        self.type = type
    
    def next(self, sequence):
        raise UnableToFulfillRequestError("No sequence defined.")


class UnitSequencerInteger(object):
    """A sequencer for Unit IDs, where id[i+1] == id[i] + 1."""
    
    def __init__(self, type=int, initial=1):
        self.type = type
        self.initial = initial
    
    def next(self, sequence):
        if sequence:
            m = max(sequence)
            if m is not None:
                return m + 1
        return self.initial


class UnitSequencerUnicode(object):
    """A sequencer for Unit IDs, where e.g. next(['abc']) == 'abd'."""
    
    def __init__(self, type=unicode, width=6,
                 range="abcdefghijklmnopqrstuvwxyz"):
        self.type = type
        self.width = width
        self.range = range
    
    def next(self, sequence):
        r = self.range
        if sequence:
            maxid = max(sequence)
            if len(maxid) != self.width:
                raise ValueError("'%s' is not of width %s." %
                                 (maxid, self.width))
            for i in range(self.width - 1, -1, -1):
                pos = r.index(maxid[i]) + 1
                if pos >= len(r) or pos < 0:
                    maxid = maxid[:i] + r[0] + maxid[i+1:]
                else:
                    maxid = maxid[:i] + r[pos] + maxid[i+1:]
                    break
            else:
                raise OverflowError("Next ID exceeds width %s." % self.width)
            return maxid
        return r[0] * self.width


class UnitProperty(object):
    """Dejavu.Unit data which will persist in storage.
    
    pre, post: Override these with functions to provide custom behaviors
        upon attribute modification. They are not called if sandbox is None.
        If you need a behavior which fires regardless, you should override
        __set__. They also are not called if the new value matches the
        existing value. Again, override __set__ if you need an exception.
    
    hints: A dictionary which provides named hints to Storage Managers
        concerning the nature of the data. A common use, for example,
        is to inform Managers that would usually store unicode strings
        as strings of length 255, that a particular value should be
        a larger object; this is done with a 'Size' mapping, such as:
        hints = {u'Size': 0}, where 0 implies no limit. Canonical storage
        hint names and implementation details may be found in storage.py
        documentation.
    """
    
    pre = None
    post = None
    
    def __init__(self, cls, key, type, index=False, hints={}):
        self.key = key
        self.type = type
        self.index = index
        self.hints = hints
        cls._properties[key] = None
        cls._property_types[key] = type
    
    def __get__(self, unit, unitclass=None):
        if unit is None:
            return self
        else:
            return unit._properties[self.key]
    
    def __set__(self, unit, value):
        if self.coerce:
            value = self.coerce(unit, value)
        
        oldvalue = unit._properties[self.key]
        # This test is expensive, but it saves us a lot of
        # unnecessary save() operations later on.
        if value != oldvalue:
            if unit.sandbox and self.pre:
                self.pre(unit, value)
            unit.dirty = True
            unit._properties[self.key] = value
            if unit.sandbox and self.post:
                self.post(unit, value)
    
    def coerce(self, unit, value):
        if value is not None and not isinstance(value, self.type):
            # Try to coerce the value.
            try:
                value = self.type(value)
            except Exception, x:
                x.args += (value, type(value))
                raise x
        return value
    
    def __delete__(self, unit):
        raise AttributeError("Unit Properties may not be deleted.")


class MetaUnit(type):
    def __init__(cls, name, bases, dct):
        cls._associations = {}
        
        # Make a copy of the parent class' _properties keys, and store
        # it in the _properties attribute of this subclass. In this
        # manner, Unit Properties should propagate down to subclasses,
        # but not back up to superclasses.
        cls._properties = dict.fromkeys(cls._properties.keys())
        cls._property_types = cls._property_types.copy()


class Unit(object):
    """A generic object, the building-block of Dejavu.
    
    These are purposefully lightweight, relying on Sandboxes to cache
    them, which in turn rely on Storage Managers to load and save them.
    
    They maintain their own "schema" via UnitProperty objects, so that the
    Storage Managers don't need to know every detail about every Unit.
    Storage Managers for simple databases, for example, will simply create
    a single flat table for each unit type. If you write a custom Storage
    Manager, you can do as you like; the only place you might run into a
    problem is if you write a custom Storage Manager for custom Unit types,
    because the knowledge between the two is indeterminate. For example,
    if we provide a standard StorageManagerForLotusNotes, and you create
    custom Units which interface with it, you should probably subclass and
    extend our StorageManagerForLotusNotes with some custom storage logic.
    
    sandbox: The sandbox in which the Unit "lives". Also serves as a flag
        indicating whether this Unit has finished the initial creation
        process. While sandbox is None, pre and post descriptor functions
        will not be called.
        
        Sandboxes receive Units during recall() and memorize();
        these processes should set the sandbox attribute.
    
    dirty: a flag indicating whether elements in the _properties dictionary
        have been modified. This flag is used by Sandboxes to optimize
        forget(): they do not ask Storage Managers to save data for Units
        which have not been modified. Because SM's may cache Units, no code
        should set this flag other than UnitProperty.__set__ and SM's.
    
    temporary: a flag indicating that the Unit should NOT be saved to
        permanent storage. This should be False for most Units; some Units
        may require an additional condition (usually a confirmation by the
        user) that their state should persist. Note that the implementation
        of 'temporary' is left to Storage Managers, not Sandboxes.
    """
    __metaclass__ = MetaUnit
    _properties = {}
    _property_types = {}
    sequencer = UnitSequencerInteger()
    
    def __init__(self, **kwargs):
        # Copy the class _properties dict into self.
        self._properties = self.__class__._properties.copy()
        
        self.sandbox = None
        self.dirty = False
        self.temporary = False
        
        for k, v in kwargs:
            setattr(self, k, v)
    
    def set_property(cls, key, type=unicode, index=False,
                     descriptor=UnitProperty):
        """Set a Unit Property for cls."""
##        if hasattr(cls, key):
##            raise ValueError(u"%s already has an attribute named '%s'."
##                             % (cls, key))
        setattr(cls, key, descriptor(cls, key, type, index))
    set_property = classmethod(set_property)
    
    def set_properties(cls, types={}, descriptor=UnitProperty):
        """Set Unit Properties for cls."""
        for key, type in types.items():
            cls.set_property(key, type, False, descriptor)
    set_properties = classmethod(set_properties)
    
    def indices(cls):
        product = []
        for key in cls.properties():
            try:
                if getattr(cls, key).index:
                    product.append(key)
            except AttributeError:
                raise StandardError(cls, key)
        return tuple(product)
    indices = classmethod(indices)
    
    def properties(cls):
        return cls._properties.iterkeys()
    properties = classmethod(properties)
    
    def property_type(cls, key):
##        return getattr(cls, key).type
        return cls._property_types[key]
    property_type = classmethod(property_type)
    
    def adjust(self, **values):
        for key, val in values.iteritems():
            setattr(self, key, val)
    
    def repress(self):
        self.sandbox.repress(self)
    
    def forget(self):
        self.sandbox.forget(self)
    
    def __copy__(self):
        newUnit = self.__class__()
        for key in self.__class__.properties():
            if key != u'ID':
                newUnit._properties[key] = self._properties[key]
        newUnit.ID = None
        newUnit.sandbox = None
        return newUnit

# The default ID type is int. If you wish to use a different type for
# the ID's of a subclass of Unit, just overwrite ID, e.g.:
#     UnitSubclass.set_property('ID', unicode, index=True)
# You will probably also want to override Unit.sequencer in the class body.
Unit.set_property(u'ID', int, index=True)



###########################################################################
##                                                                       ##
##                                Arenas                                 ##
##                                                                       ##
###########################################################################


class Arena(object):
    """A namespace/workspace for a Dejavu application."""
    
    def __init__(self):
        self.defaultStore = None
        self.stores = {}
        self.roster = containers.Prism('name', 'cls', 'store')
        self.associations = containers.Graph()
        self.engine_functions = {}
    
    def load(self, configFileName):
        """Load StorageManagers."""
        parser = ConfigParser.ConfigParser()
        # Make names case-sensitive by overriding optionxform.
        parser.optionxform = unicode
        parser.read(configFileName)
        
        stores = []
        for section in parser.sections():
            opts = dict(parser.items(section))
            stores.append((int(opts.get("Load Order", "0")), section, opts))
        stores.sort()
        
        for order, name, options in stores:
            self.add_store(name, options)
    
    def add_store(self, name, options):
        """Load and attach the requested StoreManager."""
        storage_mgr_class = xray.classes(options[u'Class'])
        self.stores[name] = store = storage_mgr_class(name, self, options)
        
        store.shutdownOrder = int(options.get('Shutdown Order', '0'))
        
        # Fill Roster, a Prism of class-associated data.
        unitClasses = options.get('Units', '')
        if unitClasses:
            for clsname in unitClasses.split(","):
                clsname = clsname.strip()
                if clsname:
                    self.roster.add(name=clsname, cls=None, store=store)
        else:
            self.defaultStore = store
    
    def shutdown(self):
        # Tell all stores to shut down.
        stores = [(x.shutdownOrder, x) for x in self.stores.itervalues()]
        stores.sort()
        for order, store in stores:
            store.shutdown()
    
    def new_sandbox(self):
        return Sandbox(self)
    
    ###########################################
    ##        Unit Class Registration        ##
    ###########################################
    
    def register(self, cls):
        """register(cls) -> Assert that Units of class 'cls' will be handled."""
        try:
            row = self.roster.row_number(name=cls.__name__)
        except ValueError:
            self.roster.add(name=cls.__name__, cls=cls, store=self.defaultStore)
        else:
            # We left cls == None in _load(). Set it now.
            self.roster.facets['cls'][row] = cls
    
    def class_by_name(self, classname):
        return self.roster.cls(name=classname)
    
    def storage(self, cls):
        return self.roster.store(cls=cls)
    
    def create_storage(self, cls):
        self.storage(cls).create_storage(cls)
    
    def associate(self, cls, key, farClass, farKey,
                  nearFactory=None, farFactory=None):
        """Associate one Unit class with another by relating attributes.
        
        cls, key: The 'near' class and its key.
        farClass, farKey: the 'far' class and its key.
        
        Far Units will be recalled if their farKey matches cls.key.
        However, if cls.key is empty or None, no Units will be recalled.
        """
        
        # Disallow overwriting of existing attributes.
        if hasattr(cls, farClass.__name__):
            raise ValueError(u"%s already has an attribute named '%s'."
                             % (cls, farClass.__name__))
        if hasattr(farClass, cls.__name__):
            raise ValueError(u"%s already has an attribute named '%s'."
                             % (farClass, cls.__name__))
        
        # Assert that both classes are registered.
        self.register(cls)
        self.register(farClass)
        
        # Add a method to cls which retrieves farClass synapses
        if nearFactory is None:
            nearFactory = _synapses_func
        func = nearFactory(key, farClass, farKey)
        setattr(cls, farClass.__name__, func)
        cls._associations[farClass] = (key, farKey)
        
        # Add a method to farClass which retrieves cls synapses
        if farFactory is None:
            farFactory = _synapses_func
        func = farFactory(farKey, cls, key)
        setattr(farClass, cls.__name__, func)
        farClass._associations[cls] = (farKey, key)
        
        # Register the association(s) in an undirected graph.
        self.associations.connect(cls, farClass)


# You can use this arena instance if you are
# deploying a single application per process.
# Otherwise, you should create your own instance
# per application.
arena = Arena()

def _synapses_func(key, farClass, farKey):
    """Produce a new synapses() function to be bound to a Unit class."""
    def synapses(self, expr=None):
        """Recall associated '%(farname)s' Units.
        
        %(farname)s Units will be recalled if their %(farkey)s matches
        self.%(key)s. However, if self.%(key)s is None,
        no Units will be recalled.
        """ % {'farname': farClass.__name__,
               'farkey': farKey,
               'key': key,
               }
        value = getattr(self, key)
        if value is None:
            return iter([])
        
        # kwargs won't take unicode keys
        f = logic.filter(**{str(farKey): value})
        if expr is not None:
            f += expr
        return self.sandbox.recall(farClass, f)
    return synapses


###########################################################################
##                                                                       ##
##                              Sandboxes                                ##
##                                                                       ##
###########################################################################


class Sandbox(object):
    """Data sandbox for Dejavu arenas.
    
    Each consumer (that is, each UI process) maintains a Sandbox for
    managing Units. Sandboxes populate themselves with Units on a lazy
    basis, allowing UI code to request data as it's needed. However, once
    obtained, such Units are persisted (usually for the lifetime of the
    thread); this important detail means that multiple requests for the
    same Units result in multiple references to the same objects, rather
    than multiple objects. Sandboxes are basically what Fowler calls
    Identity Maps.
    
    The *REALLY* important thing to understand if you're customizing this
    is that Sandboxes won't survive sharing across threads--DON'T TRY IT.
    If you need to share unit data across requests, use or make an SM which
    persists the data, and chain it with another, more normal SM.
    
    _cache(), _caches, and _stores are private for a reason--don't access
    them from interface code--tell the Sandbox to do it for you.
    """
    
    def __init__(self, arena):
        self.arena = arena
        self._caches = {}
    
    def memorize(self, unit):
        """Attach a unit to this sandbox so that it will persist."""
        cls = unit.__class__
        unit.sandbox = self
        
        # Ask the store to accept the unit, assigning it an ID if
        # necessary. The store should also set unit.dirty to False
        # if it saves the whole unit state on this call or not.
        store = self.arena.storage(cls)
        if store:
            store.reserve(unit)
        
        # Insert the unit into the cache.
        self._cache(cls)[unit.ID] = unit
        
        # Do this at the end of the func, since most on_memorize
        # will want to have an ID when called.
        if hasattr(unit, "on_memorize"):
            unit.on_memorize()
    
    def forget(self, unit):
        """Destroy the unit, both in the cache and storage."""
        cls = unit.__class__
        store = self.arena.storage(cls)
        if store:
            store.destroy(unit)
        
        del self._cache(cls)[unit.ID]
        
        if hasattr(unit, "on_forget"):
            unit.on_forget()
        
        unit.sandbox = None
    
    def recall(self, cls, expr=None, *args):
        """recall(cls, expr=None) -> Recall units of cls which match expr.
        
        If additional args are supplied:
        
        1) They shall be of the form: (cls, expr, cls, expr, cls, [expr]).
            The final expr is optional.
        2) Each such secondary cls/expr pair will be recalled; however,
            only those secondary Units which are associated with Units
            in the primary set will be returned.
        3) Instead of single Units, the yielded value will be a tuple of
            Units, in the same order as the cls args were supplied. This
            facilitates consumer code like:
            
                for invoice, price in sandbox.recall(Invoice, f, Price):
                    deal_with(invoice)
                    deal_with(price)
            
        4) If any secondary Units are found, all combinations of those
            Units, together with each primary Unit, will be returned.
        5) If no secondary Units are recalled, then a token tuple will be
            returned of the form (primary_unit, None). For example:
            
                for invoice, price in sandbox.recall(Invoice, f, Price):
                    if price is None:
                        handle_no_prices(invoice)
            
        """
        
        store = self.arena.storage(cls)
        
        if args:
            # Format extra class/expr pairs more rigorously
            pairs = []
            args = list(args)
            while args:
                c = args.pop(0)
                if self.arena.storage(c) is not store:
                    raise ValueError(u"recall() does not currently support "
                                     u"multiple classes in disparate stores.")
                if args:
                    e = args.pop(0)
                else:
                    e = None
                pairs.append((c, e))
            
            # Give up on in-memory techniques, flush it all,
            # and ask the SM to give us what we want.
            self.flush(cls)
            for c, e in pairs:
                self.flush(c)
            for units in store.recall(cls, expr, pairs):
                for unit in units:
                    if unit is not None:
                        ID = unit.ID
                        cache = self._cache(unit.__class__)
                        if ID not in cache:
                            cache[ID] = unit
                        unit.sandbox = self
                yield units
            raise StopIteration
        
        # Run through our cache first.
        cache = self._cache(cls)
        skip_cache = False
        
        if expr:
            fc = expr.func.func_code
            if (fc.co_code == '|\x00\x00i\x01\x00d\x01\x00j\x02\x00S'
                and fc.co_names[-1] == 'ID'):
                # Special-case the scenario where one Unit is
                # expected and called by ID. We should be able
                # to save a database hit.
                expectedID = fc.co_consts[-1]
                if cache.has_key(expectedID):
                    yield cache[expectedID]
                    raise StopIteration
                else:
                    skip_cache = True
        
        if not skip_cache:
            for unit in cache.itervalues():
                if expr is None or expr.evaluate(unit):
                    yield unit
        
        # Query Storage.
        if store:
            for unit in store.recall(cls, expr):
                ID = unit.ID
                # Very important that we check for existing unit, as its
                # state may have changed in memory but not in storage.
                if ID not in cache:
                    cache[ID] = unit
                    unit.sandbox = self
                    yield unit
    
    def unit(self, cls, **kwargs):
        """Recall a single Unit, else None.
        
        **kwargs will be combined into an Expression via logic.filter.
            The first Unit matching that expression is returned; if no
            Units match, None is returned.
        
        If you need a single Unit which matches a more complex
            expression, use recall().next().
        """
        expr = None
        if kwargs:
            expr = logic.filter(**kwargs)
        try:
            return self.recall(cls, expr).next()
        except StopIteration:
            return None
    
    def distinct(self, cls, fields, expr=None):
        """Recall distinct Unit property values.
        
        If only one field is specified, a list of values will be returned.
        If more than one field is specified, a zipped list will be returned.
        
        Notice that you can also use this function as a count() function
        (in fact it's the only way to do it) by using fields = ['ID'].
        """
        seen = {}
        cache = self._cache(cls)
        for unit in cache.itervalues():
            if expr is None or expr.evaluate(unit):
                row = tuple([getattr(unit, field) for field in fields])
                if row not in seen:
                    seen[row] = None
        
        store = self.arena.storage(cls)
        if store:
            for row in store.distinct(cls, fields, expr):
                if row not in seen:
                    seen[row] = None
        
        seen = seen.keys()
        seen.sort()
        if len(fields) == 1:
            seen = [x[0] for x in seen]
        return seen
    
    def count(self, cls, expr):
        return len(self.distinct(cls, ['ID'], expr))
    
    ####################################
    ##        Cache Management        ##
    ####################################
    
    def _cache(self, cls):
        """Return the cache for the specified class.
        
        This base class creates a new cache for each cls per request.
        """
        if cls not in self._caches:
            self._caches[cls] = {}
        return self._caches[cls]
    
    def purge(self, cls):
        del self._caches[cls]
    
    def flush(self, cls):
        """flush(cls) -> Repress all units of the specified class."""
        cache = self._cache(cls)
        store = self.arena.storage(cls)
        while cache:
            id, unit = cache.popitem()
            
            if hasattr(unit, "on_repress"):
                unit.on_repress()
            
            if store and unit.dirty:
                store.save(unit)
    
    def flush_all(self):
        """flush_all() -> repress() all units."""
        for cls in self._caches.iterkeys():
            self.flush(cls)
    
    def repress(self, unit):
        """repress(unit) -> Remove unit from cache (but don't destroy)."""
        if hasattr(unit, "on_repress"):
            unit.on_repress()
        
        cls = unit.__class__
        store = self.arena.storage(cls)
        if store and unit.dirty:
            store.save(unit)
        
        del self._cache(cls)[unit.ID]