root/trunk/__init__.py

import ConfigParser
import datetime


from dejavu.containers import *
from dejavu.analysis import *
from dejavu.readme import *
from dejavu import logic
import xray


###########################################################################
##                                                                       ##
##                                 Units                                 ##
##                                                                       ##
###########################################################################


# All Units currently must possess an 'ID' UnitProperty. The sequencing of
# IDs depends upon their type and the particular needs of the class. Pick
# one of these UnitSequencers to fit your subclass.

# At the moment, no ID sequences are allowed None as a value, since this
# signals a Unit which needs to be sequenced when memorized. In addition,
# you should aim to create new sequencers which generate IDs that obey
# the builtin max() and min() functions.

class UnitSequencerNull(object):
    """A null sequencer for Unit IDs. Sequencing will error.
    
    In many cases, ID values simply have no algorithmic sequence;
    for example, a set of Employee Units might use Social Security
    Numbers for IDs (which you should never, ever do ;).
    
    In other cases, sequencing will be best handled by custom algorithms
    within application code; that is, the job of abstracting the sequence
    logic would not be worth the effort.
    """
    
    def __init__(self, type=unicode):
        self.type = type
    
    def next(self, sequence):
        raise StopIteration("No sequence defined.")


class UnitSequencerInteger(object):
    """A sequencer for Unit IDs, where id[i+1] == id[i] + 1."""
    
    def __init__(self, type=int, initial=1):
        self.type = type
        self.initial = initial
    
    def next(self, sequence):
        if sequence:
            m = max(sequence)
            if m is not None:
                return m + 1
        return self.initial


class UnitSequencerUnicode(object):
    """A sequencer for Unit IDs, where e.g. next(['abc']) == 'abd'."""
    
    def __init__(self, type=unicode, width=6,
                 range="abcdefghijklmnopqrstuvwxyz"):
        self.type = type
        self.width = width
        self.range = range
    
    def next(self, sequence):
        r = self.range
        if sequence:
            maxid = max(sequence)
            if len(maxid) != self.width:
                raise OverflowError("'%s' is not of width %s." %
                                    (maxid, self.width))
            for i in range(self.width - 1, -1, -1):
                pos = r.index(maxid[i]) + 1
                if pos >= len(r) or pos < 0:
                    maxid = maxid[:i] + r[0] + maxid[i+1:]
                else:
                    maxid = maxid[:i] + r[pos] + maxid[i+1:]
                    break
            else:
                raise OverflowError("Next ID exceeds width %s." % self.width)
            return maxid
        return r[0] * self.width


class UnitProperty(object):
    """Data descriptor for Unit data which will persist in storage.
    
    pre, post: Override these with functions to provide custom behaviors
        upon attribute modification. They are not called if sandbox is None.
        If you need a behavior which fires regardless, you should override
        __set__. They also are not called if the new value matches the
        existing value. Again, override __set__ if you need an exception.
    
    hints: A dictionary which provides named hints to Storage Managers
        concerning the nature of the data. A common use, for example,
        is to inform Managers that would usually store unicode strings
        as strings of length 255, that a particular value should be
        a larger object; this is done with a 'Size' mapping, such as:
        hints = {u'Size': 0}, where 0 implies no limit. Canonical storage
        hint names and implementation details may be found in storage.py
        documentation.
    """
    
    pre = None
    post = None
    
    def __init__(self, key, type=unicode, index=False, hints={}):
        self.key = key
        self.type = type
        self.index = index
        self.hints = hints
    
    def __get__(self, unit, unitclass=None):
        if unit is None:
            # When calling on the class instead of an instance...
            return self
        else:
            return unit._properties[self.key]
    
    def __set__(self, unit, value):
        if self.coerce:
            value = self.coerce(unit, value)
        
        oldvalue = unit._properties[self.key]
        # This test is expensive, but it saves us a lot of
        # unnecessary save() operations later on.
        if value != oldvalue:
            if unit.sandbox and self.pre:
                self.pre(unit, value)
            unit.dirty = True
            unit._properties[self.key] = value
            if unit.sandbox and self.post:
                self.post(unit, value)
    
    def coerce(self, unit, value):
        if value is not None and not isinstance(value, self.type):
            # Try to coerce the value.
            try:
                value = self.type(value)
            except Exception, x:
                x.args += (value, type(value))
                raise x
        return value
    
    def __delete__(self, unit):
        raise AttributeError("Unit Properties may not be deleted.")


class MetaUnit(type):
    def __init__(cls, name, bases, dct):
        cls._associations = {}
        
        # Make a copy of the parent class' _properties keys, and store
        # it in the _properties attribute of this subclass. In this
        # manner, Unit Properties should propagate down to subclasses,
        # but not back up to superclasses.
        props = dict.fromkeys(cls._properties.keys())
        
        # Now grab any new UnitProperties defined in this class.
        # Overwrite any properties defined in superclasses.
        for key, val in dct.iteritems():
            if isinstance(val, UnitProperty):
                props[key] = val
        
        cls._properties = props


class Unit(object):
    """A generic object, the building-block of Dejavu.
    
    These are purposefully lightweight, relying on Sandboxes to cache
    them, which in turn rely on Storage Managers to load and save them.
    
    They maintain their own "schema" via UnitProperty objects, so that the
    Storage Managers don't need to know every detail about every Unit.
    Storage Managers for simple databases, for example, will simply create
    a single flat table for each unit type. If you write a custom Storage
    Manager, you can do as you like; the only place you might run into a
    problem is if you write a custom Storage Manager for custom Unit types,
    because the knowledge between the two is indeterminate. For example,
    if we provide a standard StorageManagerForLotusNotes, and you create
    custom Units which interface with it, you should probably subclass and
    extend our StorageManagerForLotusNotes with some custom storage logic.
    
    sandbox: The sandbox in which the Unit "lives". Also serves as a flag
        indicating whether this Unit has finished the initial creation
        process. While sandbox is None, pre and post descriptor functions
        will not be called.
        
        Sandboxes receive Units during recall() and memorize();
        these processes should set the sandbox attribute.
    
    dirty: a flag indicating whether elements in the _properties dictionary
        have been modified. This flag is used by Sandboxes to optimize
        forget(): they do not ask Storage Managers to save data for Units
        which have not been modified. Because SM's may cache Units, no code
        should set this flag other than UnitProperty.__set__ and SM's.
    """
    
    __metaclass__ = MetaUnit
    _properties = {}
    sequencer = UnitSequencerInteger()
    
    def __init__(self, **kwargs):
        # Copy the class _properties dict into self, setting
        # each value to None.
        self._properties = dict.fromkeys(self.__class__._properties.keys())
        
        self.sandbox = None
        self.dirty = False
        
        for k, v in kwargs.iteritems():
            setattr(self, k, v)
    
    def set_property(cls, key, type=unicode, index=False,
                     descriptor=UnitProperty):
        """Set a Unit Property for cls."""
        setattr(cls, key, descriptor(key, type, index))
        cls._properties[key] = None
    set_property = classmethod(set_property)
    
    def set_properties(cls, types={}, descriptor=UnitProperty):
        """Set Unit Properties for cls."""
        for key, type in types.items():
            cls.set_property(key, type, False, descriptor)
    set_properties = classmethod(set_properties)
    
    def indices(cls):
        """cls.indices() -> tuple of names of indexed UnitProperties."""
        product = []
        for key in cls.properties():
            try:
                if getattr(cls, key).index:
                    product.append(key)
            except AttributeError, x:
                x.args += (cls, key)
                raise x
        return tuple(product)
    indices = classmethod(indices)
    
    def properties(cls):
        """cls.properties() -> list of UnitProperty names."""
        return cls._properties.iterkeys()
    properties = classmethod(properties)
    
    def property_type(cls, key):
        """cls.property_type(key) -> type of the given UnitProperty."""
        # Retrieving from the class gives us
        # the UnitProperty object, not its value.
        return getattr(cls, key).type
    property_type = classmethod(property_type)
    
    def adjust(self, **values):
        """adjust(**values) -> Set UnitProperties by key, value pairs."""
        for key, val in values.iteritems():
            setattr(self, key, val)
    
    def repress(self):
        """repress() -> Remove this Unit from memory (do not destroy)."""
        self.sandbox.repress(self)
    
    def forget(self):
        """forget() -> Destroy this Unit."""
        self.sandbox.forget(self)
    
    def __copy__(self):
        newUnit = self.__class__()
        for key in self.__class__.properties():
            if key != u'ID':
                newUnit._properties[key] = self._properties[key]
        newUnit.ID = None
        newUnit.sandbox = None
        return newUnit
    
    def __getstate__(self):
        return (self._properties, self.dirty)
    
    def __setstate__(self, state):
        self.sandbox = None
        self._properties, self.dirty = state
    
    def add(self, unit):
        """add(unit) -> Auto-create a relationship between self and unit."""
        try:
            key, farKey = self.__class__._associations[unit.__class__]
        except KeyError:
            raise AssociationError("'%s' is not associated with '%s'"
                                   % (self.__class__, unit.__class__))
        
        nearval = getattr(self, key)
        farval = getattr(unit, farKey)
        if nearval is None:
            if farval is None:
                raise AssociationError("At least one Unit key must be set.")
            else:
                setattr(self, key, farval)
        else:
            # If far key is already set, it will simply be overwritten.
            setattr(unit, farKey, nearval)


# The default ID type is int. If you wish to use a different type for
# the ID's of a subclass of Unit, just overwrite ID, e.g.:
#     ID = UnitProperty('ID', unicode, index=True)
#       or
#     UnitSubclass.set_property('ID', unicode, index=True)
#       or even
#     UnitSubclass.ID.type = unicode
# You will probably also want to override Unit.sequencer for the class.
Unit.set_property(u'ID', int, index=True)



###########################################################################
##                                                                       ##
##                                Arenas                                 ##
##                                                                       ##
###########################################################################


class Arena(object):
    """A namespace/workspace for a Dejavu application."""
    
    def __init__(self):
        self.defaultStore = None
        self.stores = {}
        self.roster = Prism('name', 'cls', 'store')
        self.associations = Graph()
        self.engine_functions = {}
    
    def load(self, configFileName):
        """Load StorageManagers."""
        parser = ConfigParser.ConfigParser()
        # Make names case-sensitive by overriding optionxform.
        parser.optionxform = unicode
        parser.read(configFileName)
        
        stores = []
        for section in parser.sections():
            opts = dict(parser.items(section))
            stores.append((int(opts.get("Load Order", "0")), section, opts))
        stores.sort()
        
        for order, name, options in stores:
            storage_mgr_class = xray.classes(options[u'Class'])
            store = storage_mgr_class(name, self, options)
            
            unitClasses = []
            for x in options.get('Units', '').split(","):
                clsname = x.strip()
                if clsname:
                    unitClasses.append(clsname)
            
            self.add_store(name, store, unitClasses)
    
    def add_store(self, name, store, unitClasses=[]):
        """Register a StorageManager."""
        self.stores[name] = store
        
        # Fill Roster, a Prism of class-associated data.
        if unitClasses:
            for clsname in unitClasses:
                if clsname:
                    self.roster.add(name=clsname, cls=None, store=store)
        else:
            self.defaultStore = store
    
    def shutdown(self):
        """Shutdown the arena."""
        # Tell all stores to shut down.
        stores = [(x.shutdownOrder, x) for x in self.stores.itervalues()]
        stores.sort()
        for order, store in stores:
            store.shutdown()
    
    def new_sandbox(self):
        return Sandbox(self)
    
    ###########################################
    ##        Unit Class Registration        ##
    ###########################################
    
    def register(self, cls):
        """register(cls) -> Assert that Units of class 'cls' will be handled."""
        try:
            row = self.roster.row_number(name=cls.__name__)
        except ValueError:
            self.roster.add(name=cls.__name__, cls=cls, store=self.defaultStore)
        else:
            # We left cls == None in _load(). Set it now.
            self.roster.facets['cls'][row] = cls
    
    def class_by_name(self, classname):
        return self.roster.cls(name=classname)
    
    def storage(self, cls):
        return self.roster.store(cls=cls)
    
    def create_storage(self, cls):
        self.storage(cls).create_storage(cls)
    
    def associate(self, cls, key, farClass, farKey,
                  nearFactory=None, farFactory=None):
        """Associate one Unit class with another by relating attributes.
        
        cls, key: The 'near' class and its key.
        farClass, farKey: the 'far' class and its key.
        
        Far Units will be recalled if their farKey matches cls.key.
        However, if cls.key is empty or None, no Units will be recalled.
        """
        
        # Disallow overwriting of existing attributes.
        if hasattr(cls, farClass.__name__):
            raise AssociationError(u"%s already has an attribute named '%s'."
                                   % (cls, farClass.__name__))
        if hasattr(farClass, cls.__name__):
            raise AssociationError(u"%s already has an attribute named '%s'."
                                   % (farClass, cls.__name__))
        
        # Assert that both classes are registered.
        self.register(cls)
        self.register(farClass)
        
        # Add a method to cls which retrieves farClass synapses
        if nearFactory is None:
            nearFactory = _synapses_func
        func = nearFactory(key, farClass, farKey)
        setattr(cls, farClass.__name__, func)
        cls._associations[farClass] = (key, farKey)
        
        # Add a method to farClass which retrieves cls synapses
        if farFactory is None:
            farFactory = _synapses_func
        func = farFactory(farKey, cls, key)
        setattr(farClass, cls.__name__, func)
        farClass._associations[cls] = (farKey, key)
        
        # Register the association(s) in an undirected graph.
        self.associations.connect(cls, farClass)


# You can use this arena instance if you are
# deploying a single application per process.
# Otherwise, you should create your own instance
# per application.
dejavuarena = Arena()

def _synapses_func(key, farClass, farKey):
    """Produce a new synapses() function to be bound to a Unit class."""
    def synapses(self, expr=None):
        """Recall associated '%(farname)s' Units.
        
        %(farname)s Units will be recalled if their %(farkey)s matches
        self.%(key)s. However, if self.%(key)s is None,
        no Units will be recalled.
        """ % {'farname': farClass.__name__,
               'farkey': farKey,
               'key': key,
               }
        value = getattr(self, key)
        if value is None:
            return iter([])
        
        # kwargs won't take unicode keys
        f = logic.filter(**{str(farKey): value})
        if expr is not None:
            f += expr
        return self.sandbox.recall(farClass, f)
    return synapses


###########################################################################
##                                                                       ##
##                              Sandboxes                                ##
##                                                                       ##
###########################################################################


class Sandbox(object):
    """Data sandbox for Dejavu arenas.
    
    Each consumer (that is, each UI process) maintains a Sandbox for
    managing Units. Sandboxes populate themselves with Units on a lazy
    basis, allowing UI code to request data as it's needed. However, once
    obtained, such Units are persisted (usually for the lifetime of the
    thread); this important detail means that multiple requests for the
    same Units result in multiple references to the same objects, rather
    than multiple objects. Sandboxes are basically what Fowler calls
    Identity Maps.
    
    The *REALLY* important thing to understand if you're customizing this
    is that Sandboxes won't survive sharing across threads--DON'T TRY IT.
    If you need to share unit data across requests, use or make an SM which
    persists the data, and chain it with another, more normal SM.
    
    _cache(), _caches, and _stores are private for a reason--don't access
    them from interface code--tell the Sandbox to do it for you.
    """
    
    def __init__(self, arena):
        self.arena = arena
        self._caches = {}
    
    def memorize(self, unit):
        """Attach a unit to this sandbox so that it will persist."""
        cls = unit.__class__
        unit.sandbox = self
        
        # Ask the store to accept the unit, assigning it an ID if
        # necessary. The store should also set unit.dirty to False
        # if it saves the whole unit state on this call or not.
        store = self.arena.storage(cls)
        if store:
            store.reserve(unit)
        
        # Insert the unit into the cache.
        self._cache(cls)[unit.ID] = unit
        
        # Do this at the end of the func, since most on_memorize
        # will want to have an ID when called.
        if hasattr(unit, "on_memorize"):
            unit.on_memorize()
    
    def forget(self, unit):
        """Destroy the unit, both in the cache and storage."""
        cls = unit.__class__
        store = self.arena.storage(cls)
        if store:
            store.destroy(unit)
        
        del self._cache(cls)[unit.ID]
        
        if hasattr(unit, "on_forget"):
            unit.on_forget()
        
        unit.sandbox = None
    
    def recall(self, cls, expr=None, *args):
        """recall(cls, expr=None) -> Recall units of cls which match expr.
        
        If additional args are supplied:
        
        1) They shall be of the form: (cls, expr, cls, expr, cls, [expr]).
            The final expr is optional.
        2) Each such secondary cls/expr pair will be recalled; however,
            only those secondary Units which are associated with Units
            in the primary set will be returned.
        3) Instead of single Units, the yielded value will be a tuple of
            Units, in the same order as the cls args were supplied. This
            facilitates consumer code like:
            
                for invoice, price in sandbox.recall(Invoice, f, Price):
                    deal_with(invoice)
                    deal_with(price)
            
        4) If any secondary Units are found, all combinations of those
            Units, together with each primary Unit, will be returned.
        5) If no secondary Units are recalled, then a token tuple will be
            returned of the form (primary_unit, None). For example:
            
                for invoice, price in sandbox.recall(Invoice, f, Price):
                    if price is None:
                        handle_no_prices(invoice)
            
        """
        
        store = self.arena.storage(cls)
        
        if args:
            # Deal with multiple class/expr pairs.
            
            # Format extra class/expr pairs more rigorously
            pairs = []
            args = list(args)
            while args:
                c = args.pop(0)
                if self.arena.storage(c) is not store:
                    raise ValueError(u"recall() does not currently support "
                                     u"multiple classes in disparate stores.")
                if args:
                    e = args.pop(0)
                else:
                    e = None
                pairs.append((c, e))
            
            # Don't try any in-memory techniques, just flush it all,
            # and ask the SM to give us what we want.
            self.flush(cls)
            for c, e in pairs:
                self.flush(c)
            for units in store.recall(cls, expr, pairs):
                confirmed = True
                for unit in units:
                    if unit is not None:
                        ID = unit.ID
                        cache = self._cache(unit.__class__)
                        if ID not in cache:
                            cache[ID] = unit
                        unit.sandbox = self
                    if hasattr(unit, 'on_recall'):
                        try:
                            unit.on_recall()
                        except UnrecallableError:
                            confirmed = False
                if confirmed:
                    yield units
            raise StopIteration
        
        # Run through our cache first.
        cache = self._cache(cls)
        skip_cache = False
        
        if expr:
            fc = expr.func.func_code
            if (fc.co_code == '|\x00\x00i\x01\x00d\x01\x00j\x02\x00S'
                and fc.co_names[-1] == 'ID'):
                # Special-case the scenario where one Unit is
                # expected and called by ID. We should be able
                # to save a database hit.
                expectedID = fc.co_consts[-1]
                if cache.has_key(expectedID):
                    unit = cache[expectedID]
                    # Do NOT call on_recall here. That should be called
                    # only at the Sandbox-SM boundary.
                    yield unit
                    raise StopIteration
                else:
                    skip_cache = True
        
        if not skip_cache:
            for unit in cache.itervalues():
                if expr is None or expr.evaluate(unit):
                    # Do NOT call on_recall here. That should be called
                    # only at the Sandbox-SM boundary.
                    yield unit
        
        # Query Storage.
        if store:
            for unit in store.recall(cls, expr):
                ID = unit.ID
                # Very important that we check for existing unit, as its
                # state may have changed in memory but not in storage.
                if ID not in cache:
                    cache[ID] = unit
                    unit.sandbox = self
                    confirmed = True
                    if hasattr(unit, 'on_recall'):
                        try:
                            unit.on_recall()
                        except UnrecallableError:
                            confirmed = False
                    if confirmed:
                        yield unit
    
    def unit(self, cls, **kwargs):
        """Recall a single Unit, else None.
        
        **kwargs will be combined into an Expression via logic.filter.
            The first Unit matching that expression is returned; if no
            Units match, None is returned.
        
        If you need a single Unit which matches a more complex
            expression, use recall().next().
        """
        expr = None
        if kwargs:
            expr = logic.filter(**kwargs)
        try:
            return self.recall(cls, expr).next()
        except StopIteration:
            return None
    
    def distinct(self, cls, attrs, expr=None):
        """Recall distinct Unit property values.
        
        If only one attribute is specified, a list of values will be returned.
        If more than one attribute is specified, a zipped list will be returned.
        
        Notice that you can also use this function as a count() function
        (in fact it's the only way to do it) by using attrs = ['ID'].
        """
        seen = {}
        cache = self._cache(cls)
        for unit in cache.itervalues():
            if expr is None or expr.evaluate(unit):
                row = tuple([getattr(unit, attr) for attr in attrs])
                if row not in seen:
                    seen[row] = None
        
        store = self.arena.storage(cls)
        if store:
            for row in store.distinct(cls, attrs, expr):
                if row not in seen:
                    seen[row] = None
        
        seen = seen.keys()
        seen.sort()
        if len(attrs) == 1:
            seen = [x[0] for x in seen]
        return seen
    
    def count(self, cls, expr):
        return len(self.distinct(cls, ['ID'], expr))
    
    ####################################
    ##        Cache Management        ##
    ####################################
    
    def _cache(self, cls):
        """Return the cache for the specified class.
        
        This base class creates a new cache for each cls per request.
        """
        if cls not in self._caches:
            self._caches[cls] = {}
        return self._caches[cls]
    
    def purge(self, cls):
        del self._caches[cls]
    
    def flush(self, cls):
        """flush(cls) -> Repress all units of the specified class."""
        cache = self._cache(cls)
        store = self.arena.storage(cls)
        while cache:
            id, unit = cache.popitem()
            
            if hasattr(unit, "on_repress"):
                unit.on_repress()
            
            if store and unit.dirty:
                store.save(unit)
    
    def flush_all(self):
        """flush_all() -> repress() all units."""
        for cls in self._caches.iterkeys():
            self.flush(cls)
    
    def repress(self, unit):
        """repress(unit) -> Remove unit from cache (but don't destroy)."""
        if hasattr(unit, "on_repress"):
            unit.on_repress()
        
        cls = unit.__class__
        store = self.arena.storage(cls)
        if store and unit.dirty:
            store.save(unit)
        
        del self._cache(cls)[unit.ID]


###########################################################################
##                                                                       ##
##                               Errors                                  ##
##                                                                       ##
###########################################################################


class DejavuError(Exception):
    """Base class for errors which occur within Dejavu."""
    def __init__(self, *args):
        self.args = args
    
    def __str__(self):
        return u'\n'.join([unicode(eachArg) for eachArg in self.args])

class AssociationError(DejavuError):
    """Exception raised when a Unit association fails."""
    pass

class UnrecallableError(DejavuError):
    """Exception raised when a Unit was sought but not recalled."""
    pass


###########################################################################
##                                                                       ##
##                           Logic functions                             ##
##                                                                       ##
###########################################################################


def icontains(a, b):
    """Case-insensitive test b in a. Note the operand order."""
    if a is None or b is None:
        return False
    return b.lower() in a.lower()

def icontainedby(a, b):
    """Case-insensitive test a in b. Note the operand order."""
    if a is None or b is None:
        return False
    return a.lower() in b.lower()

def istartswith(a, b):
    """True if a starts with b (case-insensitive), False otherwise."""
    if a is None or b is None:
        return False
    return a.lower().startswith(b.lower())

def iendswith(a, b):
    """True if a ends with b (case-insensitive), False otherwise."""
    if a is None or b is None:
        return False
    return a.lower().endswith(b.lower())

def ieq(a, b):
    """True if a == b (case-insensitive), False otherwise."""
    if a is None or b is None:
        return False
    return (a.lower() == b.lower())

def year(value):
    """The year attribute of a date."""
    if isinstance(value, (datetime.date, datetime.datetime)):
        return value.year
    else:
        return None

def now():
    """Late-bound datetime.datetime.now(). Taint this when early binding."""
    return datetime.datetime.now()
now.bind_late = True

def today():
    """Late-bound datetime.date.today(). Taint this when early binding."""
    return datetime.date.today()
today.bind_late = True

def iscurrentweek(value):
    """If value is in the current week, return True, else False."""
    if isinstance(value, (datetime.date, datetime.datetime)):
        return datetime.date.today().strftime('%W%Y') == value.strftime('%W%Y')
    else:
        return False
iscurrentweek.bind_late = True

# Inject these functions into the logic module's globals.
class _Empty(object): pass
_d = _Empty()
for _name in ['icontains', 'icontainedby', 'istartswith', 'iendswith',
              'ieq', 'year', 'now', 'today', 'iscurrentweek']:
    setattr(_d, _name, globals()[_name])
logic.dejavu = _d