Changeset 663 for branches


Ignore:
Timestamp:
05/26/13 15:14:33 (3 years ago)
Author:
mmckerns
Message:

migrated keymap functions to keymap classes; enable memoize to use keymaps

Location:
branches/decorate
Files:
4 edited
1 moved

Legend:

Unmodified
Added
Removed
  • branches/decorate/cache.py

    r662 r663  
    11#  
    22 
    3 import collections 
     3try: 
     4    from collections import namedtuple 
     5except ImportError: 
     6    from namedtuple import namedtuple 
     7from collections import deque 
    48from random import choice #XXX: biased? 
    59from heapq import nsmallest 
     
    812from functools import update_wrapper 
    913from threading import RLock 
    10 from cache_helper import _CacheInfo, Counter, hashmap as _keymap 
    11  
     14from keymaps import hashmap as _keymap 
     15 
     16_CacheInfo = namedtuple("CacheInfo", ['hits','misses','maxsize','currsize']) 
     17 
     18class Counter(dict): 
     19    'Mapping where default values are zero' 
     20    def __missing__(self, key): 
     21        return 0 
    1222 
    1323def no_cache(*arg, **kwd): 
     
    5767    '''Infinitely-growing cache decorator. 
    5868 
    59     If *typed* is True, arguments of different types will be cached separately. 
    60     For example, f(3.0) and f(3) will be treated as distinct calls with 
    61     distinct results.  Cache typing has a memory penalty, and may also may be 
    62     ignored by some 'keymaps'. 
    63  
    6469    If *keymap* is given, it will replace the hashing algorithm for generating 
    65     cache keys.  For example, see the other hashing algorithms available in 
    66     'cache_helper.py'. With the default keymap, arguments to the cached 
    67     function must be hashable. 
     70    cache keys.  Several hashing algorithms are available in 'keymaps.py'.  With 
     71    the default keymap, arguments to the cached function must be hashable. 
     72 
     73    If the keymap retains type information, then arguments of different types 
     74    will be cached separately.  For example, f(3.0) and f(3) will be treated 
     75    as distinct calls with distinct results.  Cache typing has a memory penalty, 
     76    and may also be ignored by some 'keymaps'. 
    6877 
    6978    View the cache statistics named tuple (hits, misses, maxsize, currsize) with 
     
    7382    ''' 
    7483    maxsize = None 
    75     make_key = kwd.get('keymap', _keymap) 
    76     typed = kwd.get('typed', False) 
     84    make_key = kwd.get('keymap', None) 
     85    if make_key is None: make_key = _keymap() 
    7786 
    7887    def decorating_function(user_function): 
     
    8493 
    8594        def wrapper(*args, **kwds): 
    86             key = make_key(args, kwds, typed) 
     95            key = make_key(*args, **kwds) 
    8796 
    8897            # get cache entry or compute if not found 
     
    116125 
    117126 
    118 def lfu_cache(maxsize=100, keymap=None, typed=False): 
     127def lfu_cache(maxsize=100, keymap=None): 
    119128    '''Least-frequenty-used cache decorator. 
    120129 
     
    122131    will grow without bound. 
    123132 
    124     If *typed* is True, arguments of different types will be cached separately. 
    125     For example, f(3.0) and f(3) will be treated as distinct calls with 
    126     distinct results.  Cache typing has a memory penalty, and may also may be 
    127     ignored by some 'keymaps'. 
    128  
    129133    If *keymap* is given, it will replace the hashing algorithm for generating 
    130     cache keys.  For example, see the other hashing algorithms available in 
    131     'cache_helper.py'. With the default keymap, arguments to the cached 
    132     function must be hashable. 
     134    cache keys.  Several hashing algorithms are available in 'keymaps.py'.  With 
     135    the default keymap, arguments to the cached function must be hashable. 
     136 
     137    If the keymap retains type information, then arguments of different types 
     138    will be cached separately.  For example, f(3.0) and f(3) will be treated 
     139    as distinct calls with distinct results.  Cache typing has a memory penalty, 
     140    and may also be ignored by some 'keymaps'. 
    133141 
    134142    View the cache statistics named tuple (hits, misses, maxsize, currsize) with 
     
    140148    ''' 
    141149    if maxsize == 0: return no_cache() 
    142     if keymap is None: make_key = _keymap 
     150    if keymap is None: make_key = _keymap() 
    143151    else: make_key = keymap 
    144     if maxsize is None: return inf_cache(keymap=make_key, typed=typed) 
     152    if maxsize is None: return inf_cache(keymap=make_key) 
    145153     
    146154    def decorating_function(user_function): 
     
    153161 
    154162        def wrapper(*args, **kwds): 
    155             key = make_key(args, kwds, typed) 
     163            key = make_key(*args, **kwds) 
    156164 
    157165            # get cache entry or compute if not found 
     
    194202 
    195203 
    196 def lru_cache(maxsize=100, keymap=None, typed=False): 
     204def lru_cache(maxsize=100, keymap=None): 
    197205    '''Least-recently-used cache decorator. 
    198206 
     
    200208    will grow without bound. 
    201209 
    202     If *typed* is True, arguments of different types will be cached separately. 
    203     For example, f(3.0) and f(3) will be treated as distinct calls with 
    204     distinct results.  Cache typing has a memory penalty, and may also may be 
    205     ignored by some 'keymaps'. 
    206  
    207210    If *keymap* is given, it will replace the hashing algorithm for generating 
    208     cache keys.  For example, see the other hashing algorithms available in 
    209     'cache_helper.py'. With the default keymap, arguments to the cached 
    210     function must be hashable. 
     211    cache keys.  Several hashing algorithms are available in 'keymaps.py'.  With 
     212    the default keymap, arguments to the cached function must be hashable. 
     213 
     214    If the keymap retains type information, then arguments of different types 
     215    will be cached separately.  For example, f(3.0) and f(3) will be treated 
     216    as distinct calls with distinct results.  Cache typing has a memory penalty, 
     217    and may also be ignored by some 'keymaps'. 
    211218 
    212219    View the cache statistics named tuple (hits, misses, maxsize, currsize) with 
     
    218225    ''' 
    219226    if maxsize == 0: return no_cache() 
    220     if keymap is None: make_key = _keymap 
     227    if keymap is None: make_key = _keymap() 
    221228    else: make_key = keymap 
    222     if maxsize is None: return inf_cache(keymap=make_key, typed=typed) 
     229    if maxsize is None: return inf_cache(keymap=make_key) 
    223230    maxqueue = maxsize * 10 #XXX: user settable? confirm this works as expected 
    224231 
    225232    def decorating_function(user_function): 
    226233        cache = dict()                  # mapping of args to results 
    227         queue = collections.deque()     # order that keys have been used 
     234        queue = deque()                 # order that keys have been used 
    228235        refcount = Counter()            # times each key is in the queue 
    229236        sentinel = object()             # marker for looping around the queue 
     
    238245 
    239246        def wrapper(*args, **kwds): 
    240             key = make_key(args, kwds, typed) 
     247            key = make_key(*args, **kwds) 
    241248 
    242249            # get cache entry or compute if not found 
     
    297304 
    298305 
    299 def mru_cache(maxsize=100, keymap=None, typed=False): 
     306def mru_cache(maxsize=100, keymap=None): 
    300307    '''Most-recently-used cache decorator. 
    301308 
     
    303310    will grow without bound. 
    304311 
    305     If *typed* is True, arguments of different types will be cached separately. 
    306     For example, f(3.0) and f(3) will be treated as distinct calls with 
    307     distinct results.  Cache typing has a memory penalty, and may also may be 
    308     ignored by some 'keymaps'. 
    309  
    310312    If *keymap* is given, it will replace the hashing algorithm for generating 
    311     cache keys.  For example, see the other hashing algorithms available in 
    312     'cache_helper.py'. With the default keymap, arguments to the cached 
    313     function must be hashable. 
     313    cache keys.  Several hashing algorithms are available in 'keymaps.py'.  With 
     314    the default keymap, arguments to the cached function must be hashable. 
     315 
     316    If the keymap retains type information, then arguments of different types 
     317    will be cached separately.  For example, f(3.0) and f(3) will be treated 
     318    as distinct calls with distinct results.  Cache typing has a memory penalty, 
     319    and may also be ignored by some 'keymaps'. 
    314320 
    315321    View the cache statistics named tuple (hits, misses, maxsize, currsize) with 
     
    321327    ''' 
    322328    if maxsize == 0: return no_cache() 
    323     if keymap is None: make_key = _keymap 
     329    if keymap is None: make_key = _keymap() 
    324330    else: make_key = keymap 
    325     if maxsize is None: return inf_cache(keymap=make_key, typed=typed) 
     331    if maxsize is None: return inf_cache(keymap=make_key) 
    326332 
    327333    def decorating_function(user_function): 
    328334        cache = dict()                  # mapping of args to results 
    329         queue = collections.deque()     # order that keys have been used 
     335        queue = deque()                 # order that keys have been used 
    330336        stats = [0, 0]                  # make statistics updateable non-locally 
    331337        HITS, MISSES = 0, 1             # names for the stats fields 
     
    338344 
    339345        def wrapper(*args, **kwds): 
    340             key = make_key(args, kwds, typed) 
     346            key = make_key(*args, **kwds) 
    341347 
    342348            # get cache entry or compute if not found 
     
    379385 
    380386 
    381 def rr_cache(maxsize=100, keymap=None, typed=False): 
     387def rr_cache(maxsize=100, keymap=None): 
    382388    '''random-replacement cache decorator. 
    383389 
     
    385391    will grow without bound. 
    386392 
    387     If *typed* is True, arguments of different types will be cached separately. 
    388     For example, f(3.0) and f(3) will be treated as distinct calls with 
    389     distinct results.  Cache typing has a memory penalty, and may also may be 
    390     ignored by some 'keymaps'. 
    391  
    392393    If *keymap* is given, it will replace the hashing algorithm for generating 
    393     cache keys.  For example, see the other hashing algorithms available in 
    394     'cache_helper.py'. With the default keymap, arguments to the cached 
    395     function must be hashable. 
     394    cache keys.  Several hashing algorithms are available in 'keymaps.py'.  With 
     395    the default keymap, arguments to the cached function must be hashable. 
     396 
     397    If the keymap retains type information, then arguments of different types 
     398    will be cached separately.  For example, f(3.0) and f(3) will be treated 
     399    as distinct calls with distinct results.  Cache typing has a memory penalty, 
     400    and may also be ignored by some 'keymaps'. 
    396401 
    397402    View the cache statistics named tuple (hits, misses, maxsize, currsize) with 
     
    403408    ''' 
    404409    if maxsize == 0: return no_cache() 
    405     if keymap is None: make_key = _keymap 
     410    if keymap is None: make_key = _keymap() 
    406411    else: make_key = keymap 
    407     if maxsize is None: return inf_cache(keymap=make_key, typed=typed) 
     412    if maxsize is None: return inf_cache(keymap=make_key) 
    408413 
    409414    def decorating_function(user_function): 
     
    415420 
    416421        def wrapper(*args, **kwds): 
    417             key = make_key(args, kwds, typed) 
     422            key = make_key(*args, **kwds) 
    418423 
    419424            # get cache entry or compute if not found 
     
    451456 
    452457 
    453 def _test_hits(algorithm, maxsize=20, typed=False, rangelimit=5, tries=1000): 
    454  
    455     @algorithm(maxsize=maxsize, typed=typed) 
     458def _test_hits(algorithm, maxsize=20, keymap=None, rangelimit=5, tries=1000): 
     459 
     460    @algorithm(maxsize=maxsize, keymap=keymap) 
    456461    def f(x, y): 
    457462        return 3*x+y 
  • branches/decorate/keymaps.py

    r662 r663  
    1 # helper functions for caching 
    2 try: 
    3     from collections import namedtuple 
    4 except ImportError: 
    5     from namedtuple import namedtuple 
     1# 
    62 
    7 _CacheInfo = namedtuple("CacheInfo", ['hits','misses','maxsize','currsize']) 
     3class _Sentinel(object): 
     4    def __repr__(self): 
     5        return "<SENTINEL>" 
     6class _NoSentinel(object): 
     7    def __repr__(self): 
     8        return "<NOSENTINEL>" 
     9 
     10SENTINEL = _Sentinel() 
     11NOSENTINEL = _NoSentinel() 
     12# SENTINEL = object() 
     13# NOSENTINEL = (SENTINEL,)  #XXX: use to indicate "don't use a sentinel" ? 
    814 
    915 
    10 #XXX: convert this into a 'keymap' class... 
    11 def keymap(args, kwds, #XXX: should only take args kwds; rest are 'settings' 
    12            typed = False, 
    13            kwd_mark = (object(),), #XXX: 'nicer' kwd_mark = ("",) ?  None ? 
    14            flat = True, #XXX: if not flat, then key = (args, tuple) 
    15            fasttypes = set((int, str, frozenset, type(None))), 
    16            sorted=sorted, tuple=tuple, type=type, len=len): 
    17     'Make a cache key from optionally typed positional and keyword arguments' 
    18     if not flat: 
     16class keymap(object): 
     17 
     18    def __init__(self, typed=False, flat=True, sentinel=NOSENTINEL, **kwds): 
     19        '''initialize the key builder 
     20 
     21        typed: if True, include type information in the key 
     22        flat: if True, flatten the key to a sequence; if False, use (args, kwds) 
     23        sentinel: marker for separating args and kwds in flattened keys 
     24        ''' 
     25        self.typed = typed 
     26        self.flat = flat 
     27        self.sentinel = sentinel 
     28 
     29        # some rare kwds that allow keymap customization 
     30        self._fasttypes = kwds.get('fasttypes', set((int,str,frozenset,type(None)))) 
     31        self._sorted = kwds.get('sorted', sorted) 
     32        self._tuple = kwds.get('tuple', tuple) 
     33        self._type = kwds.get('type', type) 
     34        self._len = kwds.get('len', len) 
     35        return 
     36 
     37    def __get_sentinel(self): 
     38        if self._mark: 
     39            return self._mark[0] 
     40        return NOSENTINEL #XXX: or None? 
     41    def __sentinel(self, mark): 
     42        if mark != NOSENTINEL: 
     43            self._mark = (mark,) 
     44        else: self._mark = None 
     45 
     46    def __call__(self, *args, **kwds): 
     47        'Make cache key from optionally typed positional and keyword arguments' 
     48        if self.flat: 
     49            return self.encode(*args, **kwds) 
     50        return self.encrypt(*args, **kwds) 
     51 
     52    def encrypt(self, *args, **kwds): 
     53        """use a non-flat scheme for producing a key""" 
    1954        key = (args, kwds) #XXX: pickles larger, but is simpler to unpack 
    20         if typed: 
    21             sorted_items = sorted(kwds.items()) 
    22             key += (tuple(type(v) for v in args), \ 
    23                     tuple(type(v) for k, v in sorted_items)) 
     55        if self.typed: 
     56            sorted_items = self._sorted(kwds.items()) 
     57            key += (self._tuple(self._type(v) for v in args), \ 
     58                    self._tuple(self._type(v) for (k,v) in sorted_items)) 
    2459        return key 
    25     key = args 
    26     if kwds: 
    27         sorted_items = sorted(kwds.items()) 
    28         key += kwd_mark 
    29         for item in sorted_items: 
    30             key += item 
    31     if typed: #XXX: 'kwd_mark' between each of the 4 parts, so easy to split 
    32         key += kwd_mark + tuple(type(v) for v in args) 
     60 
     61    def encode(self, *args, **kwds): 
     62        """use a flattened scheme for producing a key""" 
     63        key = args 
    3364        if kwds: 
    34             key += kwd_mark + tuple(type(v) for k, v in sorted_items) 
    35     elif len(key) == 1 and type(key[0]) in fasttypes: 
    36         return key[0] 
    37     return key 
    38 #   return _HashedSeq(key) 
     65            sorted_items = self._sorted(kwds.items()) 
     66            if self._mark: key += self._mark 
     67            for item in sorted_items: 
     68                key += item 
     69        if self.typed: #XXX: 'mark' between each part, so easy to split 
     70            if self._mark: key += self._mark 
     71            key += self._tuple(self._type(v) for v in args) 
     72            if kwds: 
     73                if self._mark: key += self._mark 
     74                key += self._tuple(self._type(v) for (k,v) in sorted_items) 
     75        elif self._len(key) == 1 and self._type(key[0]) in self._fasttypes: 
     76            return key[0] 
     77        return key 
    3978 
    40 ''' 
    41 class _HashedSeq(list): 
    42     __slots__ = 'hashvalue' 
     79    def decrypt(self, key): 
     80        raise NotImplementedError, "Key decryption is not implemented" 
    4381 
    44     def __init__(self, tup, hash=hash): 
    45         self[:] = tup 
    46         self.hashvalue = hash(tup) 
     82    def decode(self, key): 
     83        raise NotImplementedError, "Key decoding is not implemented" 
    4784 
    48     def __hash__(self): 
    49         return self.hashvalue 
     85    def dumps(self, obj): 
     86        """a more pickle-like interface for encoding a key""" 
     87        return self.encode(obj) 
     88 
     89    def loads(self, key): 
     90        """a more pickle-like interface for decoding a key""" 
     91        return self.decode(key) 
     92 
     93    # interface 
     94    sentinel = property(__get_sentinel, __sentinel) 
     95    pass 
    5096 
    5197 
    52 def keymap(args, kwds, kwd_mark=object()): 
    53     """kwd_mark is a separator between args and kwds""" 
    54     key = args 
    55     if kwds: 
    56         key += (kwd_mark,) + tuple(sorted(kwds.items())) 
    57     return key 
    58 ''' 
     98class hashmap(keymap): 
     99    def encode(self, *args, **kwds): 
     100        return hash(keymap.encode(self, *args, **kwds)) 
     101    def encrypt(self, *args, **kwds): 
     102        return hash(keymap.encrypt(self, *args, **kwds)) 
    59103 
    60 def hashmap(*args, **kwds): 
    61     return hash(keymap(*args, **kwds)) 
     104class stringmap(keymap): 
     105   #def __init__(self, *args, **kwds): 
     106   #    keymap.__init__(self, *args, **kwds) 
     107   #    self.typed = False  #XXX: is always typed, so set typed=False 
     108    def encode(self, *args, **kwds): 
     109        return str(keymap.encode(self, *args, **kwds)) 
     110    def encrypt(self, *args, **kwds): 
     111        return str(keymap.encrypt(self, *args, **kwds)) 
    62112 
    63113import dill as pickle 
    64 def picklemap(*args, **kwds): #XXX: is always typed, so set typed=False 
    65     kwds['typed'] = kwds.get('typed', False) 
    66     return pickle.dumps(keymap(*args, **kwds)) 
    67  
    68 def stringmap(*args, **kwds): #XXX: is always typed, so set typed=False 
    69     kwds['typed'] = kwds.get('typed', False) 
    70     return str(keymap(*args, **kwds)) 
     114class picklemap(keymap): 
     115   #def __init__(self, *args, **kwds): 
     116   #    keymap.__init__(self, *args, **kwds) 
     117   #    self.typed = False  #XXX: is always typed, so set typed=False 
     118    def encode(self, *args, **kwds): 
     119        return pickle.dumps(keymap.encode(self, *args, **kwds)) 
     120    def encrypt(self, *args, **kwds): 
     121        return pickle.dumps(keymap.encrypt(self, *args, **kwds)) 
    71122 
    72123 
    73 class Counter(dict): 
    74     'Mapping where default values are zero' 
    75     def __missing__(self, key): 
    76         return 0 
    77  
    78  
     124# EOF 
  • branches/decorate/memoize.py

    r662 r663  
    22decorators that cache results to memory, to file, or to a database 
    33""" 
     4from keymaps import stringmap 
    45 
    56__all__ = ['memoize','memoized','archive_dict','db_dict'] 
     
    338339 
    339340 
    340 def memoized(memo=None, serializer=str, tol=None, deep=False, archived=False): 
     341def memoized(memo=None, keymap=None, tol=None, deep=False, archived=False): 
    341342    """Decorator that memoizes a function's return value each time it is called. 
    342343    If called later with the same arguments, the memoized value is returned, and 
     
    346347 
    347348    memo = storage hashmap (default is {}) 
    348     serializer = serializing function (e.g. pickle.dumps, but default is str) 
     349    keymap = cache key encoder (default is keymaps.stringmap(flat=False)) 
    349350    tol = integer tolerance for rounding (default is None) 
    350351    deep = boolean for rounding depth (default is False, i.e. 'shallow') 
    351352    archived = boolean for archiving (default is False, i.e. "don't archive") 
    352353    """ 
     354    if keymap is None: keymap = stringmap(flat=False) 
    353355    if memo is None: memo = archive_dict() 
    354356    elif type(memo) is dict: memo = archive_dict(memo) 
     
    367369            try: 
    368370                _args, _kwds = rounded_args(*args, **kwds) 
    369                 argstr = serializer((_args, _kwds)) 
     371                argstr = keymap(*_args, **_kwds) 
    370372                if memo.has_key(argstr): 
    371373                    return memo[argstr] 
     
    388390#FIXME: use cache maxsize algorithms... where dump if maxsize 
    389391#FIXME: can make trash_archive where archives to del 
    390 #FIXME: can have serializer be 'hash' or lambda x:x 
    391 #FIXME: should sort(kwds.items) in argstr; probably add an object to separate  
    392392 
    393393class memoize(object): 
     
    397397    Can memoize a *method* on an object. 
    398398    """ 
    399     def __init__(self, memo=None, serializer=str, tol=None, deep=False): 
     399    def __init__(self, memo=None, keymap=None, tol=None, deep=False): 
    400400#     self.func = func 
     401      if keymap is None: keymap = stringmap(flat=False) 
    401402      if memo is None: memo = archive_dict() 
    402403      elif type(memo) is dict: memo = archive_dict(memo) 
    403404      self.memo = memo 
    404       self.__serializer = serializer 
     405      self.__keymap = keymap 
    405406 
    406407      if deep: rounded = deep_round 
     
    420421        try: 
    421422          _args, _kwds = self.__rounded_args(*args, **kwds) 
    422           argstr = self.__serializer((_args, _kwds)) 
     423          argstr = self.__keymap(*_args, **_kwds) 
    423424          if self.memo.has_key(argstr): 
    424425            return self.memo[argstr] 
  • branches/decorate/surrogate.py

    r658 r663  
    2020 
    2121 
    22 import dill 
    2322from memoize import memoized 
    24 #@memoized(serializer=dill.dumps, tol=0, deep=True) # slower, but more robust 
     23from keymaps import picklemap 
     24dumps = picklemap(flat=False) 
     25#@memoized(keymap=dumps, tol=0, deep=True) # slower, but more robust 
    2526#@memoized(tol=0, deep=True) 
    26 #@memoized(serializer=dill.dumps, archived=True)    # slower, but more robust 
     27#@memoized(keymap=dumps, archived=True)    # slower, but more robust 
    2728@memoized(archived=True) 
    2829def marc_surr(x): 
  • branches/decorate/test_memoize.py

    r658 r663  
    2424#from memoize import memoize 
    2525from timer import timed 
    26 import dill 
    27  
     26from keymaps import picklemap 
     27dumps = picklemap(flat=False) 
    2828 
    2929class Spam(object): 
    3030    """A simple class with a memoized method""" 
    3131 
    32     @memoized(serializer=dill.dumps) 
     32    @memoized(keymap=dumps) 
    3333    def eggs(self, *args, **kwds): 
    3434        print 'new:', args, kwds 
     
    4949 
    5050# here caching saves time in a recursive function... 
    51 @memoized(serializer=dill.dumps) 
     51@memoized(keymap=dumps) 
    5252@timed() 
    5353def fibonacci(n): 
     
    6464 
    6565from numpy import sum, asarray 
    66 @memoized(serializer=dill.dumps, tol=3) 
     66@memoized(keymap=dumps, tol=3) 
    6767def add(*args): 
    6868    print 'new:', args 
     
    8686    return sum(x**2 - y**2) 
    8787 
    88 cost1 = memoized(serializer=dill.dumps, tol=1)(cost) 
    89 cost0 = memoized(serializer=dill.dumps, tol=0)(cost) 
    90 costD = memoized(serializer=dill.dumps, tol=0, deep=True)(cost) 
     88cost1 = memoized(keymap=dumps, tol=1)(cost) 
     89cost0 = memoized(keymap=dumps, tol=0)(cost) 
     90costD = memoized(keymap=dumps, tol=0, deep=True)(cost) 
    9191 
    9292print "rounding to one decimals..." 
     
    135135print "re_dict_memo = %s" % add.memo 
    136136 
    137 @memoized(serializer=dill.dumps) 
     137@memoized(keymap=dumps) 
    138138def add(x,y): 
    139139    return x+y 
Note: See TracChangeset for help on using the changeset viewer.