- Timestamp:
- 05/29/13 13:10:35 (3 years ago)
- Location:
- branches/decorate
- Files:
-
- 6 edited
Legend:
- Unmodified
- Added
- Removed
-
branches/decorate/archives.py
r665 r668 1 #!/usr/bin/env python 1 2 """ 2 3 custom caching dict, which archives results to memory, file, or database … … 79 80 return 80 81 def __asdict__(self): 82 """build a dictionary containing the archive contents""" 81 83 return self 82 84 def __setitem__(self, key, value): 83 85 pass 86 __setitem__.__doc__ = dict.__setitem__.__doc__ 84 87 def update(self, adict, **kwds): 85 88 pass 89 update.__doc__ = dict.update.__doc__ 86 90 def setdefault(self, key, *value): 87 91 return self.get(key, *value) 92 setdefault.__doc__ = dict.setdefault.__doc__ 88 93 def __repr__(self): 89 94 return "archive(NULL)" 95 __repr__.__doc__ = dict.__repr__.__doc__ 90 96 pass 91 97 … … 111 117 return 112 118 def __asdict__(self): 119 """build a dictionary containing the archive contents""" 113 120 if self._serialized: 114 121 try: … … 131 138 return memo 132 139 def __save__(self, memo=None): 140 """create an archive from the given dictionary""" 133 141 if memo == None: return 134 142 if self._serialized: … … 147 155 memo = self.__asdict__() 148 156 return memo[key] 157 __getitem__.__doc__ = dict.__getitem__.__doc__ 149 158 def __iter__(self): 150 159 return self.__asdict__().iterkeys() 160 __iter__.__doc__ = dict.__iter__.__doc__ 151 161 def __repr__(self): 152 162 return "archive(%s: %s)" % (self._filename, self.__asdict__()) 163 __repr__.__doc__ = dict.__repr__.__doc__ 153 164 def __setitem__(self, key, value): 154 165 memo = self.__asdict__() … … 156 167 self.__save__(memo) 157 168 return 169 __setitem__.__doc__ = dict.__setitem__.__doc__ 158 170 def clear(self): 159 171 self.__save__({}) 160 172 return 173 clear.__doc__ = dict.clear.__doc__ 161 174 #FIXME: copy, fromkeys 162 175 def get(self, key, value=None): 163 176 memo = self.__asdict__() 164 177 return memo.get(key, value) 178 get.__doc__ = dict.get.__doc__ 165 179 def has_key(self, key): 166 180 return key in self.__asdict__() 181 has_key.__doc__ = dict.has_key.__doc__ 167 182 def items(self): 168 183 return list(self.iteritems()) 184 items.__doc__ = dict.items.__doc__ 169 185 def iteritems(self): 170 186 return self.__asdict__().iteritems() 187 iteritems.__doc__ = dict.iteritems.__doc__ 171 188 iterkeys = __iter__ 172 189 def itervalues(self): 173 190 return self.__asdict__().itervalues() 191 itervalues.__doc__ = dict.itervalues.__doc__ 174 192 def keys(self): 175 193 return list(self.__iter__()) 194 keys.__doc__ = dict.keys.__doc__ 176 195 def pop(self, key, *value): 177 196 memo = self.__asdict__() … … 179 198 self.__save__(memo) 180 199 return res 200 pop.__doc__ = dict.pop.__doc__ 181 201 #FIXME: popitem 182 202 def setdefault(self, key, *value): … … 184 204 self.__setitem__(key, res) 185 205 return res 206 setdefault.__doc__ = dict.setdefault.__doc__ 186 207 def update(self, adict, **kwds): 187 208 memo = self.__asdict__() … … 189 210 self.__save__(memo) 190 211 return 212 update.__doc__ = dict.update.__doc__ 191 213 def values(self): 192 214 return list(self.itervalues()) 215 values.__doc__ = dict.values.__doc__ 193 216 pass 194 217 … … 215 238 return 216 239 def __asdict__(self): 240 """build a dictionary containing the archive contents""" 217 241 sql = "select * from %s" % self._table 218 242 res = self._curs.execute(sql) … … 225 249 if res: return res[-1][-1] # always get the last one 226 250 raise KeyError, key 251 __getitem__.__doc__ = dict.__getitem__.__doc__ 227 252 def __iter__(self): 228 253 sql = "select argstr from %s" % self._table 229 254 return (k[-1] for k in set(self._curs.execute(sql))) 255 __iter__.__doc__ = dict.__iter__.__doc__ 230 256 def __repr__(self): 231 257 return "archive(%s: %s)" % (self._table, self.__asdict__()) 258 __repr__.__doc__ = dict.__repr__.__doc__ 232 259 def __setitem__(self, key, value): #XXX: maintains 'history' of values 233 260 sql = "insert into %s values(?,?)" % self._table … … 235 262 self._conn.commit() 236 263 return 264 __setitem__.__doc__ = dict.__setitem__.__doc__ 237 265 def clear(self): 238 266 [self.pop(k) for k in self.keys()] # better delete table, add empty ? 239 267 return 268 clear.__doc__ = dict.clear.__doc__ 240 269 #FIXME: copy, fromkeys 241 270 def get(self, key, value=None): … … 243 272 if res: value = res[-1][-1] 244 273 return value 274 get.__doc__ = dict.get.__doc__ 245 275 def has_key(self, key): 246 276 return bool(self._select_key_items(key)) 277 has_key.__doc__ = dict.has_key.__doc__ 247 278 def items(self): 248 279 #return self.__asdict__().items() 249 280 return list(self.iteritems()) 281 items.__doc__ = dict.items.__doc__ 250 282 def iteritems(self): 251 283 return ((k,self.__getitem__(k)) for k in self.__iter__()) 284 iteritems.__doc__ = dict.iteritems.__doc__ 252 285 iterkeys = __iter__ 253 286 def itervalues(self): 254 287 return (self.__getitem__(k) for k in self.__iter__()) 288 itervalues.__doc__ = dict.itervalues.__doc__ 255 289 def keys(self): 256 290 #return self.__asdict__().keys() 257 291 return list(self.__iter__()) 292 keys.__doc__ = dict.keys.__doc__ 258 293 def pop(self, key, *value): 259 294 L = len(value) … … 270 305 self._conn.commit() 271 306 return _value 307 pop.__doc__ = dict.pop.__doc__ 272 308 #FIXME: popitem 273 309 def setdefault(self, key, *value): … … 283 319 self.__setitem__(key, _value) 284 320 return _value 321 setdefault.__doc__ = dict.setdefault.__doc__ 285 322 def update(self, adict, **kwds): 286 323 _dict = adict.copy() … … 288 325 [self.__setitem__(k,v) for (k,v) in _dict.items()] 289 326 return 327 update.__doc__ = dict.update.__doc__ 290 328 def values(self): 291 329 #return self.__asdict__().values() 292 330 return list(self.itervalues()) 331 values.__doc__ = dict.values.__doc__ 293 332 def _select_key_items(self, key): 294 '''Return a tuple of (key, value) pairs that match the specified key. 295 ''' 333 '''Return a tuple of (key, value) pairs that match the specified key''' 296 334 sql = "select * from %s where argstr = ?" % self._table 297 335 return tuple(self._curs.execute(sql, (key,))) -
branches/decorate/cache.py
r667 r668 1 # 1 #!/usr/bin/env python 2 # code inspired by Raymond Hettinger's LFU and LRU cache decorators 3 # on http://code.activestate.com/recipes/498245-lru-and-lfu-cache-decorators 4 # and subsequent forks as well as the version available in python3.3 5 """ 6 a selection of caching decorators 7 """ 2 8 3 9 try: … … 15 21 from archives import archive_dict 16 22 from keymaps import hashmap 23 24 __all__ = ['_CacheInfo','Counter','no_cache','inf_cache',\ 25 'lfu_cache','lru_cache','mru_cache','rr_cache'] 17 26 18 27 _CacheInfo = namedtuple("CacheInfo", ['hit','miss','load','maxsize','size']) -
branches/decorate/keymaps.py
r663 r668 1 # 1 #!/usr/bin/env python 2 """ 3 custom 'keymaps' for generating dictionary keys from function input signatures 4 """ 5 6 __all__ = ['SENTINEL','NOSENTINEL','keymap','hashmap','stringmap','picklemap'] 2 7 3 8 class _Sentinel(object): 9 """build a sentinel object for the SENTINEL singleton""" 4 10 def __repr__(self): 5 11 return "<SENTINEL>" 6 12 class _NoSentinel(object): 13 """build a sentinel object for the NOSENTINEL singleton""" 7 14 def __repr__(self): 8 15 return "<NOSENTINEL>" … … 15 22 16 23 class keymap(object): 24 """tool for converting a function's input signature to an unique key 17 25 26 This keymap does not serialize objects, but does do some formatting. 27 Since the keys are stored as raw objects, there is no information loss, 28 and thus it is easy to recover the original input signature. However, 29 to use an object as a key, the object must be hashable. 30 """ 18 31 def __init__(self, typed=False, flat=True, sentinel=NOSENTINEL, **kwds): 19 32 '''initialize the key builder … … 22 35 flat: if True, flatten the key to a sequence; if False, use (args, kwds) 23 36 sentinel: marker for separating args and kwds in flattened keys 37 38 This keymap stores function args and kwds as (args, kwds) if flat=False, 39 or a flattened (*args, zip(**kwds)) if flat=True. If typed, then 40 include a tuple of type information (args, kwds, argstypes, kwdstypes) 41 in the generated key. If a sentinel is given, the sentinel will be 42 added to a flattened key to indicate the boundary between args, keys, 43 argstypes, and kwdstypes. 24 44 ''' 25 45 self.typed = typed … … 45 65 46 66 def __call__(self, *args, **kwds): 47 ' Make cachekey from optionally typed positional and keyword arguments'67 'generate a key from optionally typed positional and keyword arguments' 48 68 if self.flat: 49 69 return self.encode(*args, **kwds) … … 51 71 52 72 def encrypt(self, *args, **kwds): 53 """use a non-flat scheme for producing a key"""73 """use a non-flat scheme for generating a key""" 54 74 key = (args, kwds) #XXX: pickles larger, but is simpler to unpack 55 75 if self.typed: … … 60 80 61 81 def encode(self, *args, **kwds): 62 """use a flattened scheme for producing a key"""82 """use a flattened scheme for generating a key""" 63 83 key = args 64 84 if kwds: … … 78 98 79 99 def decrypt(self, key): 100 """recover the stored value directly from a generated (non-flat) key""" 80 101 raise NotImplementedError, "Key decryption is not implemented" 81 102 82 103 def decode(self, key): 104 """recover the stored value directly from a generated (flattened) key""" 83 105 raise NotImplementedError, "Key decoding is not implemented" 84 106 … … 97 119 98 120 class hashmap(keymap): 121 """tool for converting a function's input signature to an unique key 122 123 This keymap generates a hash for the given object. Not all objects are 124 hashable, and generating a hash incurrs some information loss. Hashing 125 is fast, however there is not a method to recover the input signature 126 from a hash. 127 """ 99 128 def encode(self, *args, **kwds): 129 """use a flattened scheme for generating a key""" 100 130 return hash(keymap.encode(self, *args, **kwds)) 101 131 def encrypt(self, *args, **kwds): 132 """use a non-flat scheme for generating a key""" 102 133 return hash(keymap.encrypt(self, *args, **kwds)) 103 134 104 135 class stringmap(keymap): 136 """tool for converting a function's input signature to an unique key 137 138 This keymap serializes objects by converting __repr__ to a string. 139 Converting to a string is slower than hashing, however will produce a 140 key in all cases. Some forms of archival storage, like a database, 141 may require string keys. There is not a method to recover the input 142 signature from a string key that works in all cases, however this is 143 possibe for any object where __repr__ effectively mimics __init__. 144 """ 105 145 #def __init__(self, *args, **kwds): 106 146 # keymap.__init__(self, *args, **kwds) 107 147 # self.typed = False #XXX: is always typed, so set typed=False 108 148 def encode(self, *args, **kwds): 149 """use a flattened scheme for generating a key""" 109 150 return str(keymap.encode(self, *args, **kwds)) 110 151 def encrypt(self, *args, **kwds): 152 """use a non-flat scheme for generating a key""" 111 153 return str(keymap.encrypt(self, *args, **kwds)) 112 154 113 155 import dill as pickle 114 156 class picklemap(keymap): 157 """tool for converting a function's input signature to an unique key 158 159 This keymap serializes objects by pickling the object. Serializing an 160 object with pickle is relatively slower, however will reliably produce a 161 unique key for all picklable objects. Also, pickling is a reversible 162 operation, where the original input signature can be recovered from the 163 generated key. 164 """ 115 165 #def __init__(self, *args, **kwds): 116 166 # keymap.__init__(self, *args, **kwds) 117 167 # self.typed = False #XXX: is always typed, so set typed=False 118 168 def encode(self, *args, **kwds): 169 """use a flattened scheme for generating a key""" 119 170 return pickle.dumps(keymap.encode(self, *args, **kwds)) 120 171 def encrypt(self, *args, **kwds): 172 """use a non-flat scheme for generating a key""" 121 173 return pickle.dumps(keymap.encrypt(self, *args, **kwds)) 122 174 -
branches/decorate/namedtuple.py
r660 r668 1 #!/usr/bin/env python 2 # this code was written by Raymond Hettinger 3 # available at http://code.activestate.com/recipes/500261-named-tuples 4 """ 5 named tuple: a subclass of tuple with named fields 6 """ 1 7 from operator import itemgetter as _itemgetter 2 8 from keyword import iskeyword as _iskeyword -
branches/decorate/rounding.py
r665 r668 1 #!/usr/bin/env python 1 2 """ 2 3 decorators that provide rounding … … 6 7 7 8 def isiterable(x): 9 """check if an object is iterable""" 8 10 #try: 9 11 # from collections import Iterable … … 16 18 #return hasattr(x, '__len__') or hasattr(x, '__iter__') 17 19 20 #FIXME: these seem *slow*... and a bit convoluted. Maybe rewrite as classes? 18 21 19 22 def deep_round_factory(tol): 23 """helper function for deep_round (a factory for deep_round functions)""" 20 24 def deep_round(*args, **kwds): 21 25 argstype = type(args) … … 46 50 47 51 def deep_round(tol=0): 52 """decorator for rounding a function's input argument and keywords to the 53 given precision *tol*. This decorator always rounds to a floating point 54 number. 55 56 Rounding is done recursively for each element of all arguments and keywords. 57 58 For example: 59 >>> @deep_round(tol=1) 60 ... def add(x,y): 61 ... return x+y 62 ... 63 >>> add(2.54, 5.47) 64 8.0 65 >>> 66 >>> # rounds each float, regardless of depth in an object 67 >>> add([2.54, 5.47],['x','y']) 68 [2.5, 5.5, 'x', 'y'] 69 >>> 70 >>> # rounds each float, regardless of depth in an object 71 >>> add([2.54, 5.47],['x',[8.99, 'y']]) 72 [2.5, 5.5, 'x', [9.0, 'y']] 73 """ 48 74 def dec(f): 49 75 def func(*args, **kwds): … … 59 85 60 86 def simple_round_factory(tol): 87 """helper function for simple_round (a factory for simple_round functions)""" 61 88 def simple_round(*args, **kwds): 62 89 argstype = type(args) … … 71 98 72 99 def simple_round(tol=0): #NOTE: only rounds floats, nothing else 100 """decorator for rounding a function's input argument and keywords to the 101 given precision *tol*. This decorator always rounds to a floating point 102 number. 103 104 Rounding is only done for arguments or keywords that are floats. 105 106 For example: 107 >>> @simple_round(tol=1) 108 ... def add(x,y): 109 ... return x+y 110 ... 111 >>> add(2.54, 5.47) 112 8.0 113 >>> 114 >>> # does not round elements of iterables, only rounds at the top-level 115 >>> add([2.54, 5.47],['x','y']) 116 [2.54, 5.4699999999999998, 'x', 'y'] 117 >>> 118 >>> # does not round elements of iterables, only rounds at the top-level 119 >>> add([2.54, 5.47],['x',[8.99, 'y']]) 120 [2.54, 5.4699999999999998, 'x', [8.9900000000000002, 'y']] 121 """ 73 122 def dec(f): 74 123 def func(*args, **kwds): … … 84 133 85 134 def shallow_round_factory(tol): 135 """helper function for shallow_round (a factory for shallow_round functions)""" 86 136 from numpy import round as around #XXX: try or numpy makes this slow 87 137 def shallow_round(*args, **kwds): … … 103 153 104 154 def shallow_round(tol=0): #NOTE: rounds floats, lists, arrays one level deep 155 """decorator for rounding a function's input argument and keywords to the 156 given precision *tol*. This decorator always rounds to a floating point 157 number. 158 159 Rounding is done recursively for each element of all arguments and keywords, 160 however the rounding is shallow (a max of one level deep into each object). 161 162 For example: 163 >>> @shallow_round(tol=1) 164 ... def add(x,y): 165 ... return x+y 166 ... 167 >>> add(2.54, 5.47) 168 8.0 169 >>> 170 >>> # rounds each float, at the top-level or first-level of each object. 171 >>> add([2.54, 5.47],['x','y']) 172 [2.5, 5.5, 'x', 'y'] 173 >>> 174 >>> # rounds each float, at the top-level or first-level of each object. 175 >>> add([2.54, 5.47],['x',[8.99, 'y']]) 176 [2.5, 5.5, 'x', [8.9900000000000002, 'y']] 177 """ 105 178 def dec(f): 106 179 def func(*args, **kwds): -
branches/decorate/safecache.py
r667 r668 1 # 1 #!/usr/bin/env python 2 # code inspired by Raymond Hettinger's LFU and LRU cache decorators 3 # on http://code.activestate.com/recipes/498245-lru-and-lfu-cache-decorators 4 # and subsequent forks as well as the version available in python3.3 5 """ 6 'safe' versions of selected caching decorators 7 8 If a hashing error occurs, the cached function will be evaluated. 9 """ 2 10 3 11 try: … … 15 23 from archives import archive_dict 16 24 from keymaps import stringmap 17 from cache import Counter 18 19 _CacheInfo = namedtuple("CacheInfo", ['hit','miss','load','maxsize','size']) 25 from cache import _CacheInfo, Counter 26 27 __all__ = ['no_cache','inf_cache','lfu_cache',\ 28 'lru_cache','mru_cache','rr_cache'] 20 29 21 30 #XXX: what about caches that expire due to time, calls, etc... … … 825 834 if archived: 826 835 from archives import file_archive 827 f.archive(file_archive(' cache.pkl'))836 f.archive(file_archive('safecache.pkl')) 828 837 829 838 domain = range(rangelimit)
Note: See TracChangeset
for help on using the changeset viewer.