- Timestamp:
- 04/06/13 09:25:33 (3 years ago)
- Location:
- branches/decorate
- Files:
-
- 4 edited
Legend:
- Unmodified
- Added
- Removed
-
branches/decorate/memoize.py
r657 r658 1 1 """ 2 a decorator named 'memoized' that can memoize a *method* on an object. 2 decorators that cache results to memory, to file, or to a database 3 3 """ 4 5 __all__ = ['memoize','memoized','archive_dict','db_dict'] 6 4 7 def isiterable(x): 5 8 #try: … … 43 46 def dec(f): 44 47 def func(*args, **kwds): 45 _deep_round = deep_round_factory(tol) 46 _args,_kwds = _deep_round(*args, **kwds) 48 if tol is None: 49 _args,_kwds = args,kwds 50 else: 51 _deep_round = deep_round_factory(tol) 52 _args,_kwds = _deep_round(*args, **kwds) 47 53 return f(*_args, **_kwds) 48 54 return func … … 65 71 def dec(f): 66 72 def func(*args, **kwds): 67 _simple_round = simple_round_factory(tol) 68 _args,_kwds = _simple_round(*args, **kwds) 73 if tol is None: 74 _args,_kwds = args,kwds 75 else: 76 _simple_round = simple_round_factory(tol) 77 _args,_kwds = _simple_round(*args, **kwds) 69 78 return f(*_args, **_kwds) 70 79 return func … … 94 103 def dec(f): 95 104 def func(*args, **kwds): 96 _shallow_round = shallow_round_factory(tol) 97 _args,_kwds = _shallow_round(*args, **kwds) 105 if tol is None: 106 _args,_kwds = args,kwds 107 else: 108 _shallow_round = shallow_round_factory(tol) 109 _args,_kwds = _shallow_round(*args, **kwds) 98 110 return f(*_args, **_kwds) 99 111 return func … … 101 113 102 114 103 def get_archive(archive):104 import dill as pickle105 try:106 f = open(archive, 'rb')107 cache = pickle.load(f)108 f.close()109 except:110 cache = {}111 return cache112 113 114 def load_factory(memo=None):115 if memo is None: memo = {}116 def load(archive=None, *args):117 cache = get_archive(archive)118 if not args:119 memo.update(cache)120 for arg in args:121 if cache.has_key(arg):122 memo.update({arg:cache[arg]})123 cache.clear()124 return125 return load126 127 128 def dump_factory(memo=None):129 if memo is None: memo = {}130 import dill as pickle131 def dump(archive=None, *args):132 cache = get_archive(archive)133 for arg in args:134 if memo.has_key(arg):135 cache.update({arg:memo[arg]})136 if not args:137 cache.update(memo)138 try:139 f = open(archive, 'wb')140 pickle.dump(cache, f)141 except:142 pass #XXX: warning? fail?143 cache.clear()144 return145 return dump146 147 148 115 #FIXME: the below need expiration of cache due to time, calls, etc... 149 # and potentially r/w to database, file, or other caching mechanism116 # and r/w to file or other caching mechanisms 150 117 #FIXME: memoize*_round fails when decorating a class method 151 152 def memoized_nopickle_round(tol=0, deep=False):153 """Decorator that memoizes a function's return value each time it is called.154 If called later with the same arguments, the memoized value is returned, and155 not re-evaluated. This may lead to memory issues, as memo is never cleared.156 This decorator takes an integer tolerance 'tol', equal to the number of157 decimal places to which it will round off floats.158 This funciton does not pickle, and thus can only handle basic types.159 """160 memo = {}161 162 if deep: rounded = deep_round163 else: rounded = simple_round164 #else: rounded = shallow_round #FIXME: slow165 166 @rounded(tol)167 def rounded_args(*args, **kwds):168 return (args, kwds)169 170 def dec(f):171 def func(*args, **kwds):172 try:173 _args, _kwds = rounded_args(*args, **kwds)174 argstr = str((_args, _kwds))175 if not memo.has_key(argstr):176 memo[argstr] = f(*args, **kwds)177 return memo[argstr]178 except: #TypeError179 return f(*args, **kwds)180 func.memo = memo181 return func182 return dec183 184 185 def memoized_round(tol=0, deep=False):186 """Decorator that memoizes a function's return value each time it is called.187 If called later with the same arguments, the memoized value is returned, and188 not re-evaluated. This may lead to memory issues, as memo is never cleared.189 This decorator takes an integer tolerance 'tol', equal to the number of190 decimal places to which it will round off floats.191 """192 memo = {}193 194 if deep: rounded = deep_round195 else: rounded = simple_round196 #else: rounded = shallow_round #FIXME: slow197 198 @rounded(tol)199 def rounded_args(*args, **kwds):200 return (args, kwds)201 202 def dec(f):203 def func(*args, **kwds):204 try:205 _args, _kwds = rounded_args(*args, **kwds)206 #import cPickle as pickle207 import dill as pickle208 argstr = pickle.dumps((_args, _kwds))209 if not memo.has_key(argstr):210 memo[argstr] = f(*args, **kwds)211 return memo[argstr]212 except: #TypeError213 return f(*args, **kwds)214 func.memo = memo215 return func216 return dec217 218 219 118 220 119 # Want: … … 223 122 # - load an archive / update an existing archive 224 123 # - save some or all of memo to archive 225 def memoized_nopickle_archived(): 226 """Decorator that memoizes a function's return value each time it is called. 227 If called later with the same arguments, the memoized value is returned, and 228 not re-evaluated. This may lead to memory issues, as memo is never cleared. 229 This decorator takes an integer tolerance 'tol', equal to the number of 230 decimal places to which it will round off floats. 231 """ 232 memo = {} 233 load = load_factory(memo) 234 dump = dump_factory(memo) 235 236 def dec(f): 237 def func(*args, **kwds): 238 try: 239 argstr = str((args, kwds)) 240 if not memo.has_key(argstr): 241 load(argstr) 242 if not memo.has_key(argstr): 243 memo[argstr] = f(*args, **kwds) 244 return memo[argstr] 245 except: #TypeError 246 return f(*args, **kwds) 247 func.memo = memo 248 func.load = load 249 func.dump = dump 250 return func 251 return dec 252 253 254 def memoized_archived(): 255 """Decorator that memoizes a function's return value each time it is called. 256 If called later with the same arguments, the memoized value is returned, and 257 not re-evaluated. This may lead to memory issues, as memo is never cleared. 258 This decorator takes an integer tolerance 'tol', equal to the number of 259 decimal places to which it will round off floats. 260 """ 261 memo = {} 262 load = load_factory(memo) 263 dump = dump_factory(memo) 264 265 def dec(f): 266 def func(*args, **kwds): 267 try: 268 #import cPickle as pickle 269 import dill as pickle 270 argstr = pickle.dumps((args, kwds)) 271 if not memo.has_key(argstr): 272 load(argstr) 273 if not memo.has_key(argstr): 274 memo[argstr] = f(*args, **kwds) 275 return memo[argstr] #XXX: any automated dump to archive? 276 except: #TypeError 277 return f(*args, **kwds) 278 func.memo = memo 279 func.load = load #XXX: only use with no arguments? 280 func.dump = dump #XXX: only use with no arguments? 281 return func 282 return dec 124 125 class archive_dict(dict): 126 """dictionary augmented with load and dumps""" 127 def __get_archive(self, archive): 128 try: 129 f = open(archive, 'rb') 130 import dill as pickle 131 cache = pickle.load(f) 132 f.close() 133 except: 134 cache = {} 135 return cache 136 def load(self, archive=None, *args): 137 cache = self.__get_archive(archive) 138 if not args: 139 self.update(cache) 140 for arg in args: 141 if cache.has_key(arg): 142 self.update({arg:cache[arg]}) 143 cache.clear() 144 return 145 def dump(self, archive=None, *args): 146 cache = self.__get_archive(archive) 147 for arg in args: 148 if self.has_key(arg): 149 cache.update({arg:self.__getitem__(arg)}) 150 if not args: 151 cache.update(self) 152 try: 153 f = open(archive, 'wb') 154 import dill as pickle 155 pickle.dump(cache, f) 156 except: 157 pass #XXX: warning? fail? 158 cache.clear() 159 return 160 pass 161 162 163 ''' 164 class file_dict(archive_dict): 165 """dictionary-style interface to a file""" 166 def __init__(self, filename=None, serialized=True): # False 167 import os 168 """filename = full filepath""" 169 if filename is None: 170 if serialized: filename = 'memo.pkl' 171 else: filename = 'memo.py' 172 self._filename = filename 173 if not os.path.exists(filename): 174 if serialized: self.dump(filename) 175 else: open(filename, 'wb').write('memo = {}') 176 return 177 pass 178 ''' 283 179 284 180 285 181 #XXX: should inherit from object not dict ? 286 class db_dict(dict): #XXX: requires UTF-8 key 287 """decorator that can memoize to a database (API mimics a dict) 288 """ 182 class db_dict(archive_dict): #XXX: requires UTF-8 key 183 """dictionary-style interface to a database""" 289 184 def __init__(self, database=None, table=None): 290 185 """database = database url; table = table name""" 291 if database ==None: database = ':memory:'186 if database is None: database = ':memory:' 292 187 self._database = database 293 if table ==None: table = 'memo'188 if table is None: table = 'memo' 294 189 self._table = table 295 190 import sqlite3 as db … … 369 264 370 265 371 def memoized(memo=None, serializer=str ):266 def memoized(memo=None, serializer=str, tol=None, deep=False, archived=False): 372 267 """Decorator that memoizes a function's return value each time it is called. 373 268 If called later with the same arguments, the memoized value is returned, and 374 269 not re-evaluated. This may lead to memory issues, as memo is never cleared. 270 This decorator takes an integer tolerance 'tol', equal to the number of 271 decimal places to which it will round off floats. 375 272 376 273 memo = storage hashmap (default is {}) 377 274 serializer = serializing function (e.g. pickle.dumps, but default is str) 275 tol = integer tolerance for rounding (default is None) 276 deep = boolean for rounding depth (default is False, i.e. 'shallow') 277 archived = boolean for archiving (default is False, i.e. "don't archive") 378 278 """ 379 if memo == None: memo = dict() 279 if memo is None: memo = archive_dict() 280 elif type(memo) is dict: memo = archive_dict(memo) 281 # does archive make sense with databas, file, ?... (requires more thought) 282 283 if deep: rounded = deep_round 284 else: rounded = simple_round 285 #else: rounded = shallow_round #FIXME: slow 286 287 @rounded(tol) 288 def rounded_args(*args, **kwds): 289 return (args, kwds) 380 290 381 291 def dec(f): 382 292 def func(*args, **kwds): 383 293 try: 384 argstr = serializer((args, kwds)) 294 _args, _kwds = rounded_args(*args, **kwds) 295 argstr = serializer((_args, _kwds)) 385 296 if memo.has_key(argstr): 386 297 return memo[argstr] 298 if archived: 299 memo.load(argstr) 300 if memo.has_key(argstr): 301 return memo[argstr] 387 302 res = f(*args, **kwds) 388 memo[argstr] = res 303 memo[argstr] = res #XXX: any automated dump to archive? 389 304 return res 390 305 except: #TypeError 391 306 return f(*args, **kwds) 392 307 func.memo = memo 308 if archived: #XXX: archiving should be toggleable? 309 func.load = memo.load #XXX: comment out? 310 func.dump = memo.dump #XXX: comment out? 393 311 return func 394 312 return dec … … 399 317 If called later with the same arguments, the memoized value is returned, and 400 318 not re-evaluated. This may lead to memory issues, as memo is never cleared. 319 Can memoize a *method* on an object. 401 320 """ 402 321 def __init__(self, func): #memo=None, serializer=str): 403 322 self.func = func 404 #if memo ==None: memo = dict()323 #if memo is None: memo = dict() 405 324 self.memo = {} #memo 406 325 import cPickle as pickle -
branches/decorate/surrogate.py
r601 r658 20 20 21 21 22 from memoize import memoized_round, memoized_nopickle_round 23 from memoize import memoized _archived, memoized_nopickle_archived24 #@memoized _round(0, deep=True)# slower, but more robust25 #@memoized _nopickle_round(0, deep=True)26 #@memoized _archived()# slower, but more robust27 @memoized _nopickle_archived()22 import dill 23 from memoize import memoized 24 #@memoized(serializer=dill.dumps, tol=0, deep=True) # slower, but more robust 25 #@memoized(tol=0, deep=True) 26 #@memoized(serializer=dill.dumps, archived=True) # slower, but more robust 27 @memoized(archived=True) 28 28 def marc_surr(x): 29 29 """calculate perforation area using a tanh-based model surrogate -
branches/decorate/test_cached_memoize.py
r601 r658 1 from memoize import memoized_nopickle_archived as memoized 2 #from memoize import memoized_archived as memoized 1 from memoize import memoized 3 2 from timer import timed 4 3 5 4 # here caching saves time in a recursive function... 6 @memoized( )5 @memoized(archived=True) 7 6 @timed() 8 7 def fibonacci(n): -
branches/decorate/test_memoize.py
r655 r658 21 21 """ 22 22 23 #from memoize import memoized_nopickle_round as memoized 24 #from memoize import memoized_round as memoized 25 from memoize import memoized as memoized 23 from memoize import memoized 26 24 #from memoize import memoize 27 25 from timer import timed 26 import dill 28 27 29 28 … … 31 30 """A simple class with a memoized method""" 32 31 33 @memoized( )32 @memoized(serializer=dill.dumps) 34 33 def eggs(self, *args, **kwds): 35 34 print 'new:', args, kwds … … 50 49 51 50 # here caching saves time in a recursive function... 52 @memoized( )51 @memoized(serializer=dill.dumps) 53 52 @timed() 54 53 def fibonacci(n): … … 64 63 print '=' * 30 65 64 66 from memoize import memoized_round67 65 from numpy import sum, asarray 68 @memoized _round(3)66 @memoized(serializer=dill.dumps, tol=3) 69 67 def add(*args): 70 68 print 'new:', args … … 88 86 return sum(x**2 - y**2) 89 87 90 cost1 = memoized _round(1)(cost)91 cost0 = memoized _round(0)(cost)92 costD = memoized _round(0, deep=True)(cost)88 cost1 = memoized(serializer=dill.dumps, tol=1)(cost) 89 cost0 = memoized(serializer=dill.dumps, tol=0)(cost) 90 costD = memoized(serializer=dill.dumps, tol=0, deep=True)(cost) 93 91 94 92 print "rounding to one decimals..." … … 112 110 113 111 114 from memoize import memoized115 112 from memoize import db_dict 116 import dill as pickle113 import dill 117 114 @memoized(memo=db_dict()) 118 115 def add(x,y): … … 138 135 print "re_dict_memo = %s" % add.memo 139 136 140 @memoized(serializer= pickle.dumps)137 @memoized(serializer=dill.dumps) 141 138 def add(x,y): 142 139 return x+y
Note: See TracChangeset
for help on using the changeset viewer.