Skip to content

I
insights
Commit 68b5db6a by Steve Komarov
Options

removed key_override

parent bee0e50f
Showing with 91 additions and 45 deletions
src/edinsights/core/decorators.py
......@@ -9,12 +9,11 @@
import hashlib
import inspect
import json
import logging
import time
import traceback
from datetime import timedelta
from decorator import decorator
from django.core.cache import cache
......@@ -101,10 +100,15 @@ def query(category = None, name = None, description = None, args = None):
return query_factory
class NotInCacheError(Exception):
class MemoizeNotInCacheError(Exception):
pass
class MemoizeAttributeError(Exception):
pass
def mq_force_memoize(func):
def use_forcememoize(func):
"""
Forces memoization for a function func that has been decorated by
@memoize_query. This means that it will always redo the computation
......@@ -114,22 +118,39 @@ def mq_force_memoize(func):
if hasattr(func, 'force_memoize'):
return func.force_memoize
else:
return func
raise MemoizeAttributeError("Function %s does not have attribute %s" %
func.__name__, "force_memoize")
def mq_force_retrieve(func):
def use_fromcache(func):
"""
Forces retrieval from cache for a function func that has been decorated by
@memoize_query. This means that it will try to get the result from cache.
If the result is not available in cache, it will throw an exception instead
of computing the result.
"""
if hasattr(func, 'force_retrieve'):
return func.force_retrieve
if hasattr(func, 'from_cache'):
return func.from_cache
else:
return func
raise MemoizeAttributeError("Function %s does not have attribute %s" %
func.__name__, "from_cache")
def memoize_query(cache_time = 60*4, timeout = 60*15, ignores = ["<class 'pymongo.database.Database'>", "<class 'fs.osfs.OSFS'>"], key_override=None):
def use_clearcache(func):
if hasattr(func, 'clear_cache'):
return func.clear_cache
else:
raise MemoizeAttributeError("Function %s does not have attribute %s" %
func.__name__, "clear_cache")
# classes to ignore when creating a cache key
from pymongo.database import Database
import fs.osfs
from core.util import CacheHelper
import django.core.cache
DEFAULT_IGNORES = (Database, fs.osfs, CacheHelper, django.core.cache)
def memoize_query(cache_time = 60*4, timeout = 60*15, ignores = DEFAULT_IGNORES):
''' Call function only if we do not have the results for its execution already
We ignore parameters of type pymongo.database.Database and fs.osfs.OSFS. These
will be different per call, but function identically.
......@@ -139,11 +160,12 @@ def memoize_query(cache_time = 60*4, timeout = 60*15, ignores = ["<class 'pymong
'''
# Helper functions
def isuseful(a, ignores):
if str(type(a)) in ignores:
def isuseful(a):
if hasattr(a, 'memoize_ignore') and a.memoize_ignore is True:
return False
return True
def make_cache_key(f, args, kwargs):
"""
Makes a cache key out of the function name and passed arguments
......@@ -158,14 +180,11 @@ def memoize_query(cache_time = 60*4, timeout = 60*15, ignores = ["<class 'pymong
m = hashlib.new("md4")
s = str({'uniquifier': 'anevt.memoize',
'name' : f.__name__,
'module' : f.__module__,
'args': [a for a in args if isuseful(a, ignores)],
'file' : inspect.getmodule(f).__file__,
'args': [a for a in args if isuseful(a)],
'kwargs': kwargs})
m.update(s)
if key_override is not None:
key = key_override
else:
key = m.hexdigest()
key = m.hexdigest()
return key
def compute_and_cache(f, key, args, kwargs):
......@@ -175,6 +194,7 @@ def memoize_query(cache_time = 60*4, timeout = 60*15, ignores = ["<class 'pymong
# HACK: There's a slight race condition here, where we
# might recompute twice.
cache.set(key, 'Processing', timeout)
function_argspec = inspect.getargspec(f)
if function_argspec.varargs or function_argspec.args:
......@@ -206,7 +226,8 @@ def memoize_query(cache_time = 60*4, timeout = 60*15, ignores = ["<class 'pymong
def factory(f):
def opmode_default(f, *args, **kwargs):
def operation_mode_default(f, *args, **kwargs):
# Get he result from cache if possible, otherwise recompute
# and store in cache
key = make_cache_key(f, args, kwargs)
......@@ -219,7 +240,7 @@ def memoize_query(cache_time = 60*4, timeout = 60*15, ignores = ["<class 'pymong
results = compute_and_cache(f,key, args, kwargs)
return results
def opmode_forcememoize(*args, **kwargs):
def operation_mode_forcememoize(*args, **kwargs):
# Recompute and store in cache, regardless of whether key
# is in cache.
key = make_cache_key(f, args, kwargs)
......@@ -227,18 +248,24 @@ def memoize_query(cache_time = 60*4, timeout = 60*15, ignores = ["<class 'pymong
results = compute_and_cache(f, key, args, kwargs)
return results
def opmode_forceretrieve(*args, **kwargs):
def operation_mode_fromcache(*args, **kwargs):
# Retrieve from cache if possible otherwise throw an exception
key = make_cache_key(f, args, kwargs)
# print "Forcing retrieve %s %s " % (f.__name__, key)
key = make_cache_key(f, args, kwargs)
results = get_from_cache_if_possible(f, key)
if not results:
raise NotInCacheError('key %s not found in cache' % key)
raise MemoizeNotInCacheError('key %s not found in cache' % key)
return results
decfun = decorator(opmode_default,f)
decfun.force_memoize = opmode_forcememoize # activated by mq_force_memoize
decfun.force_retrieve = opmode_forceretrieve # activated by mq_force_retrieve
def operation_mode_clearcache(*args, **kwargs):
key = make_cache_key(f, args, kwargs)
return cache.delete(key)
decfun = decorator(operation_mode_default,f)
decfun.force_memoize = operation_mode_forcememoize # activated by use_forcememoize
decfun.from_cache = operation_mode_fromcache # activated by use_fromcache
decfun.clear_cache = operation_mode_clearcache
return decfun
return factory
......@@ -257,15 +284,20 @@ def cron(run_every, force_memoize=False, params={}):
def factory(f):
@periodic_task(run_every=run_every, name=f.__name__)
def run(func=None, *args, **kw):
# if the call originated from the periodic_task decorator
# func will be None. If the call originated from the rest of
# the code, func will be the same as f
# This function can be called from two distinct places. It can be
# called by the task scheduler (due to @periodic_task),
# in which case func will be None.
# It can also be called as a result of calling the function we
# are currently decorating with @cron. In this case func will be
# the same as f.
# Was it called from the task scheduler?
called_as_periodic = True if func is None else False
if called_as_periodic:
#print "called as periodic"
if force_memoize:
func = mq_force_memoize(f)
func = use_forcememoize(f)
else:
func = f
else:
......
src/edinsights/core/util.py
......@@ -136,6 +136,8 @@ def optional_parameter_call(function, optional_kwargs, passed_kwargs, arglist =
# precedence.
if arg in optional_kwargs:
args[arg] = optional_kwargs[arg](function)
#ignore default arguments in memoize
args[arg].memoize_ignore = True
elif arg in passed_kwargs:
args[arg] = passed_kwargs[arg]
else:
......
src/edinsights/periodic/__init__.py
# This module provides tests for periodic tasks using core.decorators.cron
from edinsights.core.decorators import view, mq_force_retrieve, NotInCacheError
from edinsights.periodic.tasks import big_computation, big_computation_withfm
from edinsights.core.decorators import view, use_fromcache, MemoizeNotInCacheError
from edinsights.periodic.tasks import big_computation
from edinsights.periodic.tasks import big_computation_withfm
#
@view()
def big_computation_visualizer():
return "<html>%s</html>" % mq_force_retrieve(big_computation)()
return "<html>%s</html>" % use_fromcache(big_computation)()
@view()
......@@ -13,8 +14,8 @@ def big_computation_visualizer_withfm():
try:
# returns instantly, does not perform computation if results are not
# in cache
result = mq_force_retrieve(big_computation_withfm)()
except NotInCacheError:
result = use_fromcache(big_computation_withfm)()
except MemoizeNotInCacheError:
result = "The big computation has not been performed yet"
# alternatively you can display a "please wait" message
# and run big_computation_withfm() without force_retrieve
......
src/edinsights/periodic/tasks.py
......@@ -14,17 +14,19 @@ def timestamp_to_tempfile(filename):
# methods(the support of @periodic_task for these is experimental)
# The @cron decorator should precede all other decorators
@cron(run_every=timedelta(seconds=1))
def test_cron_task():
""" Simple task that gets executed by the scheduler (celery beat).
tested by: tests.SimpleTest.test_cron
"""
timestamp_to_tempfile('test_cron_task_counter')
@cron(run_every=timedelta(seconds=1), force_memoize=False) # cron decorators should go on top
@memoize_query(60, key_override='test_cron_memoize_unique_cache_key')
def test_cron_memoize_task():
@memoize_query(60)
def test_cron_memoize_task(fs):
"""
Simple task that gets executed by the scheduler (celery beat).
Combines periodic tasks and memoization, with force_memoize=False.
......@@ -33,12 +35,13 @@ def test_cron_memoize_task():
tested by: tests.SimpleTest.test_cron_and_memoize
"""
timestamp_to_tempfile('test_cron_memoize_task')
return 42
@cron(run_every=timedelta(seconds=1), force_memoize=False) # cron decorators should go on top
@memoize_query(cache_time=60, key_override='big_computation_key')
@memoize_query(cache_time=60)
def big_computation():
"""
Simple task that gets executed by the scheduler (celery beat) and also by @view
......@@ -54,7 +57,7 @@ def big_computation():
@cron(run_every=timedelta(seconds=1), force_memoize=True) # cron decorators should go on top
@memoize_query(cache_time=60, key_override='big_computation_key_withfm')
@memoize_query(cache_time=60)
def big_computation_withfm():
"""
Simple task that gets executed by the scheduler (celery beat) and also by @view
......
src/edinsights/periodic/tests.py
......@@ -5,7 +5,7 @@ import time
from django.test import TestCase
from django.test.client import Client
from django.core.cache import cache
from core.decorators import use_clearcache
def count_timestamps(tempfilename):
with open(tempfile.gettempdir() + '/' + tempfilename, 'r') as temp_file:
......@@ -73,7 +73,11 @@ class SimpleTest(TestCase):
truncate_tempfile('test_cron_memoize_task')
# clear the cache from any previous executions of this test
cache.delete('test_cron_memoize_unique_cache_key')
# cache.delete('test_cron_memoize_unique_cache_key')
from tasks import test_cron_memoize_task
use_clearcache(test_cron_memoize_task)()
run_celery_beat(seconds=3,verbose=False)
ncalls, last_call = count_timestamps('test_cron_memoize_task')
self.assertEqual(ncalls,1) # after the first call all subsequent calls should be cached
......@@ -89,7 +93,8 @@ class SimpleTest(TestCase):
truncate_tempfile('big_computation_counter')
# delete cache from previous executions of this unit test
cache.delete('big_computation_key')
from tasks import big_computation
use_clearcache(big_computation)()
run_celery_beat(seconds=3, verbose=False)
......@@ -119,7 +124,10 @@ class SimpleTest(TestCase):
"""
truncate_tempfile('big_computation_withfm_counter')
cache.delete('big_computation_key_withfm')
from tasks import big_computation_withfm
use_clearcache(big_computation_withfm)()
run_celery_beat(seconds=3, verbose=False)
ncalls_before, lastcall_before = count_timestamps('big_computation_withfm_counter')
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment