Commit cb033b44 by pmitros

Merge pull request #65 from MITx/safe-cache-key

Use a key function that is safe for use w/ memcache

This is needed for the asset-pipeline. The pipeline has to map a requested asset name ('css/application.css') to a string that encodes the file contents ('css/application.12323423abcf.css'). It caches these associations in memcache. We have the potential for filenames with spaces in them, which memcache can't handle, so we need to escape the cache keys in such a way that memcache can handle them.

It puts these in a cache named 'staticfiles' (http://django-staticfiles.readthedocs.org/en/latest/helpers/#cachedstaticfilesstorage). In the future, it may make sense to put that cache onto the filesystem, rather than memcache, but we should do that only after performance testing.
parents f632db0e 1b5a9bc5
......@@ -7,7 +7,6 @@ Does some caching (to be explained).
'''
import hashlib
import logging
import os
import re
......@@ -16,6 +15,7 @@ import urllib
from datetime import timedelta
from lxml import etree
from util.memcache import fasthash
try: # This lets us do __name__ == ='__main__'
from django.conf import settings
......@@ -57,10 +57,6 @@ def parse_timedelta(time_str):
time_params[name] = int(param)
return timedelta(**time_params)
def fasthash(string):
m = hashlib.new("md4")
m.update(string)
return "id"+m.hexdigest()
def xpath(xml, query_string, **args):
''' Safe xpath query into an xml tree:
......@@ -122,7 +118,7 @@ def id_tag(course):
new_id = default_ids[elem.tag] + new_id
elem.set('id', new_id)
else:
elem.set('id', fasthash(etree.tostring(elem)))
elem.set('id', "id"+fasthash(etree.tostring(elem)))
def propogate_downward_tag(element, attribute_name, parent_attribute = None):
''' This call is to pass down an attribute to all children. If an element
......@@ -160,11 +156,11 @@ def user_groups(user):
cache_expiration = 60 * 60 # one hour
# Kill caching on dev machines -- we switch groups a lot
group_names = cache.get(fasthash(key))
group_names = cache.get(key)
if group_names is None:
group_names = [u.name for u in UserTestGroup.objects.filter(users=user)]
cache.set(fasthash(key), group_names, cache_expiration)
cache.set(key, group_names, cache_expiration)
return group_names
......@@ -203,7 +199,7 @@ def course_file(user,coursename=None):
cache_key = filename + "_processed?dev_content:" + str(options['dev_content']) + "&groups:" + str(sorted(groups))
if "dev" not in settings.DEFAULT_GROUPS:
tree_string = cache.get(fasthash(cache_key))
tree_string = cache.get(cache_key)
else:
tree_string = None
......@@ -211,7 +207,7 @@ def course_file(user,coursename=None):
tree = course_xml_process(etree.XML(render_to_string(filename, options, namespace = 'course')))
tree_string = etree.tostring(tree)
cache.set(fasthash(cache_key), tree_string, 60)
cache.set(cache_key, tree_string, 60)
else:
tree = etree.XML(tree_string)
......
......@@ -31,7 +31,8 @@ CACHES = {
# In staging/prod envs, the sessions also live here.
'default': {
'BACKEND': 'django.core.cache.backends.locmem.LocMemCache',
'LOCATION': 'mitx_loc_mem_cache'
'LOCATION': 'mitx_loc_mem_cache',
'KEY_FUNCTION': 'util.memcache.safe_key',
},
# The general cache is what you get if you use our util.cache. It's used for
......@@ -43,6 +44,7 @@ CACHES = {
'BACKEND': 'django.core.cache.backends.dummy.DummyCache',
'KEY_PREFIX': 'general',
'VERSION': 4,
'KEY_FUNCTION': 'util.cache.memcache_safe_key',
}
}
......
......@@ -30,12 +30,14 @@ CACHES = {
'default': {
'BACKEND': 'django.core.cache.backends.memcached.MemcachedCache',
'LOCATION': '127.0.0.1:11211',
'KEY_FUNCTION': 'util.memcache.safe_key',
},
'general': {
'BACKEND': 'django.core.cache.backends.memcached.MemcachedCache',
'LOCATION': '127.0.0.1:11211',
'KEY_PREFIX' : 'general',
'VERSION' : 5,
'KEY_FUNCTION': 'util.memcache.safe_key',
}
}
......
......@@ -30,7 +30,8 @@ CACHES = {
# In staging/prod envs, the sessions also live here.
'default': {
'BACKEND': 'django.core.cache.backends.locmem.LocMemCache',
'LOCATION': 'mitx_loc_mem_cache'
'LOCATION': 'mitx_loc_mem_cache',
'KEY_FUNCTION': 'util.memcache.safe_key',
},
# The general cache is what you get if you use our util.cache. It's used for
......@@ -42,6 +43,7 @@ CACHES = {
'BACKEND': 'django.core.cache.backends.dummy.DummyCache',
'KEY_PREFIX': 'general',
'VERSION': 4,
'KEY_FUNCTION': 'util.memcache.safe_key',
}
}
......
......@@ -54,7 +54,8 @@ CACHES = {
# In staging/prod envs, the sessions also live here.
'default': {
'BACKEND': 'django.core.cache.backends.locmem.LocMemCache',
'LOCATION': 'mitx_loc_mem_cache'
'LOCATION': 'mitx_loc_mem_cache',
'KEY_FUNCTION': 'util.memcache.safe_key',
},
# The general cache is what you get if you use our util.cache. It's used for
......@@ -66,6 +67,7 @@ CACHES = {
'BACKEND': 'django.core.cache.backends.dummy.DummyCache',
'KEY_PREFIX': 'general',
'VERSION': 4,
'KEY_FUNCTION': 'util.memcache.safe_key',
}
}
......
"""
This module provides a KEY_FUNCTION suitable for use with a memcache backend
so that we can cache any keys, not just ones that memcache would ordinarily accept
"""
from django.utils.encoding import smart_str
import hashlib
import urllib
def fasthash(string):
m = hashlib.new("md4")
m.update(string)
return m.hexdigest()
def safe_key(key, key_prefix, version):
safe_key = urllib.quote_plus(smart_str(key))
if len(safe_key) > 250:
safe_key = fasthash(safe_key)
return ":".join([key_prefix, str(version), safe_key])
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment