Commit cb033b44 by pmitros

Merge pull request #65 from MITx/safe-cache-key

Use a key function that is safe for use w/ memcache

This is needed for the asset-pipeline. The pipeline has to map a requested asset name ('css/application.css') to a string that encodes the file contents ('css/application.12323423abcf.css'). It caches these associations in memcache. We have the potential for filenames with spaces in them, which memcache can't handle, so we need to escape the cache keys in such a way that memcache can handle them.

It puts these in a cache named 'staticfiles' (http://django-staticfiles.readthedocs.org/en/latest/helpers/#cachedstaticfilesstorage). In the future, it may make sense to put that cache onto the filesystem, rather than memcache, but we should do that only after performance testing.
parents f632db0e 1b5a9bc5
...@@ -7,7 +7,6 @@ Does some caching (to be explained). ...@@ -7,7 +7,6 @@ Does some caching (to be explained).
''' '''
import hashlib
import logging import logging
import os import os
import re import re
...@@ -16,6 +15,7 @@ import urllib ...@@ -16,6 +15,7 @@ import urllib
from datetime import timedelta from datetime import timedelta
from lxml import etree from lxml import etree
from util.memcache import fasthash
try: # This lets us do __name__ == ='__main__' try: # This lets us do __name__ == ='__main__'
from django.conf import settings from django.conf import settings
...@@ -57,10 +57,6 @@ def parse_timedelta(time_str): ...@@ -57,10 +57,6 @@ def parse_timedelta(time_str):
time_params[name] = int(param) time_params[name] = int(param)
return timedelta(**time_params) return timedelta(**time_params)
def fasthash(string):
m = hashlib.new("md4")
m.update(string)
return "id"+m.hexdigest()
def xpath(xml, query_string, **args): def xpath(xml, query_string, **args):
''' Safe xpath query into an xml tree: ''' Safe xpath query into an xml tree:
...@@ -122,7 +118,7 @@ def id_tag(course): ...@@ -122,7 +118,7 @@ def id_tag(course):
new_id = default_ids[elem.tag] + new_id new_id = default_ids[elem.tag] + new_id
elem.set('id', new_id) elem.set('id', new_id)
else: else:
elem.set('id', fasthash(etree.tostring(elem))) elem.set('id', "id"+fasthash(etree.tostring(elem)))
def propogate_downward_tag(element, attribute_name, parent_attribute = None): def propogate_downward_tag(element, attribute_name, parent_attribute = None):
''' This call is to pass down an attribute to all children. If an element ''' This call is to pass down an attribute to all children. If an element
...@@ -160,11 +156,11 @@ def user_groups(user): ...@@ -160,11 +156,11 @@ def user_groups(user):
cache_expiration = 60 * 60 # one hour cache_expiration = 60 * 60 # one hour
# Kill caching on dev machines -- we switch groups a lot # Kill caching on dev machines -- we switch groups a lot
group_names = cache.get(fasthash(key)) group_names = cache.get(key)
if group_names is None: if group_names is None:
group_names = [u.name for u in UserTestGroup.objects.filter(users=user)] group_names = [u.name for u in UserTestGroup.objects.filter(users=user)]
cache.set(fasthash(key), group_names, cache_expiration) cache.set(key, group_names, cache_expiration)
return group_names return group_names
...@@ -203,7 +199,7 @@ def course_file(user,coursename=None): ...@@ -203,7 +199,7 @@ def course_file(user,coursename=None):
cache_key = filename + "_processed?dev_content:" + str(options['dev_content']) + "&groups:" + str(sorted(groups)) cache_key = filename + "_processed?dev_content:" + str(options['dev_content']) + "&groups:" + str(sorted(groups))
if "dev" not in settings.DEFAULT_GROUPS: if "dev" not in settings.DEFAULT_GROUPS:
tree_string = cache.get(fasthash(cache_key)) tree_string = cache.get(cache_key)
else: else:
tree_string = None tree_string = None
...@@ -211,7 +207,7 @@ def course_file(user,coursename=None): ...@@ -211,7 +207,7 @@ def course_file(user,coursename=None):
tree = course_xml_process(etree.XML(render_to_string(filename, options, namespace = 'course'))) tree = course_xml_process(etree.XML(render_to_string(filename, options, namespace = 'course')))
tree_string = etree.tostring(tree) tree_string = etree.tostring(tree)
cache.set(fasthash(cache_key), tree_string, 60) cache.set(cache_key, tree_string, 60)
else: else:
tree = etree.XML(tree_string) tree = etree.XML(tree_string)
......
...@@ -31,7 +31,8 @@ CACHES = { ...@@ -31,7 +31,8 @@ CACHES = {
# In staging/prod envs, the sessions also live here. # In staging/prod envs, the sessions also live here.
'default': { 'default': {
'BACKEND': 'django.core.cache.backends.locmem.LocMemCache', 'BACKEND': 'django.core.cache.backends.locmem.LocMemCache',
'LOCATION': 'mitx_loc_mem_cache' 'LOCATION': 'mitx_loc_mem_cache',
'KEY_FUNCTION': 'util.memcache.safe_key',
}, },
# The general cache is what you get if you use our util.cache. It's used for # The general cache is what you get if you use our util.cache. It's used for
...@@ -43,6 +44,7 @@ CACHES = { ...@@ -43,6 +44,7 @@ CACHES = {
'BACKEND': 'django.core.cache.backends.dummy.DummyCache', 'BACKEND': 'django.core.cache.backends.dummy.DummyCache',
'KEY_PREFIX': 'general', 'KEY_PREFIX': 'general',
'VERSION': 4, 'VERSION': 4,
'KEY_FUNCTION': 'util.cache.memcache_safe_key',
} }
} }
......
...@@ -30,12 +30,14 @@ CACHES = { ...@@ -30,12 +30,14 @@ CACHES = {
'default': { 'default': {
'BACKEND': 'django.core.cache.backends.memcached.MemcachedCache', 'BACKEND': 'django.core.cache.backends.memcached.MemcachedCache',
'LOCATION': '127.0.0.1:11211', 'LOCATION': '127.0.0.1:11211',
'KEY_FUNCTION': 'util.memcache.safe_key',
}, },
'general': { 'general': {
'BACKEND': 'django.core.cache.backends.memcached.MemcachedCache', 'BACKEND': 'django.core.cache.backends.memcached.MemcachedCache',
'LOCATION': '127.0.0.1:11211', 'LOCATION': '127.0.0.1:11211',
'KEY_PREFIX' : 'general', 'KEY_PREFIX' : 'general',
'VERSION' : 5, 'VERSION' : 5,
'KEY_FUNCTION': 'util.memcache.safe_key',
} }
} }
......
...@@ -30,7 +30,8 @@ CACHES = { ...@@ -30,7 +30,8 @@ CACHES = {
# In staging/prod envs, the sessions also live here. # In staging/prod envs, the sessions also live here.
'default': { 'default': {
'BACKEND': 'django.core.cache.backends.locmem.LocMemCache', 'BACKEND': 'django.core.cache.backends.locmem.LocMemCache',
'LOCATION': 'mitx_loc_mem_cache' 'LOCATION': 'mitx_loc_mem_cache',
'KEY_FUNCTION': 'util.memcache.safe_key',
}, },
# The general cache is what you get if you use our util.cache. It's used for # The general cache is what you get if you use our util.cache. It's used for
...@@ -42,6 +43,7 @@ CACHES = { ...@@ -42,6 +43,7 @@ CACHES = {
'BACKEND': 'django.core.cache.backends.dummy.DummyCache', 'BACKEND': 'django.core.cache.backends.dummy.DummyCache',
'KEY_PREFIX': 'general', 'KEY_PREFIX': 'general',
'VERSION': 4, 'VERSION': 4,
'KEY_FUNCTION': 'util.memcache.safe_key',
} }
} }
......
...@@ -54,7 +54,8 @@ CACHES = { ...@@ -54,7 +54,8 @@ CACHES = {
# In staging/prod envs, the sessions also live here. # In staging/prod envs, the sessions also live here.
'default': { 'default': {
'BACKEND': 'django.core.cache.backends.locmem.LocMemCache', 'BACKEND': 'django.core.cache.backends.locmem.LocMemCache',
'LOCATION': 'mitx_loc_mem_cache' 'LOCATION': 'mitx_loc_mem_cache',
'KEY_FUNCTION': 'util.memcache.safe_key',
}, },
# The general cache is what you get if you use our util.cache. It's used for # The general cache is what you get if you use our util.cache. It's used for
...@@ -66,6 +67,7 @@ CACHES = { ...@@ -66,6 +67,7 @@ CACHES = {
'BACKEND': 'django.core.cache.backends.dummy.DummyCache', 'BACKEND': 'django.core.cache.backends.dummy.DummyCache',
'KEY_PREFIX': 'general', 'KEY_PREFIX': 'general',
'VERSION': 4, 'VERSION': 4,
'KEY_FUNCTION': 'util.memcache.safe_key',
} }
} }
......
"""
This module provides a KEY_FUNCTION suitable for use with a memcache backend
so that we can cache any keys, not just ones that memcache would ordinarily accept
"""
from django.utils.encoding import smart_str
import hashlib
import urllib
def fasthash(string):
m = hashlib.new("md4")
m.update(string)
return m.hexdigest()
def safe_key(key, key_prefix, version):
safe_key = urllib.quote_plus(smart_str(key))
if len(safe_key) > 250:
safe_key = fasthash(safe_key)
return ":".join([key_prefix, str(version), safe_key])
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment