Skip to content

E
edx-platform
Commit bc822b7f by Sarina Canelake
Options

Merge pull request #10873 from edx/kill-callstackmanager 

Remove CallStackManager (PLAT-931)
parents 5ecb9af2 596e93a2
Showing with 6 additions and 271 deletions
cms/envs/test.py
......@@ -173,9 +173,6 @@ CACHES = {
},
}
# Add apps to Installed apps for testing
INSTALLED_APPS += ('openedx.core.djangoapps.call_stack_manager',)
# hide ratelimit warnings while running tests
filterwarnings('ignore', message='No request passed to the backend, unable to rate-limit')
......
lms/djangoapps/courseware/migrations/0001_initial.py
......@@ -2,7 +2,6 @@
from __future__ import unicode_literals
from django.db import migrations, models
import openedx.core.djangoapps.call_stack_manager.core
import model_utils.fields
import xmodule_django.models
import django.utils.timezone
......@@ -69,7 +68,6 @@ class Migration(migrations.Migration):
('modified', models.DateTimeField(auto_now=True, db_index=True)),
('student', models.ForeignKey(to=settings.AUTH_USER_MODEL)),
],
bases=(openedx.core.djangoapps.call_stack_manager.core.CallStackMixin, models.Model),
),
migrations.CreateModel(
name='StudentModuleHistory',
......@@ -85,7 +83,6 @@ class Migration(migrations.Migration):
options={
'get_latest_by': 'created',
},
bases=(openedx.core.djangoapps.call_stack_manager.core.CallStackMixin, models.Model),
),
migrations.CreateModel(
name='XModuleStudentInfoField',
......
lms/djangoapps/courseware/model_data.py
......@@ -45,7 +45,6 @@ from xmodule.modulestore.django import modulestore
from xblock.core import XBlockAside
from courseware.user_state_client import DjangoXBlockUserStateClient
from openedx.core.djangoapps.call_stack_manager import donottrack
log = logging.getLogger(__name__)
......@@ -992,7 +991,6 @@ class ScoresClient(object):
# @contract(user_id=int, usage_key=UsageKey, score="number|None", max_score="number|None")
@donottrack(StudentModule)
def set_score(user_id, usage_key, score, max_score):
"""
Set the score and max_score for the specified user and xblock usage.
......
lms/djangoapps/courseware/models.py
......@@ -25,7 +25,6 @@ from model_utils.models import TimeStampedModel
from student.models import user_by_anonymous_id
from submissions.models import score_set, score_reset
from openedx.core.djangoapps.call_stack_manager import CallStackManager, CallStackMixin
from xmodule_django.models import CourseKeyField, LocationKeyField, BlockTypeKeyField
log = logging.getLogger(__name__)
......@@ -72,21 +71,10 @@ class ChunkingManager(models.Manager):
return res
class ChunkingCallStackManager(CallStackManager, ChunkingManager):
"""
A derived class of ChunkingManager, and CallStackManager
Class is currently unused but remains as part of the CallStackManger work. To re-enable see comment in StudentModule
"""
pass
class StudentModule(CallStackMixin, models.Model):
class StudentModule(models.Model):
"""
Keeps student state for a particular module in a particular course.
"""
# Changed back to ChunkingManager from ChunkingCallStackManger. To re-enable CallStack Management change the line
# back to: objects = ChunkingCallStackManager() Ticket: PLAT-881
objects = ChunkingManager()
MODEL_TAGS = ['course_id', 'module_type']
......@@ -161,11 +149,11 @@ class StudentModule(CallStackMixin, models.Model):
return unicode(repr(self))
class StudentModuleHistory(CallStackMixin, models.Model):
class StudentModuleHistory(models.Model):
"""Keeps a complete history of state changes for a given XModule for a given
Student. Right now, we restrict this to problems so that the table doesn't
explode in size."""
objects = CallStackManager()
objects = ChunkingManager()
HISTORY_SAVING_TYPES = {'problem'}
class Meta(object):
......@@ -197,6 +185,9 @@ class StudentModuleHistory(CallStackMixin, models.Model):
max_grade=instance.max_grade)
history_entry.save()
def __unicode__(self):
return unicode(repr(self))
class XBlockFieldBase(models.Model):
"""
......
lms/djangoapps/courseware/user_state_client.py
......@@ -18,8 +18,6 @@ from xblock.fields import Scope, ScopeBase
from courseware.models import StudentModule, StudentModuleHistory
from edx_user_state_client.interface import XBlockUserStateClient, XBlockUserState
from openedx.core.djangoapps.call_stack_manager import donottrack
class DjangoXBlockUserStateClient(XBlockUserStateClient):
"""
......@@ -71,7 +69,6 @@ class DjangoXBlockUserStateClient(XBlockUserStateClient):
"""
self.user = user
@donottrack(StudentModule, StudentModuleHistory)
def _get_student_modules(self, username, block_keys):
"""
Retrieve the :class:`~StudentModule`s for the supplied ``username`` and ``block_keys``.
......@@ -119,7 +116,6 @@ class DjangoXBlockUserStateClient(XBlockUserStateClient):
sample_rate=self.API_DATADOG_SAMPLE_RATE,
)
@donottrack(StudentModule, StudentModuleHistory)
def get_many(self, username, block_keys, scope=Scope.user_state, fields=None):
"""
Retrieve the stored XBlock state for the specified XBlock usages.
......@@ -173,7 +169,6 @@ class DjangoXBlockUserStateClient(XBlockUserStateClient):
self._ddog_histogram(evt_time, 'get_many.blks_out', block_count)
self._ddog_histogram(evt_time, 'get_many.response_time', (finish_time - evt_time) * 1000)
@donottrack(StudentModule, StudentModuleHistory)
def set_many(self, username, block_keys_to_state, scope=Scope.user_state):
"""
Set fields for a particular XBlock.
......@@ -250,7 +245,6 @@ class DjangoXBlockUserStateClient(XBlockUserStateClient):
self._ddog_histogram(evt_time, 'set_many.blks_updated', len(block_keys_to_state))
self._ddog_histogram(evt_time, 'set_many.response_time', (finish_time - evt_time) * 1000)
@donottrack(StudentModule, StudentModuleHistory)
def delete_many(self, username, block_keys, scope=Scope.user_state, fields=None):
"""
Delete the stored XBlock state for a many xblock usages.
......@@ -291,7 +285,6 @@ class DjangoXBlockUserStateClient(XBlockUserStateClient):
finish_time = time()
self._ddog_histogram(evt_time, 'delete_many.response_time', (finish_time - evt_time) * 1000)
@donottrack(StudentModule, StudentModuleHistory)
def get_history(self, username, block_key, scope=Scope.user_state):
"""
Retrieve history of state changes for a given block for a given
......@@ -346,7 +339,6 @@ class DjangoXBlockUserStateClient(XBlockUserStateClient):
yield XBlockUserState(username, block_key, state, history_entry.created, scope)
@donottrack(StudentModule, StudentModuleHistory)
def iter_all_for_block(self, block_key, scope=Scope.user_state, batch_size=None):
"""
You get no ordering guarantees. Fetching will happen in batch_size
......@@ -357,7 +349,6 @@ class DjangoXBlockUserStateClient(XBlockUserStateClient):
raise ValueError("Only Scope.user_state is supported")
raise NotImplementedError()
@donottrack(StudentModule, StudentModuleHistory)
def iter_all_for_course(self, course_key, block_type=None, scope=Scope.user_state, batch_size=None):
"""
You get no ordering guarantees. Fetching will happen in batch_size
......
lms/envs/test.py
......@@ -516,9 +516,6 @@ FEATURES['ENABLE_EDXNOTES'] = True
# Enable teams feature for tests.
FEATURES['ENABLE_TEAMS'] = True
# Add apps to Installed apps for testing
INSTALLED_APPS += ('openedx.core.djangoapps.call_stack_manager',)
# Enable courseware search for tests
FEATURES['ENABLE_COURSEWARE_SEARCH'] = True
......
openedx/core/djangoapps/call_stack_manager/__init__.py deleted 100644 → 0
"""
Root Package for getting call stacks of various Model classes being used
"""
from __future__ import absolute_import
from .core import CallStackManager, CallStackMixin, donottrack, trackit
openedx/core/djangoapps/call_stack_manager/core.py deleted 100644 → 0
"""
Call Stack Manager deals with tracking call stacks of functions/methods/classes(Django Model Classes)
Call Stack Manager logs unique call stacks. The call stacks then can be retrieved via Splunk, or log reads.
classes:
CallStackManager - stores all stacks in global dictionary and logs
CallStackMixin - used for Model save(), and delete() method
Decorators:
@donottrack - Decorator that will halt tracking for parameterized entities,
(or halt tracking anything in case of non-parametrized decorator).
@trackit - Decorator that will start tracking decorated entity.
@track_till_now - Will log every unique call stack of parametrized entity/ entities.
TRACKING DJANGO MODEL CLASSES -
Call stacks of Model Class
in three cases-
1. QuerySet API
2. save()
3. delete()
How to use:
1. Import following in the file where class to be tracked resides
from openedx.core.djangoapps.call_stack_manager import CallStackManager, CallStackMixin
2. Override objects of default manager by writing following in any model class which you want to track-
objects = CallStackManager()
3. For tracking Save and Delete events-
Use mixin called "CallStackMixin"
For ex.
class StudentModule(models.Model, CallStackMixin):
TRACKING FUNCTIONS, and METHODS-
1. Import following-
from openedx.core.djangoapps.call_stack_manager import trackit
NOTE - @trackit is non-parameterized decorator.
FOR DISABLING TRACKING-
1. Import following at appropriate location-
from openedx.core.djangoapps.call_stack_manager import donottrack
NOTE - You need to import function/class you do not want to track.
"""
import logging
import traceback
import re
import collections
import wrapt
import types
import inspect
from django.db.models import Manager
log = logging.getLogger(__name__)
# List of regular expressions acting as filters
REGULAR_EXPS = [re.compile(x) for x in ['^.*python2.7.*$', '^.*<exec_function>.*$', '^.*exec_code_object.*$',
'^.*edxapp/src.*$', '^.*call_stack_manager.*$']]
# List keeping track of entities not to be tracked
HALT_TRACKING = []
STACK_BOOK = collections.defaultdict(list)
# Dictionary which stores call logs
# {'EntityName' : ListOf<CallStacks>}
# CallStacks is ListOf<Frame>
# Frame is a tuple ('FilePath','LineNumber','Function Name', 'Context')
# {"<class 'courseware.models.StudentModule'>" : [[(file, line number, function name, context),(---,---,---)],
# [(file, line number, function name, context),(---,---,---)]]}
def capture_call_stack(entity_name):
""" Logs customised call stacks in global dictionary STACK_BOOK and logs it.
Arguments:
entity_name - entity
"""
# Holds temporary callstack
# List with each element 4-tuple(filename, line number, function name, text)
# and filtered with respect to regular expressions
temp_call_stack = [frame for frame in traceback.extract_stack()
if not any(reg.match(frame[0]) for reg in REGULAR_EXPS)]
final_call_stack = "".join(traceback.format_list(temp_call_stack))
def _should_get_logged(entity_name): # pylint: disable=
""" Checks if current call stack of current entity should be logged or not.
Arguments:
entity_name - Name of the current entity
Returns:
True if the current call stack is to logged, False otherwise
"""
is_class_in_halt_tracking = bool(HALT_TRACKING and inspect.isclass(entity_name) and
issubclass(entity_name, tuple(HALT_TRACKING[-1])))
is_function_in_halt_tracking = bool(HALT_TRACKING and not inspect.isclass(entity_name) and
any((entity_name.__name__ == x.__name__ and
entity_name.__module__ == x.__module__)
for x in tuple(HALT_TRACKING[-1])))
is_top_none = HALT_TRACKING and HALT_TRACKING[-1] is None
# if top of STACK_BOOK is None
if is_top_none:
return False
# if call stack is empty
if not temp_call_stack:
return False
if HALT_TRACKING:
if is_class_in_halt_tracking or is_function_in_halt_tracking:
return False
else:
return temp_call_stack not in STACK_BOOK[entity_name]
else:
return temp_call_stack not in STACK_BOOK[entity_name]
if _should_get_logged(entity_name):
STACK_BOOK[entity_name].append(temp_call_stack)
if inspect.isclass(entity_name):
log.info("Logging new call stack number %s for %s:\n %s", len(STACK_BOOK[entity_name]),
entity_name, final_call_stack)
else:
log.info("Logging new call stack number %s for %s.%s:\n %s", len(STACK_BOOK[entity_name]),
entity_name.__module__, entity_name.__name__, final_call_stack)
class CallStackMixin(object):
""" Mixin class for getting call stacks when save() and delete() methods are called """
def save(self, *args, **kwargs):
""" Logs before save() and overrides respective model API save() """
capture_call_stack(type(self))
return super(CallStackMixin, self).save(*args, **kwargs)
def delete(self, *args, **kwargs):
""" Logs before delete() and overrides respective model API delete() """
capture_call_stack(type(self))
return super(CallStackMixin, self).delete(*args, **kwargs)
class CallStackManager(Manager):
""" Manager class which overrides the default Manager class for getting call stacks """
def get_queryset(self):
""" Override the default queryset API method """
capture_call_stack(self.model)
return super(CallStackManager, self).get_queryset()
def donottrack(*entities_not_to_be_tracked):
""" Decorator which halts tracking for some entities for specific functions
Arguments:
entities_not_to_be_tracked: entities which are not to be tracked
Returns:
wrapped function
"""
if not entities_not_to_be_tracked:
entities_not_to_be_tracked = None
@wrapt.decorator
def real_donottrack(wrapped, instance, args, kwargs): # pylint: disable=unused-argument
""" Takes function to be decorated and returns wrapped function
Arguments:
wrapped - The wrapped function which in turns needs to be called by wrapper function
instance - The object to which the wrapped function was bound when it was called.
args - The list of positional arguments supplied when the decorated function was called.
kwargs - The dictionary of keyword arguments supplied when the decorated function was called.
Returns:
return of wrapped function
"""
global HALT_TRACKING # pylint: disable=global-variable-not-assigned
if entities_not_to_be_tracked is None:
HALT_TRACKING.append(None)
else:
if HALT_TRACKING:
if HALT_TRACKING[-1] is None: # if @donottrack() calls @donottrack('xyz')
pass
else:
HALT_TRACKING.append(set(HALT_TRACKING[-1].union(set(entities_not_to_be_tracked))))
else:
HALT_TRACKING.append(set(entities_not_to_be_tracked))
return_value = wrapped(*args, **kwargs)
# check if the returning class is a generator
if isinstance(return_value, types.GeneratorType):
def generator_wrapper(wrapped_generator):
""" Function handling wrapped yielding values.
Argument:
wrapped_generator - wrapped function returning generator function
Returns:
Generator Wrapper
"""
try:
while True:
return_value = next(wrapped_generator)
yield return_value
finally:
HALT_TRACKING.pop()
return generator_wrapper(return_value)
else:
HALT_TRACKING.pop()
return return_value
return real_donottrack
@wrapt.decorator
def trackit(wrapped, instance, args, kwargs): # pylint: disable=unused-argument
""" Decorator which tracks logs call stacks
Arguments:
wrapped - The wrapped function which in turns needs to be called by wrapper function.
instance - The object to which the wrapped function was bound when it was called.
args - The list of positional arguments supplied when the decorated function was called.
kwargs - The dictionary of keyword arguments supplied when the decorated function was called.
Returns:
wrapped function
"""
capture_call_stack(wrapped)
return wrapped(*args, **kwargs)
openedx/core/djangoapps/call_stack_manager/models.py deleted 100644 → 0
"""
Dummy models.py file
Note -
django-nose loads models for tests, but only if the django app that the test is contained in has models itself.
This file is empty so that the unit tests can have models.
For call_stack_manager - models specific to tests are defined in tests.py
"""
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment