# Compute grades using real division, with no integer truncation
from __future__ import division

import json
import logging
import random
from collections import defaultdict

import dogstats_wrapper as dog_stats_api
from course_blocks.api import get_course_blocks
from courseware import courses
from django.conf import settings
from django.core.cache import cache
from django.test.client import RequestFactory
from opaque_keys import InvalidKeyError
from opaque_keys.edx.keys import CourseKey
from opaque_keys.edx.locator import BlockUsageLocator
from openedx.core.djangoapps.content.block_structure.api import get_course_in_cache
from openedx.core.lib.cache_utils import memoized
from courseware.model_data import FieldDataCache, ScoresClient
from openedx.core.djangoapps.signals.signals import GRADES_UPDATED
from student.models import anonymous_id_for_user
from util.db import outer_atomic
from util.module_utils import yield_dynamic_descriptor_descendants
from xblock.core import XBlock
from xmodule import graders, block_metadata_utils
from xmodule.graders import Score
from xmodule.modulestore.django import modulestore
from xmodule.modulestore.exceptions import ItemNotFoundError
from .models import StudentModule
from .module_render import get_module_for_descriptor
from .transformers.grades import GradesTransformer


log = logging.getLogger("edx.courseware")


class ProgressSummary(object):
    """
    Wrapper class for the computation of a user's scores across a course.

    Attributes
       chapters: a summary of all sections with problems in the course. It is
       organized as an array of chapters, each containing an array of sections,
       each containing an array of scores. This contains information for graded
       and ungraded problems, and is good for displaying a course summary with
       due dates, etc.

       weighted_scores: a dictionary mapping module locations to weighted Score
       objects.

       locations_to_children: a function mapping locations to their
       direct descendants.
    """
    def __init__(self, chapters, weighted_scores, locations_to_children):
        self.chapters = chapters
        self.weighted_scores = weighted_scores
        self.locations_to_children = locations_to_children

    def score_for_module(self, location):
        """
        Calculate the aggregate weighted score for any location in the course.
        This method returns a tuple containing (earned_score, possible_score).

        If the location is of 'problem' type, this method will return the
        possible and earned scores for that problem. If the location refers to a
        composite module (a vertical or section ) the scores will be the sums of
        all scored problems that are children of the chosen location.
        """
        if location in self.weighted_scores:
            score = self.weighted_scores[location]
            return score.earned, score.possible
        children = self.locations_to_children[location]
        earned = 0.0
        possible = 0.0
        for child in children:
            child_earned, child_possible = self.score_for_module(child)
            earned += child_earned
            possible += child_possible
        return earned, possible


@memoized
def block_types_with_scores():
    """
    Returns the block types that could have a score.

    Something might be a scored item if it is capable of storing a score
    (has_score=True). We also have to include anything that can have children,
    since those children might have scores. We can avoid things like Videos,
    which have state but cannot ever impact someone's grade.
    """
    return frozenset(
        cat for (cat, xblock_class) in XBlock.load_classes() if (
            getattr(xblock_class, 'has_score', False) or getattr(xblock_class, 'has_children', False)
        )
    )


def possibly_scored(usage_key):
    """
    Returns whether the given block could impact grading (i.e. scored, or has children).
    """
    return usage_key.block_type in block_types_with_scores()


def grading_context_for_course(course):
    """
    Same as grading_context, but takes in a course object.
    """
    course_structure = get_course_in_cache(course.id)
    return grading_context(course_structure)


def grading_context(course_structure):
    """
    This returns a dictionary with keys necessary for quickly grading
    a student. They are used by grades.grade()

    The grading context has two keys:
    graded_sections - This contains the sections that are graded, as
        well as all possible children modules that can affect the
        grading. This allows some sections to be skipped if the student
        hasn't seen any part of it.

        The format is a dictionary keyed by section-type. The values are
        arrays of dictionaries containing
            "section_block" : The section block
            "scored_descendant_keys" : An array of usage keys for blocks
                could possibly be in the section, for any student

    all_graded_blocks - This contains a list of all blocks that can
        affect grading a student. This is used to efficiently fetch
        all the xmodule state for a FieldDataCache without walking
        the descriptor tree again.

    """
    all_graded_blocks = []
    all_graded_sections = defaultdict(list)

    for chapter_key in course_structure.get_children(course_structure.root_block_usage_key):
        for section_key in course_structure.get_children(chapter_key):
            section = course_structure[section_key]
            scored_descendants_of_section = [section]
            if section.graded:
                for descendant_key in course_structure.post_order_traversal(
                        filter_func=possibly_scored,
                        start_node=section_key,
                ):
                    scored_descendants_of_section.append(
                        course_structure[descendant_key],
                    )

                # include only those blocks that have scores, not if they are just a parent
                section_info = {
                    'section_block': section,
                    'scored_descendants': [
                        child for child in scored_descendants_of_section
                        if getattr(child, 'has_score', None)
                    ]
                }
                section_format = getattr(section, 'format', '')
                all_graded_sections[section_format].append(section_info)
                all_graded_blocks.extend(scored_descendants_of_section)

    return {
        'all_graded_sections': all_graded_sections,
        'all_graded_blocks': all_graded_blocks,
    }


def answer_distributions(course_key):
    """
    Given a course_key, return answer distributions in the form of a dictionary
    mapping:

      (problem url_name, problem display_name, problem_id) -> {dict: answer -> count}

    Answer distributions are found by iterating through all StudentModule
    entries for a given course with type="problem" and a grade that is not null.
    This means that we only count LoncapaProblems that people have submitted.
    Other types of items like ORA or sequences will not be collected. Empty
    Loncapa problem state that gets created from running the progress page is
    also not counted.

    This method accesses the StudentModule table directly instead of using the
    CapaModule abstraction. The main reason for this is so that we can generate
    the report without any side-effects -- we don't have to worry about answer
    distribution potentially causing re-evaluation of the student answer. This
    also allows us to use the read-replica database, which reduces risk of bad
    locking behavior. And quite frankly, it makes this a lot less confusing.

    Also, we're pulling all available records from the database for this course
    rather than crawling through a student's course-tree -- the latter could
    potentially cause us trouble with A/B testing. The distribution report may
    not be aware of problems that are not visible to the user being used to
    generate the report.

    This method will try to use a read-replica database if one is available.
    """
    # dict: { module.module_state_key : (url_name, display_name) }
    state_keys_to_problem_info = {}  # For caching, used by url_and_display_name

    def url_and_display_name(usage_key):
        """
        For a given usage_key, return the problem's url and display_name.
        Handle modulestore access and caching. This method ignores permissions.

        Raises:
            InvalidKeyError: if the usage_key does not parse
            ItemNotFoundError: if there is no content that corresponds
                to this usage_key.
        """
        problem_store = modulestore()
        if usage_key not in state_keys_to_problem_info:
            problem = problem_store.get_item(usage_key)
            problem_info = (problem.url_name, problem.display_name_with_default_escaped)
            state_keys_to_problem_info[usage_key] = problem_info

        return state_keys_to_problem_info[usage_key]

    # Iterate through all problems submitted for this course in no particular
    # order, and build up our answer_counts dict that we will eventually return
    answer_counts = defaultdict(lambda: defaultdict(int))
    for module in StudentModule.all_submitted_problems_read_only(course_key):
        try:
            state_dict = json.loads(module.state) if module.state else {}
            raw_answers = state_dict.get("student_answers", {})
        except ValueError:
            log.error(
                u"Answer Distribution: Could not parse module state for StudentModule id=%s, course=%s",
                module.id,
                course_key,
            )
            continue

        try:
            url, display_name = url_and_display_name(module.module_state_key.map_into_course(course_key))
            # Each problem part has an ID that is derived from the
            # module.module_state_key (with some suffix appended)
            for problem_part_id, raw_answer in raw_answers.items():
                # Convert whatever raw answers we have (numbers, unicode, None, etc.)
                # to be unicode values. Note that if we get a string, it's always
                # unicode and not str -- state comes from the json decoder, and that
                # always returns unicode for strings.
                answer = unicode(raw_answer)
                answer_counts[(url, display_name, problem_part_id)][answer] += 1

        except (ItemNotFoundError, InvalidKeyError):
            msg = (
                "Answer Distribution: Item {} referenced in StudentModule {} " +
                "for user {} in course {} not found; " +
                "This can happen if a student answered a question that " +
                "was later deleted from the course. This answer will be " +
                "omitted from the answer distribution CSV."
            ).format(
                module.module_state_key, module.id, module.student_id, course_key
            )
            log.warning(msg)
            continue

    return answer_counts


def grade(student, course, keep_raw_scores=False, course_structure=None):
    """
    Returns the grade of the student.

    Also sends a signal to update the minimum grade requirement status.
    """
    grade_summary = _grade(student, course, keep_raw_scores, course_structure)
    responses = GRADES_UPDATED.send_robust(
        sender=None,
        username=student.username,
        grade_summary=grade_summary,
        course_key=course.id,
        deadline=course.end
    )

    for receiver, response in responses:
        log.info('Signal fired when student grade is calculated. Receiver: %s. Response: %s', receiver, response)

    return grade_summary


def _grade(student, course, keep_raw_scores, course_structure=None):
    """
    Unwrapped version of "grade"

    This grades a student as quickly as possible. It returns the
    output from the course grader, augmented with the final letter
    grade. The keys in the output are:

    - course: a CourseDescriptor
    - keep_raw_scores : if True, then value for key 'raw_scores' contains scores
      for every graded module

    More information on the format is in the docstring for CourseGrader.
    """
    if course_structure is None:
        course_structure = get_course_blocks(student, course.location)
    grading_context_result = grading_context(course_structure)
    scorable_locations = [block.location for block in grading_context_result['all_graded_blocks']]

    with outer_atomic():
        scores_client = ScoresClient.create_for_locations(course.id, student.id, scorable_locations)

    # Dict of item_ids -> (earned, possible) point tuples. This *only* grabs
    # scores that were registered with the submissions API, which for the moment
    # means only openassessment (edx-ora2)
    # We need to import this here to avoid a circular dependency of the form:
    # XBlock --> submissions --> Django Rest Framework error strings -->
    # Django translation --> ... --> courseware --> submissions
    from submissions import api as sub_api  # installed from the edx-submissions repository

    with outer_atomic():
        submissions_scores = sub_api.get_scores(
            course.id.to_deprecated_string(),
            anonymous_id_for_user(student, course.id)
        )

    totaled_scores, raw_scores = _calculate_totaled_scores(
        student, grading_context_result, submissions_scores, scores_client, keep_raw_scores
    )

    with outer_atomic():
        # Grading policy might be overriden by a CCX, need to reset it
        course.set_grading_policy(course.grading_policy)
        grade_summary = course.grader.grade(totaled_scores, generate_random_scores=settings.GENERATE_PROFILE_SCORES)

        # We round the grade here, to make sure that the grade is a whole percentage and
        # doesn't get displayed differently than it gets grades
        grade_summary['percent'] = round(grade_summary['percent'] * 100 + 0.05) / 100

        letter_grade = grade_for_percentage(course.grade_cutoffs, grade_summary['percent'])
        grade_summary['grade'] = letter_grade
        grade_summary['totaled_scores'] = totaled_scores   # make this available, eg for instructor download & debugging
        if keep_raw_scores:
            # way to get all RAW scores out to instructor
            # so grader can be double-checked
            grade_summary['raw_scores'] = raw_scores

    return grade_summary


def _calculate_totaled_scores(
        student,
        grading_context_result,
        submissions_scores,
        scores_client,
        keep_raw_scores,
):
    """
    Returns the totaled scores, which can be passed to the grader.
    """
    raw_scores = []
    totaled_scores = {}
    for section_format, sections in grading_context_result['all_graded_sections'].iteritems():
        format_scores = []
        for section_info in sections:
            section = section_info['section_block']
            section_name = block_metadata_utils.display_name_with_default(section)

            with outer_atomic():
                # Check to
                # see if any of our locations are in the scores from the submissions
                # API. If scores exist, we have to calculate grades for this section.
                should_grade_section = any(
                    unicode(descendant.location) in submissions_scores
                    for descendant in section_info['scored_descendants']
                )

                if not should_grade_section:
                    should_grade_section = any(
                        descendant.location in scores_client
                        for descendant in section_info['scored_descendants']
                    )

                # If we haven't seen a single problem in the section, we don't have
                # to grade it at all! We can assume 0%
                if should_grade_section:
                    scores = []

                    for descendant in section_info['scored_descendants']:

                        (correct, total) = get_score(
                            student,
                            descendant,
                            scores_client,
                            submissions_scores,
                        )
                        if correct is None and total is None:
                            continue

                        if settings.GENERATE_PROFILE_SCORES:  # for debugging!
                            if total > 1:
                                correct = random.randrange(max(total - 2, 1), total + 1)
                            else:
                                correct = total

                        graded = descendant.graded
                        if not total > 0:
                            # We simply cannot grade a problem that is 12/0, because we might need it as a percentage
                            graded = False

                        scores.append(
                            Score(
                                correct,
                                total,
                                graded,
                                block_metadata_utils.display_name_with_default_escaped(descendant),
                                descendant.location
                            )
                        )

                    __, graded_total = graders.aggregate_scores(scores, section_name)
                    if keep_raw_scores:
                        raw_scores += scores
                else:
                    graded_total = Score(0.0, 1.0, True, section_name, None)

                # Add the graded total to totaled_scores
                if graded_total.possible > 0:
                    format_scores.append(graded_total)
                else:
                    log.info(
                        "Unable to grade a section with a total possible score of zero. " +
                        str(section.location)
                    )

        totaled_scores[section_format] = format_scores

    return totaled_scores, raw_scores


def grade_for_percentage(grade_cutoffs, percentage):
    """
    Returns a letter grade as defined in grading_policy (e.g. 'A' 'B' 'C' for 6.002x) or None.

    Arguments
    - grade_cutoffs is a dictionary mapping a grade to the lowest
        possible percentage to earn that grade.
    - percentage is the final percent across all problems in a course
    """

    letter_grade = None

    # Possible grades, sorted in descending order of score
    descending_grades = sorted(grade_cutoffs, key=lambda x: grade_cutoffs[x], reverse=True)
    for possible_grade in descending_grades:
        if percentage >= grade_cutoffs[possible_grade]:
            letter_grade = possible_grade
            break

    return letter_grade


def progress_summary(student, course, course_structure=None):
    """
    Returns progress summary for all chapters in the course.
    """

    progress = _progress_summary(student, course, course_structure)
    if progress:
        return progress.chapters
    else:
        return None


def get_weighted_scores(student, course):
    """
    Uses the _progress_summary method to return a ProgressSummary object
    containing details of a students weighted scores for the course.
    """
    return _progress_summary(student, course)


def _progress_summary(student, course, course_structure=None):
    """
    Unwrapped version of "progress_summary".

    This pulls a summary of all problems in the course.

    Returns
    - courseware_summary is a summary of all sections with problems in the course.
    It is organized as an array of chapters, each containing an array of sections,
    each containing an array of scores. This contains information for graded and
    ungraded problems, and is good for displaying a course summary with due dates,
    etc.
    - None if the student does not have access to load the course module.

    Arguments:
        student: A User object for the student to grade
        course: A Descriptor containing the course to grade

    """
    if course_structure is None:
        course_structure = get_course_blocks(student, course.location)
    if not len(course_structure):
        return None
    scorable_locations = [block_key for block_key in course_structure if possibly_scored(block_key)]

    with outer_atomic():
        scores_client = ScoresClient.create_for_locations(course.id, student.id, scorable_locations)

    # We need to import this here to avoid a circular dependency of the form:
    # XBlock --> submissions --> Django Rest Framework error strings -->
    # Django translation --> ... --> courseware --> submissions
    from submissions import api as sub_api  # installed from the edx-submissions repository
    with outer_atomic():
        submissions_scores = sub_api.get_scores(
            unicode(course.id), anonymous_id_for_user(student, course.id)
        )

    chapters = []
    locations_to_weighted_scores = {}

    for chapter_key in course_structure.get_children(course_structure.root_block_usage_key):
        chapter = course_structure[chapter_key]
        sections = []
        for section_key in course_structure.get_children(chapter_key):
            section = course_structure[section_key]

            graded = getattr(section, 'graded', False)
            scores = []

            for descendant_key in course_structure.post_order_traversal(
                    filter_func=possibly_scored,
                    start_node=section_key,
            ):
                descendant = course_structure[descendant_key]

                (correct, total) = get_score(
                    student,
                    descendant,
                    scores_client,
                    submissions_scores,
                )
                if correct is None and total is None:
                    continue

                weighted_location_score = Score(
                    correct,
                    total,
                    graded,
                    block_metadata_utils.display_name_with_default_escaped(descendant),
                    descendant.location
                )

                scores.append(weighted_location_score)
                locations_to_weighted_scores[descendant.location] = weighted_location_score

            escaped_section_name = block_metadata_utils.display_name_with_default_escaped(section)
            section_total, _ = graders.aggregate_scores(scores, escaped_section_name)

            sections.append({
                'display_name': escaped_section_name,
                'url_name': block_metadata_utils.url_name_for_block(section),
                'scores': scores,
                'section_total': section_total,
                'format': getattr(section, 'format', ''),
                'due': getattr(section, 'due', None),
                'graded': graded,
            })

        chapters.append({
            'course': course.display_name_with_default_escaped,
            'display_name': block_metadata_utils.display_name_with_default_escaped(chapter),
            'url_name': block_metadata_utils.url_name_for_block(chapter),
            'sections': sections
        })

    return ProgressSummary(chapters, locations_to_weighted_scores, course_structure.get_children)


def weighted_score(raw_correct, raw_total, weight):
    """Return a tuple that represents the weighted (correct, total) score."""
    # If there is no weighting, or weighting can't be applied, return input.
    if weight is None or raw_total == 0:
        return (raw_correct, raw_total)
    return (float(raw_correct) * weight / raw_total, float(weight))


def get_score(user, block, scores_client, submissions_scores_cache):
    """
    Return the score for a user on a problem, as a tuple (correct, total).
    e.g. (5,7) if you got 5 out of 7 points.

    If this problem doesn't have a score, or we couldn't load it, returns (None,
    None).

    user: a Student object
    block: a BlockStructure's BlockData object
    scores_client: an initialized ScoresClient
    submissions_scores_cache: A dict of location names to (earned, possible) point tuples.
           If an entry is found in this cache, it takes precedence.
    """
    submissions_scores_cache = submissions_scores_cache or {}

    if not user.is_authenticated():
        return (None, None)

    location_url = unicode(block.location)
    if location_url in submissions_scores_cache:
        return submissions_scores_cache[location_url]

    if not getattr(block, 'has_score', False):
        # These are not problems, and do not have a score
        return (None, None)

    # Check the score that comes from the ScoresClient (out of CSM).
    # If an entry exists and has a total associated with it, we trust that
    # value. This is important for cases where a student might have seen an
    # older version of the problem -- they're still graded on what was possible
    # when they tried the problem, not what it's worth now.
    score = scores_client.get(block.location)
    if score and score.total is not None:
        # We have a valid score, just use it.
        correct = score.correct if score.correct is not None else 0.0
        total = score.total
    else:
        # This means we don't have a valid score entry and we don't have a
        # cached_max_score on hand. We know they've earned 0.0 points on this.
        correct = 0.0
        total = block.transformer_data[GradesTransformer].max_score

        # Problem may be an error module (if something in the problem builder failed)
        # In which case total might be None
        if total is None:
            return (None, None)

    return weighted_score(correct, total, block.weight)


def iterate_grades_for(course_or_id, students, keep_raw_scores=False):
    """Given a course_id and an iterable of students (User), yield a tuple of:

    (student, gradeset, err_msg) for every student enrolled in the course.

    If an error occurred, gradeset will be an empty dict and err_msg will be an
    exception message. If there was no error, err_msg is an empty string.

    The gradeset is a dictionary with the following fields:

    - grade : A final letter grade.
    - percent : The final percent for the class (rounded up).
    - section_breakdown : A breakdown of each section that makes
        up the grade. (For display)
    - grade_breakdown : A breakdown of the major components that
        make up the final grade. (For display)
    - raw_scores: contains scores for every graded module
    """
    if isinstance(course_or_id, (basestring, CourseKey)):
        course = courses.get_course_by_id(course_or_id)
    else:
        course = course_or_id

    for student in students:
        with dog_stats_api.timer('lms.grades.iterate_grades_for', tags=[u'action:{}'.format(course.id)]):
            try:
                gradeset = grade(student, course, keep_raw_scores)
                yield student, gradeset, ""
            except Exception as exc:  # pylint: disable=broad-except
                # Keep marching on even if this student couldn't be graded for
                # some reason, but log it for future reference.
                log.exception(
                    'Cannot grade student %s (%s) in course %s because of exception: %s',
                    student.username,
                    student.id,
                    course.id,
                    exc.message
                )
                yield student, {}, exc.message


def _get_mock_request(student):
    """
    Make a fake request because grading code expects to be able to look at
    the request. We have to attach the correct user to the request before
    grading that student.
    """
    request = RequestFactory().get('/')
    request.user = student
    return request


def _calculate_score_for_modules(user_id, course, modules):
    """
    Calculates the cumulative score (percent) of the given modules
    """

    # removing branch and version from exam modules locator
    # otherwise student module would not return scores since module usage keys would not match
    modules = [m for m in modules]
    locations = [
        BlockUsageLocator(
            course_key=course.id,
            block_type=module.location.block_type,
            block_id=module.location.block_id
        )
        if isinstance(module.location, BlockUsageLocator) and module.location.version
        else module.location
        for module in modules
    ]

    scores_client = ScoresClient(course.id, user_id)
    scores_client.fetch_scores(locations)

    # Iterate over all of the exam modules to get score percentage of user for each of them
    module_percentages = []
    ignore_categories = ['course', 'chapter', 'sequential', 'vertical', 'randomize', 'library_content']
    for index, module in enumerate(modules):
        if module.category not in ignore_categories and (module.graded or module.has_score):
            module_score = scores_client.get(locations[index])
            if module_score:
                correct = module_score.correct or 0
                total = module_score.total or 1
                module_percentages.append(correct / total)

    return sum(module_percentages) / float(len(module_percentages)) if module_percentages else 0


def get_module_score(user, course, module):
    """
    Collects all children of the given module and calculates the cumulative
    score for this set of modules for the given user.

    Arguments:
        user (User): The user
        course (CourseModule): The course
        module (XBlock): The module

    Returns:
        float: The cumulative score
    """
    def inner_get_module(descriptor):
        """
        Delegate to get_module_for_descriptor
        """
        field_data_cache = FieldDataCache([descriptor], course.id, user)
        return get_module_for_descriptor(
            user,
            _get_mock_request(user),
            descriptor,
            field_data_cache,
            course.id,
            course=course
        )

    modules = yield_dynamic_descriptor_descendants(
        module,
        user.id,
        inner_get_module
    )
    return _calculate_score_for_modules(user.id, course, modules)
