Add docs sketching data formats.

doc/public/Makefile

+# Makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line.
+SPHINXOPTS    =
+SPHINXBUILD   = sphinx-build
+PAPER         =
+BUILDDIR      = _build
+
+# Internal variables.
+PAPEROPT_a4     = -D latex_paper_size=a4
+PAPEROPT_letter = -D latex_paper_size=letter
+ALLSPHINXOPTS   = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .
+# the i18n builder cannot share the environment and doctrees with the others
+I18NSPHINXOPTS  = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .
+
+.PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext
+
+help:
+	@echo "Please use \`make <target>' where <target> is one of"
+	@echo "  html       to make standalone HTML files"
+	@echo "  dirhtml    to make HTML files named index.html in directories"
+	@echo "  singlehtml to make a single large HTML file"
+	@echo "  pickle     to make pickle files"
+	@echo "  json       to make JSON files"
+	@echo "  htmlhelp   to make HTML files and a HTML help project"
+	@echo "  qthelp     to make HTML files and a qthelp project"
+	@echo "  devhelp    to make HTML files and a Devhelp project"
+	@echo "  epub       to make an epub"
+	@echo "  latex      to make LaTeX files, you can set PAPER=a4 or PAPER=letter"
+	@echo "  latexpdf   to make LaTeX files and run them through pdflatex"
+	@echo "  text       to make text files"
+	@echo "  man        to make manual pages"
+	@echo "  texinfo    to make Texinfo files"
+	@echo "  info       to make Texinfo files and run them through makeinfo"
+	@echo "  gettext    to make PO message catalogs"
+	@echo "  changes    to make an overview of all changed/added/deprecated items"
+	@echo "  linkcheck  to check all external links for integrity"
+	@echo "  doctest    to run all doctests embedded in the documentation (if enabled)"
+
+clean:
+	-rm -rf $(BUILDDIR)/*
+
+html:
+	$(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html
+	@echo
+	@echo "Build finished. The HTML pages are in $(BUILDDIR)/html."
+
+dirhtml:
+	$(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml
+	@echo
+	@echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml."
+
+singlehtml:
+	$(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml
+	@echo
+	@echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml."
+
+pickle:
+	$(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle
+	@echo
+	@echo "Build finished; now you can process the pickle files."
+
+json:
+	$(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json
+	@echo
+	@echo "Build finished; now you can process the JSON files."
+
+htmlhelp:
+	$(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp
+	@echo
+	@echo "Build finished; now you can run HTML Help Workshop with the" \
+	      ".hhp project file in $(BUILDDIR)/htmlhelp."
+
+qthelp:
+	$(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp
+	@echo
+	@echo "Build finished; now you can run "qcollectiongenerator" with the" \
+	      ".qhcp project file in $(BUILDDIR)/qthelp, like this:"
+	@echo "# qcollectiongenerator $(BUILDDIR)/qthelp/edXDocs.qhcp"
+	@echo "To view the help file:"
+	@echo "# assistant -collectionFile $(BUILDDIR)/qthelp/edXDocs.qhc"
+
+devhelp:
+	$(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp
+	@echo
+	@echo "Build finished."
+	@echo "To view the help file:"
+	@echo "# mkdir -p $$HOME/.local/share/devhelp/edXDocs"
+	@echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/edXDocs"
+	@echo "# devhelp"
+
+epub:
+	$(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub
+	@echo
+	@echo "Build finished. The epub file is in $(BUILDDIR)/epub."
+
+latex:
+	$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
+	@echo
+	@echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex."
+	@echo "Run \`make' in that directory to run these through (pdf)latex" \
+	      "(use \`make latexpdf' here to do that automatically)."
+
+latexpdf:
+	$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
+	@echo "Running LaTeX files through pdflatex..."
+	$(MAKE) -C $(BUILDDIR)/latex all-pdf
+	@echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex."
+
+text:
+	$(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text
+	@echo
+	@echo "Build finished. The text files are in $(BUILDDIR)/text."
+
+man:
+	$(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man
+	@echo
+	@echo "Build finished. The manual pages are in $(BUILDDIR)/man."
+
+texinfo:
+	$(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo
+	@echo
+	@echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo."
+	@echo "Run \`make' in that directory to run these through makeinfo" \
+	      "(use \`make info' here to do that automatically)."
+
+info:
+	$(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo
+	@echo "Running Texinfo files through makeinfo..."
+	make -C $(BUILDDIR)/texinfo info
+	@echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo."
+
+gettext:
+	$(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale
+	@echo
+	@echo "Build finished. The message catalogs are in $(BUILDDIR)/locale."
+
+changes:
+	$(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes
+	@echo
+	@echo "The overview file is in $(BUILDDIR)/changes."
+
+linkcheck:
+	$(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck
+	@echo
+	@echo "Link check complete; look for any errors in the above output " \
+	      "or in $(BUILDDIR)/linkcheck/output.txt."
+
+doctest:
+	$(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest
+	@echo "Testing of doctests in the sources finished, look at the " \
+	      "results in $(BUILDDIR)/doctest/output.txt."

doc/public/conf.py

+# -*- coding: utf-8 -*-
+#
+# edX Docs documentation build configuration file, created by
+# sphinx-quickstart on Mon Feb 25 16:55:22 2013.
+#
+# This file is execfile()d with the current directory set to its containing dir.
+#
+# Note that not all possible configuration values are present in this
+# autogenerated file.
+#
+# All configuration values have a default; values that are commented out
+# serve to show the default.
+
+import sys, os
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+#sys.path.insert(0, os.path.abspath('.'))
+
+# -- General configuration -----------------------------------------------------
+
+# If your documentation needs a minimal Sphinx version, state it here.
+#needs_sphinx = '1.0'
+
+# Add any Sphinx extension module names here, as strings. They can be extensions
+# coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
+extensions = ['sphinx.ext.intersphinx', 'sphinx.ext.pngmath', 'sphinx.ext.mathjax']
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ['_templates']
+
+# The suffix of source filenames.
+source_suffix = '.rst'
+
+# The encoding of source files.
+#source_encoding = 'utf-8-sig'
+
+# The master toctree document.
+master_doc = 'index'
+
+# General information about the project.
+project = u'edX Docs'
+copyright = u'2013, edX Team'
+
+# The version info for the project you're documenting, acts as replacement for
+# |version| and |release|, also used in various other places throughout the
+# built documents.
+#
+# The short X.Y version.
+version = '0.1'
+# The full version, including alpha/beta/rc tags.
+release = '0.1'
+
+# The language for content autogenerated by Sphinx. Refer to documentation
+# for a list of supported languages.
+#language = None
+
+# There are two options for replacing |today|: either, you set today to some
+# non-false value, then it is used:
+#today = ''
+# Else, today_fmt is used as the format for a strftime call.
+#today_fmt = '%B %d, %Y'
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+exclude_patterns = ['_build']
+
+# The reST default role (used for this markup: `text`) to use for all documents.
+#default_role = None
+
+# If true, '()' will be appended to :func: etc. cross-reference text.
+#add_function_parentheses = True
+
+# If true, the current module name will be prepended to all description
+# unit titles (such as .. function::).
+#add_module_names = True
+
+# If true, sectionauthor and moduleauthor directives will be shown in the
+# output. They are ignored by default.
+#show_authors = False
+
+# The name of the Pygments (syntax highlighting) style to use.
+pygments_style = 'sphinx'
+
+# A list of ignored prefixes for module index sorting.
+#modindex_common_prefix = []
+
+
+# -- Options for HTML output ---------------------------------------------------
+
+# The theme to use for HTML and HTML Help pages.  See the documentation for
+# a list of builtin themes.
+html_theme = 'sphinxdoc'
+
+# Theme options are theme-specific and customize the look and feel of a theme
+# further.  For a list of options available for each theme, see the
+# documentation.
+#html_theme_options = {}
+
+# Add any paths that contain custom themes here, relative to this directory.
+#html_theme_path = []
+
+# The name for this set of Sphinx documents.  If None, it defaults to
+# "<project> v<release> documentation".
+#html_title = None
+
+# A shorter title for the navigation bar.  Default is the same as html_title.
+#html_short_title = None
+
+# The name of an image file (relative to this directory) to place at the top
+# of the sidebar.
+#html_logo = None
+
+# The name of an image file (within the static path) to use as favicon of the
+# docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
+# pixels large.
+#html_favicon = None
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = ['_static']
+
+# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
+# using the given strftime format.
+#html_last_updated_fmt = '%b %d, %Y'
+
+# If true, SmartyPants will be used to convert quotes and dashes to
+# typographically correct entities.
+#html_use_smartypants = True
+
+# Custom sidebar templates, maps document names to template names.
+#html_sidebars = {}
+
+# Additional templates that should be rendered to pages, maps page names to
+# template names.
+#html_additional_pages = {}
+
+# If false, no module index is generated.
+#html_domain_indices = True
+
+# If false, no index is generated.
+#html_use_index = True
+
+# If true, the index is split into individual pages for each letter.
+#html_split_index = False
+
+# If true, links to the reST sources are added to the pages.
+#html_show_sourcelink = True
+
+# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
+#html_show_sphinx = True
+
+# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
+#html_show_copyright = True
+
+# If true, an OpenSearch description file will be output, and all pages will
+# contain a <link> tag referring to it.  The value of this option must be the
+# base URL from which the finished HTML is served.
+#html_use_opensearch = ''
+
+# This is the file name suffix for HTML files (e.g. ".xhtml").
+#html_file_suffix = None
+
+# Output file base name for HTML help builder.
+htmlhelp_basename = 'edXDocsdoc'
+
+
+# -- Options for LaTeX output --------------------------------------------------
+
+latex_elements = {
+# The paper size ('letterpaper' or 'a4paper').
+#'papersize': 'letterpaper',
+
+# The font size ('10pt', '11pt' or '12pt').
+#'pointsize': '10pt',
+
+# Additional stuff for the LaTeX preamble.
+#'preamble': '',
+}
+
+# Grouping the document tree into LaTeX files. List of tuples
+# (source start file, target name, title, author, documentclass [howto/manual]).
+latex_documents = [
+  ('index', 'edXDocs.tex', u'edX Docs Documentation',
+   u'edX Team', 'manual'),
+]
+
+# The name of an image file (relative to this directory) to place at the top of
+# the title page.
+#latex_logo = None
+
+# For "manual" documents, if this is true, then toplevel headings are parts,
+# not chapters.
+#latex_use_parts = False
+
+# If true, show page references after internal links.
+#latex_show_pagerefs = False
+
+# If true, show URL addresses after external links.
+#latex_show_urls = False
+
+# Documents to append as an appendix to all manuals.
+#latex_appendices = []
+
+# If false, no module index is generated.
+#latex_domain_indices = True
+
+
+# -- Options for manual page output --------------------------------------------
+
+# One entry per manual page. List of tuples
+# (source start file, name, description, authors, manual section).
+man_pages = [
+    ('index', 'edxdocs', u'edX Docs Documentation',
+     [u'edX Team'], 1)
+]
+
+# If true, show URL addresses after external links.
+#man_show_urls = False
+
+
+# -- Options for Texinfo output ------------------------------------------------
+
+# Grouping the document tree into Texinfo files. List of tuples
+# (source start file, target name, title, author,
+#  dir menu entry, description, category)
+texinfo_documents = [
+  ('index', 'edXDocs', u'edX Docs Documentation',
+   u'edX Team', 'edXDocs', 'One line description of project.',
+   'Miscellaneous'),
+]
+
+# Documents to append as an appendix to all manuals.
+#texinfo_appendices = []
+
+# If false, no module index is generated.
+#texinfo_domain_indices = True
+
+# How to display URL addresses: 'footnote', 'no', or 'inline'.
+#texinfo_show_urls = 'footnote'
+
+
+# Example configuration for intersphinx: refer to the Python standard library.
+intersphinx_mapping = {'http://docs.python.org/': None}

doc/public/discussion_data.rst

+######################
+Discussion Forums Data
+######################
+Discussions in edX are stored in a MongoDB database as collections of JSON documents.
+
+The primary collection holding all posts and comments written by users is `contents`. There are two types of objects stored here, though they share much of the same structure. A `CommentThread` represents a comment that opens a new thread -- usually a student question of some sort. A `Comment` is a reply in the conversation started by a `CommentThread`.
+
+*****************
+Shared Attributes
+*****************
+
+The attributes that `Comment` and `CommentThread` objects share are listed below.
+
+`_id`
+-----
+  The 12-byte MongoDB unique ID for this collection. Like all MongoDB IDs, they are monotonically increasing and the first four bytes are a timestamp. 
+
+`_type`
+-------
+  `CommentThread` or `Comment` depending on the type of object.
+
+`anonymous`
+-----------
+  If true, this `Comment` or `CommentThread` will show up as written by anonymous, even to those who have moderator privileges in the forums.
+
+`anonymous_to_peers`
+--------------------
+  The idea behind this field was that `anonymous_to_peers = true` would make the the comment appear anonymous to your fellow students, but would allow the course staff to see who you were. However, that was never implemented in the UI, and only `anonymous` is actually used. The `anonymous_to_peers` field is always false.
+
+`at_position_list`
+------------------
+  No longer used. Child comments (replies) are just sorted by their `created_at` timestamp instead. 
+
+`author_id`
+-----------
+  The user who wrote this. Corresponds to the user IDs we store in our MySQL database as `auth_user.id`
+
+`body`
+------
+  Text of the comment in Markdown. UTF-8 encoded.
+
+`course_id`
+-----------
+  The full course_id of the course that this comment was made in, including org and run. This value can be seen in the URL when browsing the courseware section. Example: `BerkeleyX/Stat2.1x/2013_Spring`
+
+`created_at`
+------------
+  Timestamp in UTC. Example: `ISODate("2013-02-21T03:03:04.587Z")`
+
+`updated_at`
+------------
+  Timestamp in UTC. Example: `ISODate("2013-02-21T03:03:04.587Z")`
+
+`votes`
+-------
+  Both `CommentThread` and `Comment` objects support voting. `Comment` objects that are replies to other comments still have this attribute, even though there is no way to actually vote on them in the UI. This attribute is a dictionary that has the following inside:
+
+* `up` = list of User IDs that up-voted this comment or thread.
+* `down` = list of User IDs that down-voted this comment or thread (no longer used).
+* `up_count` = total upvotes received.
+* `down_count` = total downvotes received (no longer used).
+* `count` = total votes cast.
+* `point` = net vote, now always equal to `up_count`.
+
+A user only has one vote per `Comment` or `CommentThread`. Though it's still written to the database, the UI no longer displays an option to downvote anything.
+
+*************
+CommentThread
+*************
+The following fields are specific to `CommentThread` objects. Each thread in the forums is represented by one `CommentThread`.
+
+`closed`
+--------
+  If true, this thread was closed by a forum moderator/admin.
+
+`comment_count`
+---------------
+  The number of comment replies in this thread. This includes all replies to replies, but does not include the original comment that started the thread. So if we had::
+
+    CommentThread: "What's a good breakfast?"
+      * Comment: "Just eat cereal!"
+      * Comment: "Try a Loco Moco, it's amazing!"
+        * Comment: "A Loco Moco? Only if you want a heart attack!"
+        * Comment: "But it's worth it! Just get a spam musubi on the side."
+
+  In that exchange, the `comment_count` for the `CommentThread` is `4`.
+
+`commentable_id`
+----------------
+  We can attach a discussion to any piece of content in the course, or to top level categories like "General" and "Troubleshooting". When the `commentable_id` is a high level category, it's specified in the course's policy file. When it's a specific content piece (e.g. `600x_l5_p8`, meaning 6.00x, Lecture Sequence 5, Problem 8), it's taken from a discussion module in the course.
+
+`last_activity_at`
+------------------
+  Timestamp in UTC indicating the last time there was activity in the thread (new posts, edits, etc). Closing the thread does not affect the value in this field. 
+
+`tags_array`
+------------
+  Meant to be a list of tags that were user definable, but no longer used.
+
+`title`
+-------
+  Title of the thread, UTF-8 string.
+
+*******
+Comment
+*******
+The following fields are specific to `Comment` objects. A `Comment` is a reply to a `CommentThread` (so an answer to the question), or a reply to another `Comment` (a comment about somebody's answer). It used to be the case that `Comment` replies could nest much more deeply, but we later capped it at just these three levels (question, answer, comment) much in the way that StackOverflow does.
+
+`endorsed`
+----------
+  Boolean value, true if a forum moderator or instructor has marked that this `Comment` is a correct answer for whatever question the thread was asking. Exists for `Comments` that are replies to other `Comments`, but in that case `endorsed` is always false because there's no way to endorse such comments through the UI.
+
+`comment_thread_id`
+-------------------
+  What `CommentThread` are we a part of? All `Comment` objects have this.
+
+`parent_id`
+-----------
+  The `parent_id` is the `_id` of the `Comment` that this comment was made in reply to. Note that this only occurs in a `Comment` that is a reply to another `Comment`; it does not appear in a `Comment` that is a reply to a `CommentThread`.
+
+`parent_ids`
+------------
+  The `parent_ids` attribute appears in all `Comment` objects, and contains the `_id` of all ancestor comments. Since the UI now prevents comments from being nested more than one layer deep, it will only ever have at most one element in it. If a `Comment` has no parent, it's an empty list.

doc/public/index.rst

+.. edX documentation master file, created by
+   sphinx-quickstart on Fri Nov  2 15:43:00 2012.
+   You can adapt this file completely to your liking, but it should at least
+   contain the root `toctree` directive.
+   common-lib.rst
+   djangoapps.rst
+   xml_formats.rst
+
+
+edX Data Formats
+================
+
+Internal Formats
+----------------
+These document describe how we store course structure, student state/progress, and events internally. Useful for developers or researchers who interact with our raw data exports.
+
+.. toctree::
+   :maxdepth: 2
+
+   sql_schema.rst
+   discussion_data.rst
+   tracking_logs.rst
+
+Indices and tables
+==================
+
+.. * :ref:`genindex`
+* :ref:`modindex`
+* :ref:`search`
+

doc/public/sql_schema.rst

+##############################
+Student Info and Progress Data
+##############################
+
+The following sections detail how edX stores student state data internally, and is useful for developers and researchers who are examining database exports. This information includes demographic information collected at signup, course enrollment, course progress, and certificate status.
+
+Conventions to keep in mind:
+
+* We currently use MySQL 5.1 with InnoDB tables
+* All strings are stored as UTF-8.
+* All datetimes are stored as UTC.
+* Tables that are built into the Django framework are not documented here unless we use them in unconventional ways.
+
+All of our tables will be described below, first in summary form with field types and constraints, and then with a detailed explanation of each field. For those not familiar with the MySQL schema terminology in the table summaries:
+
+`Type`
+  This is the kind of data it is, along with the size of the field. When a numeric field has a length specified, it just means that's how many digits we want displayed -- it has no affect on the number of bytes used.
+
+  .. list-table::
+     :widths: 10 80
+     :header-rows: 1
+  
+     * - Value
+       - Meaning
+     * - `int`
+       - 4 byte integer.
+     * - `smallint`
+       - 2 byte integer, sometimes used for enumerated values.
+     * - `tinyint`
+       - 1 byte integer, but usually just used to indicate a boolean field with 0 = False and 1 = True.
+     * - `varchar`
+       - String, typically short and indexable. The length is the number of chars, not bytes (so unicode friendly).
+     * - `longtext`
+       - A long block of text, usually not indexed.
+     * - `date`
+       - Date
+     * - `datetime`
+       - Datetime in UTC, precision in seconds.
+  
+`Null`
+
+  .. list-table::
+     :widths: 10 80
+     :header-rows: 1
+  
+     * - Value
+       - Meaning
+     * - `YES`
+       - `NULL` values are allowed
+     * - `NO`
+       - `NULL` values are not allowed
+
+  .. note::
+     Django often just places blank strings instead of NULL when it wants to indicate a text value is optional. This is used more meaningful for numeric and date fields.
+
+`Key`
+  .. list-table::
+     :widths: 10 80
+     :header-rows: 1
+  
+     * - Value
+       - Meaning
+     * - `PRI`
+       - Primary key for the table, usually named `id`, unique
+     * - `UNI`
+       - Unique
+     * - `MUL`
+       - Indexed for fast lookup, but the same value can appear multiple times. A Unique index that allows `NULL` can also show up as `MUL`.
+
+
+****************
+User Information
+****************
+
+`auth_user`
+===========
+
+The `auth_user` table is built into the Django web framework that we use. It holds generic information necessary for basic login and permissions information. It has the following fields::
+
+  +------------------------------+--------------+------+-----+
+  | Field                        | Type         | Null | Key |
+  +------------------------------+--------------+------+-----+
+  | id                           | int(11)      | NO   | PRI |
+  | username                     | varchar(30)  | NO   | UNI |
+  | first_name                   | varchar(30)  | NO   |     | # Never used
+  | last_name                    | varchar(30)  | NO   |     | # Never used
+  | email                        | varchar(75)  | NO   | UNI |
+  | password                     | varchar(128) | NO   |     |
+  | is_staff                     | tinyint(1)   | NO   |     |
+  | is_active                    | tinyint(1)   | NO   |     |
+  | is_superuser                 | tinyint(1)   | NO   |     |
+  | last_login                   | datetime     | NO   |     |
+  | date_joined                  | datetime     | NO   |     |
+  | status                       | varchar(2)   | NO   |     | # No longer used
+  | email_key                    | varchar(32)  | YES  |     | # No longer used
+  | avatar_type                  | varchar(1)   | NO   |     | # No longer used
+  | country                      | varchar(2)   | NO   |     | # No longer used
+  | show_country                 | tinyint(1)   | NO   |     | # No longer used
+  | date_of_birth                | date         | YES  |     | # No longer used
+  | interesting_tags             | longtext     | NO   |     | # No longer used
+  | ignored_tags                 | longtext     | NO   |     | # No longer used
+  | email_tag_filter_strategy    | smallint(6)  | NO   |     | # No longer used
+  | display_tag_filter_strategy  | smallint(6)  | NO   |     | # No longer used
+  | consecutive_days_visit_count | int(11)      | NO   |     | # No longer used
+  +------------------------------+--------------+------+-----+
+
+`id`
+----
+  Primary key, and the value typically used in URLs that reference the user. A user has the same value for `id` here as they do in the MongoDB database's users collection. Foreign keys referencing `auth_user.id` will often be named `user_id`, but are sometimes named `student_id`.
+
+`username`
+----------
+  The unique username for a user in our system. It may contain alphanumeric, _, @, +, . and - characters. The username is the only information that the students give about themselves that we currently expose to other students. We have never allowed people to change their usernames so far, but that's not something we guarantee going forward.
+
+`first_name`
+------------
+  .. note::
+     Not used; we store a user's full name in `auth_userprofile.name` instead.
+
+`last_name`
+-----------
+  .. note::
+     Not used; we store a user's full name in `auth_userprofile.name` instead.
+
+`email`
+-------
+  Their email address. While Django by default makes this optional, we make it required, since it's the primary mechanism through which people log in. Must be unique to each user. Never shown to other users.
+
+`password`
+----------
+  A hashed version of the user's password. Depending on when the password was last set, this will either be a SHA1 hash or PBKDF2 with SHA256 (Django 1.3 uses the former and 1.4 the latter).
+
+`is_staff`
+----------
+  This value is `1` if the user is a staff member *of edX* with corresponding elevated privileges that cut across courses. It does not indicate that the person is a member of the course staff for any given course. Generally, users with this flag set to 1 are either edX program managers responsible for course delivery, or edX developers who need access for testing and debugging purposes. People who have `is_staff = 1` get instructor privileges on all courses, along with having additional debug information show up in the instructor tab.
+
+  Note that this designation has no bearing with a user's role in the forums, and confers no elevated privileges there.
+
+  Most users have a `0` for this value.
+
+`is_active`
+-----------
+  This value is `1` if the user has clicked on the activation link that was sent to them when they created their account, and `0` otherwise. Users who have `is_active = 0` generally cannot log into the system. However, when users first create their account, they are automatically logged in even though they are not active. This is to let them experience the site immediately without having to check their email. They just get a little banner at the top of their dashboard reminding them to check their email and activate their account when they have time. If they log out, they won't be able to log back in again until they've activated. However, because our sessions last a long time, it is theoretically possible for someone to use the site as a student for days without being "active".
+
+  Once `is_active` is set to `1`, the only circumstance where it would be set back to `0` would be if we decide to ban the user (which is very rare, manual operation).
+
+`is_superuser`
+--------------
+  Value is `1` if the user has admin privileges. Only the earliest developers of the system have this set to `1`, and it's no longer really used in the codebase. Set to 0 for almost everybody.
+
+`last_login`
+------------
+  A datetime of the user's last login. Should not be used as a proxy for activity, since people can use the site all the time and go days between logging in and out.
+
+`date_joined`
+-------------
+  Date that the account was created (NOT when it was activated).
+
+`(obsolete fields)`
+-------------------
+  All the following fields were added by an application called Askbot, a discussion forum package that is no longer part of the system:
+
+  * `status`
+  * `email_key`
+  * `avatar_type`
+  * `country`
+  * `show_country`
+  * `date_of_birth`
+  * `interesting_tags`
+  * `ignored_tags`
+  * `email_tag_filter_strategy`
+  * `display_tag_filter_strategy`
+  * `consecutive_days_visit_count`
+
+  Only users who were part of the prototype 6.002x course run in the Spring of 2012 would have any information in these fields. Even with those users, most of this information was never collected. Only the fields that are automatically generated have any values in them, such as tag settings.
+
+  These fields are completely unrelated to the discussion forums we currently use, and will eventually be dropped from this table.
+
+
+`auth_userprofile`
+==================
+The `auth_userprofile` table is mostly used to store user demographic information collected during the signup process. We also use it to store certain additional metadata relating to certificates. Every row in this table corresponds to one row in `auth_user`::
+
+  +--------------------+--------------+------+-----+
+  | Field              | Type         | Null | Key |
+  +--------------------+--------------+------+-----+
+  | id                 | int(11)      | NO   | PRI |
+  | user_id            | int(11)      | NO   | UNI |
+  | name               | varchar(255) | NO   | MUL |
+  | language           | varchar(255) | NO   | MUL | # Prototype course users only
+  | location           | varchar(255) | NO   | MUL | # Prototype course users only
+  | meta               | longtext     | NO   |     |
+  | courseware         | varchar(255) | NO   |     | # No longer used
+  | gender             | varchar(6)   | YES  | MUL | # Only users signed up after prototype
+  | mailing_address    | longtext     | YES  |     | # Only users signed up after prototype
+  | year_of_birth      | int(11)      | YES  | MUL | # Only users signed up after prototype
+  | level_of_education | varchar(6)   | YES  | MUL | # Only users signed up after prototype
+  | goals              | longtext     | YES  |     | # Only users signed up after prototype
+  | allow_certificate  | tinyint(1)   | NO   |     |
+  +--------------------+--------------+------+-----+
+
+There is an important split in demographic data gathered for the students who signed up during the MITx prototype phase in the spring of 2012, and those that signed up afterwards.
+
+`id`
+----
+  Primary key, not referenced anywhere else.
+
+`user_id`
+---------
+  A foreign key that maps to `auth_user.id`.
+
+`name`
+------
+  String for a user's full name. We make no constraints on language or breakdown into first/last name. The names are never shown to other students. Foreign students usually enter a romanized version of their names, but not always.
+
+  It used to be our policy to require manual approval of name changes to guard the integrity of the certificates. Students would submit a name change request and someone from the team would approve or reject as appropriate. Later, we decided to allow the name changes to take place automatically, but to log previous names in the `meta` field.
+
+`language`
+----------
+  User's preferred language, asked during the sign up process for the 6.002x prototype course given in the Spring of 2012. This information stopped being collected after the transition from MITx to edX happened, but we never removed the values from our first group of students. Sometimes written in those languages.
+
+`location`
+----------
+  User's location, asked during the sign up process for the 6.002x prototype course given in the Spring of 2012. We weren't specific, so people tended to put the city they were in, though some just specified their country and some got as specific as their street address. Again, sometimes romanized and sometimes written in their native language. Like `language`, we stopped collecting this field when we transitioned from MITx to edX, so it's only available for our first batch of students.
+
+`meta`
+------
+  An optional, freeform text field that stores JSON data. This was a hack to allow us to associate arbitrary metadata with a user. An example of the JSON that can be stored here is::
+
+    {
+      "old_names" : [
+     ["David Ormsbee", "I want to add my middle name as well.", "2012-11-15T17:28:12.658126"],
+     ["Dave Ormsbee", "Dave's too informal for a certificate.", "2013-02-07T11:15:46.524331"]
+      ],
+      "old_emails" : [["dormsbee@mitx.mit.edu", "2012-10-18T15:21:41.916389"]],
+      "6002x_exit_response" : {
+     "rating": ["6"],
+     "teach_ee": ["I do not teach EE."],
+     "improvement_textbook": ["I'd like to get the full PDF."],
+     "future_offerings": ["true"],
+     "university_comparison":
+       ["This course was <strong>on the same level</strong> as the university class."],
+     "improvement_lectures": ["More PowerPoint!"],
+     "highest_degree": ["Bachelor's degree."],
+     "future_classes": ["true"],
+     "future_updates": ["true"],
+     "favorite_parts": ["Releases, bug fixes, and askbot."]
+      }
+    }
+
+  The following are details about this metadata. Please note that the fields described below are found as JSON attributes *inside* the `meta` field, and are *not* separate database fields of their own.
+
+  `old_names`
+    A list of the previous names this user had, and the timestamps at which they submitted a request to change those names. These name change request submissions used to require a staff member to approve it before the name change took effect. This is no longer the case, though we still record their previous names.
+  
+    Note that the value stored for each entry is the name they had, not the name they requested to get changed to. People often changed their names as the time for certificate generation approached, to replace nicknames with their actual names or correct spelling/punctuation errors.
+  
+    The timestamps are UTC, like all datetimes stored in our system.
+  
+  `old_emails`
+    A list of previous emails this user had, with timestamps of when they changed them, in a format similar to `old_names`. There was never an approval process for this.
+  
+    The timestamps are UTC, like all datetimes stored in our system.
+  
+  `6002x_exit_response`
+    Answers to a survey that was sent to students after the prototype 6.002x course in the Spring of 2012. The questions and number of questions were randomly selected to measure how much survey length affected response rate. Only students from this course have this field.
+  
+
+`courseware`
+------------
+  This can be ignored. At one point, it was part of a way to do A/B tests, but it has not been used for anything meaningful since the conclusion of the prototype course in the spring of 2012.
+
+`gender`
+--------
+  Dropdown field collected during student signup. We only started collecting this information after the transition from MITx to edX, so prototype course students will have `NULL` for this field.
+
+  .. list-table::
+     :widths: 10 80
+     :header-rows: 1
+  
+     * - Value
+       - Meaning
+     * - `NULL`
+       - This student signed up before this information was collected
+     * - `''` (blank)
+       - User did not specify gender
+     * - `'f'`
+       - Female
+     * - `'m'`
+       - Male
+     * - `'o'`
+       - Other
+
+`mailing_address`
+-----------------
+  Text field collected during student signup. We only started collecting this information after the transition from MITx to edX, so prototype course students will have `NULL` for this field. Students who elected not to enter anything will have a blank string.
+
+`year_of_birth`
+---------------
+  Dropdown field collected during student signup. We only started collecting this information after the transition from MITx to edX, so prototype course students will have `NULL` for this field. Students who decided not to fill this in will also have NULL.
+
+`level_of_education`
+--------------------
+  Dropdown field collected during student signup. We only started collecting this information after the transition from MITx to edX, so prototype course students will have `NULL` for this field.
+
+  .. list-table::
+     :widths: 10 80
+     :header-rows: 1
+  
+     * - Value
+       - Meaning
+     * - `NULL` 
+       - This student signed up before this information was collected
+     * - `''` (blank)
+       - User did not specify level of education.
+     * - `'p_se'`
+       - Doctorate in science or engineering
+     * - `'p_oth'`
+       - Doctorate in another field
+     * - `'m'`
+       - Master's or professional degree
+     * - `'b'`
+       - Bachelor's degree
+     * - `'hs'`
+       - Secondary/high school
+     * - `'jhs'`
+       - Junior secondary/junior high/middle school
+     * - `'el'`
+       - Elementary/primary school
+     * - `'none'`
+       - None
+     * - `'other'`
+       - Other
+     
+`goals`
+-------
+  Text field collected during student signup in response to the prompt, "Goals in signing up for edX". We only started collecting this information after the transition from MITx to edX, so prototype course students will have `NULL` for this field. Students who elected not to enter anything will have a blank string.
+
+`allow_certificate`
+-------------------
+  Set to `1` for most students. This field is set to `0` if log analysis has revealed that this student is accessing our site from a country that the US has an embargo against. At this time, we do not issue certificates to students from those countries.
+
+
+`student_courseenrollment`
+==========================
+A row in this table represents a student's enrollment for a particular course run. If they decide to unenroll in the course, we delete their entry in this table, but we still leave all their state in `courseware_studentmodule` untouched.
+
+`id`
+----
+  Primary key.
+
+`user_id`
+---------
+  Student's ID in `auth_user.id`
+
+`course_id`
+-----------
+  The ID of the course run they're enrolling in (e.g. `MITx/6.002x/2012_Fall`). You can get this from the URL when you're viewing courseware on your browser.
+
+`created`
+---------
+  Datetime of enrollment, UTC.
+
+
+
+*******************
+Courseware Progress
+*******************
+
+Any piece of content in the courseware can store state and score in the `courseware_studentmodule` table. Grades and the user Progress page are generated by doing a walk of the course contents, searching for graded items, looking up a student's entries for those items in `courseware_studentmodule` via `(course_id, student_id, module_id)`, and then applying the grade weighting found in the course policy and grading policy files. Course policy files determine how much weight one problem has relative to another, and grading policy files determine how much categories of problems are weighted (e.g. HW=50%, Final=25%, etc.).
+
+.. warning::
+   **Modules might not be what you expect!**
+   
+   It's important to understand what "modules" are in the context of our system, as the terminology can be confusing. For the conventions of this table and many parts of our code, a "module" is a content piece that appears in the courseware. This can be nearly anything that appears when users are in the courseware tab: a video, a piece of HTML, a problem, etc. Modules can also be collections of other modules, such as sequences, verticals (modules stacked together on the same page), weeks, chapters, etc. In fact, the course itself is a top level module that contains all the other contents of the course as children. You can imagine the entire course as a tree with modules at every node.
+
+   Modules can store state, but whether and how they do so is up to the implemenation for that particular kind of module. When a user loads page, we look up all the modules they need to render in order to display it, and then we ask the database to look up state for those modules for that user. If there is corresponding entry for that user for a given module, we create a new row and set the state to an empty JSON dictionary.
+
+
+`courseware_studentmodule`
+==========================
+
+The `courseware_studentmodule` table holds all courseware state for a given user. Every student has a separate row for every piece of content in the course, making this by far our largest table::
+
+  +-------------+--------------+------+-----+
+  | Field       | Type         | Null | Key |
+  +-------------+--------------+------+-----+
+  | id          | int(11)      | NO   | PRI |
+  | module_type | varchar(32)  | NO   | MUL |
+  | module_id   | varchar(255) | NO   | MUL |
+  | student_id  | int(11)      | NO   | MUL |
+  | state       | longtext     | YES  |     |
+  | grade       | double       | YES  | MUL | # problem, selfassessment, and combinedopenended use this
+  | created     | datetime     | NO   | MUL |
+  | modified    | datetime     | NO   | MUL |
+  | max_grade   | double       | YES  |     | # problem, selfassessment, and combinedopenended use this
+  | done        | varchar(8)   | NO   | MUL | # ignore this
+  | course_id   | varchar(255) | NO   | MUL |
+  +-------------+--------------+------+-----+
+
+
+`id`
+----
+  Primary key. Rarely used though, since most lookups on this table are searches on the three tuple of `(course_id, student_id, module_id)`.
+
+`module_type`
+-------------
+
+  .. list-table::
+     :widths: 10 80
+     :header-rows: 0
+  
+     * - `chapter`
+       - The top level categories for a course. Each of these is usually labeled as a Week in the courseware, but this is just convention.
+     * - `combinedopenended`
+       - A new module type developed for grading open ended questions via self assessment, peer assessment, and machine learning.
+     * - `conditional`
+       - A new module type recently developed for 8.02x, this allows you to prevent access to certain parts of the courseware if other parts have not been completed first.
+     * - `course`
+       - The top level course module of which all course content is descended.
+     * - `problem`
+       - A problem that the user can submit solutions for. We have many different varieties.
+     * - `problemset`
+       - A collection of problems and supplementary materials, typically used for homeworks and rendered as a horizontal icon bar in the courseware. Use is inconsistent, and some courses use a `sequential` instead.
+     * - `selfassessment`
+       - Self assessment problems. An early test of the open ended grading system that is not in widespread use yet. Recently deprecated in favor of `combinedopenended`.
+     * - `sequential`
+       - A collection of videos, problems, and other materials, rendered as a horizontal icon bar in the courseware.
+     * - `timelimit`
+       - A special module that records the time you start working on a piece of courseware and enforces time limits, used for Pearson exams. This hasn't been completely generalized yet, so is not available for widespread use.
+     * - `videosequence`
+       - A collection of videos, exercise problems, and other materials, rendered as a horizontal icon bar in the courseware. Use is inconsistent, and some courses use a `sequential` instead.
+
+  There's been substantial muddling of our container types, particularly between sequentials, problemsets, and videosequences. In the beginning we only had sequentials, and these ended up being used primarily for two purposes: creating a sequence of lecture videos and exercises for instruction, and creating homework problem sets. The `problemset` and `videosequence` types were created with the hope that our system would have a better semantic understanding of what a sequence actually represented, and could at a later point choose to render them differently to the user if it was appropriate. Due to a variety of reasons, migration over to this has been spotty. They all render the same way at the moment.
+
+`module_id`
+-----------
+  Unique ID for a distinct piece of content in a course, these are recorded as URLs of the form `i4x://{org}/{course_num}/{module_type}/{module_name}`. Having URLs of this form allows us to give content a canonical representation even as we are in a state of transition between backend data stores.
+
+  .. list-table:: Breakdown of example `module_id`: `i4x://MITx/3.091x/problemset/Sample_Problems`
+     :widths: 10 20 70
+     :header-rows: 1
+     
+     * - Part
+       - Example
+       - Definition
+     * - `i4x://`
+       - 
+       - Just a convention we ran with. We had plans for the domain `i4x.org` at one point.
+     * - `org`
+       - `MITx`
+       - The organization part of the ID, indicating what organization created this piece of content.
+     * - `course_num` 
+       - `3.091x`
+       - The course number this content was created for. Note that there is no run information here, so you can't know what runs of the course this content is being used for from the `module_id` alone; you have to look at the `courseware_studentmodule.course_id` field.
+     * - `module_type` 
+       - `problemset`
+       - The module type, same value as what's in the `courseware_studentmodule.module_type` field.
+     * - `module_name`
+       - `Sample_Problems`
+       - The name given for this module by the content creators. If the module was not named, the system will generate a name based on the type and a hash of its contents (ex: `selfassessment_03c483062389`).
+
+`student_id`
+------------
+  A reference to `auth_user.id`, this is the student that this module state row belongs to.
+
+`state`
+-------
+  This is a JSON text field where different module types are free to store their state however they wish.
+
+  Container Modules: `course`, `chapter`, `problemset`, `sequential`, `videosequence`
+    The state for all of these is a JSON dictionary indicating the user's last known position within this container. This is 1-indexed, not 0-indexed, mostly because it went out that way at one point and we didn't want to later break saved navigation state for users.
+
+    Example: `{"position" : 3}`
+      When this user last interacted with this course/chapter/etc., they had clicked on the third child element. Note that the position is a simple index and not a `module_id`, so if you rearranged the order of the contents, it would not be smart enough to accomodate the changes and would point users to the wrong place.
+
+    The hierarchy goes: `course > chapter > (problemset | sequential | videosequence)`
+
+  `combinedopenended`
+    TODO: More details to come.
+
+  `conditional`
+    Conditionals don't actually store any state, so this value is always an empty JSON dictionary (`'{}'`). We should probably remove these entries altogether.
+
+  `problem`
+    There are many kinds of problems supported by the system, and they all have different state requirements. Note that one problem can have many different response fields. If a problem generates a random circuit and asks five questions about it, then all of that is stored in one row in `courseware_studentmodule`.
+
+    TODO: Write out different problem types and their state.
+
+  `selfassessment`
+    TODO: More details to come.
+
+  `timelimit`
+    This very uncommon type was only used in one Pearson exam for one course, and the format may change significantly in the future. It is currently a JSON dictionary with fields:
+
+    .. list-table::
+       :widths: 10 20 70
+       :header-rows: 1
+
+       * - JSON field
+         - Example
+         - Definition
+       * - `beginning_at`
+         - `1360590255.488154`
+         - UTC time as measured in seconds since UNIX epoch representing when the exam was started.
+       * - `ending_at`
+         - `1360596632.559758`
+         - UTC time as measured in seconds since UNIX epoch representing the time the exam will close.
+       * - `accomodation_codes`
+         - `DOUBLE`
+         - (optional) Sometimes students are given more time for accessibility reasons. Possible values are:
+         
+           * `NONE`: no time accommodation
+           * `ADDHALFTIME`: 1.5X normal time allowance
+           * `ADD30MIN`: normal time allowance + 30 minutes
+           * `DOUBLE`: 2X normal time allowance
+           * `TESTING`: extra long period (for testing/debugging)
+
+
+`grade`
+-------
+  Floating point value indicating the total unweighted grade for this problem that the student has scored. Basically how many responses they got right within the problem.
+
+  Only `problem` and `selfassessment` types use this field. All other modules set this to `NULL`. Due to a quirk in how rendering is done, `grade` can also be `NULL` for a tenth of a second or so the first time that a user loads a problem. The initial load will trigger two writes, the first of which will set the `grade` to `NULL`, and the second of which will set it to `0`.
+
+`created`
+---------
+  Datetime when this row was created (i.e. when the student first accessed this piece of content).
+
+`modified`
+---------
+  Datetime when we last updated this row. Set to be equal to `created` at first. A change in `modified` implies that there was a state change, usually in response to a user action like saving or submitting a problem, or clicking on a navigational element that records its state. However it can also be triggered if the module writes multiple times on its first load, like problems do (see note in `grade`).
+
+`max_grade`
+-----------
+  Floating point value indicating the total possible unweighted grade for this problem, or basically the number of responses that are in this problem. Though in practice it's the same for every entry with the same `module_id`, it is technically possible for it to be anything. The problems are dynamic enough where you could create a random number of responses if you wanted. This a bad idea and will probably cause grading errors, but it is possible.
+
+  Another way in which `max_grade` can differ between entries with the same `module_id` is if the problem was modified after the `max_grade` was written and the user never went back to the problem after it was updated. This might happen if a member of the course staff puts out a problem with five parts, realizes that the last part doesn't make sense, and decides to remove it. People who saw and answered it when it had five parts and never came back to it after the changes had been made will have a `max_grade` of `5`, while people who saw it later will have a `max_grade` of `4`.
+
+  These complexities in our grading system are a high priority target for refactoring in the near future.
+
+  Only `problem` and `selfassessment` types use this field. All other modules set this to `NULL`.
+
+`done`
+------
+  Ignore this field. It was supposed to be an indication whether something was finished, but was never properly used and is just `'na'` in every row.
+
+`course_id`
+-----------
+  The course that this row applies to, represented in the form org/course/run (ex: `MITx/6.002x/2012_Fall`). The same course content (same `module_id`) can be used in different courses, and a student's state needs to be tracked separately for each course.
+
+
+
+************
+Certificates
+************
+
+`certificates_generatedcertificate`
+===================================
+
+The generatedcertificate table tracks certificate state for students who have been graded after a course completes. Currently the table is only populated when a course ends and a script is run to grade students who have completed the course::
+
+  +---------------+--------------+------+-----+---------+----------------+
+  | Field         | Type         | Null | Key | Default | Extra          |
+  +---------------+--------------+------+-----+---------+----------------+
+  | id            | int(11)      | NO   | PRI | NULL    | auto_increment |
+  | user_id       | int(11)      | NO   | MUL | NULL    |                |
+  | download_url  | varchar(128) | NO   |     | NULL    |                |
+  | grade         | varchar(5)   | NO   |     | NULL    |                |
+  | course_id     | varchar(255) | NO   | MUL | NULL    |                |
+  | key           | varchar(32)  | NO   |     | NULL    |                |
+  | distinction   | tinyint(1)   | NO   |     | NULL    |                |
+  | status        | varchar(32)  | NO   |     | NULL    |                |
+  | verify_uuid   | varchar(32)  | NO   |     | NULL    |                |
+  | download_uuid | varchar(32)  | NO   |     | NULL    |                |
+  | name          | varchar(255) | NO   |     | NULL    |                |
+  | created_date  | datetime     | NO   |     | NULL    |                |
+  | modified_date | datetime     | NO   |     | NULL    |                |
+  | error_reason  | varchar(512) | NO   |     | NULL    |                |
+  +---------------+--------------+------+-----+---------+----------------+
+
+
+`user_id`, `course_id`
+----------------------
+  The table is indexed by user and course
+
+`status`
+--------
+  Status may be one of these states:
+
+  * `unavailable`
+  * `generating`
+  * `regenerating`
+  * `deleting`
+  * `deleted`
+  * `downloadable`
+  * `notpassing`
+  * `restricted`
+  * `error`
+  
+  After a course has been graded and certificates have been issued status will be one of:
+  
+  * `downloadable`
+  * `notpassing`
+  * `restricted`
+  
+  If the status is `downloadable` then the student passed the course and there will be a certificate available for download.
+
+`download_url`
+--------------
+  The `download_uuid` has the full URL to the certificate
+
+`download_uuid`, `verify_uuid`
+------------------------------
+  The two uuids are what uniquely identify the download url and the url used to download the certificate.
+
+`distinction`
+-------------
+  This was used for letters of distinction for 188.1x and is not being used for any current courses
+
+`name`
+------
+  This field records the name of the student that was set at the time the student was graded and the certificate was generated.
+
+`grade`
+-------
+  The grade of the student recorded at the time the certificate was generated. This may be different than the current grade since grading is only done once for a course when it ends.
\ No newline at end of file

doc/public/tracking_logs.rst

+#############
+Tracking Logs
+#############
+
+* Tracking logs are made available as separate tar files on S3 in the course-data bucket.
+* They are represented as JSON files that catalog all user interactions with the site.
+* To avoid filename collisions the tracking logs are organized by server name, where each directory corresponds to a server where they were stored. 
+
+*************
+Common Fields
+*************
+
+  .. list-table::
+     :widths: 10 40 10 25
+     :header-rows: 1
+     
+     * - field
+       - details
+       - type
+       - values/format
+     * - `username`
+       - username of the user who triggered the event, empty string for anonymous events (not logged in)
+       - string
+       - 
+     * - `session`
+       - key identifying the user's session, may be undefined
+       - string
+       - 32 digits key
+     * - `time`
+       - GMT time the event was triggered
+       - string	
+       - `YYYY-MM-DDThh:mm:ss.xxxxxx`
+     * - `ip`
+       - user ip address
+       - string
+       - 
+     * - `agent`
+       - users browser agent string
+       - string
+       - 
+     * - `page`
+       - page the user was visiting when the event was generated
+       - string
+       - `$URL`
+     * - event_source
+       - event source
+       - string
+       - `browser`, `server`
+     * - `event_type`
+       - type of event triggered, values depends on `event_source`
+       - string
+       - *more details listed below*
+     * - `event`
+       - specifics of the event (dependenty of the event_type)
+       - string/json
+       - *the event string may encode a JSON record*
+       
+
+*************
+Event Sources
+*************
+
+The `event_source` field identifies whether the event originated in the browser (via javascript) or on the server (during the processing of a request).
+
+Server Events
+=============
+
+  .. list-table::
+     :widths: 20 10 10 10 50
+     :header-rows: 1
+  
+     * - event_type
+       - event fields
+       - type
+       - values/format
+       - details
+     * - `show_answer`
+       - `problem_id`
+       - string
+       - 
+       - id of the problem being shown. Ex: `i4x://MITx/6.00x/problem/L15:L15_Problem_2`
+     * - `save_problem_check`
+       - `problem_id`
+       - string
+       - 
+       - id of the problem being shown
+     * -
+       - `success`
+       - string
+       - correct, incorrect
+       - whether the problem was correct
+     * -  	
+       - `attempts`
+       - integer
+       - number of attempts
+       -
+     * - 
+       - `correct_map`
+       - string/json
+       - 
+       - see details below
+     * - 
+       - `state`
+       - string/json
+       - 
+       - current problem state
+     * - 
+       - `answers`
+       - string/json
+       -
+       - students answers
+     * -
+       - `reset_problem`
+       - problem_id
+       - string
+       - id of the problem being shown
+
+
+`correct_map` details
+---------------------
+
+  .. list-table::
+     :widths: 15 10 15 10
+     :header-rows: 1
+    
+     * - correct_map fields
+       - type
+       - values/format
+       - null allowed?
+     * - hint
+       - string
+       -
+       -
+     * - hintmode
+       - boolean
+       -
+       - yes
+     * - correctness
+       - string
+       - correct, incorrect
+       -
+     * - npoints
+       - integer
+       -
+       - yes
+     * - msg
+       - string
+       -
+       -
+     * - queuestate
+       - string/json
+       - keys: key, time
+       -
+
+
+Browser Events
+==============
+
+  .. list-table::
+     :widths: 10 10 8 12 20 10
+     :header-rows: 1
+  
+     * - event_type
+       - fields
+       - type
+       - values/format
+       - details
+       - example
+     * - `book`
+       - `type`
+       - string
+       - `gotopage`	
+       -
+       -
+     * - 
+       - `old`
+       - integer	
+       - `$PAGE`
+       - from page number
+       - `2`
+     * -
+       - `new`
+       - integer
+       - `$PAGE`
+       - to page number
+       - `25`
+     * - `book`
+       - `type`
+       - string
+       - `nextpage`
+       -
+       -
+     * - 
+       - new
+       - integer
+       - `$PAGE`
+       - next page number
+       - `10`
+     * - `page_close`
+       - *empty*
+       - string
+       -
+       - 'page' field indicates which page was being closed
+       -
+     * - play_video
+       - `id`
+       - string
+       -
+       - edX id of the video being watched
+       - `i4x-HarvardX-PH207x-video-Simple_Random_Sample`
+     * -
+       - code
+       - string
+       -
+       - youtube id of the video being watched
+       - `FU3fCJNs94Y`
+     * -
+       - `currentTime`
+       - float
+       -
+       - time the video was paused at, in seconds
+       - `1.264`
+     * -
+       - `speed`
+       - string
+       - `0.75, 1.0, 1.25, 1.50`
+       - video speed being played
+       - `"1.0"`
+     * - `pause_video`
+       - `id`
+       - string
+       -
+       - edX id of the video being watched
+       -
+     * -
+       - `code`
+       - string
+       -
+       - youtube id of the video being watched
+       -
+     * -
+       - `currentTime`
+       - float
+       -
+       - time the video was paused at
+       -
+     * - 
+       - `speed`
+       - string
+       - `0.75, 1.0, 1.25, 1.50`
+       - video speed being played
+       -
+     * - `problem_check`
+       - *none*
+       - string
+       -
+       - event field contains the values of all input fields from the problem being checked (in the style of GET parameters (`key=value&key=value`))
+       -
+     * - `problem_show`
+       - `problem`
+       - string
+       - 
+       - id of the problem being checked
+       -
+     * - `seq_goto`
+       - `id`
+       - string
+       -
+       - edX id of the sequence
+       -
+     * -	
+       - `old`
+       - integer
+       -
+       - sequence element being jumped from
+       - `3`
+     * -
+       - `new`
+       - integer
+       -
+       - sequence element being jumped to
+       - `5`
+     * - `seq_next`
+       - `id`
+       - string
+       -
+       - edX id of the sequence	
+       -
+     * -	
+       - `old`
+       - integer
+       -
+       - sequence element being jumped from
+       - `4`
+     * -
+       - `new`
+       - integer
+       -
+       - sequence element being jumped to
+       - `6`
+     * - `rubric_select`
+       - `location`
+       - string
+       -
+       - location of the rubric's problem
+       - `i4x://MITx/6.00x/problem/L15:L15_Problem_2`
+     * -
+       - `category`
+       - integer
+       - 
+       - category number of the rubric selection
+       -
+     * -
+       - `value`
+       - integer
+       -
+       - value selected within the category
+       -
+     * - `(oe / peer_grading / staff_grading)`
+         `_show_problem`
+       - `location`
+       - string
+       -
+       - the location of the problem whose prompt we're showing
+       -
+     * - `(oe / peer_grading / staff_grading)`
+         `_hide_problem`
+       - `location`
+       - string
+       - 
+       - the location of the problem whose prompt we're hiding
+       -
+     * - `oe_show_full_feedback`
+       - *empty*
+       -
+       -
+       - the page where they're showing full feedback is already recorded
+       -
+     * - `oe_show_respond_to_feedback`
+       - *empty*
+       -
+       -
+       - the page where they're showing the feedback response form is already recorded
+       -
+     * - `oe_feedback_response_selected`
+       - `value`
+       - integer
+       -
+       - the value selected in the feedback response form
+       -
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+