Reorganize requirements files

.travis.yml

@@ -7,7 +7,8 @@ before_install:
   - sudo apt-get update
   - sudo xargs -a apt-packages.txt apt-get install --fix-missing
 install:
-  - pip install -r base_requirements.txt
+  - pip install -r base-requirements.txt
+  - pip install -r dev-requirements.txt
   - sudo apt-get install python-matplotlib python-numpy python-scipy python-sklearn
   - pip install nltk==2.0.3 scikit-learn==0.12.1
   - sudo mv ease/data/nltk_data /usr/share/nltk_data

base_requirements.txt → base-requirements.txt

+# Most of the requirements, except for the ones
+# that take a long time to install and have pre-built
+# binaries on Ubuntu.  This allows us to speed up the
+# test builds, since we don't need to recompile numpy/scipy/nltk/scikit
+# every time.
+
 lxml==3.0.1
-mock==0.8.0
-nose==1.2.1
 path.py==3.0
-pylint==0.26.0
 pytz==2012h
 fisher==0.1.4

dev-requirements.txt

+# Test and doc requirements
+
+coverage==3.5.3
+django-sphinx-autodoc
+mock==0.8.0
+nose==1.2.1
+pylint==0.26.0
+sphinx

pre-requirements.txt

+# Numpy needs to be installed separately from Scipy,
+# so we can't include them in the same requirements file.
+
 numpy==1.6.2

requirements.txt

--r base_requirements.txt
-coverage==3.5.3
+-r base-requirements.txt
 scipy==0.11.0
 scikit-learn==0.12.1
-sphinx
-django-sphinx-autodoc
 nltk==2.0.3

setup.py

@@ -18,15 +18,18 @@ def is_requirement(line):
         line.startswith('git+')
     )
 
-
-REQUIREMENTS = [
-    line.strip() for line in (
-        open("pre-requirements.txt").readlines() +
-        open("requirements.txt").readlines() +
-        open("base_requirements.txt").readlines()
-    )
+def load_requirements(*requirements_paths):
+    """
+    Load all requirements from the specified requirements files.
+    Returns a list of requirement strings.
+    """
+    requirements = set()
+    for path in requirements_paths:
+        requirements.update(
+            line.strip() for line in open(path).readlines()
             if is_requirement(line)
-]
+        )
+    return list(requirements)
 
 
 setup(
@@ -43,5 +46,6 @@ setup(
     keywords = "ml machine learning nlp essay education",
     url = "https://github.com/edx/ease",
     include_package_data = True,
-    install_requires=REQUIREMENTS,
+    install_requires=load_requirements("pre-requirements.txt", "requirements.txt"),
+    tests_require=load_requirements("dev-requirements.txt"),
 )