rakefile 8.48 KB
Newer Older
1 2 3
require 'rake/clean'
require 'tempfile'

4
# Build Constants
5 6
REPO_ROOT = File.dirname(__FILE__)
BUILD_DIR = File.join(REPO_ROOT, "build")
7
REPORT_DIR = File.join(REPO_ROOT, "reports")
8
LMS_REPORT_DIR = File.join(REPORT_DIR, "lms")
9 10

# Packaging constants
11
DEPLOY_DIR = "/opt/wwc"
12
PACKAGE_NAME = "mitx"
13
LINK_PATH = "/opt/wwc/mitx"
14
PKG_VERSION = "0.1"
15
COMMIT = (ENV["GIT_COMMIT"] || `git rev-parse HEAD`).chomp()[0, 10]
16
BRANCH = (ENV["GIT_BRANCH"] || `git symbolic-ref -q HEAD`).chomp().gsub('refs/heads/', '').gsub('origin/', '')
17 18 19
BUILD_NUMBER = (ENV["BUILD_NUMBER"] || "dev").chomp()

if BRANCH == "master"
20
    DEPLOY_NAME = "#{PACKAGE_NAME}-#{BUILD_NUMBER}-#{COMMIT}"
21
else
22
    DEPLOY_NAME = "#{PACKAGE_NAME}-#{BRANCH}-#{BUILD_NUMBER}-#{COMMIT}"
23 24 25
end
PACKAGE_REPO = "packages@gp.mitx.mit.edu:/opt/pkgrepo.incoming"

26 27
NORMALIZED_DEPLOY_NAME = DEPLOY_NAME.downcase().gsub(/[_\/]/, '-')
INSTALL_DIR_PATH = File.join(DEPLOY_DIR, NORMALIZED_DEPLOY_NAME)
28
PIP_REPO_REQUIREMENTS = "#{INSTALL_DIR_PATH}/repo-requirements.txt"
29
# Set up the clean and clobber tasks
30
CLOBBER.include(BUILD_DIR, REPORT_DIR, 'cover*', '.coverage', 'test_root/*_repo', 'test_root/staticfiles')
31
CLEAN.include("#{BUILD_DIR}/*.deb", "#{BUILD_DIR}/util")
32

33 34 35 36
def select_executable(*cmds)
    cmds.find_all{ |cmd| system("which #{cmd} > /dev/null 2>&1") }[0] || fail("No executables found from #{cmds.join(', ')}")
end

37 38 39 40
def django_admin(system, env, command, *args)
    django_admin = ENV['DJANGO_ADMIN_PATH'] || select_executable('django-admin.py', 'django-admin')
    return "#{django_admin} #{command} --settings=#{system}.envs.#{env} --pythonpath=. #{args.join(' ')}"
end
41

42
task :default => [:test, :pep8, :pylint]
43 44

directory REPORT_DIR
Calen Pennington committed
45

46 47 48 49 50
default_options = {
    :lms => '8000',
    :cms => '8001',
}

51 52 53
task :predjango do
    sh("find . -type f -name *.pyc -delete")
    sh('pip install -e common/lib/xmodule')
54
    sh('git submodule update --init')
55 56
end

57 58 59 60
task :clean_test_files do
    sh("git clean -fdx test_root")
end

61 62 63 64 65 66 67 68 69 70 71 72 73
[:lms, :cms, :common].each do |system|
    report_dir = File.join(REPORT_DIR, system.to_s)
    directory report_dir

    desc "Run pep8 on all #{system} code"
    task "pep8_#{system}" => report_dir do
        sh("pep8 --ignore=E501 #{system}/djangoapps #{system}/lib | tee #{report_dir}/pep8.report")
    end
    task :pep8 => "pep8_#{system}"

    desc "Run pylint on all #{system} code"
    task "pylint_#{system}" => report_dir do
        Dir["#{system}/djangoapps/*", "#{system}/lib/*"].each do |app|
74 75 76 77 78
            if File.exists? "#{app}/setup.py"
                pythonpath_prefix = "PYTHONPATH=#{app}"
            else
                pythonpath_prefix = "PYTHONPATH=#{File.dirname(app)}"
            end
79
            app = File.basename(app)
80
            sh("#{pythonpath_prefix} pylint --rcfile=.pylintrc -f parseable #{app} | tee #{report_dir}/#{app}.pylint.report")
81 82 83 84 85
        end
    end
    task :pylint => "pylint_#{system}"
end

86
$failed_tests = 0
87

88
def run_tests(system, report_dir, stop_on_failure=true)
89 90
    ENV['NOSE_XUNIT_FILE'] = File.join(report_dir, "nosetests.xml")
    ENV['NOSE_COVER_HTML_DIR'] = File.join(report_dir, "cover")
91 92 93 94 95 96
    sh(django_admin(system, :test, 'test', *Dir["#{system}/djangoapps/*"].each)) do |ok, res|
        if !ok and stop_on_failure
            abort "Test failed!"
        end
        $failed_tests += 1 unless ok
    end
97 98
end

99
TEST_TASKS = []
100

101
[:lms, :cms].each do |system|
102
    report_dir = File.join(REPORT_DIR, system.to_s)
103 104
    directory report_dir

105
    # Per System tasks
106
    desc "Run all django tests on our djangoapps for the #{system}"
107
    task "test_#{system}", [:stop_on_failure] => ["clean_test_files", "#{system}:collectstatic:test", "fasttest_#{system}"]
108

109 110
    # Have a way to run the tests without running collectstatic -- useful when debugging without
    # messing with static files.
111 112 113
    task "fasttest_#{system}", [:stop_on_failure] => [report_dir, :predjango] do |t, args|
        args.with_defaults(:stop_on_failure => 'true')
        run_tests(system, report_dir, args.stop_on_failure)
114
    end
115

116
    TEST_TASKS << "test_#{system}"
117 118 119 120 121

    desc <<-desc
        Start the #{system} locally with the specified environment (defaults to dev).
        Other useful environments are devplus (for dev testing with a real local database)
        desc
122
    task system, [:env, :options] => [:predjango] do |t, args|
123
        args.with_defaults(:env => 'dev', :options => default_options[system])
124 125
        sh(django_admin(system, args.env, 'runserver', args.options))
    end
126 127

    # Per environment tasks
128 129 130
    Dir["#{system}/envs/*.py"].each do |env_file|
        env = File.basename(env_file).gsub(/\.py/, '')
        desc "Attempt to import the settings file #{system}.envs.#{env} and report any errors"
131
        task "#{system}:check_settings:#{env}" => :predjango do
132 133
            sh("echo 'import #{system}.envs.#{env}' | #{django_admin(system, env, 'shell')}")
        end
134 135 136

        desc "Run collectstatic in the specified environment"
        task "#{system}:collectstatic:#{env}" => :predjango do
137
            sh("#{django_admin(system, env, 'collectstatic', '--noinput')}")
138
        end
139
    end
140 141 142 143 144
end

Dir["common/lib/*"].each do |lib|
    task_name = "test_#{lib}"

145
    report_dir = File.join(REPORT_DIR, task_name.gsub('/', '_'))
146 147 148
    directory report_dir

    desc "Run tests for common lib #{lib}"
149
    task task_name => report_dir do
150
        ENV['NOSE_XUNIT_FILE'] = File.join(report_dir, "nosetests.xml")
151
        sh("nosetests #{lib} --cover-erase --with-xunit --with-xcoverage --cover-html --cover-inclusive --cover-package #{File.basename(lib)} --cover-html-dir #{File.join(report_dir, "cover")}")
152
    end
153 154 155 156 157 158 159 160 161 162 163
    TEST_TASKS << task_name
end

task :test do
    TEST_TASKS.each do |task|
        Rake::Task[task].invoke(false)
    end

    if $failed_tests > 0
        abort "Tests failed!"
    end
164
end
165

166 167
task :runserver => :lms

168
desc "Run django-admin <action> against the specified system and environment"
169
task "django-admin", [:action, :system, :env, :options] => [:predjango] do |t, args|
170 171 172 173
    args.with_defaults(:env => 'dev', :system => 'lms', :options => '')
    sh(django_admin(args.system, args.env, args.action, args.options))
end

174
task :package do
175
    FileUtils.mkdir_p(BUILD_DIR)
176

177
    Dir.chdir(BUILD_DIR) do
178 179
        afterremove = Tempfile.new('afterremove')
        afterremove.write <<-AFTERREMOVE.gsub(/^\s*/, '')
John Jarvis committed
180
        #! /bin/bash
181 182
        set -e
        set -x
183

184
        # to be a little safer this rm is executed
John Jarvis committed
185
        # as the makeitso user
186

John Jarvis committed
187
        if [[ -d "#{INSTALL_DIR_PATH}" ]]; then
188
            sudo rm -rf "#{INSTALL_DIR_PATH}"
189 190 191 192 193
        fi

        AFTERREMOVE
        afterremove.close()
        FileUtils.chmod(0755, afterremove.path)
194 195 196 197 198 199

        postinstall = Tempfile.new('postinstall')
        postinstall.write <<-POSTINSTALL.gsub(/^\s*/, '')
        #! /bin/sh
        set -e
        set -x
200

201

202
        service gunicorn stop || echo "Unable to stop gunicorn. Continuing"
203
        rm -f #{LINK_PATH}
204
        ln -s #{INSTALL_DIR_PATH} #{LINK_PATH}
205
        chown makeitso:makeitso #{LINK_PATH}
206

207 208 209 210 211 212
        # install python modules that are in the package
        if [ -r #{PIP_REPO_REQUIREMENTS} ]; then
            cd #{INSTALL_DIR_PATH}
            pip install -r #{PIP_REPO_REQUIREMENTS}
        fi

213
        chown -R makeitso:makeitso #{INSTALL_DIR_PATH}
214

215
        # Delete mako temp files
216
        rm -rf /tmp/tmp*mako
217

218
        service gunicorn start || echo "Unable to start gunicorn. Continuing"
219 220 221 222
        POSTINSTALL
        postinstall.close()
        FileUtils.chmod(0755, postinstall.path)

223
        args = ["fakeroot", "fpm", "-s", "dir", "-t", "deb",
224
            "--after-install=#{postinstall.path}",
225
            "--after-remove=#{afterremove.path}",
226
            "--prefix=#{INSTALL_DIR_PATH}",
227 228 229
            "--exclude=**/build/**",
            "--exclude=**/rakefile",
            "--exclude=**/.git/**",
230
            "--exclude=**/*.pyc",
231
            "--exclude=**/reports/**",
232
            "-C", "#{REPO_ROOT}",
233
            "--provides=#{PACKAGE_NAME}",
234
            "--name=#{NORMALIZED_DEPLOY_NAME}",
235
            "--version=#{PKG_VERSION}",
236
            "-a", "all",
237
            "."]
238 239 240
        system(*args) || raise("fpm failed to build the .deb")
    end
end
241 242

task :publish => :package do
243
    sh("scp #{BUILD_DIR}/#{NORMALIZED_DEPLOY_NAME}_#{PKG_VERSION}*.deb #{PACKAGE_REPO}")
244
end
245 246 247 248 249 250 251 252 253 254 255 256

namespace :cms do
  desc "Import course data within the given DATA_DIR variable"
  task :import do
    if ENV['DATA_DIR']
      sh(django_admin(:cms, :dev, :import, ENV['DATA_DIR']))
    else
      raise "Please specify a DATA_DIR variable that point to your data directory.\n" +
        "Example: \`rake cms:import DATA_DIR=../data\`"
    end
  end
end