rakefile

require 'rake/clean'
require 'tempfile'
require 'net/http'
require 'launchy'
require 'colorize'
require 'erb'
require 'tempfile'

# Build Constants
REPO_ROOT = File.dirname(__FILE__)
BUILD_DIR = File.join(REPO_ROOT, "build")
REPORT_DIR = File.join(REPO_ROOT, "reports")
LMS_REPORT_DIR = File.join(REPORT_DIR, "lms")

# Packaging constants
DEPLOY_DIR = "/opt/wwc"
PACKAGE_NAME = "mitx"
LINK_PATH = "/opt/wwc/mitx"
PKG_VERSION = "0.1"
COMMIT = (ENV["GIT_COMMIT"] || `git rev-parse HEAD`).chomp()[0, 10]
BRANCH = (ENV["GIT_BRANCH"] || `git symbolic-ref -q HEAD`).chomp().gsub('refs/heads/', '').gsub('origin/', '')
BUILD_NUMBER = (ENV["BUILD_NUMBER"] || "dev").chomp()

# Set up the clean and clobber tasks
CLOBBER.include(BUILD_DIR, REPORT_DIR, 'test_root/*_repo', 'test_root/staticfiles')
CLEAN.include("#{BUILD_DIR}/*.deb", "#{BUILD_DIR}/util")

def select_executable(*cmds)
    cmds.find_all{ |cmd| system("which #{cmd} > /dev/null 2>&1") }[0] || fail("No executables found from #{cmds.join(', ')}")
end

def django_admin(system, env, command, *args)
    django_admin = ENV['DJANGO_ADMIN_PATH'] || select_executable('django-admin.py', 'django-admin')
    return "#{django_admin} #{command} --traceback --settings=#{system}.envs.#{env} --pythonpath=. #{args.join(' ')}"
end

# Runs Process.spawn, and kills the process at the end of the rake process
# Expects the same arguments as Process.spawn
def background_process(*command)
    pid = Process.spawn({}, *command, {:pgroup => true})

    at_exit do
        puts "Ending process and children"
        pgid = Process.getpgid(pid)
        begin
            Timeout.timeout(5) do
                puts "Terminating process group #{pgid}"
                Process.kill(:SIGTERM, -pgid)
                puts "Waiting on process group #{pgid}"
                Process.wait(-pgid)
                puts "Done waiting on process group #{pgid}"
            end
        rescue Timeout::Error
            puts "Killing process group #{pgid}"
            Process.kill(:SIGKILL, -pgid)
            puts "Waiting on process group #{pgid}"
            Process.wait(-pgid)
            puts "Done waiting on process group #{pgid}"
        end
    end
end

def django_for_jasmine(system, django_reload)
    if !django_reload
        reload_arg = '--noreload'
    end

    port = 10000 + rand(40000)
    jasmine_url = "http://localhost:#{port}/_jasmine/"

    background_process(*django_admin(system, 'jasmine', 'runserver', '-v', '0', port.to_s, reload_arg).split(' '))

    up = false
    start_time = Time.now
    until up do
        if Time.now - start_time > 30
            abort "Timed out waiting for server to start to run jasmine tests"
        end
        begin
            response = Net::HTTP.get_response(URI(jasmine_url))
            puts response.code
            up = response.code == '200'
        rescue => e
            puts e.message
        ensure
            puts('Waiting server to start')
            sleep(0.5)
        end
    end
    yield jasmine_url
end

def template_jasmine_runner(lib)
    coffee_files = Dir["#{lib}/**/js/**/*.coffee", "common/static/coffee/src/**/*.coffee"]
    if !coffee_files.empty?
        sh("node_modules/.bin/coffee -c #{coffee_files.join(' ')}")
    end
    phantom_jasmine_path = File.expand_path("common/test/phantom-jasmine")
    common_js_root = File.expand_path("common/static/js")
    common_coffee_root = File.expand_path("common/static/coffee/src")

    # Get arrays of spec and source files, ordered by how deep they are nested below the library
    # (and then alphabetically) and expanded from a relative to an absolute path
    spec_glob = File.join("#{lib}", "**", "spec", "**", "*.js")
    src_glob = File.join("#{lib}", "**", "src", "**", "*.js")
    js_specs = Dir[spec_glob].sort_by {|p| [p.split('/').length, p]} .map {|f| File.expand_path(f)}
    js_source = Dir[src_glob].sort_by {|p| [p.split('/').length, p]} .map {|f| File.expand_path(f)}

    template = ERB.new(File.read("#{lib}/jasmine_test_runner.html.erb"))
    template_output = "#{lib}/jasmine_test_runner.html"
    File.open(template_output, 'w') do |f|
        f.write(template.result(binding))
    end
    yield File.expand_path(template_output)
end


def report_dir_path(dir)
    return File.join(REPORT_DIR, dir.to_s)
end

def compile_assets(watch=false, debug=false)
    xmodule_cmd = 'xmodule_assets common/static/xmodule'
    if watch
        xmodule_cmd = "watchmedo shell-command \
                       --patterns='*.js;*.coffee;*.sass;*.scss;*.css' \
                       --recursive \
                       --command='#{xmodule_cmd}' \
                       common/lib/xmodule"
    end
    coffee_cmd = "node_modules/.bin/coffee #{watch ? '--watch' : ''} --compile */static"
    sass_cmd = "sass #{debug ? '--debug-info' : '--style compressed'} " +
                     "--load-path ./common/static/sass " +
                     "--require ./common/static/sass/bourbon/lib/bourbon.rb " +
                     "#{watch ? '--watch' : '--update --force'} */static"

    [xmodule_cmd, coffee_cmd, sass_cmd].each do |cmd|
        if watch
            background_process(cmd)
        else
            pid = Process.spawn(cmd)
            puts "Waiting for `#{cmd}` to complete (pid #{pid})"
            Process.wait(pid)
            puts "Completed"
            if !$?.exited? || $?.exitstatus != 0
                abort "`#{cmd}` failed"
            end
        end
    end
end

task :default => [:test, :pep8, :pylint]

directory REPORT_DIR

default_options = {
    :lms => '8000',
    :cms => '8001',
}

desc "Install all prerequisites needed for the lms and cms"
task :install_prereqs => [:install_node_prereqs, :install_ruby_prereqs, :install_python_prereqs]

desc "Install all node prerequisites for the lms and cms"
task :install_node_prereqs do
    sh('npm install')
end

desc "Install all ruby prerequisites for the lms and cms"
task :install_ruby_prereqs do
    sh('bundle install')
end

desc "Install all python prerequisites for the lms and cms"
task :install_python_prereqs do
    sh('pip install -r requirements.txt')
    # Check for private-requirements.txt: used to install our libs as working dirs,
    # or personal-use tools.
    if File.file?("private-requirements.txt")
        sh('pip install -r private-requirements.txt')
    end
end

task :predjango do
    sh("find . -type f -name *.pyc -delete")
    sh('pip install -q --no-index -r local-requirements.txt')
end

task :clean_test_files do
    sh("git clean -fqdx test_root")
end

[:lms, :cms, :common].each do |system|
    report_dir = report_dir_path(system)
    directory report_dir

    desc "Run pep8 on all #{system} code"
    task "pep8_#{system}" => report_dir do
        sh("pep8 #{system} | tee #{report_dir}/pep8.report")
    end
    task :pep8 => "pep8_#{system}"

    desc "Run pylint on all #{system} code"
    task "pylint_#{system}" => report_dir do
        apps = Dir["#{system}/*.py", "#{system}/djangoapps/*", "#{system}/lib/*"].map do |app|
            File.basename(app)
        end.select do |app|
            app !=~ /.pyc$/
        end.map do |app|
            if app =~ /.py$/
                app.gsub('.py', '')
            else
                app
            end
        end

        pythonpath_prefix = "PYTHONPATH=#{system}:#{system}/djangoapps:#{system}/lib:common/djangoapps:common/lib"
        sh("#{pythonpath_prefix} pylint --rcfile=.pylintrc -f parseable #{apps.join(' ')} | tee #{report_dir}/pylint.report")
    end
    task :pylint => "pylint_#{system}"

end

$failed_tests = 0

def run_under_coverage(cmd, root)
    cmd0, cmd_rest = cmd.split(" ", 2)
    # We use "python -m coverage" so that the proper python will run the importable coverage
    # rather than the coverage that OS path finds.
    cmd = "python -m coverage run --rcfile=#{root}/.coveragerc `which #{cmd0}` #{cmd_rest}"
    return cmd
end

def run_tests(system, report_dir, stop_on_failure=true)
    ENV['NOSE_XUNIT_FILE'] = File.join(report_dir, "nosetests.xml")
    dirs = Dir["common/djangoapps/*"] + Dir["#{system}/djangoapps/*"]
    cmd = django_admin(system, :test, 'test', '--logging-clear-handlers', *dirs.each)
    sh(run_under_coverage(cmd, system)) do |ok, res|
        if !ok and stop_on_failure
            abort "Test failed!"
        end
        $failed_tests += 1 unless ok
    end
end

TEST_TASK_DIRS = []

task :fastlms do
    # this is >2 times faster that rake [lms], and does not need web, good for local dev
    django_admin = ENV['DJANGO_ADMIN_PATH'] || select_executable('django-admin.py', 'django-admin')
    sh("#{django_admin} runserver --traceback --settings=lms.envs.dev   --pythonpath=.")
end

[:lms, :cms].each do |system|
    report_dir = report_dir_path(system)

    # Per System tasks
    desc "Run all django tests on our djangoapps for the #{system}"
    task "test_#{system}", [:stop_on_failure] => ["clean_test_files", :predjango, "#{system}:gather_assets:test", "fasttest_#{system}"]

    # Have a way to run the tests without running collectstatic -- useful when debugging without
    # messing with static files.
    task "fasttest_#{system}", [:stop_on_failure] => [report_dir, :predjango] do |t, args|
        args.with_defaults(:stop_on_failure => 'true')
        run_tests(system, report_dir, args.stop_on_failure)
    end

    task :fasttest => "fasttest_#{system}"

    TEST_TASK_DIRS << system

    desc <<-desc
        Start the #{system} locally with the specified environment (defaults to dev).
        Other useful environments are devplus (for dev testing with a real local database)
        desc
    task system, [:env, :options] => [:predjango] do |t, args|
        args.with_defaults(:env => 'dev', :options => default_options[system])

        # Compile all assets first
        compile_assets(watch=false, debug=true)

        # Listen for any changes to assets
        compile_assets(watch=true, debug=true)

        sh(django_admin(system, args.env, 'runserver', args.options))
    end

    # Per environment tasks
    Dir["#{system}/envs/**/*.py"].each do |env_file|
        env = env_file.gsub("#{system}/envs/", '').gsub(/\.py/, '').gsub('/', '.')
        desc "Attempt to import the settings file #{system}.envs.#{env} and report any errors"
        task "#{system}:check_settings:#{env}" => :predjango do
            sh("echo 'import #{system}.envs.#{env}' | #{django_admin(system, env, 'shell')}")
        end

        desc "Compile coffeescript and sass, and then run collectstatic in the specified environment"
        task "#{system}:gather_assets:#{env}" do
            compile_assets()
            sh("#{django_admin(system, env, 'collectstatic', '--noinput')} > /dev/null") do |ok, status|
                if !ok
                    abort "collectstatic failed!"
                end
            end
        end
    end

    desc "Open jasmine tests for #{system} in your default browser"
    task "browse_jasmine_#{system}" do
        compile_assets()
        django_for_jasmine(system, true) do |jasmine_url|
            Launchy.open(jasmine_url)
            puts "Press ENTER to terminate".red
            $stdin.gets
        end
    end

    desc "Use phantomjs to run jasmine tests for #{system} from the console"
    task "phantomjs_jasmine_#{system}" do
        compile_assets()
        phantomjs = ENV['PHANTOMJS_PATH'] || 'phantomjs'
        django_for_jasmine(system, false) do |jasmine_url|
            sh("#{phantomjs} common/test/phantom-jasmine/lib/run_jasmine_test.coffee #{jasmine_url}")
        end
    end
end

desc "Reset the relational database used by django. WARNING: this will delete all of your existing users"
task :resetdb, [:env] do |t, args|
    args.with_defaults(:env => 'dev')
    sh(django_admin(:lms, args.env, 'syncdb'))
    sh(django_admin(:lms, args.env, 'migrate'))
end

desc "Update the relational database to the latest migration"
task :migrate, [:env] do |t, args|
    args.with_defaults(:env => 'dev')
    sh(django_admin(:lms, args.env, 'migrate'))
end

desc "Run tests for the internationalization library"
task :test_i18n do
  test = File.join(REPO_ROOT, "i18n", "tests")
  sh("nosetests #{test}")
end

Dir["common/lib/*"].select{|lib| File.directory?(lib)}.each do |lib|
    task_name = "test_#{lib}"

    report_dir = report_dir_path(lib)

    desc "Run tests for common lib #{lib}"
    task task_name => report_dir do
        ENV['NOSE_XUNIT_FILE'] = File.join(report_dir, "nosetests.xml")
        cmd = "nosetests #{lib}"
        sh(run_under_coverage(cmd, lib)) do |ok, res|
            $failed_tests += 1 unless ok
        end
    end
    TEST_TASK_DIRS << lib

    desc "Run tests for common lib #{lib} (without coverage)"
    task "fasttest_#{lib}" do
        sh("nosetests #{lib}")
    end

    desc "Open jasmine tests for #{lib} in your default browser"
    task "browse_jasmine_#{lib}" do
        template_jasmine_runner(lib) do |f|
            sh("python -m webbrowser -t 'file://#{f}'")
            puts "Press ENTER to terminate".red
            $stdin.gets
        end
    end

    desc "Use phantomjs to run jasmine tests for #{lib} from the console"
    task "phantomjs_jasmine_#{lib}" do
        phantomjs = ENV['PHANTOMJS_PATH'] || 'phantomjs'
        template_jasmine_runner(lib) do |f|
            sh("#{phantomjs} common/test/phantom-jasmine/lib/run_jasmine_test.coffee #{f}")
        end
    end
end

task :report_dirs

TEST_TASK_DIRS.each do |dir|
    report_dir = report_dir_path(dir)
    directory report_dir
    task :report_dirs => [REPORT_DIR, report_dir]
end

task :test do
    TEST_TASK_DIRS.each do |dir|
        Rake::Task["test_#{dir}"].invoke(false)
    end

    if $failed_tests > 0
        abort "Tests failed!"
    end
end

namespace :coverage do
    desc "Build the html coverage reports"
    task :html => :report_dirs do
        TEST_TASK_DIRS.each do |dir|
            report_dir = report_dir_path(dir)

            if !File.file?("#{report_dir}/.coverage")
                next
            end

            sh("coverage html --rcfile=#{dir}/.coveragerc")
        end
    end

    desc "Build the xml coverage reports"
    task :xml => :report_dirs do
        TEST_TASK_DIRS.each do |dir|
            report_dir = report_dir_path(dir)

            if !File.file?("#{report_dir}/.coverage")
                next
            end
            # Why doesn't the rcfile control the xml output file properly??
            sh("coverage xml -o #{report_dir}/coverage.xml --rcfile=#{dir}/.coveragerc")
        end
    end
end

task :runserver => :lms

desc "Run django-admin <action> against the specified system and environment"
task "django-admin", [:action, :system, :env, :options] do |t, args|
    args.with_defaults(:env => 'dev', :system => 'lms', :options => '')
    sh(django_admin(args.system, args.env, args.action, args.options))
end

desc "Set the staff bit for a user"
task :set_staff, [:user, :system, :env] do |t, args|
    args.with_defaults(:env => 'dev', :system => 'lms', :options => '')
    sh(django_admin(args.system, args.env, 'set_staff', args.user))
end

namespace :cms do
  desc "Clone existing MongoDB based course"
  task :clone do

    if ENV['SOURCE_LOC'] and ENV['DEST_LOC']
      sh(django_admin(:cms, :dev, :clone, ENV['SOURCE_LOC'], ENV['DEST_LOC']))
    else
      raise "You must pass in a SOURCE_LOC and DEST_LOC parameters"
    end
  end

  desc "Delete existing MongoDB based course"
  task :delete_course do

    if ENV['LOC'] and ENV['COMMIT']
        sh(django_admin(:cms, :dev, :delete_course, ENV['LOC'], ENV['COMMIT']))
    elsif ENV['LOC']
      sh(django_admin(:cms, :dev, :delete_course, ENV['LOC']))
    else
      raise "You must pass in a LOC parameter"
    end
  end

  desc "Import course data within the given DATA_DIR variable"
  task :import do
    if ENV['DATA_DIR'] and ENV['COURSE_DIR']
      sh(django_admin(:cms, :dev, :import, ENV['DATA_DIR'], ENV['COURSE_DIR']))
    elsif ENV['DATA_DIR']
      sh(django_admin(:cms, :dev, :import, ENV['DATA_DIR']))
    else
      raise "Please specify a DATA_DIR variable that point to your data directory.\n" +
        "Example: \`rake cms:import DATA_DIR=../data\`"
    end
  end

  desc "Imports all the templates from the code pack"
  task :update_templates do
    sh(django_admin(:cms, :dev, :update_templates))
  end

  desc "Import course data within the given DATA_DIR variable"
  task :xlint do
    if ENV['DATA_DIR'] and ENV['COURSE_DIR']
      sh(django_admin(:cms, :dev, :xlint, ENV['DATA_DIR'], ENV['COURSE_DIR']))
    elsif ENV['DATA_DIR']
      sh(django_admin(:cms, :dev, :xlint, ENV['DATA_DIR']))
    else
      raise "Please specify a DATA_DIR variable that point to your data directory.\n" +
        "Example: \`rake cms:import DATA_DIR=../data\`"
    end
  end

  desc "Export course data to a tar.gz file"
  task :export do
    if ENV['COURSE_ID'] and ENV['OUTPUT_PATH']
      sh(django_admin(:cms, :dev, :export, ENV['COURSE_ID'], ENV['OUTPUT_PATH']))
    else
      raise "Please specify a COURSE_ID and OUTPUT_PATH.\n" +
        "Example: \`rake cms:export COURSE_ID=MITx/12345/name OUTPUT_PATH=foo.tar.gz\`"
    end
  end
end

desc "Build a properties file used to trigger autodeploy builds"
task :autodeploy_properties do
    File.open("autodeploy.properties", "w") do |file|
        file.puts("UPSTREAM_NOOP=false")
        file.puts("UPSTREAM_BRANCH=#{BRANCH}")
        file.puts("UPSTREAM_JOB=#{PACKAGE_NAME}")
        file.puts("UPSTREAM_REVISION=#{COMMIT}")
    end
end

# --- Internationalization tasks

desc "Extract localizable strings from sources"
task :extract_dev_strings do
  sh(File.join(REPO_ROOT, "i18n", "extract.py"))
end

desc "Compile localizable strings from sources. With optional flag 'extract', will extract strings first."
task :generate_i18n do
  if ARGV.last.downcase == 'extract'
    Rake::Task["extract_dev_strings"].execute
  end
  sh(File.join(REPO_ROOT, "i18n", "generate.py"))
end

desc "Simulate international translation by generating dummy strings corresponding to source strings."
task :dummy_i18n do
  source_files = Dir["#{REPO_ROOT}/conf/locale/en/LC_MESSAGES/*.po"]
  dummy_locale = 'fr'
  cmd = File.join(REPO_ROOT, "i18n", "make_dummy.py")
  for file in source_files do
    sh("#{cmd} #{file} #{dummy_locale}")
  end
end

# --- Develop and public documentation ---
desc "Invoke sphinx 'make build' to generate docs."
task :builddocs, [:options] do |t, args|
    if args.options == 'pub'
        path = "doc/public"
    else
        path = "docs"
    end

    Dir.chdir(path) do
        sh('make html')
    end
end

desc "Show docs in browser (mac and ubuntu)."
task :showdocs, [:options] do |t, args|
    if args.options == 'pub'
        path = "doc/public"
    else
        path = "docs"
    end

    Dir.chdir("#{path}/build/html") do
        if RUBY_PLATFORM.include? 'darwin'  #  mac os
            sh('open index.html')
        elsif RUBY_PLATFORM.include? 'linux'  # make more ubuntu specific?
            sh('sensible-browser index.html')  # ubuntu
        else
            raise "\nUndefined how to run browser on your machine.
Please use 'rake builddocs' and then manually open
'mitx/#{path}/build/html/index.html."
        end
    end
end

desc "Build docs and show them in browser"
task :doc, [:options] =>  :builddocs do |t, args|
    Rake::Task["showdocs"].invoke(args.options)
end
# --- Develop and public documentation ---