Merge pull request #2079 from edx/ned/fingerprint-site-packages

Fingerprint the filenames in site-packages to decide when to install req...

rakefiles/helpers.rb

@@ -14,16 +14,31 @@ def report_dir_path(dir)
     return File.join(REPORT_DIR, dir.to_s)
 end
 
-def when_changed(unchanged_message, *files)
-    Rake::Task[PREREQS_MD5_DIR].invoke
-    cache_file = File.join(PREREQS_MD5_DIR, files.join('-').gsub(/\W+/, '-')) + '.md5'
+def compute_fingerprint(files, dirs)
     digest = Digest::MD5.new()
+
+    # Digest the contents of all the files.
     Dir[*files].select{|file| File.file?(file)}.each do |file|
         digest.file(file)
     end
-    if !File.exists?(cache_file) or digest.hexdigest != File.read(cache_file)
+
+    # Digest the names of the files in all the dirs.
+    dirs.each do |dir|
+        file_names = Dir.entries(dir).sort.join(" ")
+        digest.update(file_names)
+    end
+
+    digest.hexdigest
+end
+
+# Hash the contents of all the files, and the names of files in the dirs.
+# Run the block if they've changed.
+def when_changed(unchanged_message, files, dirs=[])
+    Rake::Task[PREREQS_MD5_DIR].invoke
+    cache_file = File.join(PREREQS_MD5_DIR, files[0].gsub(/\W+/, '-').sub(/-+$/, '')) + '.md5'
+    if !File.exists?(cache_file) or compute_fingerprint(files, dirs) != File.read(cache_file)
         yield
-        File.write(cache_file, digest.hexdigest)
+        File.write(cache_file, compute_fingerprint(files, dirs))
     elsif !unchanged_message.empty?
         puts unchanged_message
     end

rakefiles/prereqs.rake

 require './rakefiles/helpers.rb'
 
-
 PREREQS_MD5_DIR = ENV["PREREQ_CACHE_DIR"] || File.join(REPO_ROOT, '.prereqs_cache')
 
 CLOBBER.include(PREREQS_MD5_DIR)
@@ -13,7 +12,7 @@ task :install_prereqs => [:install_node_prereqs, :install_ruby_prereqs, :install
 desc "Install all node prerequisites for the lms and cms"
 task :install_node_prereqs => "ws:migrate" do
     unchanged = 'Node requirements unchanged, nothing to install'
-    when_changed(unchanged, 'package.json') do
+    when_changed(unchanged, ['package.json']) do
         sh('npm install')
     end unless ENV['NO_PREREQ_INSTALL']
 end
@@ -21,20 +20,21 @@ end
 desc "Install all ruby prerequisites for the lms and cms"
 task :install_ruby_prereqs => "ws:migrate" do
     unchanged = 'Ruby requirements unchanged, nothing to install'
-    when_changed(unchanged, 'Gemfile') do
+    when_changed(unchanged, ['Gemfile']) do
         sh('bundle install')
     end unless ENV['NO_PREREQ_INSTALL']
 end
 
 desc "Install all python prerequisites for the lms and cms"
 task :install_python_prereqs => "ws:migrate" do
+    site_packages_dir = `python -c 'import os; import distutils.sysconfig as dusc; print dusc.get_python_lib()'`.chomp
     unchanged = 'Python requirements unchanged, nothing to install'
-    when_changed(unchanged, 'requirements/**/*') do
+    when_changed(unchanged, ['requirements/**/*'], [site_packages_dir]) do
         ENV['PIP_DOWNLOAD_CACHE'] ||= '.pip_download_cache'
         sh('pip install --exists-action w -r requirements/edx/base.txt')
         sh('pip install --exists-action w -r requirements/edx/post.txt')
-        # Check for private-requirements.txt: used to install our libs as working dirs,
-        # or personal-use tools.
+        # requirements/private.txt is used to install our libs as
+        # working dirs, or for personal-use tools.
         if File.file?("requirements/private.txt")
             sh('pip install -r requirements/private.txt')
         end