Merge pull request #2790 from open-craft/tim/improve-pipeline-workflow

Automate preparatory steps for running pipeline acceptance tests

playbooks/roles/analytics_pipeline/files/acceptance.json

+{
+  "connection_user": "hadoop",
+  "credentials_file_url": "/edx/etc/edx-analytics-pipeline/output.json",
+  "exporter_output_bucket": "",
+  "geolocation_data": "/var/tmp/geolocation-data.mmdb",
+  "hive_user": "hadoop",
+  "host": "localhost",
+  "identifier": "local-devstack",
+  "manifest_input_format": "org.edx.hadoop.input.ManifestTextInputFormat",
+  "oddjob_jar": "hdfs://localhost:9000/edx-analytics-pipeline/packages/edx-analytics-hadoop-util.jar",
+  "tasks_branch": "origin/HEAD",
+  "tasks_log_path": "/tmp/acceptance/",
+  "tasks_output_url": "hdfs://localhost:9000/acceptance-test-output/",
+  "tasks_repo": "/edx/app/analytics_pipeline/analytics_pipeline",
+  "vertica_creds_url": "",
+  "wheel_url": "https://edx-wheelhouse.s3-website-us-east-1.amazonaws.com/Ubuntu/precise"
+}

playbooks/roles/analytics_pipeline/tasks/main.yml

@@ -10,9 +10,9 @@
 #
 #
 # Tasks for role analytics_pipeline
-# 
+#
 # Overview:
-# 
+#
 # Prepare the machine to run the edX Analytics Data Pipeline. The pipeline currently "installs itself"
 # via an ansible playbook that is not included in the edx/configuration repo. However, in order to
 # run the pipeline in a devstack environment, some configuration needs to be performed. In a production
@@ -24,7 +24,7 @@
 # hadoop_master: ensures hadoop services are installed
 # hive: the pipeline makes extensive usage of hive, so that needs to be installed as well
 # sqoop: similarly to hive, the pipeline uses this tool extensively
-# 
+#
 # Example play:
 #
 # - name: Deploy all dependencies of edx-analytics-pipeline to the node
@@ -174,3 +174,22 @@
   tags:
     - install
     - install:configuration
+
+- name: store configuration for acceptance tests
+  copy: >
+    src=acceptance.json
+    dest=/var/tmp/acceptance.json
+    mode=644
+  tags:
+    - install
+    - install:configuration
+
+- name: grant access to table storing test data in output database
+  mysql_user: >
+    user={{ ANALYTICS_PIPELINE_OUTPUT_DATABASE.user }}
+    password={{ ANALYTICS_PIPELINE_OUTPUT_DATABASE.password }}
+    priv=acceptance%.*:ALL
+    append_privs=yes
+  tags:
+    - install
+    - install:configuration

vagrant/release/analyticstack/Vagrantfile

@@ -46,6 +46,7 @@ MOUNT_DIRS = {
   :forum => {:repo => "cs_comments_service", :local => "/edx/app/forum/cs_comments_service", :owner => "forum"},
   :insights => {:repo => "insights", :local => "/edx/app/insights/edx_analytics_dashboard", :owner => "insights"},
   :analytics_api => {:repo => "analytics_api", :local => "/edx/app/analytics_api/analytics_api", :owner => "analytics_api"},
+  :analytics_pipeline => {:repo => "edx-analytics-pipeline", :local => "/edx/app/analytics_pipeline/analytics_pipeline", :owner => "hadoop"},
   # This src directory won't have useful permissions. You can set them from the
   # vagrant user in the guest OS. "sudo chmod 0777 /edx/src" is useful.
   :src => {:repo => "src", :local => "/edx/src", :owner => "root"},