Merge pull request #3472 from edx/brian/update_hadoop

Upgrade Hadoop and Hive

playbooks/roles/analytics_pipeline/tasks/main.yml

@@ -229,3 +229,10 @@
   tags:
     - install
     - install:configuration
+
+- name: Initialize Hive metadata store schema
+  shell: ". {{ HADOOP_COMMON_CONF_DIR }}/hadoop-env.sh && {{ HIVE_HOME }}/bin/schematool -dbType mysql -initSchema"
+  become_user: "{{ hadoop_common_user }}"
+  tags:
+    - install
+    - install:base

playbooks/roles/hadoop_common/defaults/main.yml

@@ -11,7 +11,7 @@
 # Defaults for role hadoop_common
 #
 
-HADOOP_COMMON_VERSION: 2.3.0
+HADOOP_COMMON_VERSION: 2.7.2
 HADOOP_COMMON_USER_HOME: "{{ COMMON_APP_DIR }}/hadoop"
 HADOOP_COMMON_HOME: "{{ HADOOP_COMMON_USER_HOME }}/hadoop"
 HADOOP_COMMON_DATA: "{{ COMMON_DATA_DIR }}/hadoop"
@@ -31,16 +31,19 @@ hadoop_common_group: hadoop
 hadoop_common_temporary_dir: /var/tmp
 hadoop_common_dist:
   filename: "hadoop-{{ HADOOP_COMMON_VERSION }}.tar.gz"
-  url: "https://archive.apache.org/dist/hadoop/core/hadoop-{{ HADOOP_COMMON_VERSION }}/hadoop-{{ HADOOP_COMMON_VERSION }}.tar.gz"
-  sha256sum: 3fad58b525a47cf74458d0996564a2151c5a28baa1f92383e7932774deef5023
+  url: "https://archive.apache.org/dist/hadoop/common/hadoop-{{ HADOOP_COMMON_VERSION }}/hadoop-{{ HADOOP_COMMON_VERSION }}.tar.gz"
+  sha256sum: 49AD740F85D27FA39E744EB9E3B1D9442AE63D62720F0AABDAE7AA9A718B03F7
+
 hadoop_common_protobuf_dist:
   filename: "protobuf-{{ HADOOP_COMMON_PROTOBUF_VERSION }}.tar.gz"
   url: "https://github.com/google/protobuf/releases/download/v{{ HADOOP_COMMON_PROTOBUF_VERSION }}/protobuf-{{ HADOOP_COMMON_PROTOBUF_VERSION }}.tar.gz"
   sha256sum: c55aa3dc538e6fd5eaf732f4eb6b98bdcb7cedb5b91d3b5bdcf29c98c293f58e
+
 hadoop_common_native_dist:
-  filename: "release-{{ HADOOP_COMMON_VERSION }}.tar.gz"
-  url: "https://github.com/apache/hadoop-common/archive/release-{{ HADOOP_COMMON_VERSION }}.tar.gz"
-  sha256sum: a8e1b49d4e891255d465e9449346ac7fb259bb35dce07d9f0df3b46fac3e9bd0
+  filename: "hadoop-{{ HADOOP_COMMON_VERSION }}-src.tar.gz"
+  url: "https://archive.apache.org/dist/hadoop/common/hadoop-{{ HADOOP_COMMON_VERSION }}/hadoop-{{ HADOOP_COMMON_VERSION }}-src.tar.gz"
+  sha256sum: 7d48e61b5464a76543fecf5655d06215cf8674d248b28bc74f613bc8aa047d33
+
 hadoop_common_java_home: "{{ oraclejdk_link }}"
 hadoop_common_env: "{{ HADOOP_COMMON_HOME }}/hadoop_env"
 

playbooks/roles/hadoop_common/tasks/main.yml

@@ -191,7 +191,7 @@
 - name: native lib built
   shell: "mvn package -X -Pnative -DskipTests"
   args:
-    chdir: "{{ hadoop_common_temporary_dir }}/hadoop-common-release-{{ HADOOP_COMMON_VERSION }}/hadoop-common-project"
+    chdir: "{{ hadoop_common_temporary_dir }}/hadoop-{{ HADOOP_COMMON_VERSION }}-src/hadoop-common-project"
   environment:
     LD_LIBRARY_PATH: /usr/local/lib
   when: not native_libs_built.stat.exists
@@ -207,7 +207,7 @@
 - name: new native libs installed
   shell: "chown {{ hadoop_common_user }}:{{ hadoop_common_group }} {{ item }} && cp {{ item }} {{ HADOOP_COMMON_HOME }}/lib/native/{{ item }}"
   args:
-    chdir: "{{ hadoop_common_temporary_dir }}/hadoop-common-release-{{ HADOOP_COMMON_VERSION }}/hadoop-common-project/hadoop-common/target/native/target/usr/local/lib"
+    chdir: "{{ hadoop_common_temporary_dir }}/hadoop-{{ HADOOP_COMMON_VERSION }}-src/hadoop-common-project/hadoop-common/target/native/target/usr/local/lib"
   with_items:
     - libhadoop.a
     - libhadoop.so

playbooks/roles/hive/defaults/main.yml

@@ -11,7 +11,8 @@
 # Defaults for role hive
 # 
 
-HIVE_VERSION: 0.11.0
+HIVE_VERSION: 1.0.0
+# This should match the value for SQOOP_MYSQL_CONNECTOR_VERSION in sqoop role.
 HIVE_MYSQL_CONNECTOR_VERSION: 5.1.29
 HIVE_HOME: "{{ HADOOP_COMMON_USER_HOME }}/hive"
 HIVE_CONF: "{{ HIVE_HOME }}/conf"
@@ -38,7 +39,7 @@ HIVE_SITE_DEFAULT_CONFIG:
   javax.jdo.option.ConnectionDriverName: "com.mysql.jdbc.Driver"
   javax.jdo.option.ConnectionUserName: "{{ HIVE_METASTORE_DATABASE.user }}"
   javax.jdo.option.ConnectionPassword: "{{ HIVE_METASTORE_DATABASE.password }}"
-  datanucleus.autoCreateSchema: "true"
+  datanucleus.autoCreateSchema: "false"
   hive.metastore.schema.verification: "true"
 
 #
@@ -54,13 +55,16 @@ HIVE_SITE_EXTRA_CONFIG: {}
 hive_role_name: hive
 hive_temporary_dir: /var/tmp
 hive_dist:
-  filename: "hive-{{ HIVE_VERSION }}-bin.tar.gz"
-  url: "https://archive.apache.org/dist/hive/hive-{{ HIVE_VERSION }}/hive-{{ HIVE_VERSION }}-bin.tar.gz"
-  sha256sum: c22ee328438e80a8ee4b66979dba69650511a73f8b6edf2d87d93c74283578e5
+  filename: "apache-hive-{{ HIVE_VERSION }}-bin.tar.gz"
+  url: "https://archive.apache.org/dist/hive/hive-{{ HIVE_VERSION }}/apache-hive-{{ HIVE_VERSION }}-bin.tar.gz"
+  sha256sum: 4dab651288fee1d234e674efd3573a7441a8748e9a555e453999a75abc553b53
+  untarred_filename: "apache-hive-{{ HIVE_VERSION }}-bin"
 hive_mysql_connector_dist:
-  filename: "mysql-connector-java-{{ SQOOP_MYSQL_CONNECTOR_VERSION }}.tar.gz"
-  url: "http://dev.mysql.com/get/Downloads/Connector-J/mysql-connector-java-{{ SQOOP_MYSQL_CONNECTOR_VERSION }}.tar.gz"
+  filename: "mysql-connector-java-{{ HIVE_MYSQL_CONNECTOR_VERSION }}.tar.gz"
+  url: "http://dev.mysql.com/get/Downloads/Connector-J/mysql-connector-java-{{ HIVE_MYSQL_CONNECTOR_VERSION }}.tar.gz"
   sha256sum: 04ad83b655066b626daaabb9676a00f6b4bc43f0c234cbafafac1209dcf1be73
+  untarred_filename: "mysql-connector-java-{{ HIVE_MYSQL_CONNECTOR_VERSION }}"
+  jarfilename: "mysql-connector-java-{{ HIVE_MYSQL_CONNECTOR_VERSION }}-bin.jar"
 
 #
 # OS packages

playbooks/roles/hive/tasks/main.yml

@@ -33,14 +33,14 @@
   when: not extracted_dir.stat.exists
 
 - name: distribution extracted
-  shell: "tar -xzf {{ hive_temporary_dir }}/{{ hive_dist.filename }} && chown -R {{ hadoop_common_user }}:{{ hadoop_common_group }} hive-{{ HIVE_VERSION }}-bin"
+  shell: "tar -xzf {{ hive_temporary_dir }}/{{ hive_dist.filename }} && chown -R {{ hadoop_common_user }}:{{ hadoop_common_group }} {{ hive_dist.untarred_filename }}"
   args:
     chdir: "{{ HADOOP_COMMON_USER_HOME }}"
   when: not extracted_dir.stat.exists
 
 - name: versioned directory symlink created
   file:
-    src: "{{ HADOOP_COMMON_USER_HOME }}/hive-{{ HIVE_VERSION }}-bin"
+    src: "{{ HADOOP_COMMON_USER_HOME }}/{{ hive_dist.untarred_filename }}"
     dest: "{{ HIVE_HOME }}"
     owner: "{{ hadoop_common_user }}"
     group: "{{ hadoop_common_group }}"
@@ -67,7 +67,7 @@
     state: directory
 
 - name: hive mysql connector installed
-  shell: "cp mysql-connector-java-{{ HIVE_MYSQL_CONNECTOR_VERSION }}-bin.jar {{ HIVE_LIB }} && chown {{ hadoop_common_user }}:{{ hadoop_common_group }} {{ HIVE_LIB }}/mysql-connector-java-{{ HIVE_MYSQL_CONNECTOR_VERSION }}-bin.jar"
+  shell: "cp {{ hive_mysql_connector_dist.jarfilename }} {{ HIVE_LIB }} && chown {{ hadoop_common_user }}:{{ hadoop_common_group }} {{ HIVE_LIB }}/{{ hive_mysql_connector_dist.jarfilename }}"
   args:
     chdir: "/{{ hive_temporary_dir }}/mysql-connector-java-{{ HIVE_MYSQL_CONNECTOR_VERSION }}"
   when: not extracted_dir.stat.exists