Move hive metastore to mysql

playbooks/roles/edxlocal/tasks/main.yml

@@ -144,6 +144,20 @@
     priv='{{ ANALYTICS_API_DATABASES.reports.NAME }}.*:ALL'
   when: ANALYTICS_API_SERVICE_CONFIG is defined
 
+- name: create a database for thie hive metastore
+  mysql_db: >
+    db={{ HIVE_METASTORE_DATABASE.name }}
+    state=present
+    encoding=latin1
+  when: HIVE_METASTORE_DATABASE is defined
+
+- name: setup the edxapp db user
+  mysql_user: >
+    name={{ HIVE_METASTORE_DATABASE.user }}
+    password={{ HIVE_METASTORE_DATABASE.password }}
+    priv='{{ HIVE_METASTORE_DATABASE.name }}.*:ALL'
+  when: HIVE_METASTORE_DATABASE is defined
+
 - name: setup the edx-notes-api db user
   mysql_user: >
     name={{ EDX_NOTES_API_MYSQL_DB_USER }}

playbooks/roles/hive/defaults/main.yml

@@ -12,8 +12,17 @@
 # 
 
 HIVE_VERSION: 0.11.0
+HIVE_MYSQL_CONNECTOR_VERSION: 5.1.29
 HIVE_HOME: "{{ HADOOP_COMMON_USER_HOME }}/hive"
 HIVE_CONF: "{{ HIVE_HOME }}/conf"
+HIVE_LIB: "{{ HIVE_HOME }}/lib"
+
+HIVE_METASTORE_DATABASE:
+  user: edx_hive
+  password: edx
+  name: edx_hive_metastore
+  host: 127.0.0.1
+  port: 3306
 
 
 #
@@ -25,6 +34,10 @@ hive_dist:
   filename: "hive-{{ HIVE_VERSION }}-bin.tar.gz"
   url: "https://archive.apache.org/dist/hive/hive-{{ HIVE_VERSION }}/hive-{{ HIVE_VERSION }}-bin.tar.gz"
   sha256sum: c22ee328438e80a8ee4b66979dba69650511a73f8b6edf2d87d93c74283578e5
+hive_mysql_connector_dist:
+  filename: "mysql-connector-java-{{ SQOOP_MYSQL_CONNECTOR_VERSION }}.tar.gz"
+  url: "http://dev.mysql.com/get/Downloads/Connector-J/mysql-connector-java-{{ SQOOP_MYSQL_CONNECTOR_VERSION }}.tar.gz"
+  sha256sum: 04ad83b655066b626daaabb9676a00f6b4bc43f0c234cbafafac1209dcf1be73
 
 #
 # OS packages

playbooks/roles/hive/tasks/main.yml

@@ -43,6 +43,31 @@
     dest={{ HIVE_HOME }}
     owner={{ hadoop_common_user }} group={{ hadoop_common_group }} state=link
 
+- name: hive mysql connector distribution downloaded
+  get_url: >
+    url={{ hive_mysql_connector_dist.url }}
+    sha256sum={{ hive_mysql_connector_dist.sha256sum }}
+    dest={{ hive_temporary_dir }}
+  when: not extracted_dir.stat.exists
+
+- name: hive mysql connector distribution extracted
+  shell: >
+    chdir={{ hive_temporary_dir }}
+    tar -xzf {{ hive_temporary_dir }}/{{ hive_mysql_connector_dist.filename }}
+  when: not extracted_dir.stat.exists
+
+- name: hive lib exists
+  file: >
+    path={{ HIVE_LIB }}
+    owner={{ hadoop_common_user }} group={{ hadoop_common_group }} state=directory
+
+- name: hive mysql connector installed
+  shell: >
+    chdir=/{{ hive_temporary_dir }}/mysql-connector-java-{{ HIVE_MYSQL_CONNECTOR_VERSION }}
+    cp mysql-connector-java-{{ HIVE_MYSQL_CONNECTOR_VERSION }}-bin.jar {{ HIVE_LIB }} &&
+    chown {{ hadoop_common_user }}:{{ hadoop_common_group }} {{ HIVE_LIB }}/mysql-connector-java-{{ HIVE_MYSQL_CONNECTOR_VERSION }}-bin.jar
+  when: not extracted_dir.stat.exists
+
 - name: configuration installed
   template: >
     src={{ item }}.j2
@@ -50,6 +75,7 @@
     mode=0640 owner={{ hadoop_common_user }} group={{ hadoop_common_group }}
   with_items:
     - hive-env.sh
+    - hive-site.xml
 
 - name: env vars sourced in hadoop env
   lineinfile: >

playbooks/roles/hive/templates/hive-site.xml.j2

+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<configuration>
+  <property>
+    <name>javax.jdo.option.ConnectionURL</name>  
+    <value>jdbc:mysql://{{ HIVE_METASTORE_DATABASE.host }}:{{ HIVE_METASTORE_DATABASE.port }}/{{ HIVE_METASTORE_DATABASE.name }}</value>
+  </property>
+  <property>
+    <name>javax.jdo.option.ConnectionDriverName</name>  
+    <value>com.mysql.jdbc.Driver</value>
+  </property>
+  <property>
+    <name>javax.jdo.option.ConnectionUserName</name>  
+    <value>{{ HIVE_METASTORE_DATABASE.user }}</value>
+  </property>
+  <property>
+    <name>javax.jdo.option.ConnectionPassword</name>  
+    <value>{{ HIVE_METASTORE_DATABASE.password }}</value>
+  </property>
+  <property>
+    <name>datanucleus.autoCreateSchema</name>  
+    <value>true</value>
+  </property>
+  <property>
+    <name>hive.metastore.schema.verification</name>  
+    <value>true</value>
+  </property>
+</configuration>

vagrant/base/analyticstack/Vagrantfile

@@ -41,6 +41,8 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
   config.vm.network :forwarded_port, guest: 18080, host: 18080
   config.vm.network :forwarded_port, guest: 8100, host: 8100  # Analytics Data API
   config.vm.network :forwarded_port, guest: 8110, host: 8110  # Insights
+  config.vm.network :forwarded_port, guest: 50070, host: 50070  # HDFS Admin UI
+  config.vm.network :forwarded_port, guest: 8088, host: 8088  # Hadoop Resource Manager
   config.ssh.insert_key = true
 
   config.vm.synced_folder  ".", "/vagrant", disabled: true

vagrant/base/devstack/Vagrantfile

@@ -44,8 +44,10 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
   config.vm.network :forwarded_port, guest: 8765, host: 8765
   config.vm.network :forwarded_port, guest: 9200, host: 9200
   config.vm.network :forwarded_port, guest: 18080, host: 18080
-  config.vm.network :forwarded_port, guest: 8100, host: 8100  # Insights
-  config.vm.network :forwarded_port, guest: 8110, host: 8110  # Analytics Data API
+  config.vm.network :forwarded_port, guest: 8100, host: 8100  # Analytics Data API
+  config.vm.network :forwarded_port, guest: 8110, host: 8110  # Insights
+  config.vm.network :forwarded_port, guest: 50070, host: 50070  # HDFS Admin UI
+  config.vm.network :forwarded_port, guest: 8088, host: 8088  # Hadoop Resource Manager
   config.ssh.insert_key = true
 
   config.vm.synced_folder  ".", "/vagrant", disabled: true

vagrant/release/analyticstack/Vagrantfile

@@ -37,8 +37,10 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
   config.vm.network :forwarded_port, guest: 18080, host: 18080
   config.vm.network :forwarded_port, guest: 8765, host: 8765
   config.vm.network :forwarded_port, guest: 9200, host: 9200
-  config.vm.network :forwarded_port, guest: 8100, host: 8100
-  config.vm.network :forwarded_port, guest: 8110, host: 8110
+  config.vm.network :forwarded_port, guest: 8100, host: 8100  # Analytics Data API
+  config.vm.network :forwarded_port, guest: 8110, host: 8110  # Insights
+  config.vm.network :forwarded_port, guest: 50070, host: 50070  # HDFS Admin UI
+  config.vm.network :forwarded_port, guest: 8088, host: 8088  # Hadoop Resource Manager
   config.ssh.insert_key = true
 
   config.vm.synced_folder  ".", "/vagrant", disabled: true

vagrant/release/devstack/Vagrantfile

@@ -126,8 +126,10 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
   config.vm.network :forwarded_port, guest: 18080, host: 18080
   config.vm.network :forwarded_port, guest: 8765, host: 8765
   config.vm.network :forwarded_port, guest: 9200, host: 9200
-  config.vm.network :forwarded_port, guest: 8100, host: 8100
-  config.vm.network :forwarded_port, guest: 8110, host: 8110
+  config.vm.network :forwarded_port, guest: 8100, host: 8100  # Analytics Data API
+  config.vm.network :forwarded_port, guest: 8110, host: 8110  # Insights
+  config.vm.network :forwarded_port, guest: 50070, host: 50070  # HDFS Admin UI
+  config.vm.network :forwarded_port, guest: 8088, host: 8088  # Hadoop Resource Manager
   config.ssh.insert_key = true
 
   config.vm.synced_folder  ".", "/vagrant", disabled: true