Merge pull request #3808 from edx/brian/ficus.master/update-analyticstack-for-xenial

Make changes for running analyticstack on 16.04.

playbooks/roles/analytics_pipeline/tasks/main.yml

@@ -127,21 +127,46 @@
     - install
     - install:app-requirements
 
-- name: Ensure hdfs services are started
+- name: reload systemd configuration
+  command: systemctl daemon-reload
+  tags:
+    - install
+    - install:configuration
+
+- name: enable Hadoop services
+  service:
+    name: "{{ item }}"
+    enabled: yes
+  with_items: "{{ hadoop_common_services }}"
+  tags:
+    - install
+    - install:configuration
+
+- name: start Hadoop services
   service:
-    name: hdfs
+    name: "{{ item }}"
     state: started
+  with_items: "{{ hadoop_common_services }}"
   tags:
     - manage
     - manage:start
 
-- name: Ensure map reduce services are started
+- name: stop Hadoop services
   service:
-    name: yarn
-    state: started
+    name: "{{ item }}"
+    state: stopped
+  with_items: "{{ hadoop_common_services }}"
+  tags:
+    - manage:stop
+
+- name: restart Hadoop services
+  service:
+    name: "{{ item }}"
+    state: restarted
+  with_items: "{{ hadoop_common_services }}"
   tags:
-    - manage
     - manage:start
+    - manage:restart
 
 - name: Ensure package dir exists in HDFS
   shell: ". {{ HADOOP_COMMON_CONF_DIR }}/hadoop-env.sh && hdfs dfs -mkdir -p /edx-analytics-pipeline/packages/"
@@ -156,6 +181,10 @@
   tags:
     - install
     - install:app-requirements
+  register: libcp
+  until: libcp|success
+  retries: 6
+  delay: 10
 
 - name: Ensure the data directory exists
   shell: ". {{ HADOOP_COMMON_CONF_DIR }}/hadoop-env.sh && hdfs dfs -mkdir -p {{ ANALYTICS_PIPELINE_HDFS_DATA_DIR }}"

playbooks/roles/hadoop_common/defaults/main.yml

@@ -49,7 +49,7 @@ hadoop_common_env: "{{ HADOOP_COMMON_HOME }}/hadoop_env"
 #
 
 hadoop_common_debian_pkgs:
-  - llvm-gcc
+  - gcc
   - build-essential
   - make
   - cmake
@@ -79,8 +79,23 @@ hadoop_common_redhat_pkgs: []
 #   yarn.nodemanager.vmem-pmem-ratio: 2.1
 
 mapred_site_config: {}
+
 yarn_site_config:
   yarn.log-aggregation-enable: true
   # 24 hour log retention
   yarn.log-aggregation.retain-seconds: 86400
+  # Checking virtual memory usage causes too many spurious failures.
+  yarn.nodemanager.vmem-check-enabled: false
+
+HADOOP_CORE_SITE_EXTRA_CONFIG: {}
+HDFS_SITE_EXTRA_CONFIG: {}
 
+# Define all the services here, assuming we have a single-node Hadoop
+# cluster.
+hadoop_common_services:
+  - hdfs-namenode
+  - hdfs-datanode
+  - yarn-resourcemanager
+  - yarn-nodemanager
+  - yarn-proxyserver
+  - mapreduce-historyserver

playbooks/roles/hadoop_common/tasks/main.yml

@@ -28,6 +28,16 @@
     state: present
   with_items: "{{ hadoop_common_debian_pkgs }}"
 
+# Update procps to get fix to
+#  https://bugs.launchpad.net/ubuntu/+source/procps/+bug/1637026, which
+#  causes hadoop jobs to fail on Xenial.
+- name: update system packages
+  apt:
+    pkg: "{{ item }}"
+    state: latest
+  with_items:
+    - procps
+
 - name: ensure group exists
   group:
     name: "{{ hadoop_common_group }}"
@@ -108,16 +118,17 @@
     - hdfs-site.xml
     - yarn-site.xml
 
-- name: upstart scripts installed
+- name: service definitions installed
   template:
-    src: "{{ item }}.j2"
-    dest: "/etc/init/{{ item }}"
+    src: "etc/systemd/system/{{ item }}.service.j2"
+    dest: "/etc/systemd/system/{{ item }}.service"
     mode: 0640
     owner: root
     group: root
-  with_items:
-    - hdfs.conf
-    - yarn.conf
+  with_items: "{{ hadoop_common_services }}"
+  tags:
+    - install
+    - install:configuration
 
 - name: hadoop env file exists
   file:
@@ -196,7 +207,7 @@
 - name: new native libs installed
   shell: "chown {{ hadoop_common_user }}:{{ hadoop_common_group }} {{ item }} && cp {{ item }} {{ HADOOP_COMMON_HOME }}/lib/native/{{ item }}"
   args:
-    chdir={{ hadoop_common_temporary_dir }}/hadoop-common-release-{{ HADOOP_COMMON_VERSION }}/hadoop-common-project/hadoop-common/target/native/target/usr/local/lib
+    chdir: "{{ hadoop_common_temporary_dir }}/hadoop-common-release-{{ HADOOP_COMMON_VERSION }}/hadoop-common-project/hadoop-common/target/native/target/usr/local/lib"
   with_items:
     - libhadoop.a
     - libhadoop.so

playbooks/roles/hadoop_common/templates/etc/systemd/system/hdfs-datanode.service.j2

+[Unit]
+Description=Hadoop Distributed File System Data Node
+
+[Service]
+Type=forking
+ExecStart={{ HADOOP_COMMON_HOME }}/sbin/hadoop-daemon.sh --script hdfs start datanode
+ExecStop={{ HADOOP_COMMON_HOME }}/sbin/hadoop-daemon.sh --script hdfs stop datanode
+User={{ hadoop_common_user }}
+Group={{ hadoop_common_group }}
+
+[Install]
+WantedBy=multi-user.target

playbooks/roles/hadoop_common/templates/etc/systemd/system/hdfs-namenode.service.j2

+[Unit]
+Description=Hadoop Distributed File System Name Node
+
+[Service]
+Type=forking
+ExecStart={{ HADOOP_COMMON_HOME }}/sbin/hadoop-daemon.sh --script hdfs start namenode
+ExecStop={{ HADOOP_COMMON_HOME }}/sbin/hadoop-daemon.sh --script hdfs stop namenode
+User={{ hadoop_common_user }}
+Group={{ hadoop_common_group }}
+
+[Install]
+WantedBy=multi-user.target

playbooks/roles/hadoop_common/templates/etc/systemd/system/mapreduce-historyserver.service.j2

+[Unit]
+Description=Hadoop MapReduce History Server
+
+[Service]
+Type=forking
+ExecStart={{ HADOOP_COMMON_HOME }}/sbin/mr-jobhistory-daemon.sh start historyserver
+ExecStop={{ HADOOP_COMMON_HOME }}/sbin/mr-jobhistory-daemon.sh stop historyserver
+User={{ hadoop_common_user }}
+Group={{ hadoop_common_group }}
+
+[Install]
+WantedBy=multi-user.target

playbooks/roles/hadoop_common/templates/etc/systemd/system/yarn-nodemanager.service.j2

+[Unit]
+Description=Hadoop YARN Node Manager
+
+[Service]
+Type=forking
+ExecStart={{ HADOOP_COMMON_HOME }}/sbin/yarn-daemon.sh start nodemanager
+ExecStop={{ HADOOP_COMMON_HOME }}/sbin/yarn-daemon.sh stop nodemanager
+User={{ hadoop_common_user }}
+Group={{ hadoop_common_group }}
+
+[Install]
+WantedBy=multi-user.target

playbooks/roles/hadoop_common/templates/etc/systemd/system/yarn-proxyserver.service.j2

+[Unit]
+Description=Hadoop YARN Proxy Server
+
+[Service]
+Type=forking
+ExecStart={{ HADOOP_COMMON_HOME }}/sbin/yarn-daemon.sh start proxyserver
+ExecStop={{ HADOOP_COMMON_HOME }}/sbin/yarn-daemon.sh stop proxyserver
+User={{ hadoop_common_user }}
+Group={{ hadoop_common_group }}
+
+[Install]
+WantedBy=multi-user.target

playbooks/roles/hadoop_common/templates/etc/systemd/system/yarn-resourcemanager.service.j2

+[Unit]
+Description=Hadoop YARN Resource Manager
+
+[Service]
+Type=forking
+ExecStart={{ HADOOP_COMMON_HOME }}/sbin/yarn-daemon.sh start resourcemanager
+ExecStop={{ HADOOP_COMMON_HOME }}/sbin/yarn-daemon.sh stop resourcemanager
+User={{ hadoop_common_user }}
+Group={{ hadoop_common_group }}
+
+[Install]
+WantedBy=multi-user.target

playbooks/vagrant-analytics.yml

@@ -13,6 +13,9 @@
     EDXAPP_OAUTH_ENFORCE_SECURE: false
     EDXAPP_LMS_BASE_SCHEME: http
     ECOMMERCE_DJANGO_SETTINGS_MODULE: "ecommerce.settings.devstack"
+    # When provisioning your devstack, we apply security updates
+    COMMON_SECURITY_UPDATES: true
+    SECURITY_UPGRADE_ON_ANSIBLE: true
   roles:
     - common
     - vhost
@@ -21,19 +24,24 @@
     - edxlocal
     - memcache
     - mongo
-    - { role: 'rabbitmq', rabbitmq_ip: '127.0.0.1' }
+    - role: rabbitmq
+      rabbitmq_ip: 127.0.0.1
     - edxapp
     - oraclejdk
     - elasticsearch
     - forum
     - ecommerce
-    - ecomworker
+    - role: ecomworker
+      ECOMMERCE_WORKER_BROKER_HOST: 127.0.0.1
     - programs
     - role: notifier
       NOTIFIER_DIGEST_TASK_INTERVAL: "5"
+    # The following two are added, relative to devstack
     - analytics_api
     - insights
+    # Leaving out browsers and browsermob-proxy, relative to devstack.
     - local_dev
     - demo
+    # adding this in, relative to devstack.
     - analytics_pipeline
     - oauth_client_setup

vagrant/base/analyticstack/Vagrantfile

-Vagrant.require_version ">= 1.6.5"
+Vagrant.require_version ">= 1.8.7"
 unless Vagrant.has_plugin?("vagrant-vbguest")
   raise "Please install the vagrant-vbguest plugin by running `vagrant plugin install vagrant-vbguest`"
 end
 
 VAGRANTFILE_API_VERSION = "2"
 
-MEMORY = 4096
+# Analyticstack requires more memory than Devstack.
+MEMORY = 6144
 CPU_COUNT = 2
 
 vm_guest_ip = "192.168.33.10"
@@ -33,11 +34,10 @@ VERSION_VARS = [
 
 Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
 
-  # Creates a devstack from a base Ubuntu 12.04 image for virtualbox
-  config.vm.box     = "precise64"
-  config.vm.box_url = "http://files.vagrantup.com/precise64.box"
+  # Creates a devstack from a base Ubuntu 16.04 image for virtualbox
+  config.vm.box = "boxcutter/ubuntu1604"
 
-  config.vm.network :private_network, ip: vm_guest_ip
+  config.vm.network :private_network, ip: vm_guest_ip, nic_type: "virtio"
 
   # If you want to run the box but don't need network ports, set VAGRANT_NO_PORTS=1.
   # This is useful if you want to run more than one box at once.
@@ -79,6 +79,9 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
   # See: https://github.com/mitchellh/vagrant/issues/1188
   config.vm.provision "shell", inline: 'echo \'LC_ALL="en_US.UTF-8"\' > /etc/default/locale'
 
+  # Get ready for ansible on this box.
+  config.vm.provision "shell", path: '../../../util/install/ansible-bootstrap.sh'
+
   # Use vagrant-vbguest plugin to make sure Guest Additions are in sync
   config.vbguest.auto_reboot = true
   config.vbguest.auto_update = true

vagrant/release/analyticstack/Vagrantfile

-Vagrant.require_version ">= 1.6.5"
+Vagrant.require_version ">= 1.8.7"
 unless Vagrant.has_plugin?("vagrant-vbguest")
   raise "Please install the vagrant-vbguest plugin by running `vagrant plugin install vagrant-vbguest`"
 end
 
 VAGRANTFILE_API_VERSION = "2"
 
-MEMORY = 4096
+# Analyticstack requires more memory than Devstack.
+MEMORY = 6144
 CPU_COUNT = 2
 
 # These are versioning variables in the roles. Each can be overridden, first
@@ -49,8 +50,13 @@ end
 # to a name and a file path, which are used for retrieving
 # a Vagrant box from the internet.
 openedx_releases = {
+  "open-release/ficus.master" => "ficus-analyticstack-2017-04-20",
+  "open-release/ficus.3" => "ficus-analyticstack-2017-04-20",
+
   "open-release/eucalyptus" => "eucalyptus-analyticstack-2016-09-04",
   "open-release/eucalyptus.2" => "eucalyptus-analyticstack-2016-09-04",
+  "open-release/eucalyptus.master" => "eucalyptus-analyticstack-2016-09-04",
+
   "named-release/dogwood.3" => {
     :name => "analyticstack", :file => "dogwood-analyticstack-2016-03-15.box",
   },
@@ -63,6 +69,7 @@ openedx_releases.default = {
 }
 
 openedx_release = ENV['OPENEDX_RELEASE']
+boxname = ENV['OPENEDX_BOXNAME']
 
 # Build -e override lines for each overridable variable.
 extra_vars_lines = ""
@@ -92,18 +99,24 @@ SCRIPT
 
 Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
 
-  reldata = openedx_releases[openedx_release]
-  if Hash == reldata.class
-    boxname = openedx_releases[openedx_release][:name]
-    boxfile = openedx_releases[openedx_release].fetch(:file, "#{boxname}.box")
-  else
-    boxname = reldata
+  boxfile = ""
+  if not boxname
+    reldata = openedx_releases[openedx_release]
+    if Hash == reldata.class
+      boxname = openedx_releases[openedx_release][:name]
+      boxfile = openedx_releases[openedx_release].fetch(:file, "")
+    else
+      boxname = reldata
+    end
+  end
+  if boxfile == ""
     boxfile = "#{boxname}.box"
   end
 
   # Creates an edX devstack VM from an official release
   config.vm.box     = boxname
   config.vm.box_url = "http://files.edx.org/vagrant-images/#{boxfile}"
+  config.vm.box_check_update = false
 
   config.vm.network :private_network, ip: "192.168.33.10"
 
@@ -151,6 +164,12 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
     # Allow DNS to work for Ubuntu 12.10 host
     # http://askubuntu.com/questions/238040/how-do-i-fix-name-service-for-vagrant-client
     vb.customize ["modifyvm", :id, "--natdnshostresolver1", "on"]
+
+    # Virtio is faster, but the box needs to have support for it.  We didn't
+    # have support in the boxes before Ficus.
+    if !(boxname.include?("dogwood") || boxname.include?("eucalyptus"))
+      vb.customize ['modifyvm', :id, '--nictype1', 'virtio']
+    end
   end
 
   # Use vagrant-vbguest plugin to make sure Guest Additions are in sync