Merge pull request #2786 from proversity-org/config/mapred

Provide for Yarn/MapReduce memory allocation (Hadoop)

playbooks/roles/hadoop_common/defaults/main.yml

@@ -9,7 +9,7 @@
 #
 ##
 # Defaults for role hadoop_common
-# 
+#
 
 HADOOP_COMMON_VERSION: 2.3.0
 HADOOP_COMMON_USER_HOME: "{{ COMMON_APP_DIR }}/hadoop"
@@ -60,3 +60,23 @@ hadoop_common_debian_pkgs:
   - maven
 
 hadoop_common_redhat_pkgs: []
+
+#
+# MapReduce/Yarn memory config (defaults for m1.medium)
+# http://docs.aws.amazon.com/ElasticMapReduce/latest/DeveloperGuide/TaskConfiguration_H2.html
+#
+# mapred_site_config:
+#   mapreduce.map.memory_mb: 768
+#   mapreduce.map.java.opts: '-Xmx512M'
+#   mapreduce.reduce.memory.mb: 1024
+#   mapreduce.reduce.java.opts: '-Xmx768M'
+
+# yarn_site_config:
+#   yarn.app.mapreduce.am.resource.mb: 1024
+#   yarn.scheduler.minimum-allocation-mb: 32
+#   yarn.scheduler.maximum-allocation-mb: 2048
+#   yarn.nodemanager.resource.memory-mb: 2048
+#   yarn.nodemanager.vmem-pmem-ratio: 2.1
+
+mapred_site_config: {}
+yarn_site_config: {}

playbooks/roles/hadoop_common/templates/mapred-site.xml.j2

@@ -6,4 +6,14 @@
     <name>mapreduce.framework.name</name>
     <value>yarn</value>
   </property>
+
+{% if mapred_site_config is defined %}
+{% for key,value in mapred_site_config.iteritems() %}
+  <property>
+    <name>{{ key }}}</name>
+    <value>{{ value }}</value>
+  </property>
+{% endfor %}
+{% endif %}
+
 </configuration>
\ No newline at end of file

playbooks/roles/hadoop_common/templates/yarn-site.xml.j2

@@ -5,9 +5,19 @@
     <name>yarn.nodemanager.aux-services</name>
     <value>mapreduce_shuffle</value>
   </property>
-  
+
   <property>
     <name>yarn.nodemanager.aux-services.mapreduce.shuffle.class</name>
     <value>org.apache.hadoop.mapred.ShuffleHandler</value>
   </property>
+
+{% if yarn_site_config is defined %}
+{% for key,value in yarn_site_config.iteritems() %}
+  <property>
+    <name>{{ key }}}</name>
+    <value>{{ value }}</value>
+  </property>
+{% endfor %}
+{% endif %}
+
 </configuration>
\ No newline at end of file