squash and rebase to upstream/master

allow for mapred/yarn resource config

use default mapreduce memory settings

corrected defaults for m1.medium, added yarn scheduler vars

testing with increased yarn am resource

remove task.io settings, defaults to AWS recommended

testing new hadoop conditional configuration

iter dict for conditional config

playbooks/roles/hadoop_common/defaults/main.yml

@@ -9,7 +9,7 @@
 #
 ##
 # Defaults for role hadoop_common
-# 
+#
 
 HADOOP_COMMON_VERSION: 2.3.0
 HADOOP_COMMON_USER_HOME: "{{ COMMON_APP_DIR }}/hadoop"
@@ -60,3 +60,23 @@ hadoop_common_debian_pkgs:
   - maven
 
 hadoop_common_redhat_pkgs: []
+
+#
+# MapReduce/Yarn memory config (defaults for m1.medium)
+# http://docs.aws.amazon.com/ElasticMapReduce/latest/DeveloperGuide/TaskConfiguration_H2.html
+#
+# mapred_site_config:
+#   mapreduce.map.memory_mb: 768
+#   mapreduce.map.java.opts: '-Xmx512M'
+#   mapreduce.reduce.memory.mb: 1024
+#   mapreduce.reduce.java.opts: '-Xmx768M'
+
+# yarn_site_config:
+#   yarn.app.mapreduce.am.resource.mb: 1024
+#   yarn.scheduler.minimum-allocation-mb: 32
+#   yarn.scheduler.maximum-allocation-mb: 2048
+#   yarn.nodemanager.resource.memory-mb: 2048
+#   yarn.nodemanager.vmem-pmem-ratio: 2.1
+
+mapred_site_config: {}
+yarn_site_config: {}

playbooks/roles/hadoop_common/templates/mapred-site.xml.j2

@@ -6,4 +6,14 @@
     <name>mapreduce.framework.name</name>
     <value>yarn</value>
   </property>
+
+{% if mapred_site_config is defined %}
+{% for key,value in mapred_site_config.iteritems() %}
+  <property>
+    <name>{{ key }}}</name>
+    <value>{{ value }}</value>
+  </property>
+{% endfor %}
+{% endif %}
+
 </configuration>
\ No newline at end of file

playbooks/roles/hadoop_common/templates/yarn-site.xml.j2

@@ -5,9 +5,19 @@
     <name>yarn.nodemanager.aux-services</name>
     <value>mapreduce_shuffle</value>
   </property>
-  
+
   <property>
     <name>yarn.nodemanager.aux-services.mapreduce.shuffle.class</name>
     <value>org.apache.hadoop.mapred.ShuffleHandler</value>
   </property>
+
+{% if yarn_site_config is defined %}
+{% for key,value in yarn_site_config.iteritems() %}
+  <property>
+    <name>{{ key }}}</name>
+    <value>{{ value }}</value>
+  </property>
+{% endfor %}
+{% endif %}
+
 </configuration>
\ No newline at end of file