Merge remote-tracking branch 'origin' into jarv/fabric-for-deploys

AUTHORS

+John Jarvis <jarv@edx.org>
+Sef Kloninger <sef@kloninger.com>
+Joe Blaylock <jrbl@jrbl.org>
+Vik Paruchuri <vik@edx.org>
+Jason Bau <jbau@stanford.edu>
+Ed Zarecor <ed@edx.org>
+John Kern <kern3020@gmail.com>

cloudformation_templates/edx-reference-architecture.json

@@ -47,7 +47,12 @@
     "WebServerPort":{
       "Description":"The TCP port for the Web Server",
       "Type":"Number",
-      "Default":"8888"
+      "Default":"80"
+    },
+    "CacheNodePort":{
+      "Description":"The TCP port for the nodes in the Elasticache cluster",
+      "Type":"Number",
+      "Default":"11211"
     },
     "BastionInstanceType":{
       "Description":"Bastion Host EC2 instance type",
@@ -1571,6 +1576,10 @@
                 "'\n",
                 "  exit 1\n",
                 "}\n",
+                "for dev in /dev/xvdc /dev/xvdd; do sudo echo w | fdisk $dev; sudo mkfs -t ext4 $dev;done;\n",
+                "sudo mkdir /mnt/logs\n",
+                "sudo mount /dev/xvdc /mnt/logs\n",
+                "sudo mount /dev/xvdd /opt\n",
                 "apt-get -y update\n",
                 "apt-get -y install python-setuptools\n",
                 "echo \"Python Tools installed\" - `date` >> /home/ubuntu/cflog.txt\n",
@@ -1616,13 +1625,13 @@
         },
         "BlockDeviceMappings":[
           {
-            "DeviceName":"/dev/xvdb",
+            "DeviceName":"/dev/xvdc",
             "Ebs":{
               "VolumeSize":"50"
             }
           },
           {
-            "DeviceName":"/dev/xvdc",
+            "DeviceName":"/dev/xvdd",
             "Ebs":{
               "VolumeSize":"50"
             }
@@ -1672,8 +1681,8 @@
         "LaunchConfigurationName":{
           "Ref":"EdxappServer"
         },
-        "MinSize":"1",
-        "MaxSize":"6",
+        "MinSize":"2",
+        "MaxSize":"2",
         "DesiredCapacity":{
           "Ref":"EdxappDesiredCapacity"
         },
@@ -1761,6 +1770,10 @@
     "EdxappELB":{
       "Type":"AWS::ElasticLoadBalancing::LoadBalancer",
       "Properties":{
+        "LBCookieStickinessPolicy" : [{
+          "PolicyName" : "EdxappStickinessPolicy",
+          "CookieExpirationPeriod" : "180"
+        } ],
         "SecurityGroups":[
           {
             "Ref":"EdxappELBSecurityGroup"
@@ -1785,6 +1798,9 @@
         "Subnets":[
           {
             "Ref":"PublicSubnet01"
+          },
+          {
+            "Ref":"PublicSubnet02"
           }
         ]
       }
@@ -1992,13 +2008,13 @@
         },
         "BlockDeviceMappings":[
           {
-            "DeviceName":"/dev/xvdb",
+            "DeviceName":"/dev/xvdc",
             "Ebs":{
               "VolumeSize":"50"
             }
           },
           {
-            "DeviceName":"/dev/xvdc",
+            "DeviceName":"/dev/xvdd",
             "Ebs":{
               "VolumeSize":"50"
             }
@@ -2048,7 +2064,7 @@
         "LaunchConfigurationName":{
           "Ref":"XqueueServer"
         },
-        "MinSize":"1",
+        "MinSize":"2",
         "MaxSize":"2",
         "DesiredCapacity":{
           "Ref":"XqueueDesiredCapacity"
@@ -2162,9 +2178,12 @@
           "Timeout":"5"
         },
         "Subnets":[
-          {
-            "Ref":"PublicSubnet01"
-          }
+            {
+		"Ref":"XqueueSubnet01"
+            },
+            {
+		"Ref":"XqueueSubnet02"
+            }
         ]
       }
     },
@@ -2365,13 +2384,13 @@
         },
         "BlockDeviceMappings":[
           {
-            "DeviceName":"/dev/xvdb",
+            "DeviceName":"/dev/xvdc",
             "Ebs":{
               "VolumeSize":"50"
             }
           },
           {
-            "DeviceName":"/dev/xvdc",
+            "DeviceName":"/dev/xvdd",
             "Ebs":{
               "VolumeSize":"50"
             }
@@ -2421,7 +2440,7 @@
         "LaunchConfigurationName":{
           "Ref":"RabbitMQServer"
         },
-        "MinSize":"1",
+        "MinSize":"2",
         "MaxSize":"2",
         "DesiredCapacity":{
           "Ref":"RabbitMQDesiredCapacity"
@@ -2537,10 +2556,10 @@
         },
         "Subnets":[
           {
-            "Ref":"PublicSubnet01"
+            "Ref":"RabbitSubnet01"
           },
           {
-            "Ref":"PublicSubnet02"
+            "Ref":"RabbitSubnet02"
           }
         ]
       }
@@ -2750,13 +2769,13 @@
         },
         "BlockDeviceMappings":[
           {
-            "DeviceName":"/dev/xvdb",
+            "DeviceName":"/dev/xvdc",
             "Ebs":{
               "VolumeSize":"50"
             }
           },
           {
-            "DeviceName":"/dev/xvdc",
+            "DeviceName":"/dev/xvdd",
             "Ebs":{
               "VolumeSize":"50"
             }
@@ -2806,7 +2825,7 @@
         "LaunchConfigurationName":{
           "Ref":"XServer"
         },
-        "MinSize":"1",
+        "MinSize":"2",
         "MaxSize":"2",
         "DesiredCapacity":{
           "Ref":"XServerDesiredCapacity"
@@ -2922,10 +2941,10 @@
         },
         "Subnets":[
           {
-            "Ref":"PublicSubnet01"
+            "Ref":"XServerSubnet01"
           },
           {
-            "Ref":"PublicSubnet02"
+            "Ref":"XServerSubnet02"
           }
         ]
       }
@@ -3087,10 +3106,10 @@
         "SecurityGroupIngress":[
           {
             "IpProtocol":"tcp",
-            "FromPort":"12345",
-            "ToPort":"12345",
+            "FromPort": { "Ref":"CacheNodePort" },
+            "ToPort": { "Ref":"CacheNodePort" },
             "SourceSecurityGroupId":{
-              "Ref":"EdxappELBSecurityGroup"
+              "Ref":"EdxappServerSecurityGroup"
             }
           }
         ]
@@ -3174,7 +3193,7 @@
             "--cache-node-type ", { "Ref":"CacheNodeType" }, " ",
             "--engine memcached ",
             "--engine-version 1.4.5 ",
-            "--port 12345 ",
+            "--port ", { "Ref":"CacheNodePort"}, " ",
             "--preferred-availability-zone ",
             { "Fn::FindInMap":[
                 "MapRegionsToAvailZones",
@@ -3184,7 +3203,8 @@
             }, " ",
             "--cache-parameter-group-name default.memcached1.4 ",
             "--auto-minor-version-upgrade true ",
-            "--cache-subnet-group-name ", { "Ref":"EdxVPC" },"-ElasticCacheSubnetGroup"
+            "--cache-subnet-group-name ", { "Ref":"EdxVPC" },"-ElasticCacheSubnetGroup ",
+	    "--security-group-ids ", { "Ref":"CacheSecurityGroup" }
           ]
         ]
       }

playbooks/group_vars/all

@@ -3,6 +3,7 @@ app_base_dir: /opt/wwc
 log_base_dir: /mnt/logs
 venv_dir: /opt/edx
 platform_code_dir: $app_base_dir/edx-platform
+os_name: ubuntu
 
 # these pathes are relative to the playbook dir
 # directory for secret settings (keys, etc)

playbooks/roles/gunicorn/templates/lms-preview.conf.j2

@@ -18,10 +18,10 @@ env LANG=en_US.UTF-8
 env DJANGO_SETTINGS_MODULE=lms.envs.aws
 env SERVICE_VARIANT="lms-preview"
   
-chdir ${app_base_dir}/mitx
+chdir {{platform_code_dir}}
 setuid www-data
 
-exec ${venv_dir}/bin/gunicorn --preload -b 127.0.0.1:$PORT -w $WORKERS --timeout=300 --pythonpath=${app_base_dir}/mitx lms.wsgi
+exec ${venv_dir}/bin/gunicorn --preload -b 127.0.0.1:$PORT -w $WORKERS --timeout=300 --pythonpath={{platform_code_dir}} lms.wsgi
 
 post-start script
   while true

playbooks/roles/lms/tasks/main.yml

@@ -76,7 +76,7 @@
 
 ## Install the debian package requirements system-wide
 - name: store remote apt_repos list for ansible use
-  command: cat {{platform_code_dir}}/apt-repos.txt
+  command: cat {{ apt_sources_file }}
   register: apt_repos_list
   tags:
   - lms
@@ -103,7 +103,7 @@
   - install
 
 - name: store remote apt_packages list for ansible use
-  command: cat {{platform_code_dir}}/apt-packages.txt
+  command: cat {{ apt_packages_file }}
   register: apt_packages_list
   tags:
   - lms
@@ -118,25 +118,42 @@
   - cms
   - install
 
-# Install the python requirements into $venv_dir 
-- name : install python pre-requirements
-  pip: requirements="{{platform_code_dir}}/pre-requirements.txt" virtualenv="{{venv_dir}}" state=present
+
+# Install the python modules into {{ venv_dir }}
+- name : install base python packages using the shell
+  # Need to use shell rather than pip so that we can maintain the context of our current working directory; some 
+  # requirements are pathed relative to the edx-platform repo. Using the pip from inside the virtual environment implicitly
+  # installs everything into that virtual environment.
+  shell: cd {{ platform_code_dir }} && {{ venv_dir }}/bin/pip install --use-mirrors -r {{ base_requirements_file }}
+  tags:
+  - lms
+  - cms
+  - install
+  
+  
+# Install the python post requirements into {{ venv_dir }}
+- name : install python post-requirements
+  pip: requirements="{{ post_requirements_file }}" virtualenv="{{ venv_dir }}" state=present
   tags:
   - lms
   - cms
   - install
 
-# Install the python modules into $venv_dir 
-- name : install python packages using the shell
-  #pip: requirements="{{platform_code_dir}}/requirements.txt" virtualenv="{{venv_dir}}" 
+# Install the final python modules into {{ venv_dir }}
+- name : install python post-post requirements using the shell
   # Need to use shell rather than pip so that we can maintain the context of our current working directory; some 
   # requirements are pathed relative to the edx-platform repo. Using the pip from inside the virtual environment implicitly
   # installs everything into that virtual environment.
-  shell: cd {{platform_code_dir}} && {{venv_dir}}/bin/pip install --use-mirrors -r {{platform_code_dir}}/requirements.txt
+  shell: cd {{ platform_code_dir }} && {{ venv_dir }}/bin/pip install --use-mirrors -r {{ item }}
+  with_items:
+  - "{{ repo_requirements_file }}"
+  - "{{ github_requirements_file }}"
+  - "{{ local_requirements_file }}"
   tags:
   - lms
   - cms
   - install
   
+
 # Creates LMS upstart file
 - include: ../../gunicorn/tasks/upstart.yml service_variant=lms

playbooks/roles/lms/vars/main.yml

@@ -4,6 +4,16 @@
 lms_auth_config: {}
 lms_env_config: {}
 lms_source_repo: git@github.com:edx/edx-platform.git
+
+apt_sources_file:  "{{ platform_code_dir }}/requirements/system/{{os_name}}/apt-repos.txt"
+apt_packages_file: "{{ platform_code_dir }}/requirements/system/{{os_name}}/apt-packages.txt"
+
+local_requirements_file:  "{{ platform_code_dir }}/requirements/local.txt"
+post_requirements_file:   "{{ platform_code_dir }}/requirements/post.txt"
+base_requirements_file:   "{{ platform_code_dir }}/requirements/base.txt"
+github_requirements_file: "{{ platform_code_dir }}/requirements/github.txt"
+repo_requirements_file:   "{{ platform_code_dir }}/requirements/repo.txt"
+
 lms_debian_pkgs:
   - apparmor-utils
   - aspell