Merge pull request #4227 from edx/nadeem/OPS2489

Convert sandbox from rabbit to redis OPS-2489

playbooks/edx-east/edx_continuous_integration.yml

@@ -25,6 +25,7 @@
     - edxapp
     - testcourses
     - { role: 'rabbitmq', rabbitmq_ip: '127.0.0.1' }
+    - role: redis
     - oraclejdk
     - elasticsearch
     - forum

playbooks/edx-east/redis.yml

+- name: Deploy redis
+  hosts: all
+  become: True
+  gather_facts: True
+  roles:
+    - aws
+    - redis
+    - role: datadog
+      when: COMMON_ENABLE_DATADOG
+    - role: splunkforwarder
+      when: COMMON_ENABLE_SPLUNKFORWARDER
+    - role: newrelic
+      when: COMMON_ENABLE_NEWRELIC
+    - role: newrelic_infrastructure
+      when: COMMON_ENABLE_NEWRELIC_INFRASTRUCTURE

playbooks/roles/flower/defaults/main.yml

@@ -7,6 +7,7 @@ FLOWER_BROKER_HOST: "127.0.0.1"
 FLOWER_BROKER_PORT: 5672
 FLOWER_ADDRESS: "0.0.0.0"
 FLOWER_PORT: "5555"
+FLOWER_BROKER_TRANSPORT: 'amqp'
 
 FLOWER_OAUTH2_KEY: "A Client ID from Google's OAUTH2 provider"
 FLOWER_OAUTH2_SECRET: "A Client Secret from Google's OAUTH2 provider"
@@ -24,10 +25,11 @@ flower_venv_bin: "{{ flower_venv_dir }}/bin"
 
 flower_python_reqs:
   - "flower==0.8.3"
+  - "redis==2.10.6"
 
 flower_deploy_path: "{{ flower_venv_bin }}:/usr/local/sbin:/usr/local/bin:/usr/bin:/sbin:/bin"
 
-flower_broker: "amqp://{{ FLOWER_BROKER_USERNAME }}:{{ FLOWER_BROKER_PASSWORD }}@{{ FLOWER_BROKER_HOST }}:{{ FLOWER_BROKER_PORT }}"
+flower_broker: "{{ FLOWER_BROKER_TRANSPORT }}://{{ FLOWER_BROKER_USERNAME }}:{{ FLOWER_BROKER_PASSWORD }}@{{ FLOWER_BROKER_HOST }}:{{ FLOWER_BROKER_PORT }}"
 
 flower_environment:
   PATH: "{{ flower_deploy_path }}"

playbooks/roles/redis/defaults/main.yml

@@ -19,7 +19,6 @@ REDIS_MAX_MEMORY_POLICY: "noeviction"
 # vars are namespace with the module name.
 #
 redis_role_name: redis
-redis_ppa: "ppa:chris-lea/redis-server"
 
 redis_user: redis
 redis_group: redis

playbooks/roles/redis/tasks/main.yml

@@ -21,16 +21,12 @@
 #
 #
 
-- name: Add the redis ppa
-  apt_repository:
-    repo: "{{ redis_ppa }}"
-    state: present
-
 - name: Install redis system packages
   apt:
     name: "{{ item }}"
     install_recommends: yes
     state: present
+    update_cache: yes
   with_items: "{{ redis_debian_pkgs }}"
   notify:
     - reload redis

playbooks/roles/redis/templates/etc/redis/redis.conf.j2

@@ -35,51 +35,17 @@
 # include /path/to/local.conf
 # include /path/to/other.conf
 
-################################## NETWORK #####################################
+################################ GENERAL  #####################################
 
-# By default, if no "bind" configuration directive is specified, Redis listens
-# for connections from all the network interfaces available on the server.
-# It is possible to listen to just one or multiple selected interfaces using
-# the "bind" configuration directive, followed by one or more IP addresses.
-#
-# Examples:
-#
-# bind 192.168.1.100 10.0.0.1
-# bind 127.0.0.1 ::1
-#
-# ~~~ WARNING ~~~ If the computer running Redis is directly exposed to the
-# internet, binding to all the interfaces is dangerous and will expose the
-# instance to everybody on the internet. So by default we uncomment the
-# following bind directive, that will force Redis to listen only into
-# the IPv4 lookback interface address (this means Redis will be able to
-# accept connections only from clients running into the same computer it
-# is running).
-#
-# IF YOU ARE SURE YOU WANT YOUR INSTANCE TO LISTEN TO ALL THE INTERFACES
-# JUST COMMENT THE FOLLOWING LINE.
-# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-bind {{ REDIS_BIND_IP }}
+# By default Redis does not run as a daemon. Use 'yes' if you need it.
+# Note that Redis will write a pid file in /var/run/redis.pid when daemonized.
+daemonize yes
 
-# Protected mode is a layer of security protection, in order to avoid that
-# Redis instances left open on the internet are accessed and exploited.
-#
-# When protected mode is on and if:
-#
-# 1) The server is not binding explicitly to a set of addresses using the
-#    "bind" directive.
-# 2) No password is configured.
-#
-# The server only accepts connections from clients connecting from the
-# IPv4 and IPv6 loopback addresses 127.0.0.1 and ::1, and from Unix domain
-# sockets.
-#
-# By default protected mode is enabled. You should disable it only if
-# you are sure you want clients from other hosts to connect to Redis
-# even if no authentication is configured, nor a specific set of interfaces
-# are explicitly listed using the "bind" directive.
-protected-mode yes
+# When running daemonized, Redis writes a pid file in /var/run/redis.pid by
+# default. You can specify a custom pid file location here.
+pidfile /var/run/redis/redis-server.pid
 
-# Accept connections on the specified port, default is 6379 (IANA #815344).
+# Accept connections on the specified port, default is 6379.
 # If port 0 is specified Redis will not listen on a TCP socket.
 port 6379
 
@@ -92,14 +58,22 @@ port 6379
 # in order to get the desired effect.
 tcp-backlog 511
 
-# Unix socket.
+# By default Redis listens for connections from all the network interfaces
+# available on the server. It is possible to listen to just one or multiple
+# interfaces using the "bind" configuration directive, followed by one or
+# more IP addresses.
+#
+# Examples:
 #
+# bind 192.168.1.100 10.0.0.1
+bind {{ REDIS_BIND_IP }}
+
 # Specify the path for the Unix socket that will be used to listen for
 # incoming connections. There is no default, so Redis will not listen
 # on a unix socket when not specified.
 #
-# unixsocket /tmp/redis.sock
-unixsocketperm 755
+# unixsocket /var/run/redis/redis.sock
+# unixsocketperm 700
 
 # Close the connection after a client is idle for N seconds (0 to disable)
 timeout 0
@@ -117,38 +91,9 @@ timeout 0
 # Note that to close the connection the double of the time is needed.
 # On other kernels the period depends on the kernel configuration.
 #
-# A reasonable value for this option is 300 seconds, which is the new
-# Redis default starting with Redis 3.2.1.
+# A reasonable value for this option is 60 seconds.
 tcp-keepalive 0
 
-################################# GENERAL #####################################
-
-# By default Redis does not run as a daemon. Use 'yes' if you need it.
-# Note that Redis will write a pid file in /var/run/redis.pid when daemonized.
-daemonize yes
-
-# If you run Redis from upstart or systemd, Redis can interact with your
-# supervision tree. Options:
-#   supervised no      - no supervision interaction
-#   supervised upstart - signal upstart by putting Redis into SIGSTOP mode
-#   supervised systemd - signal systemd by writing READY=1 to $NOTIFY_SOCKET
-#   supervised auto    - detect upstart or systemd method based on
-#                        UPSTART_JOB or NOTIFY_SOCKET environment variables
-# Note: these supervision methods only signal "process is ready."
-#       They do not enable continuous liveness pings back to your supervisor.
-supervised no
-
-# If a pid file is specified, Redis writes it where specified at startup
-# and removes it at exit.
-#
-# When the server runs non daemonized, no pid file is created if none is
-# specified in the configuration. When the server is daemonized, the pid file
-# is used even if not specified, defaulting to "/var/run/redis.pid".
-#
-# Creating a pid file is best effort: if Redis is not able to create it
-# nothing bad happens, the server will start and run normally.
-pidfile /var/run/redis/redis.pid
-
 # Specify the server verbosity level.
 # This can be one of:
 # debug (a lot of information, useful for development/testing)
@@ -160,11 +105,11 @@ loglevel notice
 # Specify the log file name. Also the empty string can be used to force
 # Redis to log on the standard output. Note that if you use standard
 # output for logging but daemonize, logs will be sent to /dev/null
-logfile ""
+logfile /var/log/redis/redis-server.log
 
 # To enable logging to the system logger, just set 'syslog-enabled' to yes,
 # and optionally update the other syslog parameters to suit your needs.
-syslog-enabled yes
+# syslog-enabled no
 
 # Specify the syslog identity.
 # syslog-ident redis
@@ -234,7 +179,7 @@ rdbcompression yes
 rdbchecksum yes
 
 # The filename where to dump the DB
-dbfilename redis.rdb
+dbfilename dump.rdb
 
 # The working directory.
 #
@@ -435,35 +380,6 @@ slave-priority 100
 # By default min-slaves-to-write is set to 0 (feature disabled) and
 # min-slaves-max-lag is set to 10.
 
-# A Redis master is able to list the address and port of the attached
-# slaves in different ways. For example the "INFO replication" section
-# offers this information, which is used, among other tools, by
-# Redis Sentinel in order to discover slave instances.
-# Another place where this info is available is in the output of the
-# "ROLE" command of a masteer.
-#
-# The listed IP and address normally reported by a slave is obtained
-# in the following way:
-#
-#   IP: The address is auto detected by checking the peer address
-#   of the socket used by the slave to connect with the master.
-#
-#   Port: The port is communicated by the slave during the replication
-#   handshake, and is normally the port that the slave is using to
-#   list for connections.
-#
-# However when port forwarding or Network Address Translation (NAT) is
-# used, the slave may be actually reachable via different IP and port
-# pairs. The following two options can be used by a slave in order to
-# report to its master a specific set of IP and port, so that both INFO
-# and ROLE will report those values.
-#
-# There is no need to use both the options if you need to override just
-# the port or the IP address.
-#
-# slave-announce-ip 5.5.5.5
-# slave-announce-port 1234
-
 ################################## SECURITY ###################################
 
 # Require clients to issue AUTH <PASSWORD> before processing any other
@@ -918,36 +834,11 @@ notify-keyspace-events ""
 hash-max-ziplist-entries 512
 hash-max-ziplist-value 64
 
-# Lists are also encoded in a special way to save a lot of space.
-# The number of entries allowed per internal list node can be specified
-# as a fixed maximum size or a maximum number of elements.
-# For a fixed maximum size, use -5 through -1, meaning:
-# -5: max size: 64 Kb  <-- not recommended for normal workloads
-# -4: max size: 32 Kb  <-- not recommended
-# -3: max size: 16 Kb  <-- probably not recommended
-# -2: max size: 8 Kb   <-- good
-# -1: max size: 4 Kb   <-- good
-# Positive numbers mean store up to _exactly_ that number of elements
-# per list node.
-# The highest performing option is usually -2 (8 Kb size) or -1 (4 Kb size),
-# but if your use case is unique, adjust the settings as necessary.
-list-max-ziplist-size -2
-
-# Lists may also be compressed.
-# Compress depth is the number of quicklist ziplist nodes from *each* side of
-# the list to *exclude* from compression.  The head and tail of the list
-# are always uncompressed for fast push/pop operations.  Settings are:
-# 0: disable all list compression
-# 1: depth 1 means "don't start compressing until after 1 node into the list,
-#    going from either the head or tail"
-#    So: [head]->node->node->...->node->[tail]
-#    [head], [tail] will always be uncompressed; inner nodes will compress.
-# 2: [head]->[next]->node->node->...->node->[prev]->[tail]
-#    2 here means: don't compress head or head->next or tail->prev or tail,
-#    but compress all nodes between them.
-# 3: [head]->[next]->[next]->node->node->...->node->[prev]->[prev]->[tail]
-# etc.
-list-compress-depth 0
+# Similarly to hashes, small lists are also encoded in a special way in order
+# to save a lot of space. The special representation is only used when
+# you are under the following limits:
+list-max-ziplist-entries 512
+list-max-ziplist-value 64
 
 # Sets have a special encoding in just one case: when a set is composed
 # of just strings that happen to be integers in radix 10 in the range

util/jenkins/ansible-provision.sh

@@ -127,7 +127,7 @@ fi
 
 if [[ -z $ami ]]; then
   if [[ $server_type == "full_edx_installation" ]]; then
-    ami="ami-0d600177"
+    ami="ami-d17c14ab"
   elif [[ $server_type == "ubuntu_16.04" || $server_type == "full_edx_installation_from_scratch" ]]; then
     ami="ami-aa2ea6d0"
   fi
@@ -353,6 +353,7 @@ EOF
         # additional tasks that need to be run if the
         # entire edx stack is brought up from an AMI
         run_ansible rabbitmq.yml -i "${deploy_host}," $extra_var_arg --user ubuntu
+        run_ansible redis.yml -i "${deploy_host}," $extra_var_arg --user ubuntu
         run_ansible restart_supervisor.yml -i "${deploy_host}," $extra_var_arg --user ubuntu
     fi
 fi