added backoff decorator

ecommerce/core/constants.py

@@ -31,6 +31,9 @@ COURSE_ENTITLEMENT_PRODUCT_CLASS_NAME = 'Course Entitlement'
 # Discovery Service constants
 DEFAULT_CATALOG_PAGE_SIZE = 100
 
+# Backoff for api calls switch
+BACKOFF_FOR_API_CALLS_SWITCH = u'enable_backoff_for_enterprise_api_calls'
+
 
 class Status(object):
     """Health statuses."""

ecommerce/core/migrations/0046_backoff_for_api_calls_switch.py

+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.conf import settings
+from django.db import migrations
+
+from ecommerce.core.constants import BACKOFF_FOR_API_CALLS_SWITCH
+
+
+def create_switch(apps, schema_editor):
+    """Create a switch for backoff and retry of api calls."""
+    Switch = apps.get_model('waffle', 'Switch')
+    Switch.objects.get_or_create(name=BACKOFF_FOR_API_CALLS_SWITCH, defaults={'active': False})
+
+
+def remove_switch(apps, schema_editor):
+    """Remove backoff for api calls switch."""
+    Switch = apps.get_model('waffle', 'Switch')
+    Switch.objects.filter(name=BACKOFF_FOR_API_CALLS_SWITCH).delete()
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0045_auto_20180510_0823'),
+    ]
+    operations = [
+        migrations.RunPython(create_switch, remove_switch)
+    ]

ecommerce/enterprise/api.py

@@ -4,11 +4,14 @@ Methods for fetching enterprise API data.
 import logging
 from urllib import urlencode
 
+import backoff
+import waffle
 from django.conf import settings
 from requests.exceptions import ConnectionError, Timeout
-from slumber.exceptions import SlumberHttpBaseException
+from slumber.exceptions import SlumberBaseException, SlumberHttpBaseException
 
 from ecommerce.cache_utils.utils import TieredCache
+from ecommerce.core.constants import BACKOFF_FOR_API_CALLS_SWITCH
 from ecommerce.core.utils import get_cache_key
 
 logger = logging.getLogger(__name__)
@@ -162,12 +165,44 @@ def fetch_enterprise_learner_data(site, user):
     api = site.siteconfiguration.enterprise_api_client
     endpoint = getattr(api, api_resource_name)
     querystring = {'username': user.username}
-    response = endpoint().get(**querystring)
 
+    if waffle.switch_is_active(BACKOFF_FOR_API_CALLS_SWITCH):
+        response = _get_with_backoff(endpoint, **querystring)
+    else:
+        response = endpoint().get(**querystring)
     TieredCache.set_all_tiers(cache_key, response, settings.ENTERPRISE_API_CACHE_TIMEOUT)
     return response
 
 
+@backoff.on_exception(backoff.expo,
+                      (ConnectionError,
+                       SlumberBaseException),
+                      max_tries=3,
+                      max_time=30)
+def _get_with_backoff(endpoint, **querystring):
+    """
+    Resilient helper function for fetch_enterprise_learner_data(site, user)
+    In case of the below mentioned exceptions, the endpoint request is sent again after
+    an exponential backoff interval
+
+    Arguments:
+        endpoint: api endpoint
+        querystring: Dict containing query parameters
+
+    Returns:
+        dict: see docstring for fetch_enterprise_learner_data(site, user)
+
+    Raises:
+        ConnectionError: requests exception "ConnectionError", raised if if ecommerce is unable to connect
+            to enterprise api server.
+        SlumberBaseException: base slumber exception "SlumberBaseException", raised if API response contains
+            http error status like 4xx, 5xx etc.
+        Timeout: requests exception "Timeout", raised if enterprise API is taking too long for returning
+            a response. This exception is raised for both connection timeout and read timeout.
+    """
+    return endpoint().get(**querystring)
+
+
 def catalog_contains_course_runs(site, course_run_ids, enterprise_customer_uuid, enterprise_customer_catalog_uuid=None):
     """
     Determine if course runs are associated with the EnterpriseCustomer.

ecommerce/enterprise/tests/test_api.py

 import ddt
 import httpretty
+import waffle
 from django.conf import settings
 from mock import patch
 from oscar.core.loading import get_model
+from requests.exceptions import ConnectionError
+from slumber.exceptions import SlumberBaseException
 
 from ecommerce.cache_utils.utils import TieredCache
+from ecommerce.core.constants import BACKOFF_FOR_API_CALLS_SWITCH
 from ecommerce.core.tests import toggle_switch
 from ecommerce.core.utils import get_cache_key
 from ecommerce.courses.tests.factories import CourseFactory
@@ -100,6 +104,35 @@ class EnterpriseAPITests(EnterpriseServiceMockMixin, DiscoveryTestMixin, TestCas
         enterprise_api.fetch_enterprise_learner_data(self.request.site, self.learner)
         self._assert_num_requests(expected_number_of_requests)
 
+    @patch("slumber.Resource.get")
+    @ddt.data(ConnectionError, SlumberBaseException)
+    def test_get_response_backoff(self, exception, mocked_get):
+        """
+        Test the helper function _get_with_backoff(endpoint, **querystring) backoff and
+        retry in case of request exceptions
+        """
+        mocked_get.side_effect = exception
+
+        self.mock_access_token_response()
+        self.mock_enterprise_learner_api()
+        with self.assertRaises(exception):
+            with patch('waffle.switch_is_active', return_value=True):
+                enterprise_api.fetch_enterprise_learner_data(self.request.site, self.learner)
+
+        self.assertEqual(mocked_get.call_count, 3)
+
+    @patch("ecommerce.enterprise.api._get_with_backoff")
+    def test_waffle_backoff_disabled(self, mocked_get):
+        """
+        Test that the helper function _get_with_backoff(endpoint, **querystring) backoff and
+        retry is not called if waffle flag BACKOFF_FOR_API_CALLS_SWITCH is not active
+        """
+        self.mock_access_token_response()
+        self.mock_enterprise_learner_api()
+        enterprise_api.fetch_enterprise_learner_data(self.request.site, self.learner)
+        self.assertEquals(waffle.switch_is_active(BACKOFF_FOR_API_CALLS_SWITCH), False)
+        self.assertEquals(mocked_get.call_count, 0)
+
     def test_fetch_enterprise_learner_entitlements(self):
         """
         Verify that method "fetch_enterprise_learner_data" returns a proper

requirements/base.txt

@@ -5,6 +5,7 @@
 #############################
 -e git+https://github.com/edx/analytics-python.git@1.2.11#egg=analytics-python==1.2.11
 
+backoff==1.5.0
 coreapi==2.3.1
 django==1.11.11
 django-compressor==2.2