Commit a18bce81 by David Ormsbee Committed by Diana Huang

Basic cleanup of code to determine whether a user has a LinkedIn account.

parent 1a5eb086
# Create your models here. # Create your models here.
...@@ -8,11 +8,14 @@ import uuid ...@@ -8,11 +8,14 @@ import uuid
from django.conf import settings from django.conf import settings
from django.core.management.base import CommandError from django.core.management.base import CommandError
import requests
from ...models import LinkedInToken from ...models import LinkedInToken
class LinkedInError(Exception):
pass
class LinkedinAPI(object): class LinkedInAPI(object):
""" """
Encapsulates the LinkedIn API. Encapsulates the LinkedIn API.
""" """
...@@ -74,9 +77,14 @@ class LinkedinAPI(object): ...@@ -74,9 +77,14 @@ class LinkedinAPI(object):
""" """
Make an HTTP call to the LinkedIn JSON API. Make an HTTP call to the LinkedIn JSON API.
""" """
if settings.LINKEDIN_API.get('TEST_MODE'):
raise LinkedInError(
"Attempting to make real API call while in test mode - "
"Mock LinkedInAPI.call_json_api instead."
)
try: try:
request = urllib2.Request(url, headers={'x-li-format': 'json'}) request = urllib2.Request(url, headers={'x-li-format': 'json'})
response = urllib2.urlopen(request).read() response = urllib2.urlopen(request, timeout=5).read()
return json.loads(response) return json.loads(response)
except urllib2.HTTPError, error: except urllib2.HTTPError, error:
self.http_error(error, "Error calling LinkedIn API") self.http_error(error, "Error calling LinkedIn API")
......
...@@ -12,13 +12,14 @@ from django.utils import timezone ...@@ -12,13 +12,14 @@ from django.utils import timezone
from optparse import make_option from optparse import make_option
from ...models import LinkedIn from util.query import use_read_replica_if_available
from . import LinkedinAPI from linkedin.models import LinkedIn
from . import LinkedInAPI
FRIDAY = 4 FRIDAY = 4
def get_call_limits(): def get_call_limits(force_unlimited=False):
""" """
Returns a tuple of: (max_checks, checks_per_call, time_between_calls) Returns a tuple of: (max_checks, checks_per_call, time_between_calls)
...@@ -40,7 +41,7 @@ def get_call_limits(): ...@@ -40,7 +41,7 @@ def get_call_limits():
lastfriday -= datetime.timedelta(days=1) lastfriday -= datetime.timedelta(days=1)
safeharbor_begin = lastfriday.replace(hour=18, minute=0) safeharbor_begin = lastfriday.replace(hour=18, minute=0)
safeharbor_end = safeharbor_begin + datetime.timedelta(days=2, hours=11) safeharbor_end = safeharbor_begin + datetime.timedelta(days=2, hours=11)
if safeharbor_begin < now < safeharbor_end: if force_unlimited or (safeharbor_begin < now < safeharbor_end):
return -1, 80, 1 return -1, 80, 1
elif now.hour >= 18 or now.hour < 5: elif now.hour >= 18 or now.hour < 5:
return 500, 80, 1 return 500, 80, 1
...@@ -62,33 +63,38 @@ class Command(BaseCommand): ...@@ -62,33 +63,38 @@ class Command(BaseCommand):
dest='recheck', dest='recheck',
default=False, default=False,
help='Check users that have been checked in the past to see if ' help='Check users that have been checked in the past to see if '
'they have joined or left LinkedIn since the last check'), 'they have joined or left LinkedIn since the last check'
),
make_option( make_option(
'--force', '--force',
action='store_true', action='store_true',
dest='force', dest='force',
default=False, default=False,
help='Disregard the parameters provided by LinkedIn about when it ' help='Disregard the parameters provided by LinkedIn about when it '
'is appropriate to make API calls.')) 'is appropriate to make API calls.'
)
)
def handle(self, *args, **options): def handle(self, *args, **options):
""" """
Check users. Check users.
""" """
api = LinkedinAPI(self) api = LinkedInAPI(self)
recheck = options.pop('recheck', False) recheck = options.get('recheck', False)
force = options.pop('force', False) force = options.get('force', False)
if force: max_checks, checks_per_call, time_between_calls = get_call_limits(force)
max_checks, checks_per_call, time_between_calls = -1, 80, 1
else: if not max_checks:
max_checks, checks_per_call, time_between_calls = get_call_limits() raise CommandError("No checks allowed during this time.")
if not max_checks:
raise CommandError("No checks allowed during this time.") def user_batches_to_check():
"""Generate batches of users we should query against LinkedIn."""
def batch_users():
"Generator to lazily generate batches of users to query."
count = 0 count = 0
batch = [] batch = []
users = use_read_replica_if_available(
None
)
for user in User.objects.all(): for user in User.objects.all():
if not hasattr(user, 'linkedin'): if not hasattr(user, 'linkedin'):
LinkedIn(user=user).save() LinkedIn(user=user).save()
...@@ -98,8 +104,9 @@ class Command(BaseCommand): ...@@ -98,8 +104,9 @@ class Command(BaseCommand):
if len(batch) == checks_per_call: if len(batch) == checks_per_call:
yield batch yield batch
batch = [] batch = []
count += 1 count += 1
if max_checks != 1 and count == max_checks: if max_checks != -1 and count >= max_checks:
self.stderr.write( self.stderr.write(
"WARNING: limited to checking only %d users today." "WARNING: limited to checking only %d users today."
% max_checks) % max_checks)
...@@ -107,20 +114,21 @@ class Command(BaseCommand): ...@@ -107,20 +114,21 @@ class Command(BaseCommand):
if batch: if batch:
yield batch yield batch
def do_batch(batch): def update_linkedin_account_status(users):
"Process a batch of users." """
emails = (u.email for u in batch) Given a an iterable of User objects, check their email addresses
for user, has_account in zip(batch, api.batch(emails)): to see if they have LinkedIn email addresses and save that
information to our database.
"""
emails = (u.email for u in users)
for user, has_account in zip(users, api.batch(emails)):
linkedin = user.linkedin linkedin = user.linkedin
if linkedin.has_linkedin_account != has_account: if linkedin.has_linkedin_account != has_account:
linkedin.has_linkedin_account = has_account linkedin.has_linkedin_account = has_account
linkedin.save() linkedin.save()
batches = batch_users() for i, user_batch in enumerate(user_batches_to_check()):
try: if i > 0:
do_batch(batches.next()) # may raise StopIteration # Sleep between LinkedIn API web service calls
for batch in batches:
time.sleep(time_between_calls) time.sleep(time_between_calls)
do_batch(batch) update_linkedin_account_status(user_batch)
except StopIteration:
pass
...@@ -3,7 +3,7 @@ Log into LinkedIn API. ...@@ -3,7 +3,7 @@ Log into LinkedIn API.
""" """
from django.core.management.base import BaseCommand from django.core.management.base import BaseCommand
from . import LinkedinAPI from . import LinkedInAPI
class Command(BaseCommand): class Command(BaseCommand):
...@@ -19,7 +19,7 @@ class Command(BaseCommand): ...@@ -19,7 +19,7 @@ class Command(BaseCommand):
def handle(self, *args, **options): def handle(self, *args, **options):
""" """
""" """
api = LinkedinAPI(self) api = LinkedInAPI(self)
print "Let's log into your LinkedIn account." print "Let's log into your LinkedIn account."
print "Start by visiting this url:" print "Start by visiting this url:"
print api.authorization_url() print api.authorization_url()
......
...@@ -16,7 +16,7 @@ from certificates.models import GeneratedCertificate ...@@ -16,7 +16,7 @@ from certificates.models import GeneratedCertificate
from courseware.courses import get_course_by_id from courseware.courses import get_course_by_id
from ...models import LinkedIn from ...models import LinkedIn
from . import LinkedinAPI from . import LinkedInAPI
class Command(BaseCommand): class Command(BaseCommand):
...@@ -43,7 +43,7 @@ class Command(BaseCommand): ...@@ -43,7 +43,7 @@ class Command(BaseCommand):
def __init__(self): def __init__(self):
super(Command, self).__init__() super(Command, self).__init__()
self.api = LinkedinAPI(self) self.api = LinkedInAPI(self)
def handle(self, *args, **options): def handle(self, *args, **options):
whitelist = self.api.config.get('EMAIL_WHITELIST') whitelist = self.api.config.get('EMAIL_WHITELIST')
......
...@@ -4,11 +4,11 @@ import StringIO ...@@ -4,11 +4,11 @@ import StringIO
from django.core.management.base import CommandError from django.core.management.base import CommandError
from django.test import TestCase from django.test import TestCase
from linkedin.management.commands import LinkedinAPI from linkedin.management.commands import LinkedInAPI
from linkedin.models import LinkedInToken from linkedin.models import LinkedInToken
class LinkedinAPITests(TestCase): class LinkedInAPITests(TestCase):
def setUp(self): def setUp(self):
patcher = mock.patch('linkedin.management.commands.uuid.uuid4') patcher = mock.patch('linkedin.management.commands.uuid.uuid4')
...@@ -17,7 +17,7 @@ class LinkedinAPITests(TestCase): ...@@ -17,7 +17,7 @@ class LinkedinAPITests(TestCase):
self.addCleanup(patcher.stop) self.addCleanup(patcher.stop)
def make_one(self): def make_one(self):
return LinkedinAPI(DummyCommand()) return LinkedInAPI(DummyCommand())
@mock.patch('django.conf.settings.LINKEDIN_API', None) @mock.patch('django.conf.settings.LINKEDIN_API', None)
def test_ctor_no_api_config(self): def test_ctor_no_api_config(self):
......
...@@ -69,7 +69,7 @@ class FindUsersTests(TestCase): ...@@ -69,7 +69,7 @@ class FindUsersTests(TestCase):
@mock.patch(MODULE + 'time') @mock.patch(MODULE + 'time')
@mock.patch(MODULE + 'User') @mock.patch(MODULE + 'User')
@mock.patch(MODULE + 'LinkedinAPI') @mock.patch(MODULE + 'LinkedInAPI')
@mock.patch(MODULE + 'get_call_limits') @mock.patch(MODULE + 'get_call_limits')
def test_command_success_recheck_no_limits(self, get_call_limits, apicls, def test_command_success_recheck_no_limits(self, get_call_limits, apicls,
usercls, time): usercls, time):
...@@ -93,7 +93,7 @@ class FindUsersTests(TestCase): ...@@ -93,7 +93,7 @@ class FindUsersTests(TestCase):
@mock.patch(MODULE + 'time') @mock.patch(MODULE + 'time')
@mock.patch(MODULE + 'User') @mock.patch(MODULE + 'User')
@mock.patch(MODULE + 'LinkedinAPI') @mock.patch(MODULE + 'LinkedInAPI')
@mock.patch(MODULE + 'get_call_limits') @mock.patch(MODULE + 'get_call_limits')
def test_command_success_no_recheck_no_limits(self, get_call_limits, apicls, def test_command_success_no_recheck_no_limits(self, get_call_limits, apicls,
usercls, time): usercls, time):
...@@ -123,7 +123,7 @@ class FindUsersTests(TestCase): ...@@ -123,7 +123,7 @@ class FindUsersTests(TestCase):
@mock.patch(MODULE + 'time') @mock.patch(MODULE + 'time')
@mock.patch(MODULE + 'User') @mock.patch(MODULE + 'User')
@mock.patch(MODULE + 'LinkedinAPI') @mock.patch(MODULE + 'LinkedInAPI')
@mock.patch(MODULE + 'get_call_limits') @mock.patch(MODULE + 'get_call_limits')
def test_command_success_no_recheck_no_users(self, get_call_limits, apicls, def test_command_success_no_recheck_no_users(self, get_call_limits, apicls,
usercls, time): usercls, time):
...@@ -149,7 +149,7 @@ class FindUsersTests(TestCase): ...@@ -149,7 +149,7 @@ class FindUsersTests(TestCase):
@mock.patch(MODULE + 'time') @mock.patch(MODULE + 'time')
@mock.patch(MODULE + 'User') @mock.patch(MODULE + 'User')
@mock.patch(MODULE + 'LinkedinAPI') @mock.patch(MODULE + 'LinkedInAPI')
@mock.patch(MODULE + 'get_call_limits') @mock.patch(MODULE + 'get_call_limits')
def test_command_success_recheck_with_limit(self, get_call_limits, apicls, def test_command_success_recheck_with_limit(self, get_call_limits, apicls,
usercls, time): usercls, time):
...@@ -178,7 +178,7 @@ class FindUsersTests(TestCase): ...@@ -178,7 +178,7 @@ class FindUsersTests(TestCase):
self.assertTrue(command.stderr.getvalue().startswith("WARNING")) self.assertTrue(command.stderr.getvalue().startswith("WARNING"))
@mock.patch(MODULE + 'User') @mock.patch(MODULE + 'User')
@mock.patch(MODULE + 'LinkedinAPI') @mock.patch(MODULE + 'LinkedInAPI')
@mock.patch(MODULE + 'get_call_limits') @mock.patch(MODULE + 'get_call_limits')
def test_command_success_recheck_with_force(self, get_call_limits, apicls, def test_command_success_recheck_with_force(self, get_call_limits, apicls,
usercls): usercls):
...@@ -199,6 +199,7 @@ class FindUsersTests(TestCase): ...@@ -199,6 +199,7 @@ class FindUsersTests(TestCase):
"Mock LinkedIn API." "Mock LinkedIn API."
return [email % 2 == 0 for email in emails] return [email % 2 == 0 for email in emails]
api.batch = dummy_batch api.batch = dummy_batch
get_call_limits.return_value = (-1, 80, 1)
fut(force=True) fut(force=True)
self.assertEqual([u.linkedin.has_linkedin_account for u in users], self.assertEqual([u.linkedin.has_linkedin_account for u in users],
[i % 2 == 0 for i in xrange(10)]) [i % 2 == 0 for i in xrange(10)])
......
...@@ -266,6 +266,7 @@ LINKEDIN_API = { ...@@ -266,6 +266,7 @@ LINKEDIN_API = {
'COMPANY_NAME': 'edX', 'COMPANY_NAME': 'edX',
'COMPANY_ID': '0000000', 'COMPANY_ID': '0000000',
'EMAIL_FROM': 'The Team <team@test.foo>', 'EMAIL_FROM': 'The Team <team@test.foo>',
'TEST_MODE': True
} }
################### Make tests faster ################### Make tests faster
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment