Merge pull request #23 from edx/yro/refactor_newrelic

Eliminate NR decorators

bin/deliver

@@ -25,7 +25,6 @@ class DeliverCli:
             os.path.abspath(__file__)
         ))
         self.auth_yaml = os.path.join(self.ROOT_DIR, 'instance_config.yaml')
-        self.new_relic_config = os.path.join(self.ROOT_DIR, 'veda_newrelic.ini')
         self.celery_daemon = os.path.join(self.ROOT_DIR, 'control', 'celeryapp.py')
 
     def get_args(self):
@@ -62,8 +61,7 @@ class DeliverCli:
         if auth_dict is not None:
             os.system(
                 ' '.join((
-                    'NEW_RELIC_CONFIG_FILE=' + self.new_relic_config,
-                    'newrelic-admin run-program python',
+                    'python',
                     self.celery_daemon,
                     'worker',
                     '--loglevel=info',

control/veda_deliver.py

@@ -10,15 +10,8 @@ from os.path import expanduser
 import requests
 import datetime
 import ftplib
-import newrelic.agent
 import shutil
 
-newrelic.agent.initialize(
-    os.path.join(
-        os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
-        'veda_newrelic.ini'
-    )
-)
 
 try:
     boto.config.add_section('Boto')
@@ -84,7 +77,6 @@ class VedaDelivery:
             except yaml.YAMLError as exc:
                 return None
 
-    @newrelic.agent.background_task()
     def run(self):
         """
         Check the destination, route via available methods,

control/veda_encode.py

@@ -4,7 +4,6 @@ import sys
 import django
 import yaml
 import uuid
-import newrelic.agent
 
 
 """
@@ -16,13 +15,6 @@ Get a list of needed encodes from VEDA
 from control_env import *
 from dependencies.shotgun_api3 import Shotgun
 
-newrelic.agent.initialize(
-    os.path.join(
-        os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
-        'veda_newrelic.ini'
-    )
-)
-
 
 class VedaEncode():
 
@@ -62,7 +54,6 @@ class VedaEncode():
             except yaml.YAMLError as exc:
                 return None
 
-    @newrelic.agent.background_task()
     def determine_encodes(self):
         self.match_profiles()
 

control/veda_file_discovery.py

@@ -3,7 +3,6 @@ import os.path
 import boto
 import boto.s3
 from boto.exception import S3ResponseError, S3DataError
-import newrelic.agent
 import yaml
 
 try:
@@ -12,12 +11,6 @@ except:
     pass
 boto.config.set('Boto', 'http_socket_timeout', '100')
 
-newrelic.agent.initialize(
-    os.path.join(
-        os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
-        'veda_newrelic.ini'
-    )
-)
 """
 multi-point videofile discovery
 Currently:
@@ -61,7 +54,6 @@ class FileDiscovery(object):
         self.ftp_faillog = "/Users/Shared/edX1/LG/FailedTransfers.log"
         self.node_work_directory = kwargs.get('node_work_directory', WORK_DIRECTORY)
 
-    @newrelic.agent.background_task()
     def about_video_ingest(self):
         """
         Crawl VEDA Upload bucket
@@ -136,7 +128,6 @@ class FileDiscovery(object):
 
         reset_queries()
 
-    @newrelic.agent.background_task()
     def studio_s3_ingest(self):
         """
         Ingest files from studio upload endpoint

control/veda_file_ingest.py

@@ -12,14 +12,7 @@ from django.db import reset_queries
 import uuid
 import yaml
 import hashlib
-import newrelic.agent
 
-newrelic.agent.initialize(
-    os.path.join(
-        os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
-        'veda_newrelic.ini'
-    )
-)
 
 """
 Discovered file ingest/insert/job triggering
@@ -111,7 +104,6 @@ class VedaIngest:
             except yaml.YAMLError as exc:
                 return None
 
-    @newrelic.agent.background_task()
     def insert(self):
         """
         NOTE:
@@ -245,7 +237,6 @@ class VedaIngest:
                         else:
                             self.video_proto.resolution = '1920x1080'
 
-    @newrelic.agent.background_task()
     def database_record(self):
         """
         Start DB Inserts, Get Information
@@ -393,7 +384,6 @@ class VedaIngest:
         )
         VAC.call()
 
-    @newrelic.agent.background_task()
     def abvid_report(self):
         if self.video_proto.abvid_serial is None:
             return None
@@ -406,7 +396,6 @@ class VedaIngest:
         R.upload_status()
         self.complete = True
 
-    @newrelic.agent.background_task()
     def rename(self):
         """
         Rename to VEDA ID,

control/veda_heal.py

@@ -6,14 +6,6 @@ from datetime import timedelta
 import yaml
 import uuid
 
-import newrelic.agent
-
-newrelic.agent.initialize(
-    os.path.join(
-        os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
-        'veda_newrelic.ini'
-    )
-)
 
 """
 Heal Process
@@ -76,7 +68,6 @@ class VedaHeal():
             except yaml.YAMLError as exc:
                 return None
 
-    @newrelic.agent.background_task()
     def discovery(self):
         self.video_query = Video.objects.filter(
             video_trans_start__lt=self.current_time - timedelta(
@@ -122,7 +113,6 @@ class VedaHeal():
                         queue=cel_queue
                     )
 
-    @newrelic.agent.background_task()
     def determine_fault(self, video_object):
         """
         Is there anything to do with this?
@@ -222,7 +212,6 @@ class VedaHeal():
             self.val_status = 'transcode_queue'
         return encode_list
 
-    @newrelic.agent.background_task()
     def purge(self):
         """
         Purge Work Directory

control/veda_hotstore.py

@@ -8,7 +8,6 @@ import yaml
 import boto.s3
 from boto.s3.key import Key
 from boto.exception import S3ResponseError
-import newrelic.agent
 from os.path import expanduser
 
 from veda_utils import ErrorObject
@@ -19,12 +18,6 @@ except:
     pass
 boto.config.set('Boto', 'http_socket_timeout', '100')
 
-newrelic.agent.initialize(
-    os.path.join(
-        os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
-        'veda_newrelic.ini'
-    )
-)
 homedir = expanduser("~")
 
 
@@ -62,7 +55,6 @@ class Hotstore(object):
             except yaml.YAMLError as exc:
                 return None
 
-    @newrelic.agent.background_task()
     def upload(self):
         if self.auth_dict is None:
             return False

control/veda_utils.py

@@ -6,14 +6,7 @@ import datetime
 import boto.ses
 import hashlib
 import subprocess
-import newrelic.agent
 
-newrelic.agent.initialize(
-    os.path.join(
-        os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
-        'veda_newrelic.ini'
-    )
-)
 
 """
 Let's do some quick and dirty error handling & logging
@@ -50,7 +43,6 @@ class EmailAlert():
             except yaml.YAMLError as exc:
                 return None
 
-    @newrelic.agent.background_task()
     def email(self):
         email_subject = '[ VEDA ALERTING ]'
         email_subject += ' : ' + self.subject
@@ -165,7 +157,6 @@ class Report():
             except yaml.YAMLError as exc:
                 return None
 
-    @newrelic.agent.background_task()
     def upload_status(self):
         if self.upload_serial is None:
             return None
@@ -286,7 +277,6 @@ class Metadata():
         self.freezing_bug = False
         self.val_status = None
 
-    @newrelic.agent.background_task()
     def _METADATA(self):
         """
         use st filesize for filesize
@@ -324,7 +314,6 @@ class Metadata():
                     if self.video_proto.resolution is None:
                         self.video_proto.resolution = vid_breakout[3].strip()
 
-    @newrelic.agent.background_task()
     def _FAULT(self, video_object):
         if self.video_object is None:
             return []

control/veda_val.py

@@ -6,16 +6,9 @@ import ast
 import json
 import datetime
 import yaml
-import newrelic.agent
 
 requests.packages.urllib3.disable_warnings()
 
-newrelic.agent.initialize(
-    os.path.join(
-        os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
-        'veda_newrelic.ini'
-    )
-)
 
 """
 Send data to VAL, either Video ID data or endpoint URLs
@@ -69,7 +62,6 @@ class VALAPICall():
         )
         self.auth_dict = self._AUTH()
 
-    @newrelic.agent.background_task()
     def call(self):
         if self.auth_dict is None:
             print 'No AUTH'
@@ -102,7 +94,6 @@ class VALAPICall():
                 )
                 return None
 
-    @newrelic.agent.background_task()
     def val_tokengen(self):
         """
         Generate a API token for VAL

control/veda_video_validation.py

@@ -4,14 +4,6 @@ import sys
 import subprocess
 import fnmatch
 import django
-import newrelic.agent
-
-newrelic.agent.initialize(
-    os.path.join(
-        os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
-        'veda_newrelic.ini'
-    )
-)
 
 """
 VEDA Intake/Product Final Testing Suite
@@ -43,7 +35,6 @@ class Validation():
         seconds_duration = (((hours * 60) + minutes) * 60) + seconds
         return seconds_duration
 
-    @newrelic.agent.background_task()
     def validate(self):
         """
         Test #1 - assumes file is in 'work' directory of node

youtube_callback/daemon.py

@@ -7,7 +7,6 @@ import sys
 import datetime
 from datetime import timedelta
 import django
-import newrelic.agent
 
 from django.utils.timezone import utc
 
@@ -24,13 +23,6 @@ os.environ['DJANGO_SETTINGS_MODULE'] = 'VEDA.settings'
 
 django.setup()
 
-newrelic.agent.initialize(
-    os.path.join(
-        os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
-        'veda_newrelic.ini'
-    )
-)
-
 """
 Defaults
 """
@@ -46,7 +38,6 @@ def get_course(course_id):
     return course
 
 
-@newrelic.agent.background_task()
 def generate_course_list():
 
     course_list = []
@@ -79,7 +70,6 @@ def generate_course_list():
     return course_list
 
 
-@newrelic.agent.background_task()
 def weed_dupes(course_list, course):
     for c in course_list:
         if c.yt_logon == course.yt_logon:
@@ -87,7 +77,6 @@ def weed_dupes(course_list, course):
     return True
 
 
-@newrelic.agent.background_task()
 def determine_missing_url(course_object):
 
     video_query = Video.objects.filter(

youtube_callback/sftp_id_retrieve.py

@@ -7,7 +7,6 @@ from datetime import timedelta
 import django
 from django.utils.timezone import utc
 import fnmatch
-import newrelic.agent
 import os
 from os.path import expanduser
 import pysftp
@@ -27,13 +26,6 @@ from control.veda_val import VALAPICall
 from control.veda_utils import ErrorObject, Metadata, VideoProto
 from youtube_callback.daemon import get_course
 
-newrelic.agent.initialize(
-    os.path.join(
-        os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
-        'veda_newrelic.ini'
-    )
-)
-
 """
 Defaults:
 """
@@ -42,7 +34,6 @@ workdir = os.path.join(homedir, 'download_data_holding')
 YOUTUBE_LOOKBACK_DAYS = 15
 
 
-@newrelic.agent.background_task()
 def callfunction(course):
     """
 
@@ -64,7 +55,6 @@ def callfunction(course):
             urlpatch(upload_data)
 
 
-@newrelic.agent.background_task()
 def xml_downloader(course):
     """
 
@@ -96,7 +86,6 @@ def xml_downloader(course):
         return None
 
 
-@newrelic.agent.background_task()
 def crawl_sftp(d, s1):
     """
     crawl the sftp dir and dl the XML files for parsing
@@ -147,7 +136,6 @@ def crawl_sftp(d, s1):
     s1.cwd('..')
 
 
-@newrelic.agent.background_task()
 def domxml_parser(file):
     """
 
@@ -209,7 +197,6 @@ def domxml_parser(file):
     return upload_data
 
 
-@newrelic.agent.background_task()
 def urlpatch(upload_data):
     """