--- backend/backend/daemons/backend.py | 11 ++++++----- backend/backend/daemons/job_grab.py | 7 ++++--- backend/backend/frontend.py | 9 ++++++--- backend/run/copr_run_job_grab.py | 4 +--- 4 files changed, 17 insertions(+), 14 deletions(-)
diff --git a/backend/backend/daemons/backend.py b/backend/backend/daemons/backend.py index 8818f71..934a566 100644 --- a/backend/backend/daemons/backend.py +++ b/backend/backend/daemons/backend.py @@ -52,11 +52,11 @@ class CoprBackend(object):
self.task_queues = {}
- self.frontend_client = FrontendClient(self.opts) - self.is_running = False - self.log = get_redis_logger(self.opts, "backend.main", "backend")
+ self.frontend_client = FrontendClient(self.opts, self.log) + self.is_running = False + def clean_task_queues(self): """ Make sure there is nothing in our task queues @@ -162,7 +162,7 @@ class CoprBackend(object):
try: self.log.info("Rescheduling unfinished builds before stop") - self.frontend_client.reschedule_all_running() + self.frontend_client.reschedule_all_running(5) except RequestException as err: self.log.exception(err) return @@ -178,7 +178,8 @@ class CoprBackend(object):
try: self.log.info("Rescheduling old unfinished builds") - self.frontend_client.reschedule_all_running() + # 120*5 = 10 minutes + self.frontend_client.reschedule_all_running(120) except RequestException as err: self.log.exception(err) return diff --git a/backend/backend/daemons/job_grab.py b/backend/backend/daemons/job_grab.py index 558ae57..21871ae 100644 --- a/backend/backend/daemons/job_grab.py +++ b/backend/backend/daemons/job_grab.py @@ -13,6 +13,8 @@ from requests import get, RequestException from retask.task import Task from retask.queue import Queue
+from backend.frontend import FrontendClient + from ..actions import Action from ..constants import JOB_GRAB_TASK_END_PUBSUB from ..helpers import get_redis_connection, get_redis_logger @@ -32,12 +34,11 @@ class CoprJobGrab(object):
:param Munch opts: backend config - :type frontend_client: FrontendClient :param lock: :py:class:`multiprocessing.Lock` global backend lock
"""
- def __init__(self, opts, frontend_client): + def __init__(self, opts): """ base class initialization """
self.opts = opts @@ -51,13 +52,13 @@ class CoprJobGrab(object):
self.added_jobs_dict = dict() # task_id -> task dict
- self.frontend_client = frontend_client
self.rc = None self.channel = None self.ps_thread = None
self.log = get_redis_logger(self.opts, "backend.job_grab", "job_grab") + self.frontend_client = FrontendClient(self.opts, self.log)
def connect_queues(self): """ diff --git a/backend/backend/frontend.py b/backend/backend/frontend.py index 52c91b6..6f442d3 100644 --- a/backend/backend/frontend.py +++ b/backend/backend/frontend.py @@ -8,12 +8,13 @@ class FrontendClient(object): Object to send data back to fronted """
- def __init__(self, opts): + def __init__(self, opts, logger=None): super(FrontendClient, self).__init__() self.frontend_url = "{}/backend".format(opts.frontend_base_url) self.frontend_auth = opts.frontend_auth
self.msg = None + self.log = logger
def _post_to_frontend(self, data, url_path): """ @@ -43,6 +44,8 @@ class FrontendClient(object): """ for i in range(max_repeats): try: + if i and self.log: + self.log.warning("failed to post data to frontend, repeat #{0}".format(i)) return self._post_to_frontend(data, url_path) except RequestException: time.sleep(5) @@ -74,7 +77,7 @@ class FrontendClient(object): data = {"build_id": build_id, "chroot": chroot_name} self._post_to_frontend_repeatedly(data, "reschedule_build_chroot")
- def reschedule_all_running(self): - response = self._post_to_frontend({}, "reschedule_all_running") + def reschedule_all_running(self, attempts): + response = self._post_to_frontend_repeatedly({}, "reschedule_all_running", attempts) if response.status_code != 200: raise RequestException("Failed to reschedule all running jobs") diff --git a/backend/run/copr_run_job_grab.py b/backend/run/copr_run_job_grab.py index b39fabe..d1ce730 100755 --- a/backend/run/copr_run_job_grab.py +++ b/backend/run/copr_run_job_grab.py @@ -11,13 +11,11 @@ sys.path.append("/usr/share/copr/")
from backend.helpers import get_backend_opts from backend.daemons.job_grab import CoprJobGrab -from backend.frontend import FrontendClient
def main(): opts = get_backend_opts() - fc = FrontendClient(opts) - grabber = CoprJobGrab(opts, frontend_client=fc) + grabber = CoprJobGrab(opts) grabber.run()
copr-devel@lists.fedorahosted.org