Hi,
Here is a hotfix for fedimg. Right now Rawhide messages are not
getting processed. This hotfix will fix the issue.
PR for the same[1]
[1]
https://github.com/fedora-infra/fedimg/pull/103
+1s
From b52a1dd1eda25bfbade026a1ebde93839fa3379b Mon Sep 17 00:00:00 2001
From: Sayan Chowdhury <sayan.chowdhury2012(a)gmail.com>
Date: Thu, 19 Apr 2018 22:05:27 +0530
Subject: [PATCH 1/2] fedimg: Add the intial files for PR#103
Signed-off-by: Sayan Chowdhury <sayan.chowdhury2012(a)gmail.com>
---
files/hotfix/fedimg/consumers.py | 160 +++++++++++++++++++++++++++++++++++++++
roles/fedimg/tasks/main.yml | 8 ++
2 files changed, 168 insertions(+)
create mode 100644 files/hotfix/fedimg/consumers.py
diff --git a/files/hotfix/fedimg/consumers.py b/files/hotfix/fedimg/consumers.py
new file mode 100644
index 0000000..ce4d662
--- /dev/null
+++ b/files/hotfix/fedimg/consumers.py
@@ -0,0 +1,160 @@
+# -*- coding: utf-8 -*-
+# This file is part of fedimg.
+# Copyright (C) 2014-2017 Red Hat, Inc.
+#
+# fedimg is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# fedimg is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public
+# License along with fedimg; if not, see
http://www.gnu.org/licenses,
+# or write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+#
+# Authors: David Gay <dgay(a)redhat.com>
+# Sayan Chowdhury <sayanchowdhury(a)fedoraproject.org>
+"""
+This is the `fedmsg consumer`_ that subscribes to the topic emitted after the
+completion of the nightly and production compose. The consumer on receving the
+message uploads the image using the API of the cloud providers.
+"""
+
+import logging
+import multiprocessing.pool
+
+import fedmsg.consumers
+import fedmsg.encoding
+import fedfind.release
+import fedfind.exceptions
+
+import fedimg.uploader
+
+from fedimg.config import PROCESS_COUNT, STATUS_FILTER
+from fedimg.utils import get_rawxz_urls, get_value_from_dict
+
+_log = logging.getLogger(__name__)
+
+
+class FedimgConsumer(fedmsg.consumers.FedmsgConsumer):
+ """
+ A `fedmsg consumer`_ that listens to the pungi compose topics and kicks
+ of the process to upload the images to various cloud providers.
+
+ Attributes:
+ topic (str): The topics this consumer is subscribed to. Set to
+ ``org.fedoraproject.prod.pungi.compose.status.change``.
+ config_key (str): The key to set to ``True`` in the fedmsg config to
+ enable this consumer. The key is ``fedimgconsumer.prod.enabled``.
+ """
+ topic = ['org.fedoraproject.prod.pungi.compose.status.change']
+ config_key = "fedimgconsumer.prod.enabled"
+
+ def __init__(self, *args, **kwargs):
+ _log.info("FedimgConsumer initializing")
+ super(FedimgConsumer, self).__init__(*args, **kwargs)
+
+ # Threadpool for upload jobs
+ _log.info("Creating thread pool of %s process", PROCESS_COUNT)
+ self.upload_pool = multiprocessing.pool.ThreadPool(
+ processes=PROCESS_COUNT
+ )
+ _log.info("FedimgConsumer initialized")
+
+ def consume(self, msg):
+ """
+ This is called when we receive a message matching our topics.
+
+ Args:
+ msg (dict): The raw message from fedmsg.
+ """
+ _log.info('Received %r %r', msg['topic'],
msg['body']['msg_id'])
+
+ msg_info = msg['body']['msg']
+ if msg_info['status'] not in STATUS_FILTER:
+ _log.debug('%s is not valid status' % msg_info['status'])
+ return
+
+ location = msg_info['location']
+ compose_id = msg_info['compose_id']
+ try:
+ compose_metadata =
fedfind.release.get_release(cid=compose_id).metadata
+ except fedfind.exceptions.UnsupportedComposeError:
+ LOG.debug("%r is unsupported compose" % compose_id)
+ return
+
+
+ # Till F27, both cloud-base and atomic images were available
+ # under variant CloudImages. With F28 and onward releases,
+ # cloud-base image compose moved to cloud variant and atomic images
+ # moved under atomic variant.
+ prev_rel = ['26', '27']
+ if msg_info['release_version'] in prev_rel:
+ images_meta = get_value_from_dict(
+ compose_metadata, 'images', 'payload', 'images',
'CloudImages',
+ 'x86_64')
+ else:
+ images_meta = get_value_from_dict(
+ compose_metadata, 'images', 'payload', 'images',
+ 'Cloud', 'x86_64')
+ images_meta.extend(get_value_from_dict(
+ compose_metadata, 'images', 'payload',
+ 'images', 'AtomicHost', 'x86_64'))
+
+ images_meta = get_value_from_dict(
+ compose_metadata,
+ 'images',
+ 'payload',
+ 'images',
+ 'CloudImages',
+ 'x86_64'
+ )
+
+ if images_meta is None:
+ _log.debug('No compatible image found to process')
+ return
+
+ upload_urls = get_rawxz_urls(location, images_meta)
+ if len(upload_urls) > 0:
+ _log.info("Start processing compose id: %s", compose_id)
+ fedimg.uploader.upload(
+ pool=self.upload_pool,
+ urls=upload_urls,
+ compose_id=compose_id,
+ push_notifications=True
+ )
+
+
+class FedimgStagingConsumer(FedimgConsumer):
+ """
+ A `fedmsg consumer`_ that listens to the staging pungi compose topics and
+ kicks of the process to upload the images to various cloud providers.
+
+ Attributes:
+ topic (str): The topics this consumer is subscribed to. Set to
+ ``org.fedoraproject.stg.pungi.compose.status.change``.
+ config_key (str): The key to set to ``True`` in the fedmsg config to
+ enable this consumer. The key is ``fedimgconsumer.stg.enabled``.
+ """
+ topic = ['org.fedoraproject.stg.pungi.compose.status.change']
+ config_key = "fedimgconsumer.stg.enabled"
+
+
+class FedimgDevConsumer(FedimgConsumer):
+ """
+ A `fedmsg consumer`_ that listens to the dev pungi compose topics and
+ kicks of the process to upload the images to various cloud providers.
+
+ Attributes:
+ topic (str): The topics this consumer is subscribed to. Set to
+ ``org.fedoraproject.dev.pungi.compose.status.change``.
+ config_key (str): The key to set to ``True`` in the fedmsg config to
+ enable this consumer. The key is ``fedimgconsumer.dev.enabled``.
+ """
+ topic = ['org.fedoraproject.dev.pungi.compose.status.change']
+ config_key = "fedimgconsumer.dev.enabled"
diff --git a/roles/fedimg/tasks/main.yml b/roles/fedimg/tasks/main.yml
index b0f18bf..2e35608 100644
--- a/roles/fedimg/tasks/main.yml
+++ b/roles/fedimg/tasks/main.yml
@@ -130,3 +130,11 @@
tags:
- fedimg
- hotfix
+
+- name: hotfix - copy the consumers.py over to the site-packages
+ copy: src="{{ files }}/hotfix/fedimg/consumers.py"
dest=/usr/lib/python2.7/site-packages/fedimg/consumers.py
+ notify:
+ - restart fedmsg-hub
+ tags:
+ - fedimg
+ - hotfix
--
2.9.4
From 1bf3db45c8154d0a6a0f29836ec14ce24fa404c1 Mon Sep 17 00:00:00 2001
From: Sayan Chowdhury <sayan.chowdhury2012(a)gmail.com>
Date: Thu, 19 Apr 2018 22:07:12 +0530
Subject: [PATCH 2/2] fedimg: Add the patch for the PR#103, fix processing
Rawhide messages
Signed-off-by: Sayan Chowdhury <sayan.chowdhury2012(a)gmail.com>
---
files/hotfix/fedimg/consumers.py | 11 +----------
1 file changed, 1 insertion(+), 10 deletions(-)
diff --git a/files/hotfix/fedimg/consumers.py b/files/hotfix/fedimg/consumers.py
index ce4d662..8195321 100644
--- a/files/hotfix/fedimg/consumers.py
+++ b/files/hotfix/fedimg/consumers.py
@@ -85,7 +85,7 @@ class FedimgConsumer(fedmsg.consumers.FedmsgConsumer):
try:
compose_metadata =
fedfind.release.get_release(cid=compose_id).metadata
except fedfind.exceptions.UnsupportedComposeError:
- LOG.debug("%r is unsupported compose" % compose_id)
+ _log.debug("%r is unsupported compose" % compose_id)
return
@@ -106,15 +106,6 @@ class FedimgConsumer(fedmsg.consumers.FedmsgConsumer):
compose_metadata, 'images', 'payload',
'images', 'AtomicHost', 'x86_64'))
- images_meta = get_value_from_dict(
- compose_metadata,
- 'images',
- 'payload',
- 'images',
- 'CloudImages',
- 'x86_64'
- )
-
if images_meta is None:
_log.debug('No compatible image found to process')
return
--
2.9.4
--
Sayan Chowdhury <
https://sayanchowdhury.dgplug.org/>
Senior Software Engineer, Fedora Engineering - Emerging Platform
GPG Fingerprint : 0F16 E841 E517 225C 7D13 AB3C B023 9931 9CD0 5C8B
Proud to work at The Open Organization!