[releng] find_unblocked_orphans: Cache complete pkgdb info
by Till Maas
commit e6f7b5f47d4019b095d23781fe251d618385544b
Author: Till Maas <opensource(a)till.name>
Date: Sat Oct 18 10:49:24 2014 +0200
find_unblocked_orphans: Cache complete pkgdb info
scripts/find_unblocked_orphans.py | 111 ++++++++++++++++++++-----------------
1 files changed, 61 insertions(+), 50 deletions(-)
---
diff --git a/scripts/find_unblocked_orphans.py b/scripts/find_unblocked_orphans.py
index d485d6d..bf94aea 100755
--- a/scripts/find_unblocked_orphans.py
+++ b/scripts/find_unblocked_orphans.py
@@ -179,35 +179,46 @@ def write_cache(data, filename, cachedir='~/.cache'):
pickle.dump(data, pickle_file)
-def get_people(package, branch=RAWHIDE_RELEASE["branch"]):
- def associated(pkginfo, exclude=None):
- """
-
- :param exclude: People to exclude, e.g. the point of contact.
- :type exclude: list
- """
- other_people = set()
- for acl in pkginfo.get("acls", []):
- if acl["status"] == "Approved":
- fas_name = acl["fas_name"]
- if fas_name != "group::provenpackager" and \
- fas_name not in exclude:
- other_people.add(fas_name)
- return sorted(other_people)
+class PKGDBInfo(object):
+ def __init__(self, package, branch=RAWHIDE_RELEASE["branch"]):
+ self.package = package
+ self.branch = branch
- try:
- pkginfo = pkgdb.get_package(package, branches=branch)
- except Exception as e:
- sys.stderr.write(
- "Error getting maintainer of package {} on branch {}\n".format(
- package, branch))
- # FIXME: Write proper traceback
- sys.stderr.write(str(e))
- return []
- pkginfo = pkginfo["packages"][0]
- people_ = [pkginfo["point_of_contact"]]
- people_.extend(associated(pkginfo, exclude=people_))
- return people_
+ try:
+ pkginfo = pkgdb.get_package(package, branches=branch)
+ except Exception as e:
+ sys.stderr.write(
+ "Error getting pkgdb info for {} on {}\n".format(
+ package, branch))
+ # FIXME: Write proper traceback
+ sys.stderr.write(str(e))
+ self.pkginfo = None
+ return
+
+ self.pkginfo = pkginfo["packages"][0]
+
+ def get_people(self):
+ def associated(pkginfo, exclude=None):
+ """
+
+ :param exclude: People to exclude, e.g. the point of contact.
+ :type exclude: list
+ """
+ other_people = set()
+ for acl in pkginfo.get("acls", []):
+ if acl["status"] == "Approved":
+ fas_name = acl["fas_name"]
+ if fas_name != "group::provenpackager" and \
+ fas_name not in exclude:
+ other_people.add(fas_name)
+ return sorted(other_people)
+
+ if self.pkginfo is not None:
+ people_ = [self.pkginfo["point_of_contact"]]
+ people_.extend(associated(self.pkginfo, exclude=people_))
+ return people_
+ else:
+ return []
def setup_yum(repo=RAWHIDE_RELEASE["repo"],
@@ -285,9 +296,9 @@ class DepChecker(object):
yumbase = setup_yum(repo=RELEASES[release]["repo"],
source_repo=RELEASES[release]["source_repo"])
self.yumbase = yumbase
- self.people_queue = Queue()
- self.people_cache = "orphans-people-{}.pickle".format(release)
- self.people_dict = get_cache(self.people_cache, default={})
+ self.pkgdbinfo_queue = Queue()
+ self.pkgdb_cache = "orphans-pkgdb-{}.pickle".format(release)
+ self.pkgdb_dict = get_cache(self.pkgdb_cache, default={})
def create_mapping(self):
src_by_bin = {} # Dict of source pkg objects by binary package objects
@@ -394,19 +405,19 @@ class DepChecker(object):
prov)
return OrderedDict(sorted(dependent_packages.items()))
- def people_worker(self):
+ def pkgdb_worker(self):
branch = RELEASES[self.release]["branch"]
while True:
- package = self.people_queue.get()
- if package not in self.people_dict:
- people_ = get_people(package, branch)
- self.people_dict[package] = people_
- self.people_queue.task_done()
+ package = self.pkgdbinfo_queue.get()
+ if package not in self.pkgdb_dict:
+ pkginfo = PKGDBInfo(package, branch)
+ self.pkgdb_dict[package] = pkginfo
+ self.pkgdbinfo_queue.task_done()
def recursive_deps(self, packages, max_deps=20):
# Start threads to get information about (co)maintainers for packages
for i in range(0, 2):
- people_thread = Thread(target=self.people_worker)
+ people_thread = Thread(target=self.pkgdb_worker)
people_thread.daemon = True
people_thread.start()
# keep pylint silent
@@ -414,7 +425,7 @@ class DepChecker(object):
# get a list of all rpm_pkgs that are to be removed
rpm_pkg_names = []
for name in packages:
- self.people_queue.put(name)
+ self.pkgdbinfo_queue.put(name)
# Empty list if pkg is only for a different arch
bin_pkgs = self.by_src.get(name, [])
rpm_pkg_names.extend([p.name for p in bin_pkgs])
@@ -456,7 +467,7 @@ class DepChecker(object):
).setdefault(pkg, set()).add(dep)
for srpm_name in new_srpm_names:
- self.people_queue.put(srpm_name)
+ self.pkgdbinfo_queue.put(srpm_name)
ignore.extend(new_names)
if allow_more:
@@ -473,9 +484,9 @@ class DepChecker(object):
" completed\n".format(max_deps, name))
sys.stderr.write("Waiting for (co)maintainer information...")
- self.people_queue.join()
+ self.pkgdbinfo_queue.join()
sys.stderr.write("done\n")
- write_cache(self.people_dict, self.people_cache)
+ write_cache(self.pkgdb_dict, self.pkgdb_cache)
return dep_map
# This function was stolen from pungi
@@ -497,7 +508,7 @@ class DepChecker(object):
sys.exit(1)
-def maintainer_table(packages, people_dict):
+def maintainer_table(packages, pkgdb_dict):
affected_people = {}
if with_texttable:
@@ -509,7 +520,7 @@ def maintainer_table(packages, people_dict):
table = ""
for package_name in packages:
- people = people_dict[package_name]
+ people = pkgdb_dict[package_name].get_people()
for p in people:
affected_people.setdefault(p, set()).add(package_name)
p = ', '.join(people)
@@ -524,14 +535,14 @@ def maintainer_table(packages, people_dict):
return table, affected_people
-def dependency_info(dep_map, affected_people, people_dict):
+def dependency_info(dep_map, affected_people, pkgdb_dict):
info = ""
for package_name, subdict in dep_map.items():
if subdict:
info += "Depending on: {} ({})\n".format(package_name,
len(subdict.keys()))
for fedora_package, dependent_packages in subdict.items():
- people = people_dict[fedora_package]
+ people = pkgdb_dict[fedora_package].get_people()
for p in people:
affected_people.setdefault(p, set()).add(package_name)
p = ", ".join(people)
@@ -555,13 +566,13 @@ def maintainer_info(affected_people):
return info
-def package_info(packages, dep_map, people_dict, orphans=None, failed=None):
+def package_info(packages, dep_map, pkgdb_dict, orphans=None, failed=None):
info = ""
- table, affected_people = maintainer_table(packages, people_dict)
+ table, affected_people = maintainer_table(packages, pkgdb_dict)
info += table
info += "\n\nThe following packages require above mentioned packages:\n"
- info += dependency_info(dep_map, affected_people, people_dict)
+ info += dependency_info(dep_map, affected_people, pkgdb_dict)
info += "Affected (co)maintainers\n"
info += maintainer_info(affected_people)
@@ -656,7 +667,7 @@ def main():
# TODO: add app args to either depsolve or not
dep_map = depchecker.recursive_deps(unblocked)
sys.stderr.write('done\n')
- info, addresses = package_info(unblocked, dep_map, depchecker.people_dict,
+ info, addresses = package_info(unblocked, dep_map, depchecker.pkgdb_dict,
orphans=orphans, failed=failed)
text += "\n"
text += info
9 years, 7 months
while true; do sleep 1h; rpm-ostree compose tree...; done
by Colin Walters
Hi, right now the Atomic host is kind of missing its most important part
- the ability to receive updates.
I am actually kind of confused as to what the blockers are, because it
seems like it'd be basically a two line shell script in $SUBJECT run as
a daemon.
I've been doing this on the atomic01.qa.fedoraproject.org and pushing it
out to alt.fp.org, but I'd like to figure out how we move forward on the
official compose.
I think Dennis said something about filing a ticket for a location?
Where is that?
Are there any other blockers? Storage? Tool documentation?
9 years, 7 months
[releng] find_unblocked_orphans: Clarify instructions
by Till Maas
commit 7c7bc5bb25269e6e59a72bf51e1815f3d91d09d8
Author: Till Maas <opensource(a)till.name>
Date: Thu Oct 16 18:30:26 2014 +0200
find_unblocked_orphans: Clarify instructions
scripts/find_unblocked_orphans.py | 4 ++--
1 files changed, 2 insertions(+), 2 deletions(-)
---
diff --git a/scripts/find_unblocked_orphans.py b/scripts/find_unblocked_orphans.py
index b47ef2f..d485d6d 100755
--- a/scripts/find_unblocked_orphans.py
+++ b/scripts/find_unblocked_orphans.py
@@ -105,8 +105,8 @@ should be retired, please do so now with a proper reason:
https://fedoraproject.org/wiki/How_to_remove_a_package_at_end_of_life
Note: If you received this mail directly you (co)maintain one of the affected
-packages or a package that depends on one. Please orphan the affected package
-or retire your package to avoid broken dependencies.
+packages or a package that depends on one. Please adopt the affected package or
+retire your depending package to avoid broken dependencies.
"""
FOOTER = """-- \nThe script creating this output is run and developed by Fedora
9 years, 7 months
Fedora Release Engineering meeting summary for 2014-10-13
by Dennis Gilmore
-----BEGIN PGP SIGNED MESSAGE-----
Hash: SHA1
======================================
#fedora-meeting-1: RELENG (2014-10-13)
======================================
Meeting started by dgilmore-bne at 14:34:27 UTC. The full logs are
available at
http://meetbot.fedoraproject.org/fedora-meeting-1/2014-10-13/releng.2014-...
.
Meeting summary
- ---------------
* init process (dgilmore-bne, 14:34:47)
* Secondary Architectures updates (dgilmore-bne, 14:41:06)
* Secondary Architectures update - ppc (dgilmore-bne, 14:41:11)
* Secondary Architectures update - s390 (dgilmore-bne, 14:48:41)
* Secondary Architectures update - arm (dgilmore-bne, 14:50:25)
* #5931 [Proposal] Move new branch and unretire requests to pkgdb2
(dgilmore-bne, 14:58:59)
* LINK: https://fedorahosted.org/rel-eng/ticket/5931 (dgilmore-bne,
14:59:20)
* open floor (dgilmore-bne, 15:28:34)
Meeting ended at 15:33:57 UTC.
Action Items
- ------------
Action Items, by person
- -----------------------
* **UNASSIGNED**
* (none)
People Present (lines said)
- ---------------------------
* dgilmore-bne (66)
* pingou (29)
* pbrobinson (22)
* nirik (8)
* zodbot (4)
* sharkcz (4)
* pbabinca (3)
* bochecha (1)
* tyll (0)
* masta (0)
* dgilmore (0)
Generated by `MeetBot`_ 0.1.4
.. _`MeetBot`: http://wiki.debian.org/MeetBot
-----BEGIN PGP SIGNATURE-----
Version: GnuPG v2
iQIcBAEBAgAGBQJUPHcrAAoJEH7ltONmPFDR/cUP/28Yw2H4e80/NSGa2j+oAeMg
BOqANdkt2aV98gz6Ut/p7a8SAYUpSbzD7+CILomSBfdEnLeswlwSzvKF2VizjvOD
jNOK1teKmhCogyy5c3b4puuFZ5Kp1MwXtmjTotj7dBu/DSUL8VRQf6twJwpBwUn3
6Rtx4IVzqdgTestGeT0706HDDiLAE+sQbPa1w9/tfqoYp6L5ZgcGrqEe9DQdjIwF
0LukTjGyRrEC0PgTMYsQOd2sS5K55a/5Xvix+JqTn71psdNPU0HomlBf/QuWcuon
KzosN6+/HxY5OAIXPiNEUJnq2bC8E+ClNQ+UMXRQ1rEp9MFSyTeTp5bk5g5HiisP
0vpNBdPUoPjn/fdnMlY+b1Ers6HD3aDNGjCAoEoqmsVG2tPMyhXmLthNKZX6MugP
MPITb3sfSFuHHo1750LHIAhV+IQh5c93OtrypV/XeHKFLQ5R//9T5B3nEzKNdE8H
I5jvx9yj4wwbUJQesqHHxiyDbNDadZdY7Slzf6yD6Cco4KekD/5klEp/ysoN3iQj
zlr7phaJPz5Q4fwJgROBuSDGDnq64kA4FcS08WowNshlRxbf6+Ck7d9sbCsbe1kN
KWGOkZ0SDBn9yynlUtLVxy1IWDOFuQCGUVlwEBHS56VzvV7sbaMcWhCo4hUVLIQd
8LVwX9E14Z5IxcF/+euP
=YQmS
-----END PGP SIGNATURE-----
9 years, 7 months
[releng] Fix line widths
by Till Maas
commit 45be26a7d7308840abeaef942822591a202a8860
Author: Till Maas <opensource(a)till.name>
Date: Fri Oct 10 20:18:54 2014 +0200
Fix line widths
scripts/find_unblocked_orphans.py | 22 +++++++++++++---------
1 files changed, 13 insertions(+), 9 deletions(-)
---
diff --git a/scripts/find_unblocked_orphans.py b/scripts/find_unblocked_orphans.py
index 9687377..b47ef2f 100755
--- a/scripts/find_unblocked_orphans.py
+++ b/scripts/find_unblocked_orphans.py
@@ -71,7 +71,8 @@ RAWHIDE_RELEASE = dict(
BRANCHED_RELEASE = dict(
repo='https://kojipkgs.fedoraproject.org/mash/branched/i386/os',
- source_repo='https://kojipkgs.fedoraproject.org/mash/branched/source/SRPMS',
+ source_repo='https://kojipkgs.fedoraproject.org/mash/branched/source/'
+ 'SRPMS',
tag='f21',
branch='f21',
mailto='devel(a)lists.fedoraproject.org',
@@ -98,14 +99,14 @@ According to https://fedoraproject.org/wiki/Schedule branching will
occur not earlier than 2014-07-08. The packages will be retired shortly before.
"""
-HEADER = """The following packages are orphaned and might be retired eventually,
-unless someone adopts them. If you know for sure that the package should be
-retired, please do so now with a proper reason:
+HEADER = """The following packages are orphaned and might be retired
+eventually, unless someone adopts them. If you know for sure that the package
+should be retired, please do so now with a proper reason:
https://fedoraproject.org/wiki/How_to_remove_a_package_at_end_of_life
Note: If you received this mail directly you (co)maintain one of the affected
-packages or a package that depends on one. Please orphan the affected package or
-retire your package to avoid broken dependencies.
+packages or a package that depends on one. Please orphan the affected package
+or retire your package to avoid broken dependencies.
"""
FOOTER = """-- \nThe script creating this output is run and developed by Fedora
@@ -338,7 +339,8 @@ class DepChecker(object):
rpms = self.by_src[srpmname]
except KeyError:
# If we don't have a package in the repo, there is nothing to do
- sys.stderr.write("Package {0} not found in repo\n".format(srpmname))
+ sys.stderr.write(
+ "Package {0} not found in repo\n".format(srpmname))
rpms = []
# provides of all packages built from ``srpmname``
@@ -490,7 +492,8 @@ class DepChecker(object):
arch='src')[0]
return srpmpo
except IndexError:
- print >> sys.stderr, "Error: Cannot find a source rpm for %s" % srpm
+ sys.stderr.write(
+ "Error: Cannot find a source rpm for {}\n".format(srpm))
sys.exit(1)
@@ -579,7 +582,8 @@ def package_info(packages, dep_map, people_dict, orphans=None, failed=None):
orphans_breaking_deps = [o for o in orphans if
o in dep_map and dep_map[o]]
- info += wrap_and_format("Orphans (dependend on)", orphans_breaking_deps)
+ info += wrap_and_format("Orphans (dependend on)",
+ orphans_breaking_deps)
orphans_not_breaking_deps = [o for o in orphans if
o not in dep_map or not dep_map[o]]
9 years, 7 months
discussion on Fedora Council Engineering representative
by Matthew Miller
Spamming many lists to make sure this doesn't go unnoticed. (In case my
accidentally posting to both devel and devel-announce separately doesn't do
it.)
Please see the thread titled "Engineering Representiatve for the new Fedora
Council" on the devel list — this affects and should have the input of
everyone involved in the broad category of "building and shipping the
distro" efforts in Fedora. Thanks!
--
Matthew Miller
<mattdm(a)fedoraproject.org>
Fedora Project Leader
9 years, 7 months
Pkgdb2 - Update on new branch/package process
by Pierre-Yves Chibon
Dear all,
Sorry for the long silence about this.
Here is where we are now (in git, nothing merged/in prod atm):
* user can request new packages on pkgdb
* user can request a new branch for a package on pkgdb
* these actions are recorded and stored in pkgdb
Available at:
http://209.132.184.188/admin/actions/
and
http://209.132.184.188/api/admin/actions/
* there is a new pkgdb-admin part of pkgdb-cli allowing to list actions and
act of them (approve (ie: process)/deny)
-> the pkgdb UI only updates the status of the action, while the cli
pkgdb-admin actually process the request (as well as updating the status
of the action at the end)
-> pkgdb-admin would be the place where we would add all the checks we want
to do before processing a package/branch request
fedmsg messages will be broadcasted for new branch.
This part is all there and basically ready :)
The second part is about listening to fedmsg message about new branches and
adjusting the git repos accordingly.
This is where things are a little more hairy.
Currently, we have fedmsg-pkgdb-sync-git [1] that basically listens for pkgdb
messages on the bus and run the ansible playbook: run_pkgdb_sync_git.yml [2]
(which in turn runs pkgdb_sync_git_branches.py) when needed.
The problem which I am facing is in the case of new EPEL branch creation.
These branches are created from another one. For example in the case of the
epel7 branch we used (are we still?) to branch either from either el6 or f19.
With that in mind, I allowed in pkgdb to specify from which branch one would
like to get the new branch.
The problem is that this information is not available to fedmsg-pkgdb-sync-git
and even if it was, we would not be able to pass it along to the ansible
playbook.
So afaics the options are:
1/ We hard-code somewhere from which branch each branch should come (ie: f*
comes from master, el6 comes from ?, epel7 comes from f19...)
2/ We manage to create the branch in such a way that they point to the first
commit in the repo (suggestion from Till), from there the maintainers should
be able to fast-forward to the branch they think should apply
3/ We manage to just create the branch in gitolite and just let the maintainer
populate it by merging the branch they think should apply
I have a preference for #3, but our tests with bochecha for it didn't quite
succeed.
But in all 3 options, I basically can drop the `branch_from` field when
requesting a new branch in pkgdb.
So what is our preference and how do we want to proceed from here?
Thanks,
Pierre
[1] https://github.com/fedora-infra/fedmsg-pkgdb-sync-git
[2] https://infrastructure.fedoraproject.org/infra/ansible/playbooks/run_pkgd...
9 years, 7 months
[releng] find_unblocked_orphans: refactor a little
by Till Maas
commit ff5c84609f0d8f8b03850d64d08dc2323a0eecfc
Author: Till Maas <opensource(a)till.name>
Date: Thu Oct 9 19:17:41 2014 +0200
find_unblocked_orphans: refactor a little
scripts/find_unblocked_orphans.py | 32 +++++++++++++++++---------------
1 files changed, 17 insertions(+), 15 deletions(-)
---
diff --git a/scripts/find_unblocked_orphans.py b/scripts/find_unblocked_orphans.py
index 2760397..9687377 100755
--- a/scripts/find_unblocked_orphans.py
+++ b/scripts/find_unblocked_orphans.py
@@ -552,27 +552,20 @@ def maintainer_info(affected_people):
return info
-def package_info(packages, release, orphans=None, failed=None):
- sys.stderr.write("Setting up dependency checker...")
- depchecker = DepChecker(release)
- sys.stderr.write("done\n")
+def package_info(packages, dep_map, people_dict, orphans=None, failed=None):
info = ""
- sys.stderr.write('Calculating dependencies...')
- # Create yum object and depsolve out if requested.
- # TODO: add app args to either depsolve or not
- dep_map = depchecker.recursive_deps(packages)
- sys.stderr.write('done\n')
- table, affected_people = maintainer_table(packages, depchecker.people_dict)
+ table, affected_people = maintainer_table(packages, people_dict)
info += table
info += "\n\nThe following packages require above mentioned packages:\n"
- info += dependency_info(dep_map, affected_people, depchecker.people_dict)
+ info += dependency_info(dep_map, affected_people, people_dict)
info += "Affected (co)maintainers\n"
info += maintainer_info(affected_people)
wrapper = textwrap.TextWrapper(
- break_long_words=False, subsequent_indent=" ", break_on_hyphens=False
+ break_long_words=False, subsequent_indent=" ",
+ break_on_hyphens=False
)
def wrap_and_format(label, pkgs):
@@ -624,7 +617,8 @@ def main():
parser.add_argument("--skip-orphans", dest="skip_orphans",
help="Do not look for orphans",
default=False, action="store_true")
- parser.add_argument("--release", choices=RELEASES.keys(), default="rawhide")
+ parser.add_argument("--release", choices=RELEASES.keys(),
+ default="rawhide")
parser.add_argument("--mailto", default=None,
help="Send mail to this address (for testing)")
parser.add_argument(
@@ -650,8 +644,16 @@ def main():
sys.stderr.write('done\n')
text = HEADER.format(RELEASES[args.release]["tag"].upper())
- info, addresses = package_info(unblocked, args.release, orphans=orphans,
- failed=failed)
+ sys.stderr.write("Setting up dependency checker...")
+ depchecker = DepChecker(args.release)
+ sys.stderr.write("done\n")
+ sys.stderr.write('Calculating dependencies...')
+ # Create yum object and depsolve out if requested.
+ # TODO: add app args to either depsolve or not
+ dep_map = depchecker.recursive_deps(unblocked)
+ sys.stderr.write('done\n')
+ info, addresses = package_info(unblocked, dep_map, depchecker.people_dict,
+ orphans=orphans, failed=failed)
text += "\n"
text += info
text += FOOTER
9 years, 7 months