gotmax23 pushed to ansible-packaging (f35). "Add
ansible_collection.py helper script (..more)"
by notificationsï¼ fedoraproject.org
Notification time stamped 2022-09-30 21:11:11 UTC
From bc337bd8e48760c9670613391c7f0145542cded5 Mon Sep 17 00:00:00 2001
From: Maxwell G <gotmax(a)e.email>
Date: Sep 26 2022 04:07:26 +0000
Subject: Add ansible_collection.py helper script
The %collection_namespace and %collection_name macros are clunky. This
script preforms two tasks:
- install: extract the namespace, name, and version from galaxy.yml,
run `ansible-galaxy install`, and write a filelist for use with
%files -f.
- test: create a temporary build directory with the required
directory structure for ansible-test to work and run `ansible-test
units`.
---
diff --git a/ansible-packaging.spec b/ansible-packaging.spec
index 50e256d..4e2478c 100644
--- a/ansible-packaging.spec
+++ b/ansible-packaging.spec
@@ -9,7 +9,12 @@ Source0: ansible-generator
Source1: ansible.attr
Source2: macros.ansible
Source3: macros.ansible-srpm
-Source4: COPYING
+Source4: ansible_collection.py
+
+Source100: COPYING
+
+# Needed for ansible_collection.py
+Requires: %{py3_dist pyyaml}
# Require ansible-core for building. Collections still have a boolean runtime
# dependency on either ansible 2.9 OR ansible-core.
@@ -67,9 +72,10 @@ cp -a %{sources} .
%install
install -Dpm0644 -t %{buildroot}%{_fileattrsdir} ansible.attr
-install -Dpm0644 -t %{buildroot}%{_rpmmacrodir} macros.ansible
-install -Dpm0644 -t %{buildroot}%{_rpmmacrodir} macros.ansible-srpm
+install -Dpm0644 -t %{buildroot}%{_rpmmacrodir} macros.ansible
+install -Dpm0644 -t %{buildroot}%{_rpmmacrodir} macros.ansible-srpm
install -Dpm0755 -t %{buildroot}%{_rpmconfigdir} ansible-generator
+install -Dpm0755 -t %{buildroot}%{_rpmconfigdir} ansible_collection.py
%files
@@ -77,6 +83,7 @@ install -Dpm0755 -t %{buildroot}%{_rpmconfigdir} ansible-generator
%{_fileattrsdir}/ansible.attr
%{_rpmmacrodir}/macros.ansible
%{_rpmconfigdir}/ansible-generator
+%{_rpmconfigdir}/ansible_collection.py
%files -n ansible-srpm-macros
diff --git a/ansible_collection.py b/ansible_collection.py
new file mode 100755
index 0000000..18f04a9
--- /dev/null
+++ b/ansible_collection.py
@@ -0,0 +1,138 @@
+#!/usr/bin/python3
+# SPDX-License-Identifier: GPL-3.0-or-later
+# SPDX-FileCopyrightText 2022 Maxwell G <gotmax(a)e.email>
+
+"""
+This script uses Ansible Collection metadata from galaxy.yml to figure out the
+namespace, name, and version of the collection being packaged.
+
+``ansible_collection.py install`` (used by %ansible_collecton_install) uses
+this information to find and install the collection artifact that was just
+built with %ansible_collection_build. It also generates a files list for use
+with `%files -f`.
+
+``ansible_collection.py test`` (used by %ansible_test_unit) parses galaxy.yml
+to determine the collection namespace and name that's needed to create the
+directory structure that ansible-test expects. After creating a temporary build
+directory with the needed structure, the script runs ansible-test units with
+the provided arguments.
+"""
+
+import argparse
+import shutil
+import subprocess
+import sys
+from pathlib import Path
+from tempfile import TemporaryDirectory
+from typing import Any, Dict, Optional, Sequence, Union
+
+from yaml import CSafeLoader, load
+
+
+class CollectionError(Exception):
+ pass
+
+
+class AnsibleCollection:
+ def __init__(self, collection_srcdir: Optional[Path] = None) -> None:
+ self.collection_srcdir = collection_srcdir or Path.cwd()
+ self.data = self._load_data()
+ self.namespace = self.data["namespace"]
+ self.name = self.data["name"]
+ self.version = self.data["version"]
+
+ def _load_data(self) -> Dict[str, Any]:
+ path = self.collection_srcdir / "galaxy.yml"
+ if not path.exists():
+ raise CollectionError(f"{path} does not exist!")
+ print(f"Loading collection metadata from {path}")
+
+ with open(path, encoding="utf-8") as file:
+ return load(file, Loader=CSafeLoader)
+
+ def install(self, destdir: Union[str, Path]) -> None:
+ args = (
+ "ansible-galaxy",
+ "collection",
+ "install",
+ "-n",
+ "-p",
+ str(destdir),
+ f"{self.namespace}-{self.name}-{self.version}.tar.gz",
+ )
+ print(f"Running: {args}")
+ print()
+ subprocess.run(args, check=True, cwd=self.collection_srcdir)
+
+ def write_filelist(self, filelist: Path) -> None:
+ filelist.parent.mkdir(parents=True, exist_ok=True)
+ contents = "%{ansible_collections_dir}/" + self.namespace
+ print(f"Writing filelist to {filelist}")
+ with open(filelist, "w", encoding="utf-8") as file:
+ file.write(contents)
+
+ def unit_test(self, extra_args: Sequence) -> None:
+ with TemporaryDirectory() as temp:
+ temppath = Path(temp) / "ansible_collections" / self.namespace / self.name
+ shutil.copytree(
+ self.collection_srcdir,
+ temppath,
+ )
+ args = ("ansible-test", "units", *extra_args)
+ print(f"Running: {args}")
+ print()
+ subprocess.run(args, cwd=temppath, check=True)
+
+
+def parseargs() -> argparse.Namespace:
+ parser = argparse.ArgumentParser(
+ "Install and test Ansible Collections in an rpmbuild environment"
+ )
+ subparsers = parser.add_subparsers(dest="action")
+ install_parser = subparsers.add_parser(
+ "install",
+ help="Run ansible-galaxy collection install and write filelist",
+ )
+ install_parser.add_argument(
+ "--collections-dir",
+ required=True,
+ help="Collection destination directory",
+ type=Path,
+ )
+ install_parser.add_argument(
+ "--filelist",
+ type=Path,
+ required=True,
+ help="%%{ansible_collection_filelist}",
+ )
+
+ test_parser = subparsers.add_parser(
+ "test",
+ help="Run ansible-test unit after creating the necessary directory structure",
+ )
+ test_parser.add_argument(
+ "extra_args", nargs="*", help="Extra arguments to pass to ansible-test"
+ )
+ args = parser.parse_args()
+ # add_subparsers does not support required on Python 3.6
+ if not args.action:
+ parser.print_usage()
+ sys.exit(2)
+ return args
+
+
+def main():
+ args = parseargs()
+ collection = AnsibleCollection()
+ if args.action == "install":
+ collection.install(args.collections_dir)
+ collection.write_filelist(args.filelist)
+ elif args.action == "test":
+ collection.unit_test(args.extra_args)
+
+
+if __name__ == "__main__":
+ try:
+ main()
+ except CollectionError as err:
+ sys.exit(err)
https://src.fedoraproject.org/rpms/ansible-packaging/c/bc337bd8e48760c967...
1Â year, 6Â months
gotmax23 pushed to ansible-packaging (f36). "Update changelog"
by notificationsï¼ fedoraproject.org
Notification time stamped 2022-09-30 21:08:43 UTC
From 7ea0e618c901051d80d2bddb8e87ade5631b4980 Mon Sep 17 00:00:00 2001
From: Maxwell G <gotmax(a)e.email>
Date: Sep 30 2022 20:33:25 +0000
Subject: Update changelog
---
diff --git a/ansible-packaging.spec b/ansible-packaging.spec
index 26b648f..dd929fb 100644
--- a/ansible-packaging.spec
+++ b/ansible-packaging.spec
@@ -1,6 +1,6 @@
Name: ansible-packaging
Version: 1
-Release: 7%{?dist}
+Release: 8.1%{?dist}
Summary: RPM packaging macros and generators for Ansible collections
License: GPL-3.0-or-later
@@ -149,6 +149,17 @@ echo "Ensure macro prefers the collection namespace and name passed as an argume
%changelog
+* Sat Sep 24 2022 Maxwell G <gotmax(a)e.email> - 1-8.1
+- Refactor %%ansible_collection_url, %%ansible_collection_install,
+ %%ansible_test_unit.
+- Specfiles no longer need to define %%collection_namespace or %%collection_name
+ for the macros to work.
+- Add new %%ansible_collections_dir, %%ansible_roles_dir, and
+ %ansible_collection_filelist macros.
+- Prepare to deprecate %%ansible_collection_files
+- Undefine %%_package_note_file to stop that file from leaking into collection
+ artifacts.
+
* Mon Aug 01 2022 Maxwell G <gotmax(a)e.email> - 1-7
- Implement %%ansible_test_unit and add ansible-packaging-tests metapackage.
- Require ansible-core at buildtime
https://src.fedoraproject.org/rpms/ansible-packaging/c/7ea0e618c901051d80...
1Â year, 6Â months
gotmax23 pushed to ansible-packaging (f36). "Escape macro in
changelog"
by notificationsï¼ fedoraproject.org
Notification time stamped 2022-09-30 21:08:43 UTC
From f85cd2c4b6a4181991f08dc6930876634409613b Mon Sep 17 00:00:00 2001
From: Maxwell G <gotmax(a)e.email>
Date: Sep 30 2022 20:40:24 +0000
Subject: Escape macro in changelog
---
diff --git a/ansible-packaging.spec b/ansible-packaging.spec
index dd929fb..2bfb057 100644
--- a/ansible-packaging.spec
+++ b/ansible-packaging.spec
@@ -155,7 +155,7 @@ echo "Ensure macro prefers the collection namespace and name passed as an argume
- Specfiles no longer need to define %%collection_namespace or %%collection_name
for the macros to work.
- Add new %%ansible_collections_dir, %%ansible_roles_dir, and
- %ansible_collection_filelist macros.
+ %%ansible_collection_filelist macros.
- Prepare to deprecate %%ansible_collection_files
- Undefine %%_package_note_file to stop that file from leaking into collection
artifacts.
https://src.fedoraproject.org/rpms/ansible-packaging/c/f85cd2c4b6a4181991...
1Â year, 6Â months
gotmax23 pushed to ansible-packaging (f36). "Undefine
%%_package_note_file (..more)"
by notificationsï¼ fedoraproject.org
Notification time stamped 2022-09-30 21:08:43 UTC
From fc8efd22baac9128ba46ce0ce164c8fbc15cf5f3 Mon Sep 17 00:00:00 2001
From: Maxwell G <gotmax(a)e.email>
Date: Sep 30 2022 20:33:25 +0000
Subject: Undefine %%_package_note_file
to stop that file from leaking into collection artifacts
---
diff --git a/macros.ansible b/macros.ansible
index 57c89fe..db2269a 100644
--- a/macros.ansible
+++ b/macros.ansible
@@ -7,7 +7,10 @@
%ansible_collection_build() ansible-galaxy collection build
+# On F36, package-notes-srpm-macros inserts a package note file into
+# %%{buildsubdir} that ends up getting included in the collection builds.
%ansible_collection_install() %{shrink:
+%undefine _package_note_file
%{_rpmconfigdir}/ansible_collection.py install
--collections-dir %{buildroot}%{ansible_collections_dir}
--filelist %{ansible_collection_filelist}
https://src.fedoraproject.org/rpms/ansible-packaging/c/fc8efd22baac9128ba...
1Â year, 6Â months
gotmax23 pushed to ansible-packaging (f36). "%ansible_collection_url:
Don't require control macros (..more)"
by notificationsï¼ fedoraproject.org
Notification time stamped 2022-09-30 21:08:43 UTC
From c67bd58e7937afa0cff3e9ce94e4d5d3a3a14a3d Mon Sep 17 00:00:00 2001
From: Maxwell G <gotmax(a)e.email>
Date: Sep 30 2022 20:33:25 +0000
Subject: %ansible_collection_url: Don't require control macros
Reimplement %ansible_collection_url to accept the collection namespace
and name as arguments instead of requiring oblique control macros. This
also adds some basic tests to ensure that the macro behaves properly.
---
diff --git a/ansible-packaging.spec b/ansible-packaging.spec
index 4e2478c..26b648f 100644
--- a/ansible-packaging.spec
+++ b/ansible-packaging.spec
@@ -77,6 +77,57 @@ install -Dpm0644 -t %{buildroot}%{_rpmmacrodir} macros.ansible-srpm
install -Dpm0755 -t %{buildroot}%{_rpmconfigdir} ansible-generator
install -Dpm0755 -t %{buildroot}%{_rpmconfigdir} ansible_collection.py
+%check
+# TODO: Currently, this only tests %%{ansible_collection_url}.
+
+rpm_eval() {
+ default_macros_path="$(rpm --showrc | grep 'Macro path' | awk -F ': ' '{print $2}')"
+ rpm --macros="${default_macros_path}:%{buildroot}%{_rpmmacrodir}/macros.*" "$@"
+}
+
+errors() {
+ error="error: %%ansible_collection_url: You must pass the collection namespace as the first arg and the collection name as the second"
+ "$@" && exit 1
+ "$@" |& grep -q "${error}"
+}
+
+echo "Ensure macro fails when only collection_namespace macro is defined"
+errors rpm_eval -D 'collection_namespace cc' -E '%%ansible_collection_url'
+
+echo
+echo "Ensure macro fails when only collection_name macro is defined"
+errors rpm_eval -D 'collection_name cc' -E '%%ansible_collection_url'
+
+echo
+echo "Ensure macro fails when second argument is missing"
+errors rpm_eval -E '%%ansible_collection_url a'
+
+echo
+echo "Ensure macro fails when second argument is missing"
+errors rpm_eval -D 'collection_name b' -E '%%ansible_collection_url a'
+
+echo
+echo "Ensure macro fails when neither the control macros nor macro arguments are passed"
+errors rpm_eval -E '%%ansible_collection_url'
+
+
+echo
+echo
+echo "Ensure macro works when both arguments are passed and no control macros are set"
+[[ $(rpm_eval -E '%%ansible_collection_url community general') == \
+ "https://galaxy.ansible.com/community/general" ]]
+
+echo
+echo "Ensure macro works with the control macros"
+[[ $(rpm_eval -D 'collection_namespace ansible' -D 'collection_name posix' \
+ -E '%%ansible_collection_url') == "https://galaxy.ansible.com/ansible/posix" ]]
+
+echo
+echo "Ensure macro prefers the collection namespace and name passed as an argument over the control macros"
+[[ $(rpm_eval -D 'collection_namespace ansible' -D 'collection_name posix' \
+ -E '%%ansible_collection_url community general') == "https://galaxy.ansible.com/community/general" ]]
+
+
%files
%license COPYING
diff --git a/macros.ansible-srpm b/macros.ansible-srpm
index 9b208b4..0adacd7 100644
--- a/macros.ansible-srpm
+++ b/macros.ansible-srpm
@@ -1 +1,25 @@
-%ansible_collection_url() https://galaxy.ansible.com/%{collection_namespace}/%{collection_name}
+# Note(gotmax23): I'm trying to get rid of the need for control macros in favor
+# of a metadata based approach. %%ansible_collection_url is the only macro that
+# requires manually specifying the collection namespace and name, as it is used
+# at the SRPM build stage.
+#
+# Currently, this macro supports either passing this information as arguments
+# or defining the control macros. In order to reduce confusion, this is not an
+# either or approach. Both arguments must be passed OR both control macros must
+# be defined.
+
+%ansible_collection_url() %{lua:
+ local namespace_name = nil
+ if rpm.expand("%collection_namespace") ~= "%collection_namespace"
+ and rpm.expand("%collection_name") ~= "%collection_name" then
+ namespace_name = rpm.expand("%collection_namespace") .. "/" .. rpm.expand("%collection_name")
+ end
+ if rpm.expand("%1") ~= "%1" and rpm.expand("%2") ~= "%2" then
+ namespace_name = rpm.expand("%1") .. "/" .. rpm.expand("%2")
+ end
+ if not namespace_name then
+ rpm.expand("%{error:%%ansible_collection_url: You must pass the collection " ..
+ "namespace as the first arg and the collection name as the second}")
+ end
+ print("https://galaxy.ansible.com/" .. namespace_name)
+}
https://src.fedoraproject.org/rpms/ansible-packaging/c/c67bd58e7937afa0cf...
1Â year, 6Â months
gotmax23 pushed to ansible-packaging (f36). "Check if artifact exists
before trying to install it"
by notificationsï¼ fedoraproject.org
Notification time stamped 2022-09-30 21:08:43 UTC
From 5e5c534c320247ac15c4825d251e9130f9408291 Mon Sep 17 00:00:00 2001
From: Maxwell G <gotmax(a)e.email>
Date: Sep 30 2022 20:14:22 +0000
Subject: Check if artifact exists before trying to install it
---
diff --git a/ansible_collection.py b/ansible_collection.py
index 18f04a9..17c6777 100755
--- a/ansible_collection.py
+++ b/ansible_collection.py
@@ -51,6 +51,14 @@ class AnsibleCollection:
return load(file, Loader=CSafeLoader)
def install(self, destdir: Union[str, Path]) -> None:
+ artifact = self.collection_srcdir / Path(
+ f"{self.namespace}-{self.name}-{self.version}.tar.gz"
+ )
+ if not artifact.exists() and not artifact.is_file():
+ raise CollectionError(
+ f"{artifact} does not exist! Did you run %ansible_collection_build?"
+ )
+
args = (
"ansible-galaxy",
"collection",
@@ -58,11 +66,12 @@ class AnsibleCollection:
"-n",
"-p",
str(destdir),
- f"{self.namespace}-{self.name}-{self.version}.tar.gz",
+ str(artifact),
)
print(f"Running: {args}")
print()
subprocess.run(args, check=True, cwd=self.collection_srcdir)
+ print()
def write_filelist(self, filelist: Path) -> None:
filelist.parent.mkdir(parents=True, exist_ok=True)
@@ -134,5 +143,5 @@ def main():
if __name__ == "__main__":
try:
main()
- except CollectionError as err:
+ except (CollectionError, subprocess.CalledProcessError) as err:
sys.exit(err)
https://src.fedoraproject.org/rpms/ansible-packaging/c/5e5c534c320247ac15...
1Â year, 6Â months
gotmax23 pushed to ansible-packaging (f36). "Add new
%ansible_collection_filelist macro (..more)"
by notificationsï¼ fedoraproject.org
Notification time stamped 2022-09-30 21:08:43 UTC
From b1de39d9c64cabf7c40c29d445e135b0e41a3f0b Mon Sep 17 00:00:00 2001
From: Maxwell G <gotmax(a)e.email>
Date: Sep 26 2022 04:07:27 +0000
Subject: Add new %ansible_collection_filelist macro
%ansible_collection_files cannot be adapted to dynamically determine the
collection namespace. We need to use a file list for that.
%ansible_collection_install writes out a file list to
%ansible_collection_filelist. Packagers would pass
`%{ansible_collection_filelist}` to `%files -f`.
Eventually, I'd like to deprecate %ansible_collection_files. That won't
happen until the new approach has gotten more testing and adoption.
---
diff --git a/macros.ansible b/macros.ansible
index 0c81432..57c89fe 100644
--- a/macros.ansible
+++ b/macros.ansible
@@ -18,5 +18,11 @@
--python-interpreter %{__python3} --local %{?*}
}
+# TODO: Officially deprecate this macro and add the following line to the macro
+# def after the new approach has gotten more testing and adoption:
+# %%{warn: %%{ansible_collection_files} is deprecated. Use %%files -f %%{ansible_collection_filelist} instead.}
+%ansible_collection_files %{shrink:
+%{ansible_collections_dir}/%{collection_namespace}/
+}
-%ansible_collection_files %{_datadir}/ansible/collections/ansible_collections/%{collection_namespace}/
+%ansible_collection_filelist %{__ansible_builddir}/ansible_collection_files
https://src.fedoraproject.org/rpms/ansible-packaging/c/b1de39d9c64cabf7c4...
1Â year, 6Â months
gotmax23 pushed to ansible-packaging (f36). "Flush stdout before
subprocess.run (..more)"
by notificationsï¼ fedoraproject.org
Notification time stamped 2022-09-30 21:08:43 UTC
From e3d150ec904179fd2d529763491692ee3b192c44 Mon Sep 17 00:00:00 2001
From: Maxwell G <gotmax(a)e.email>
Date: Sep 30 2022 20:14:22 +0000
Subject: Flush stdout before subprocess.run
This commit adds sys.stdout.flush() before subprocess.run() in
ansible_collection.py. Without this, the print statements are shown
after the command output when building in mock.
---
diff --git a/ansible_collection.py b/ansible_collection.py
index 17c6777..70093be 100755
--- a/ansible_collection.py
+++ b/ansible_collection.py
@@ -70,6 +70,9 @@ class AnsibleCollection:
)
print(f"Running: {args}")
print()
+ # Without this, the print statements are shown after the command
+ # output when building in mock.
+ sys.stdout.flush()
subprocess.run(args, check=True, cwd=self.collection_srcdir)
print()
@@ -90,6 +93,9 @@ class AnsibleCollection:
args = ("ansible-test", "units", *extra_args)
print(f"Running: {args}")
print()
+ # Without this, the print statements are shown after the command
+ # output when building in mock.
+ sys.stdout.flush()
subprocess.run(args, cwd=temppath, check=True)
https://src.fedoraproject.org/rpms/ansible-packaging/c/e3d150ec904179fd2d...
1Â year, 6Â months
gotmax23 pushed to ansible-packaging (f36). "Outsource
%ansible_test_unit logic to helper script (..more)"
by notificationsï¼ fedoraproject.org
Notification time stamped 2022-09-30 21:08:43 UTC
From 78dcf4c992c5c3189675c7f579864c9140cf5898 Mon Sep 17 00:00:00 2001
From: Maxwell G <gotmax(a)e.email>
Date: Sep 26 2022 04:07:27 +0000
Subject: Outsource %ansible_test_unit logic to helper script
We want %ansible_test_unit to retrieve the collection name{,space} from
galaxy.yml. This removes the need for the %collection_name{,space}
macros and utilizes ansible_collection.py's more robust approach to
running the tests.
---
diff --git a/macros.ansible b/macros.ansible
index 856688e..0c81432 100644
--- a/macros.ansible
+++ b/macros.ansible
@@ -13,12 +13,10 @@
--filelist %{ansible_collection_filelist}
}
+%ansible_test_unit() %{shrink:
+%{_rpmconfigdir}/ansible_collection.py test --
+--python-interpreter %{__python3} --local %{?*}
+}
-%ansible_test_unit() %{expand:\\\
-mkdir -p ../ansible_collections/%{collection_namespace}
-cp -a $(pwd) ../ansible_collections/%{collection_namespace}/%{collection_name}
-pushd ../ansible_collections/%{collection_namespace}/%{collection_name}
-ansible-test units --python-interpreter %{__python3} --local %{?*}
-popd}
%ansible_collection_files %{_datadir}/ansible/collections/ansible_collections/%{collection_namespace}/
https://src.fedoraproject.org/rpms/ansible-packaging/c/78dcf4c992c5c31896...
1Â year, 6Â months