This replaces the method previously used in lib/autotest/site_packages.py. To avoid non-standard file permissions for tests in /usr/share/autotest/client/site_tests, this patch creates site_autotest.py to perform additional steps when preparing an autotest client for jobs. In site_autotest.py, we extend the existing BaseAutotest._install() method to perform several tasks prior to the standard autotest client preparation. Those tasks include:
* Optionally install additional yum repositories (based on optional global_config.ini option) * Install, or upgrade, autoqa * Copy all files in /etc/autoqa to the client
This patch also updates autoqa.spec and Makefile to include the new file. --- Makefile | 3 +- autoqa.spec | 3 +- lib/autotest/site_autotest.py | 42 +++++++++++++++++++++++++++++++++++++++++ lib/autotest/site_packages.py | 34 --------------------------------- 4 files changed, 46 insertions(+), 36 deletions(-) create mode 100644 lib/autotest/site_autotest.py delete mode 100644 lib/autotest/site_packages.py
diff --git a/Makefile b/Makefile index f8d65ad..4a2134a 100644 --- a/Makefile +++ b/Makefile @@ -37,7 +37,8 @@ install: build for t in tests/*; do cp -a $$t $(PREFIX)$(TEST_DIR); done install -d $(PREFIX)$(AUTOTEST_DIR)/client/{bin,common_lib} install -m 0644 lib/autotest/site_utils.py $(PREFIX)$(AUTOTEST_DIR)/client/bin/ - install -m 0644 lib/autotest/site_packages.py $(PREFIX)$(AUTOTEST_DIR)/client/common_lib/ + install -d $(PREFIX)$(AUTOTEST_DIR)/server + install -m 0644 lib/autotest/site_autotest.py $(PREFIX)$(AUTOTEST_DIR)/server/ ( cd lib/python; $(PYTHON) setup.py install --skip-build --root $(PREFIX)/ )
build: lib/python/build diff --git a/autoqa.spec b/autoqa.spec index a2189bd..b8112b6 100644 --- a/autoqa.spec +++ b/autoqa.spec @@ -15,6 +15,7 @@ Requires: autotest Requires: koji Requires: python-fedora Requires: mod_wsgi +BuildRequires: autotest BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-buildroot BuildArch: noarch
@@ -59,7 +60,7 @@ rm -rf $RPM_BUILD_ROOT %{testdir} %{hookdir} %{_datadir}/autotest/client/bin/site_utils.py* -%{_datadir}/autotest/client/common_lib/site_packages.py* +%{_datadir}/autotest/server/site_autotest.py* %{python_sitelib}/autoqa*
diff --git a/lib/autotest/site_autotest.py b/lib/autotest/site_autotest.py new file mode 100644 index 0000000..04bbd2c --- /dev/null +++ b/lib/autotest/site_autotest.py @@ -0,0 +1,42 @@ +# site_autotest.py - site-specific autotest extensions for use in autoqa +# +# this must be copied to autotest/server/, because it is imported by +# autotest/server/server.py +# +# Author: James Laska jlaska@redhat.com + +import os +import logging +import common +from autotest_lib.server.autotest import BaseAutotest +from autotest_lib.client.common_lib import global_config + +class SiteAutotest(BaseAutotest): + def _install(self, host=None, autodir=None, use_autoserv=True, + use_packaging=True): + + # First, enable any requested yum repositories + logging.info("Installing autoqa on %s", host.hostname) + c = global_config.global_config + repository_cfg_urls = c.get_config_value( + "FEDORA.AUTOQA", "repository_cfg_urls", type=list, default=list()) + + for url in repository_cfg_urls: + logging.debug("Enabling %s package repo" % os.path.basename(url)) + host.run("cd /etc/yum.repos.d && (test -f %s || curl -s -O %s)" % \ + (os.path.basename(url), url)) + + # Next, ensure autoqa is installed and up-to-date + logging.debug("Installing or upgrading autoqa package") + host.run("yum -y install autoqa") + + # Next, copy any autoqa config files + logging.debug("Copying autoqa configuration files") + host.send_file('/etc/autoqa', '/etc/autoqa', delete_dest=True) + + result = host.run("rpm -q autoqa") + logging.info("Installation of %s completed" % result.stdout) + + # Finally, continue installation by calling our parent + super(SiteAutotest, self)._install(host, autodir, use_autoserv, use_packaging) + diff --git a/lib/autotest/site_packages.py b/lib/autotest/site_packages.py deleted file mode 100644 index 280e064..0000000 --- a/lib/autotest/site_packages.py +++ /dev/null @@ -1,34 +0,0 @@ -# site_packages.py - site-specific autotest packages methods for use in autoqa -# -# this must be copied to autotest/client/common_lib, because it is imported by -# autotest/client/common_lib/packages.py -# -# Author: Kamil Paral kparal@redhat.com - -import os, glob, shutil -from autotest_lib.client.common_lib import base_packages - -class SitePackageManager(base_packages.BasePackageManager): - def tar_package(self, pkg_name, src_dir, dest_dir, exclude_string=None): - ''' - Same as common_lib.base_packages.tar_package, but it also copies all - AutoQA config files into src_dir for our site_tests. - ''' - autoqa_test = (os.path.basename(os.path.dirname(src_dir)) == 'site_tests') - configs = glob.glob('/etc/autoqa/*') - - # if this is our test, copy all config files to src_dir to be tarred - # and transferred to client - if autoqa_test: - for config in configs: - shutil.copy(config, src_dir) - - # run the parent method - result = super(SitePackageManager, self).tar_package(pkg_name, src_dir, dest_dir, exclude_string) - - # remove all the config files - if autoqa_test: - for config in configs: - os.remove(os.path.join(src_dir, os.path.basename(config))) - - return result
----- Original Message -----
This replaces the method previously used in lib/autotest/site_packages.py. To avoid non-standard file permissions for tests in /usr/share/autotest/client/site_tests, this patch creates site_autotest.py to perform additional steps when preparing an autotest client for jobs. In site_autotest.py, we extend the existing BaseAutotest._install() method to perform several tasks prior to the standard autotest client preparation. Those tasks include:
- Optionally install additional yum repositories (based on optional
global_config.ini option)
- Install, or upgrade, autoqa
- Copy all files in /etc/autoqa to the client
This patch also updates autoqa.spec and Makefile to include the new file.
Outstanding, you win the 'Patch Of The Week' prize! :-) This is really great, it's much better hack than mine was.
I have tested a little and found out following: 1. rsync must be installed on both server and the client in order to work, otherwise it falls back to scp. Maybe we could add rsync as autotest dependency? 2. In order for scp to work, openssh-clients must be installed on both server and the client. (And of course also openssh-server on the client). 3. There are some differences how scp and rsync handle destination directories when they exist/don't exist. The only reliable way when the transfer worked well for both scp and rsync is to delete and re-create the destination directory prior to the transfer. This is probably a bug in autotest, it should handle it internally.
This is my current implementation:
class SiteAutotest(BaseAutotest): def _install(self, host=None, autodir=None, use_autoserv=True, use_packaging=True):
# NOTE: Due to inconsistent rsync/scp behavior inside autotest, all # the transferred directories must end with a slash and must be removed # and re-created at the destination machine prior to the transfer
# copy autoqa config files logging.debug("Copying autoqa configuration files") dir_ = '/etc/autoqa/' host.run('rm -rf %s; mkdir -p %s' % (dir_, dir_)) host.send_file(dir_, dir_)
# copy autoqa library logging.debug("Copying autoqa library") dir_ = '/usr/lib/python2.7/site-packages/autoqa/' host.run('rm -rf %s; mkdir -p %s' % (dir_, dir_)) host.send_file(dir_, dir_)
logging.info("Installation of autoqa completed")
# Finally, continue installation by calling our parent super(SiteAutotest, self)._install(host, autodir, use_autoserv, use_packaging)
You surely noticed that I don't install autoqa package anymore. That's right, I believe we don't want to install that package. Automatic installation has this drawbacks: I. For the production server: When we release a new autoqa release, all the clients will be immediately updated to this new version, before we have even updated the production server. That can cause quite some problems. II. For the staging server: We couldn't operate a staging server like this. We need to have unreleased code deployed to the clients, not the latest release.
Therefore I believe we need to find a way to ensure that autoqa versions on the server and on the client always match. Easiest solution is simply to copy the config files and the autoqa library, as I have done in the code above. But there are a few issues to solve:
a) The destination directory of autoqa library depends on the python version used. We can't hardcode it. Maybe we can transfer the library source code and do 'python setup.py install'? b) The autoqa package dependencies are not installed that way. We need to find a way to ensure all the dependencies are installed.
Any ideas?
On Mon, 2011-01-24 at 08:27 -0500, Kamil Paral wrote:
----- Original Message -----
This replaces the method previously used in lib/autotest/site_packages.py. To avoid non-standard file permissions for tests in /usr/share/autotest/client/site_tests, this patch creates site_autotest.py to perform additional steps when preparing an autotest client for jobs. In site_autotest.py, we extend the existing BaseAutotest._install() method to perform several tasks prior to the standard autotest client preparation. Those tasks include:
- Optionally install additional yum repositories (based on optional
global_config.ini option)
- Install, or upgrade, autoqa
- Copy all files in /etc/autoqa to the client
This patch also updates autoqa.spec and Makefile to include the new file.
Outstanding, you win the 'Patch Of The Week' prize! :-) This is really great, it's much better hack than mine was.
Haha, no ... this is just another hack :)
I have tested a little and found out following:
- rsync must be installed on both server and the client in order
to work, otherwise it falls back to scp. Maybe we could add rsync as autotest dependency?
Good call, I can do that.
- In order for scp to work, openssh-clients must be installed
on both server and the client. (And of course also openssh-server on the client).
Sounds like another autotest-server dependency?
- There are some differences how scp and rsync handle destination
directories when they exist/don't exist. The only reliable way when the transfer worked well for both scp and rsync is to delete and re-create the destination directory prior to the transfer. This is probably a bug in autotest, it should handle it internally.
Agreed, it seems autotest expects to handle this for us as well. More comments on this below.
This is my current implementation:
class SiteAutotest(BaseAutotest): def _install(self, host=None, autodir=None, use_autoserv=True, use_packaging=True):
# NOTE: Due to inconsistent rsync/scp behavior inside autotest, all # the transferred directories must end with a slash and must be removed # and re-created at the destination machine prior to the transfer # copy autoqa config files logging.debug("Copying autoqa configuration files") dir_ = '/etc/autoqa/' host.run('rm -rf %s; mkdir -p %s' % (dir_, dir_)) host.send_file(dir_, dir_) # copy autoqa library logging.debug("Copying autoqa library") dir_ = '/usr/lib/python2.7/site-packages/autoqa/'
I don't think this will work since it hard-codes the python version string. Oh nm ... you noted this below as well.
host.run('rm -rf %s; mkdir -p %s' % (dir_, dir_)) host.send_file(dir_, dir_)
The delete_dest=True argument for autotest host.send_file() attempts to handle this for us. In fact, they use the same/similar commands that you list above.
delete_dest: if this is true, the command will also clear out any old files at dest that are not in the source
I didn't do exhaustive testing to ensure that delete_dest was honored, but can queue that up shortly. I'd like to have the built-in autotest delete_dest= support working, but we can certainly use the above if the autotest support is broken or not patchable in the short-term.
logging.info("Installation of autoqa completed") # Finally, continue installation by calling our parent super(SiteAutotest, self)._install(host, autodir, use_autoserv, use_packaging)
You surely noticed that I don't install autoqa package anymore. That's right, I believe we don't want to install that package. Automatic installation has this drawbacks: I. For the production server: When we release a new autoqa release, all the clients will be immediately updated to this new version, before we have even updated the production server. That can cause quite some problems.
Good point. I don't see this as an entirely bad thing, we want to run the latest stable content ... but you are right, they should match on the server+client. I could adjust this support so that the autoqa version installed matches the version installed on the server. Hrmm, no that's not good either. I could adjust it so that it *only* installs the package ... no upgrades. That would solve the first-time setup use case ... and still leave upgrading clients to the administrator.
II. For the staging server: We couldn't operate a staging server like this. We need to have unreleased code deployed to the clients, not the latest release.
I believe we can address this ... see below.
Therefore I believe we need to find a way to ensure that autoqa versions on the server and on the client always match. Easiest solution is simply to copy the config files and the autoqa library, as I have done in the code above. But there are a few issues to solve:
a) The destination directory of autoqa library depends on the python version used. We can't hardcode it. Maybe we can transfer the library source code and do 'python setup.py install'? b) The autoqa package dependencies are not installed that way. We need to find a way to ensure all the dependencies are installed.
Any ideas?
How about just using packaging instead of recreating all the "packaging" tasks manually. Seems like it would be beneficial to only maintain one "packaging" scenario. My assumption with the original patch was that all code would be packaged and published in a repo somewhere, even for a staging instance. For staging code, some cronjob would git checkout, attempt a build and update the 'testing' repo [1] with the build results. This mimics the behavior for official Fedora updates and lets existing mechanisms deal with packaging.
The goal with the original patchset was from an idea you gave me. Basically, making it *really* effortless to add a new system and start scheduling tests to it. I was able to hide all the package install and configuration into the site_autotest.py. While it seemed to work, I'm not convinced we want to put all AutoQA setup into this method. I liked it, but as coded, anyone with the autoqa and autotest-server packages installed will get autoqa installed+configured on *all* of their test clients. I may need to make this support optional by way of a global_config.ini setting.
So my recommendation would be ... 1. Adjust autotest-server %requires to add rsync and openssh-{clients,server} 2. Adjust patchset to make autoqa install+config optional based on global_config.ini setting 3. Adjust patchset to only install autoqa ... not upgrade 4. Investigate whether delete_dest=True is working ... file bug/fix as needed 5. Further discuss how best to deploy staging packages ... my thought is to just use rpms and repos.
How's that sound?
Thanks, James
[1] http://repos.fedorapeople.org/repos/fedora-qa/autoqa/fedora-autoqa-testing.r...
----- Original Message -----
host.run('rm -rf %s; mkdir -p %s' % (dir_, dir_)) host.send_file(dir_, dir_)
The delete_dest=True argument for autotest host.send_file() attempts to handle this for us. In fact, they use the same/similar commands that you list above.
delete_dest: if this is true, the command will also clear out any old files at dest that are not in the source
I didn't do exhaustive testing to ensure that delete_dest was honored, but can queue that up shortly. I'd like to have the built-in autotest delete_dest= support working, but we can certainly use the above if the autotest support is broken or not patchable in the short-term.
It's simple. Try to do: host.send_file('/etc/autoqa', '/etc/autoqa', delete_dest=True)
This won't work with rsync, it will create /etc/autoqa/autoqa even if the destination doesn't exist. On the other hand, it will work with scp if the destination doesn't exist, but it won't work if the destination does exist (it will create /etc/autoqa/autoqa).
So let's try: host.send_file('/etc/autoqa/', '/etc/autoqa/', delete_dest=True)
This will work with rsync in both cases (destination exists and doesn't exist). But scp will fail if the destination doesn't exist.
Broooken! :-)
The easiest workaround is to remove and re-create the destination manually and always use slashes.
How about just using packaging instead of recreating all the "packaging" tasks manually. Seems like it would be beneficial to only maintain one "packaging" scenario. My assumption with the original patch was that all code would be packaged and published in a repo somewhere, even for a staging instance. For staging code, some cronjob would git checkout, attempt a build and update the 'testing' repo [1] with the build results. This mimics the behavior for official Fedora updates and lets existing mechanisms deal with packaging.
The goal with the original patchset was from an idea you gave me. Basically, making it *really* effortless to add a new system and start scheduling tests to it. I was able to hide all the package install and configuration into the site_autotest.py. While it seemed to work, I'm not convinced we want to put all AutoQA setup into this method. I liked it, but as coded, anyone with the autoqa and autotest-server packages installed will get autoqa installed+configured on *all* of their test clients. I may need to make this support optional by way of a global_config.ini setting.
I can imagine this could work just fine for production and staging environment. But there is one use case missing, and that is the development environment. Basically, when I'm developing AutoQA on my server+client(s), I don't want to replicate every change I make onto all those machines. That's really tedious. Also I'm often quite confused where to make this change (on the server/on the client/on all machines). It depends on which part of AutoQA I'm currently working on (hooks/harness/library/tests).
My dream is to do any change on the server only, and it gets propagated to all clients. I can do that using rsync, but yes, it's not as clean as using rpm packages (which you can't use for development anyway).
I'll have to think more about it, tomorrow.
So my recommendation would be ...
- Adjust autotest-server %requires to add rsync and
openssh-{clients,server}
Actually only the autotest client requires openssh-server. (But it is kind of obvious because you need to register it with the server and for that you need openssh-server installed.) Both the server and the client require openssh-clients (and preferably also rsync).
On Mon, 2011-01-24 at 12:28 -0500, Kamil Paral wrote:
----- Original Message -----
host.run('rm -rf %s; mkdir -p %s' % (dir_, dir_)) host.send_file(dir_, dir_)
The delete_dest=True argument for autotest host.send_file() attempts to handle this for us. In fact, they use the same/similar commands that you list above.
delete_dest: if this is true, the command will also clear out any old files at dest that are not in the source
I didn't do exhaustive testing to ensure that delete_dest was honored, but can queue that up shortly. I'd like to have the built-in autotest delete_dest= support working, but we can certainly use the above if the autotest support is broken or not patchable in the short-term.
It's simple. Try to do: host.send_file('/etc/autoqa', '/etc/autoqa', delete_dest=True)
This won't work with rsync, it will create /etc/autoqa/autoqa even if the destination doesn't exist. On the other hand, it will work with scp if the destination doesn't exist, but it won't work if the destination does exist (it will create /etc/autoqa/autoqa).
So let's try: host.send_file('/etc/autoqa/', '/etc/autoqa/', delete_dest=True)
This will work with rsync in both cases (destination exists and doesn't exist). But scp will fail if the destination doesn't exist.
Broooken! :-)
Okay, I'll try this soon ... I'd like to fix autotest if possible.
The easiest workaround is to remove and re-create the destination manually and always use slashes.
That'll work if needed.
How about just using packaging instead of recreating all the "packaging" tasks manually. Seems like it would be beneficial to only maintain one "packaging" scenario. My assumption with the original patch was that all code would be packaged and published in a repo somewhere, even for a staging instance. For staging code, some cronjob would git checkout, attempt a build and update the 'testing' repo [1] with the build results. This mimics the behavior for official Fedora updates and lets existing mechanisms deal with packaging.
The goal with the original patchset was from an idea you gave me. Basically, making it *really* effortless to add a new system and start scheduling tests to it. I was able to hide all the package install and configuration into the site_autotest.py. While it seemed to work, I'm not convinced we want to put all AutoQA setup into this method. I liked it, but as coded, anyone with the autoqa and autotest-server packages installed will get autoqa installed+configured on *all* of their test clients. I may need to make this support optional by way of a global_config.ini setting.
I can imagine this could work just fine for production and staging environment. But there is one use case missing, and that is the development environment. Basically, when I'm developing AutoQA on my server+client(s), I don't want to replicate every change I make onto all those machines. That's really tedious. Also I'm often quite confused where to make this change (on the server/on the client/on all machines). It depends on which part of AutoQA I'm currently working on (hooks/harness/library/tests).
Ah, good use case. Can we think of any reason that the autoqa code on the server shouldn't match the code on the clients? Eventually, we will have mixed client environment (i386/x86_64/$other using f15, f14, f13 epel5 etc...).
This implies that our code must work for all supported releases. Meaning, we can't have autoqa-0.4.4-1.el5 but autoqa-0.4.4-2 for f14. I don't have a problem with this, I typically build new packages for all supported releases. But just want to confirm
My dream is to do any change on the server only, and it gets propagated to all clients. I can do that using rsync, but yes, it's not as clean as using rpm packages (which you can't use for development anyway).
I'll have to think more about it, tomorrow.
This is a cool idea. Let's do it!
So my recommendation would be ...
- Adjust autotest-server %requires to add rsync and
openssh-{clients,server}
Actually only the autotest client requires openssh-server. (But it is kind of obvious because you need to register it with the server and for that you need openssh-server installed.) Both the server and the client require openssh-clients (and preferably also rsync).
Okay
Thanks, James
----- Original Message -----
Okay, I'll try this soon ... I'd like to fix autotest if possible.
Sure, we should try to fix the upstream, this is just a workaround.
Reported here: http://autotest.kernel.org/ticket/45
I can imagine this could work just fine for production and staging environment. But there is one use case missing, and that is the development environment. Basically, when I'm developing AutoQA on my server+client(s), I don't want to replicate every change I make onto all those machines. That's really tedious. Also I'm often quite confused where to make this change (on the server/on the client/on all machines). It depends on which part of AutoQA I'm currently working on (hooks/harness/library/tests).
Ah, good use case. Can we think of any reason that the autoqa code on the server shouldn't match the code on the clients? Eventually, we will have mixed client environment (i386/x86_64/$other using f15, f14, f13 epel5 etc...).
This implies that our code must work for all supported releases. Meaning, we can't have autoqa-0.4.4-1.el5 but autoqa-0.4.4-2 for f14. I don't have a problem with this, I typically build new packages for all supported releases. But just want to confirm
That is a good question. Currently I believe our code is exactly the same for all architectures/operating systems. But in the future, who knows.
Gosh, I would like to use the rpm versioned approach for the production server, as you mentioned. But I would also like to use more agile (and painless) method for staging and development environment.
I have had a brief look at autotest. It has some support for sending and installing packages tests. Maybe we could use that. I'll have to study it more [1].
[1] http://autotest.kernel.org/wiki/PackagingExample
I have started to discuss this issue with Josef, he has some interesting ideas. I'll send further comments later on.
Below is a patch that makes maintenance-free clients. Installs everything automatically, transfers config files and the library. You can also specify further packages to install. The patch is also available at copylib branch. I admit it's little hackish, but it's just a proof-of-concept.
For the current release I would recommend use this site_autotest.py approach just for copying config files. That means everything else would stay the same as it was before (manual installation of autoqa on all clients needed). For the next release we can discuss the best approach and implement it.
==================
From 1888da9e982c31774379c80bb12cf287d10c90fe Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Kamil=20P=C3=A1ral?= kparal@redhat.com Date: Tue, 25 Jan 2011 11:37:50 +0100 Subject: [PATCH] copy config files and autoqa library to the client
This will copy config files and autoqa library to the autotest client prior to the test execution. It will also install any required packages (like autoqa dependencies etc). That means the autotest clients are now completely maintanance-free. --- Makefile | 9 ++++++- autoqa.spec | 7 ++++- client-deps-custom.conf | 2 + client-deps.conf | 4 +++ lib/autotest/site_autotest.py | 53 +++++++++++++++++++++++++++++++++++++++++ lib/autotest/site_packages.py | 34 -------------------------- 6 files changed, 72 insertions(+), 37 deletions(-) create mode 100644 client-deps-custom.conf create mode 100644 client-deps.conf create mode 100644 lib/autotest/site_autotest.py delete mode 100644 lib/autotest/site_packages.py
diff --git a/Makefile b/Makefile index f8d65ad..26bbd23 100644 --- a/Makefile +++ b/Makefile @@ -25,6 +25,8 @@ install: build [ -f $(PREFIX)/etc/autoqa/autoqa.conf ] || install -m 0644 autoqa.conf $(PREFIX)/etc/autoqa/ [ -f $(PREFIX)/etc/autoqa/fas.conf ] || install -m 0640 -g autotest fas.conf $(PREFIX)/etc/autoqa install -m 0644 repoinfo.conf $(PREFIX)/etc/autoqa/ + install -m 0644 client-deps.conf $(PREFIX)/etc/autoqa/ + [ -f $(PREFIX)/etc/autoqa/client-deps-custom.conf ] || install -m 0644 client-deps-custom.conf $(PREFIX)/etc/autoqa install -d $(PREFIX)$(HOOK_DIR) for h in hooks/*; do cp -a $$h $(PREFIX)$(HOOK_DIR); done ## git-post-receive setup @@ -37,8 +39,13 @@ install: build for t in tests/*; do cp -a $$t $(PREFIX)$(TEST_DIR); done install -d $(PREFIX)$(AUTOTEST_DIR)/client/{bin,common_lib} install -m 0644 lib/autotest/site_utils.py $(PREFIX)$(AUTOTEST_DIR)/client/bin/ - install -m 0644 lib/autotest/site_packages.py $(PREFIX)$(AUTOTEST_DIR)/client/common_lib/ + install -d $(PREFIX)$(AUTOTEST_DIR)/server + install -m 0644 lib/autotest/site_autotest.py $(PREFIX)$(AUTOTEST_DIR)/server/ ( cd lib/python; $(PYTHON) setup.py install --skip-build --root $(PREFIX)/ ) + ## populate client-deps.conf + # we don't need autotest and mod_wsgi on the client, filter them out + grep '^Requires: ' autoqa.spec | cut -f2 -d ' ' | grep -Ev '(autotest|mod_wsgi)' \ + >> $(PREFIX)/etc/autoqa/client-deps.conf
build: lib/python/build
diff --git a/autoqa.spec b/autoqa.spec index a2189bd..8b8d13f 100644 --- a/autoqa.spec +++ b/autoqa.spec @@ -15,6 +15,7 @@ Requires: autotest Requires: koji Requires: python-fedora Requires: mod_wsgi +BuildRequires: autotest BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-buildroot BuildArch: noarch
@@ -36,7 +37,7 @@ make build PYTHON=%{__python} %install rm -rf $RPM_BUILD_ROOT make install PREFIX=$RPM_BUILD_ROOT TEST_DIR=%{testdir} HOOK_DIR=%{hookdir} PYTHON=%{__python} -install -m 644 autoqa.conf repoinfo.conf $RPM_BUILD_ROOT%{_sysconfdir}/autoqa/ +install -m 644 autoqa.conf repoinfo.conf client-deps.conf client-deps-custom.conf $RPM_BUILD_ROOT%{_sysconfdir}/autoqa/ install -m 640 -g autotest fas.conf $RPM_BUILD_ROOT%{_sysconfdir}/autoqa/
@@ -50,7 +51,9 @@ rm -rf $RPM_BUILD_ROOT %dir %{_sysconfdir}/autoqa %config(noreplace) %{_sysconfdir}/autoqa/autoqa.conf %config(noreplace) %{_sysconfdir}/autoqa/fas.conf +%config(noreplace) %{_sysconfdir}/autoqa/client-deps-custom.conf %config %{_sysconfdir}/autoqa/repoinfo.conf +%config %{_sysconfdir}/autoqa/client-deps.conf %config(noreplace) %{testdir}/rats_sanity/irb.cfg %config(noreplace) %{_sysconfdir}/httpd/conf.d/autoqa-git-hook.conf %dir %attr(0775,root,autotest) %{_localstatedir}/cache/autoqa @@ -59,7 +62,7 @@ rm -rf $RPM_BUILD_ROOT %{testdir} %{hookdir} %{_datadir}/autotest/client/bin/site_utils.py* -%{_datadir}/autotest/client/common_lib/site_packages.py* +%{_datadir}/autotest/server/site_autotest.py* %{python_sitelib}/autoqa*
diff --git a/client-deps-custom.conf b/client-deps-custom.conf new file mode 100644 index 0000000..8b5e60f --- /dev/null +++ b/client-deps-custom.conf @@ -0,0 +1,2 @@ +# Custom list of packages to install on autotest clients +# Provide one package name per line diff --git a/client-deps.conf b/client-deps.conf new file mode 100644 index 0000000..c42ec36 --- /dev/null +++ b/client-deps.conf @@ -0,0 +1,4 @@ +# Auto-generated list of AutoQA dependencies, DO NOT EDIT +# For specifying custom packages to install on the client, edit client-deps-custom.conf +openssh-clients +rsync diff --git a/lib/autotest/site_autotest.py b/lib/autotest/site_autotest.py new file mode 100644 index 0000000..8e52e90 --- /dev/null +++ b/lib/autotest/site_autotest.py @@ -0,0 +1,53 @@ +# site_autotest.py - site-specific autotest extensions for use in autoqa +# +# this must be copied to autotest/server/, because it is imported by +# autotest/server/server.py +# +# Author: James Laska jlaska@redhat.com + +import os +import logging +import common +from autotest_lib.server.autotest import BaseAutotest + +class SiteAutotest(BaseAutotest): + def _install(self, host=None, autodir=None, use_autoserv=True, + use_packaging=True): + + # NOTE: Due to inconsistent rsync/scp behavior inside autotest, all + # the transferred directories must end with a slash and must be removed + # and re-created at the destination machine prior to the transfer + + # install required packages + logging.debug("Installing autoqa dependencies and other required packages") + packages = [] + for conf in ['/etc/autoqa/client-deps.conf', '/etc/autoqa/client-deps-custom.conf']: + packages += [l.strip() for l in file(conf).readlines() if not l.strip().startswith('#')] + if packages: + host.run('yum install -y %s' % ' '.join(packages)) + + # copy autoqa config files + logging.debug("Copying autoqa configuration files") + srcdir = '/etc/autoqa/' + destdir = srcdir + host.run('rm -rf %(dir)s; mkdir -p %(dir)s' % {'dir': destdir}) + host.send_file(srcdir, destdir) + + # copy autoqa library + logging.debug("Copying autoqa library") + import autoqa + srcdir = os.path.dirname(autoqa.__file__) + assert srcdir, "Can't find autoqa library on the server" + srcdir += '/' # trailing slash is important + site_pkgs = host.run('python -c 'import sys,re; print filter(lambda s: re.match("/usr/lib/.*/site-packages$",s),sys.path)[0]'').stdout + site_pkgs = site_pkgs.strip() # remove trailing newline + assert site_pkgs, "Can't find python site-packages directory on the client" + destdir = site_pkgs + '/autoqa/' + host.run('rm -rf %(dir)s; mkdir -p %(dir)s' % {'dir': destdir}) + host.send_file(srcdir, destdir) + + logging.info("Installation of autoqa completed") + + # Finally, continue installation by calling our parent + super(SiteAutotest, self)._install(host, autodir, use_autoserv, use_packaging) + diff --git a/lib/autotest/site_packages.py b/lib/autotest/site_packages.py deleted file mode 100644 index 280e064..0000000 --- a/lib/autotest/site_packages.py +++ /dev/null @@ -1,34 +0,0 @@ -# site_packages.py - site-specific autotest packages methods for use in autoqa -# -# this must be copied to autotest/client/common_lib, because it is imported by -# autotest/client/common_lib/packages.py -# -# Author: Kamil Paral kparal@redhat.com - -import os, glob, shutil -from autotest_lib.client.common_lib import base_packages - -class SitePackageManager(base_packages.BasePackageManager): - def tar_package(self, pkg_name, src_dir, dest_dir, exclude_string=None): - ''' - Same as common_lib.base_packages.tar_package, but it also copies all - AutoQA config files into src_dir for our site_tests. - ''' - autoqa_test = (os.path.basename(os.path.dirname(src_dir)) == 'site_tests') - configs = glob.glob('/etc/autoqa/*') - - # if this is our test, copy all config files to src_dir to be tarred - # and transferred to client - if autoqa_test: - for config in configs: - shutil.copy(config, src_dir) - - # run the parent method - result = super(SitePackageManager, self).tar_package(pkg_name, src_dir, dest_dir, exclude_string) - - # remove all the config files - if autoqa_test: - for config in configs: - os.remove(os.path.join(src_dir, os.path.basename(config))) - - return result
On Tue, 2011-01-25 at 07:23 -0500, Kamil Paral wrote:
----- Original Message -----
Okay, I'll try this soon ... I'd like to fix autotest if possible.
Sure, we should try to fix the upstream, this is just a workaround.
Reported here: http://autotest.kernel.org/ticket/45
I can imagine this could work just fine for production and staging environment. But there is one use case missing, and that is the development environment. Basically, when I'm developing AutoQA on my server+client(s), I don't want to replicate every change I make onto all those machines. That's really tedious. Also I'm often quite confused where to make this change (on the server/on the client/on all machines). It depends on which part of AutoQA I'm currently working on (hooks/harness/library/tests).
Ah, good use case. Can we think of any reason that the autoqa code on the server shouldn't match the code on the clients? Eventually, we will have mixed client environment (i386/x86_64/$other using f15, f14, f13 epel5 etc...).
This implies that our code must work for all supported releases. Meaning, we can't have autoqa-0.4.4-1.el5 but autoqa-0.4.4-2 for f14. I don't have a problem with this, I typically build new packages for all supported releases. But just want to confirm
That is a good question. Currently I believe our code is exactly the same for all architectures/operating systems. But in the future, who knows.
Gosh, I would like to use the rpm versioned approach for the production server, as you mentioned. But I would also like to use more agile (and painless) method for staging and development environment.
I have had a brief look at autotest. It has some support for sending and installing packages tests. Maybe we could use that. I'll have to study it more [1].
[1] http://autotest.kernel.org/wiki/PackagingExample
I have started to discuss this issue with Josef, he has some interesting ideas. I'll send further comments later on.
Below is a patch that makes maintenance-free clients. Installs everything automatically, transfers config files and the library. You can also specify further packages to install. The patch is also available at copylib branch. I admit it's little hackish, but it's just a proof-of-concept.
For the current release I would recommend use this site_autotest.py approach just for copying config files. That means everything else would stay the same as it was before (manual installation of autoqa on all clients needed). For the next release we can discuss the best approach and implement it.
==================
From 1888da9e982c31774379c80bb12cf287d10c90fe Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Kamil=20P=C3=A1ral?= kparal@redhat.com Date: Tue, 25 Jan 2011 11:37:50 +0100 Subject: [PATCH] copy config files and autoqa library to the client
This will copy config files and autoqa library to the autotest client prior to the test execution. It will also install any required packages (like autoqa dependencies etc). That means the autotest clients are now completely maintanance-free.
Makefile | 9 ++++++- autoqa.spec | 7 ++++- client-deps-custom.conf | 2 + client-deps.conf | 4 +++
What are your thoughts around usage for the base and the -custom.conf? Is this a mechanism for site administrators to define additional dependencies they want installed on autotest clients? Is this expected to be temporary until the autotest Packaging support you mentioned is explored/implemented?
Do we really need two config files? Can this just be a single option in an existing config file? Or, do you anticipate additional autotest-specific tunables? Perhaps autotest.conf might be appropriate?
lib/autotest/site_autotest.py | 53 +++++++++++++++++++++++++++++++++++++++++ lib/autotest/site_packages.py | 34 -------------------------- 6 files changed, 72 insertions(+), 37 deletions(-) create mode 100644 client-deps-custom.conf create mode 100644 client-deps.conf create mode 100644 lib/autotest/site_autotest.py delete mode 100644 lib/autotest/site_packages.py
diff --git a/Makefile b/Makefile index f8d65ad..26bbd23 100644 --- a/Makefile +++ b/Makefile @@ -25,6 +25,8 @@ install: build [ -f $(PREFIX)/etc/autoqa/autoqa.conf ] || install -m 0644 autoqa.conf $(PREFIX)/etc/autoqa/ [ -f $(PREFIX)/etc/autoqa/fas.conf ] || install -m 0640 -g autotest fas.conf $(PREFIX)/etc/autoqa install -m 0644 repoinfo.conf $(PREFIX)/etc/autoqa/
- install -m 0644 client-deps.conf $(PREFIX)/etc/autoqa/
- [ -f $(PREFIX)/etc/autoqa/client-deps-custom.conf ] || install -m 0644 client-deps-custom.conf $(PREFIX)/etc/autoqa install -d $(PREFIX)$(HOOK_DIR) for h in hooks/*; do cp -a $$h $(PREFIX)$(HOOK_DIR); done ## git-post-receive setup
@@ -37,8 +39,13 @@ install: build for t in tests/*; do cp -a $$t $(PREFIX)$(TEST_DIR); done install -d $(PREFIX)$(AUTOTEST_DIR)/client/{bin,common_lib} install -m 0644 lib/autotest/site_utils.py $(PREFIX)$(AUTOTEST_DIR)/client/bin/
- install -m 0644 lib/autotest/site_packages.py $(PREFIX)$(AUTOTEST_DIR)/client/common_lib/
- install -d $(PREFIX)$(AUTOTEST_DIR)/server
- install -m 0644 lib/autotest/site_autotest.py $(PREFIX)$(AUTOTEST_DIR)/server/ ( cd lib/python; $(PYTHON) setup.py install --skip-build --root $(PREFIX)/ )
- ## populate client-deps.conf
- # we don't need autotest and mod_wsgi on the client, filter them out
- grep '^Requires: ' autoqa.spec | cut -f2 -d ' ' | grep -Ev '(autotest|mod_wsgi)' \
>> $(PREFIX)/etc/autoqa/client-deps.conf
build: lib/python/build
diff --git a/autoqa.spec b/autoqa.spec index a2189bd..8b8d13f 100644 --- a/autoqa.spec +++ b/autoqa.spec @@ -15,6 +15,7 @@ Requires: autotest Requires: koji Requires: python-fedora Requires: mod_wsgi +BuildRequires: autotest BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-buildroot BuildArch: noarch
@@ -36,7 +37,7 @@ make build PYTHON=%{__python} %install rm -rf $RPM_BUILD_ROOT make install PREFIX=$RPM_BUILD_ROOT TEST_DIR=%{testdir} HOOK_DIR=%{hookdir} PYTHON=%{__python} -install -m 644 autoqa.conf repoinfo.conf $RPM_BUILD_ROOT%{_sysconfdir}/autoqa/ +install -m 644 autoqa.conf repoinfo.conf client-deps.conf client-deps-custom.conf $RPM_BUILD_ROOT%{_sysconfdir}/autoqa/ install -m 640 -g autotest fas.conf $RPM_BUILD_ROOT%{_sysconfdir}/autoqa/
@@ -50,7 +51,9 @@ rm -rf $RPM_BUILD_ROOT %dir %{_sysconfdir}/autoqa %config(noreplace) %{_sysconfdir}/autoqa/autoqa.conf %config(noreplace) %{_sysconfdir}/autoqa/fas.conf +%config(noreplace) %{_sysconfdir}/autoqa/client-deps-custom.conf %config %{_sysconfdir}/autoqa/repoinfo.conf +%config %{_sysconfdir}/autoqa/client-deps.conf %config(noreplace) %{testdir}/rats_sanity/irb.cfg %config(noreplace) %{_sysconfdir}/httpd/conf.d/autoqa-git-hook.conf %dir %attr(0775,root,autotest) %{_localstatedir}/cache/autoqa @@ -59,7 +62,7 @@ rm -rf $RPM_BUILD_ROOT %{testdir} %{hookdir} %{_datadir}/autotest/client/bin/site_utils.py* -%{_datadir}/autotest/client/common_lib/site_packages.py* +%{_datadir}/autotest/server/site_autotest.py* %{python_sitelib}/autoqa*
diff --git a/client-deps-custom.conf b/client-deps-custom.conf new file mode 100644 index 0000000..8b5e60f --- /dev/null +++ b/client-deps-custom.conf @@ -0,0 +1,2 @@ +# Custom list of packages to install on autotest clients +# Provide one package name per line diff --git a/client-deps.conf b/client-deps.conf new file mode 100644 index 0000000..c42ec36 --- /dev/null +++ b/client-deps.conf @@ -0,0 +1,4 @@ +# Auto-generated list of AutoQA dependencies, DO NOT EDIT +# For specifying custom packages to install on the client, edit client-deps-custom.conf +openssh-clients +rsync
These are autotest-server dependencies, right?
diff --git a/lib/autotest/site_autotest.py b/lib/autotest/site_autotest.py new file mode 100644 index 0000000..8e52e90 --- /dev/null +++ b/lib/autotest/site_autotest.py @@ -0,0 +1,53 @@ +# site_autotest.py - site-specific autotest extensions for use in autoqa +# +# this must be copied to autotest/server/, because it is imported by +# autotest/server/server.py +# +# Author: James Laska jlaska@redhat.com
+import os +import logging +import common +from autotest_lib.server.autotest import BaseAutotest
+class SiteAutotest(BaseAutotest):
- def _install(self, host=None, autodir=None, use_autoserv=True,
use_packaging=True):
# NOTE: Due to inconsistent rsync/scp behavior inside autotest, all
# the transferred directories must end with a slash and must be removed
# and re-created at the destination machine prior to the transfer
# install required packages
logging.debug("Installing autoqa dependencies and other required packages")
packages = []
for conf in ['/etc/autoqa/client-deps.conf', '/etc/autoqa/client-deps-custom.conf']:
packages += [l.strip() for l in file(conf).readlines() if not l.strip().startswith('#')]
if packages:
host.run('yum install -y %s' % ' '.join(packages))
# copy autoqa config files
logging.debug("Copying autoqa configuration files")
srcdir = '/etc/autoqa/'
destdir = srcdir
host.run('rm -rf %(dir)s; mkdir -p %(dir)s' % {'dir': destdir})
host.send_file(srcdir, destdir)
# copy autoqa library
logging.debug("Copying autoqa library")
import autoqa
srcdir = os.path.dirname(autoqa.__file__)
assert srcdir, "Can't find autoqa library on the server"
srcdir += '/' # trailing slash is important
site_pkgs = host.run('python -c \'import sys,re; print filter(lambda s: re.match("/usr/lib/.*/site-packages$",s),sys.path)[0]\'').stdout
site_pkgs = site_pkgs.strip() # remove trailing newline
assert site_pkgs, "Can't find python site-packages directory on the client"
destdir = site_pkgs + '/autoqa/'
host.run('rm -rf %(dir)s; mkdir -p %(dir)s' % {'dir': destdir})
host.send_file(srcdir, destdir)
logging.info("Installation of autoqa completed")
# Finally, continue installation by calling our parent
super(SiteAutotest, self)._install(host, autodir, use_autoserv, use_packaging)
diff --git a/lib/autotest/site_packages.py b/lib/autotest/site_packages.py deleted file mode 100644 index 280e064..0000000 --- a/lib/autotest/site_packages.py +++ /dev/null @@ -1,34 +0,0 @@ -# site_packages.py - site-specific autotest packages methods for use in autoqa -# -# this must be copied to autotest/client/common_lib, because it is imported by -# autotest/client/common_lib/packages.py -# -# Author: Kamil Paral kparal@redhat.com
-import os, glob, shutil -from autotest_lib.client.common_lib import base_packages
-class SitePackageManager(base_packages.BasePackageManager):
- def tar_package(self, pkg_name, src_dir, dest_dir, exclude_string=None):
'''
Same as common_lib.base_packages.tar_package, but it also copies all
AutoQA config files into src_dir for our site_tests.
'''
autoqa_test = (os.path.basename(os.path.dirname(src_dir)) == 'site_tests')
configs = glob.glob('/etc/autoqa/*')
# if this is our test, copy all config files to src_dir to be tarred
# and transferred to client
if autoqa_test:
for config in configs:
shutil.copy(config, src_dir)
Nice check! I wasn't sure how we'd do something like this, but you got it :)
I don't think we can do shutil.copy() to copy config files into the test directories. I mean, it certainly works, but it requires funky permissions on the autotest packaging so that the 'autotest' user can write into the site_tests directory. Can we changed this to your modified rsync/scp method instead?
# run the parent method
result = super(SitePackageManager, self).tar_package(pkg_name, src_dir, dest_dir, exclude_string)
# remove all the config files
if autoqa_test:
for config in configs:
os.remove(os.path.join(src_dir, os.path.basename(config)))
Huh, I never knew about os.remove() ... I've always used os.unlink(). 'Remove' seems like a better name anyway.
return result
----- Original Message -----
Makefile | 9 ++++++- autoqa.spec | 7 ++++- client-deps-custom.conf | 2 + client-deps.conf | 4 +++
What are your thoughts around usage for the base and the -custom.conf? Is this a mechanism for site administrators to define additional dependencies they want installed on autotest clients? Is this expected to be temporary until the autotest Packaging support you mentioned is explored/implemented?
Do we really need two config files? Can this just be a single option in an existing config file? Or, do you anticipate additional autotest-specific tunables? Perhaps autotest.conf might be appropriate?
I have supposed that client-deps.conf would be automatically replaced on update and would hold all autoqa dependencies we want to install on the client. Thus it shouldn't be edited by the administrator (it won't keep changes between updates).
On the other hand client-deps-custom.conf may contain some additional packages the maintainer wants to have installed (I have used vim and mc for example). This config file will be retained on autoqa upgrade (using %config(noreplace) option in spec file).
This is just a quick patch used for debate, not really intended for merging into master.
--- /dev/null +++ b/client-deps.conf @@ -0,0 +1,4 @@ +# Auto-generated list of AutoQA dependencies, DO NOT EDIT +# For specifying custom packages to install on the client, edit client-deps-custom.conf +openssh-clients +rsync
These are autotest-server dependencies, right?
Rather autotest(-client), but correct.
- # if this is our test, copy all config files to src_dir to be
tarred
- # and transferred to client
- if autoqa_test:
- for config in configs:
- shutil.copy(config, src_dir)
Nice check! I wasn't sure how we'd do something like this, but you got it :)
I don't think we can do shutil.copy() to copy config files into the test directories. I mean, it certainly works, but it requires funky permissions on the autotest packaging so that the 'autotest' user can write into the site_tests directory. Can we changed this to your modified rsync/scp method instead?
Hey, James, this file is *removed* :-) My patch is based on master, so basically some of the changes are the same as in your patch (e.g. removing site_packages.py).
Once again, this patch is sent just for further reference and inspiring new ideas. I would like to deal with it only in the next+1 autoqa release, and currently I began to incline to your proposal of RPM packaging approach.
For the next autoqa release, I would like to use site_autotest.py just to transfer config files (and install rsync and scp). That's non-controversial and we need it. The rest can wait. Patch will follow soon.
On Tue, 2011-01-25 at 08:56 -0500, Kamil Paral wrote:
----- Original Message -----
Makefile | 9 ++++++- autoqa.spec | 7 ++++- client-deps-custom.conf | 2 + client-deps.conf | 4 +++
What are your thoughts around usage for the base and the -custom.conf? Is this a mechanism for site administrators to define additional dependencies they want installed on autotest clients? Is this expected to be temporary until the autotest Packaging support you mentioned is explored/implemented?
Do we really need two config files? Can this just be a single option in an existing config file? Or, do you anticipate additional autotest-specific tunables? Perhaps autotest.conf might be appropriate?
I have supposed that client-deps.conf would be automatically replaced on update and would hold all autoqa dependencies we want to install on the client. Thus it shouldn't be edited by the administrator (it won't keep changes between updates).
On the other hand client-deps-custom.conf may contain some additional packages the maintainer wants to have installed (I have used vim and mc for example). This config file will be retained on autoqa upgrade (using %config(noreplace) option in spec file).
This is just a quick patch used for debate, not really intended for merging into master.
--- /dev/null +++ b/client-deps.conf @@ -0,0 +1,4 @@ +# Auto-generated list of AutoQA dependencies, DO NOT EDIT +# For specifying custom packages to install on the client, edit client-deps-custom.conf +openssh-clients +rsync
These are autotest-server dependencies, right?
Rather autotest(-client), but correct.
- # if this is our test, copy all config files to src_dir to be
tarred
- # and transferred to client
- if autoqa_test:
- for config in configs:
- shutil.copy(config, src_dir)
Nice check! I wasn't sure how we'd do something like this, but you got it :)
I don't think we can do shutil.copy() to copy config files into the test directories. I mean, it certainly works, but it requires funky permissions on the autotest packaging so that the 'autotest' user can write into the site_tests directory. Can we changed this to your modified rsync/scp method instead?
Hey, James, this file is *removed* :-) My patch is based on master, so basically some of the changes are the same as in your patch (e.g. removing site_packages.py).
Once again, this patch is sent just for further reference and inspiring new ideas. I would like to deal with it only in the next+1 autoqa release, and currently I began to incline to your proposal of RPM packaging approach.
For the next autoqa release, I would like to use site_autotest.py just to transfer config files (and install rsync and scp). That's non-controversial and we need it. The rest can wait. Patch will follow soon.
Oh gosh!!! I'm really sorry, I completely misread the change. That'll teach me not to read patches before coffee.
Looking back at the patch. Everything looks good to me and your strategy seems perfect. Let's focus on the config file management for 0.4.4 and extend this support to allow for config/install-free clients.
Thanks, James
On Tue, 2011-01-25 at 07:23 -0500, Kamil Paral wrote:
----- Original Message -----
Okay, I'll try this soon ... I'd like to fix autotest if possible.
Sure, we should try to fix the upstream, this is just a workaround.
Reported here: http://autotest.kernel.org/ticket/45
Will look into this as soon as I can... right now I'm packing for FUDCon, but will soon get to it.
I can imagine this could work just fine for production and staging environment. But there is one use case missing, and that is the development environment. Basically, when I'm developing AutoQA on my server+client(s), I don't want to replicate every change I make onto all those machines. That's really tedious. Also I'm often quite confused where to make this change (on the server/on the client/on all machines). It depends on which part of AutoQA I'm currently working on (hooks/harness/library/tests).
Ah, good use case. Can we think of any reason that the autoqa code on the server shouldn't match the code on the clients? Eventually, we will have mixed client environment (i386/x86_64/$other using f15, f14, f13 epel5 etc...).
This implies that our code must work for all supported releases. Meaning, we can't have autoqa-0.4.4-1.el5 but autoqa-0.4.4-2 for f14. I don't have a problem with this, I typically build new packages for all supported releases. But just want to confirm
That is a good question. Currently I believe our code is exactly the same for all architectures/operating systems. But in the future, who knows.
Gosh, I would like to use the rpm versioned approach for the production server, as you mentioned. But I would also like to use more agile (and painless) method for staging and development environment.
I have had a brief look at autotest. It has some support for sending and installing packages tests. Maybe we could use that. I'll have to study it more [1].
[1] http://autotest.kernel.org/wiki/PackagingExample
I have started to discuss this issue with Josef, he has some interesting ideas. I'll send further comments later on.
Below is a patch that makes maintenance-free clients. Installs everything automatically, transfers config files and the library. You can also specify further packages to install. The patch is also available at copylib branch. I admit it's little hackish, but it's just a proof-of-concept.
For the current release I would recommend use this site_autotest.py approach just for copying config files. That means everything else would stay the same as it was before (manual installation of autoqa on all clients needed). For the next release we can discuss the best approach and implement it.
==================
From 1888da9e982c31774379c80bb12cf287d10c90fe Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Kamil=20P=C3=A1ral?= kparal@redhat.com Date: Tue, 25 Jan 2011 11:37:50 +0100 Subject: [PATCH] copy config files and autoqa library to the client
This will copy config files and autoqa library to the autotest client prior to the test execution. It will also install any required packages (like autoqa dependencies etc). That means the autotest clients are now completely maintanance-free.
Makefile | 9 ++++++- autoqa.spec | 7 ++++- client-deps-custom.conf | 2 + client-deps.conf | 4 +++ lib/autotest/site_autotest.py | 53 +++++++++++++++++++++++++++++++++++++++++ lib/autotest/site_packages.py | 34 -------------------------- 6 files changed, 72 insertions(+), 37 deletions(-) create mode 100644 client-deps-custom.conf create mode 100644 client-deps.conf create mode 100644 lib/autotest/site_autotest.py delete mode 100644 lib/autotest/site_packages.py
diff --git a/Makefile b/Makefile index f8d65ad..26bbd23 100644 --- a/Makefile +++ b/Makefile @@ -25,6 +25,8 @@ install: build [ -f $(PREFIX)/etc/autoqa/autoqa.conf ] || install -m 0644 autoqa.conf $(PREFIX)/etc/autoqa/ [ -f $(PREFIX)/etc/autoqa/fas.conf ] || install -m 0640 -g autotest fas.conf $(PREFIX)/etc/autoqa install -m 0644 repoinfo.conf $(PREFIX)/etc/autoqa/
- install -m 0644 client-deps.conf $(PREFIX)/etc/autoqa/
- [ -f $(PREFIX)/etc/autoqa/client-deps-custom.conf ] || install -m 0644 client-deps-custom.conf $(PREFIX)/etc/autoqa install -d $(PREFIX)$(HOOK_DIR) for h in hooks/*; do cp -a $$h $(PREFIX)$(HOOK_DIR); done ## git-post-receive setup
@@ -37,8 +39,13 @@ install: build for t in tests/*; do cp -a $$t $(PREFIX)$(TEST_DIR); done install -d $(PREFIX)$(AUTOTEST_DIR)/client/{bin,common_lib} install -m 0644 lib/autotest/site_utils.py $(PREFIX)$(AUTOTEST_DIR)/client/bin/
- install -m 0644 lib/autotest/site_packages.py $(PREFIX)$(AUTOTEST_DIR)/client/common_lib/
- install -d $(PREFIX)$(AUTOTEST_DIR)/server
- install -m 0644 lib/autotest/site_autotest.py $(PREFIX)$(AUTOTEST_DIR)/server/ ( cd lib/python; $(PYTHON) setup.py install --skip-build --root $(PREFIX)/ )
- ## populate client-deps.conf
- # we don't need autotest and mod_wsgi on the client, filter them out
- grep '^Requires: ' autoqa.spec | cut -f2 -d ' ' | grep -Ev '(autotest|mod_wsgi)' \
>> $(PREFIX)/etc/autoqa/client-deps.conf
build: lib/python/build
diff --git a/autoqa.spec b/autoqa.spec index a2189bd..8b8d13f 100644 --- a/autoqa.spec +++ b/autoqa.spec @@ -15,6 +15,7 @@ Requires: autotest Requires: koji Requires: python-fedora Requires: mod_wsgi +BuildRequires: autotest BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-buildroot BuildArch: noarch
@@ -36,7 +37,7 @@ make build PYTHON=%{__python} %install rm -rf $RPM_BUILD_ROOT make install PREFIX=$RPM_BUILD_ROOT TEST_DIR=%{testdir} HOOK_DIR=%{hookdir} PYTHON=%{__python} -install -m 644 autoqa.conf repoinfo.conf $RPM_BUILD_ROOT%{_sysconfdir}/autoqa/ +install -m 644 autoqa.conf repoinfo.conf client-deps.conf client-deps-custom.conf $RPM_BUILD_ROOT%{_sysconfdir}/autoqa/ install -m 640 -g autotest fas.conf $RPM_BUILD_ROOT%{_sysconfdir}/autoqa/
@@ -50,7 +51,9 @@ rm -rf $RPM_BUILD_ROOT %dir %{_sysconfdir}/autoqa %config(noreplace) %{_sysconfdir}/autoqa/autoqa.conf %config(noreplace) %{_sysconfdir}/autoqa/fas.conf +%config(noreplace) %{_sysconfdir}/autoqa/client-deps-custom.conf %config %{_sysconfdir}/autoqa/repoinfo.conf +%config %{_sysconfdir}/autoqa/client-deps.conf %config(noreplace) %{testdir}/rats_sanity/irb.cfg %config(noreplace) %{_sysconfdir}/httpd/conf.d/autoqa-git-hook.conf %dir %attr(0775,root,autotest) %{_localstatedir}/cache/autoqa @@ -59,7 +62,7 @@ rm -rf $RPM_BUILD_ROOT %{testdir} %{hookdir} %{_datadir}/autotest/client/bin/site_utils.py* -%{_datadir}/autotest/client/common_lib/site_packages.py* +%{_datadir}/autotest/server/site_autotest.py* %{python_sitelib}/autoqa*
diff --git a/client-deps-custom.conf b/client-deps-custom.conf new file mode 100644 index 0000000..8b5e60f --- /dev/null +++ b/client-deps-custom.conf @@ -0,0 +1,2 @@ +# Custom list of packages to install on autotest clients +# Provide one package name per line diff --git a/client-deps.conf b/client-deps.conf new file mode 100644 index 0000000..c42ec36 --- /dev/null +++ b/client-deps.conf @@ -0,0 +1,4 @@ +# Auto-generated list of AutoQA dependencies, DO NOT EDIT +# For specifying custom packages to install on the client, edit client-deps-custom.conf +openssh-clients +rsync diff --git a/lib/autotest/site_autotest.py b/lib/autotest/site_autotest.py new file mode 100644 index 0000000..8e52e90 --- /dev/null +++ b/lib/autotest/site_autotest.py @@ -0,0 +1,53 @@ +# site_autotest.py - site-specific autotest extensions for use in autoqa +# +# this must be copied to autotest/server/, because it is imported by +# autotest/server/server.py +# +# Author: James Laska jlaska@redhat.com
+import os +import logging +import common +from autotest_lib.server.autotest import BaseAutotest
+class SiteAutotest(BaseAutotest):
- def _install(self, host=None, autodir=None, use_autoserv=True,
use_packaging=True):
# NOTE: Due to inconsistent rsync/scp behavior inside autotest, all
# the transferred directories must end with a slash and must be removed
# and re-created at the destination machine prior to the transfer
# install required packages
logging.debug("Installing autoqa dependencies and other required packages")
packages = []
for conf in ['/etc/autoqa/client-deps.conf', '/etc/autoqa/client-deps-custom.conf']:
packages += [l.strip() for l in file(conf).readlines() if not l.strip().startswith('#')]
if packages:
host.run('yum install -y %s' % ' '.join(packages))
# copy autoqa config files
logging.debug("Copying autoqa configuration files")
srcdir = '/etc/autoqa/'
destdir = srcdir
host.run('rm -rf %(dir)s; mkdir -p %(dir)s' % {'dir': destdir})
host.send_file(srcdir, destdir)
# copy autoqa library
logging.debug("Copying autoqa library")
import autoqa
srcdir = os.path.dirname(autoqa.__file__)
assert srcdir, "Can't find autoqa library on the server"
srcdir += '/' # trailing slash is important
site_pkgs = host.run('python -c \'import sys,re; print filter(lambda s: re.match("/usr/lib/.*/site-packages$",s),sys.path)[0]\'').stdout
site_pkgs = site_pkgs.strip() # remove trailing newline
assert site_pkgs, "Can't find python site-packages directory on the client"
destdir = site_pkgs + '/autoqa/'
host.run('rm -rf %(dir)s; mkdir -p %(dir)s' % {'dir': destdir})
host.send_file(srcdir, destdir)
logging.info("Installation of autoqa completed")
# Finally, continue installation by calling our parent
super(SiteAutotest, self)._install(host, autodir, use_autoserv, use_packaging)
diff --git a/lib/autotest/site_packages.py b/lib/autotest/site_packages.py deleted file mode 100644 index 280e064..0000000 --- a/lib/autotest/site_packages.py +++ /dev/null @@ -1,34 +0,0 @@ -# site_packages.py - site-specific autotest packages methods for use in autoqa -# -# this must be copied to autotest/client/common_lib, because it is imported by -# autotest/client/common_lib/packages.py -# -# Author: Kamil Paral kparal@redhat.com
-import os, glob, shutil -from autotest_lib.client.common_lib import base_packages
-class SitePackageManager(base_packages.BasePackageManager):
- def tar_package(self, pkg_name, src_dir, dest_dir, exclude_string=None):
'''
Same as common_lib.base_packages.tar_package, but it also copies all
AutoQA config files into src_dir for our site_tests.
'''
autoqa_test = (os.path.basename(os.path.dirname(src_dir)) == 'site_tests')
configs = glob.glob('/etc/autoqa/*')
# if this is our test, copy all config files to src_dir to be tarred
# and transferred to client
if autoqa_test:
for config in configs:
shutil.copy(config, src_dir)
# run the parent method
result = super(SitePackageManager, self).tar_package(pkg_name, src_dir, dest_dir, exclude_string)
# remove all the config files
if autoqa_test:
for config in configs:
os.remove(os.path.join(src_dir, os.path.basename(config)))
return result
On Mon, 2011-01-24 at 09:59 -0500, James Laska wrote:
On Mon, 2011-01-24 at 08:27 -0500, Kamil Paral wrote:
Outstanding, you win the 'Patch Of The Week' prize! :-) This is really great, it's much better hack than mine was.
Haha, no ... this is just another hack :)
^ It's a good one, you have to admit :)
a) The destination directory of autoqa library depends on the python version used. We can't hardcode it. Maybe we can transfer the library source code and do 'python setup.py install'?
Even though James suggested taking another approach, in the meantime I worked on this one and this is the result I currently have:
# copy autoqa library logging.debug("Copying autoqa library") import autoqa srcdir = os.path.dirname(autoqa.__file__) assert srcdir, "Can't find autoqa library on the server" srcdir += '/' # trailing slash is important site_pkgs = host.run('python -c 'import sys,re; print filter(lambda s: re.match("/usr/lib/.*/site-packages$",s),sys.path)[0]'').stdout site_pkgs = site_pkgs.strip() # remove trailing newline assert site_pkgs, "Can't find python site-packages directory on the client" destdir = site_pkgs + '/autoqa/' host.run('rm -rf %(dir)s; mkdir -p %(dir)s' % {'dir': destdir}) host.send_file(srcdir, destdir)
Too hackish?
On Mon, 2011-01-24 at 11:52 -0500, Kamil Paral wrote:
a) The destination directory of autoqa library depends on the python version used. We can't hardcode it. Maybe we can transfer the library source code and do 'python setup.py install'?
Even though James suggested taking another approach, in the meantime I worked on this one and this is the result I currently have:
# copy autoqa library logging.debug("Copying autoqa library") import autoqa srcdir = os.path.dirname(autoqa.__file__) assert srcdir, "Can't find autoqa library on the server" srcdir += '/' # trailing slash is important site_pkgs = host.run('python -c \'import sys,re; print filter(lambda s: re.match("/usr/lib/.*/site-packages$",s),sys.path)[0]\'').stdout site_pkgs = site_pkgs.strip() # remove trailing newline assert site_pkgs, "Can't find python site-packages directory on the client" destdir = site_pkgs + '/autoqa/' host.run('rm -rf %(dir)s; mkdir -p %(dir)s' % {'dir': destdir}) host.send_file(srcdir, destdir)
Too hackish?
Hmm, I like python setup.py install better because it is better to hand off work to code that knows where to find the python site-packages dir using a more robust approach.
autoqa-devel@lists.fedorahosted.org