commit 7119e8ed32bd1f497563d1d4bc989a6f39765701 Author: Michal Srb msrb@redhat.com Date: Mon Jul 7 16:03:34 2014 +0200
Another try to speed RPM generators up a bit
depgenerators/maven.req | 16 +++++++++-- depgenerators/osgi.req | 25 +++++++++++----- python/javapackages/common/osgi.py | 53 ++++++++++++++++++++++------------- 3 files changed, 63 insertions(+), 31 deletions(-) --- diff --git a/depgenerators/maven.req b/depgenerators/maven.req index 642bbf0..a770d99 100755 --- a/depgenerators/maven.req +++ b/depgenerators/maven.req @@ -109,8 +109,10 @@ class TagBuilder(object): self.provided_artifacts = pickle.load(cachefile) cachefile.close() except IOError: - metadata_paths = self.gather_metadata_files() - metadata = Metadata(metadata_paths) + if not metadata_paths: + metadata_paths = self.gather_metadata_files() + if not metadata: + metadata = Metadata(metadata_paths) self.provided_artifacts.extend(metadata.get_provided_artifacts()) cachefile = open('.provided_artifacts.cache', 'w') pickle.dump(self.provided_artifacts, cachefile) @@ -120,15 +122,23 @@ class TagBuilder(object): self.provided_osgi = pickle.load(cachefile) cachefile.close() except IOError: + if not metadata_paths: + metadata_paths = self.gather_metadata_files() + if not metadata: + metadata = Metadata(metadata_paths) self.provided_osgi.extend(metadata.get_osgi_provides()) cachefile = open('.provided_osgi.cache', 'w') - pickle.dump(self.provided_artifacts, cachefile) + pickle.dump(self.provided_osgi, cachefile) cachefile.close() try: cachefile = open('.skipped_artifacts.cache', 'r') self.skipped_artifacts = pickle.load(cachefile) cachefile.close() except IOError: + if not metadata_paths: + metadata_paths = self.gather_metadata_files() + if not metadata: + metadata = Metadata(metadata_paths) self.skipped_artifacts.extend(metadata.get_skipped_artifacts()) cachefile = open('.skipped_artifacts.cache', 'w') pickle.dump(self.skipped_artifacts, cachefile) diff --git a/depgenerators/osgi.req b/depgenerators/osgi.req index e64ba1b..df41cb5 100755 --- a/depgenerators/osgi.req +++ b/depgenerators/osgi.req @@ -32,6 +32,7 @@
import javapackages.common.osgi as osgi import sys +import pickle import os
class TagBuilder(object): @@ -47,14 +48,22 @@ class TagBuilder(object):
# get all provided bundles provided = [] - for dirpath, _, filenames in os.walk(buildroot): - for filename in filenames: - fpath = os.path.abspath(os.path.join(dirpath, filename)) - if _check_path(fpath): - manifest = osgi.open_manifest(fpath) - if not manifest: - continue - provided.append(osgi.OsgiProvideInfo.from_manifest(manifest)) + try: + cachefile = open('.provided_bundles.cache', 'r') + provided = pickle.load(cachefile) + cachefile.close() + except IOError: + for dirpath, _, filenames in os.walk(buildroot): + for filename in filenames: + fpath = os.path.abspath(os.path.join(dirpath, filename)) + if _check_path(fpath): + manifest = osgi.open_manifest(fpath) + if not manifest: + continue + provided.append(osgi.OsgiProvideInfo.from_manifest(manifest)) + cachefile = open('.provided_bundles.cache', 'w') + pickle.dump(provided, cachefile) + cachefile.close()
# print requires for path in paths: diff --git a/python/javapackages/common/osgi.py b/python/javapackages/common/osgi.py index 4f208d6..5e4efa7 100644 --- a/python/javapackages/common/osgi.py +++ b/python/javapackages/common/osgi.py @@ -32,6 +32,7 @@ # Authors: Alexander Kurtakov akurtako@redhat.com
import os +import pickle import zipfile from zipfile import ZipFile
@@ -145,24 +146,36 @@ def look_for_path_in_metadata(path): except KeyError: raise Exception("RPM_BUILD_ROOT environment is not set")
- for dirpath, dirnames, filenames in os.walk(buildroot): - for filename in filenames: - fpath = os.path.abspath(os.path.join(dirpath, filename)) - # FIXME: add path to metadata directory to config file? - if "/maven-metadata/" in fpath: - try: - mdata = Metadata(fpath) - artifacts = mdata.get_provided_artifacts() - for a in artifacts: - path = os.path.abspath(path) - if path.startswith(buildroot): - path = path[len(buildroot):] - path = os.path.join('/', path) - if a.path: - if (os.path.abspath(a.path) == path or - (path.startswith(os.path.abspath(a.path)) and - os.path.realpath(buildroot + path))): - return True - except MetadataInvalidException: - pass + artifacts = [] + try: + cachefile = open('.provided_artifacts.cache', 'r') + artifacts = pickle.load(cachefile) + cachefile.close() + except IOError: + metadata_paths = [] + for dirpath, dirnames, filenames in os.walk(buildroot): + for filename in filenames: + fpath = os.path.abspath(os.path.join(dirpath, filename)) + # FIXME: add path to metadata directory to config file? + if "/maven-metadata/" in fpath: + metadata_paths.append(fpath) + try: + mdata = Metadata(metadata_paths) + artifacts = mdata.get_provided_artifacts() + cachefile = open('.provided_artifacts.cache', 'w') + pickle.dump(artifacts, cachefile) + cachefile.close() + except MetadataInvalidException: + pass + + for a in artifacts: + path = os.path.abspath(path) + if path.startswith(buildroot): + path = path[len(buildroot):] + path = os.path.join('/', path) + if a.path: + if (os.path.abspath(a.path) == path or + (path.startswith(os.path.abspath(a.path)) and + os.path.realpath(buildroot + path))): + return True return False