From: Mathieu Bridon bochecha@daitauha.fr
The current format for the 'sources file is as follows:
$hash $filename
We're eventually going to move to:
$hashtype $hash $filename
This commit just prepares us for that, allowing to parse both formats.
If the 'sources' file doesn't contain the $hashtype, it is assumed to be the one defined in the config file as lookasidehash. --- src/pyrpkg/__init__.py | 10 +++++++++- src/pyrpkg/sources.py | 9 +++++---- test/test_sources.py | 2 +- 3 files changed, 15 insertions(+), 6 deletions(-)
diff --git a/src/pyrpkg/__init__.py b/src/pyrpkg/__init__.py index 5c4dbc5..f9606fe 100644 --- a/src/pyrpkg/__init__.py +++ b/src/pyrpkg/__init__.py @@ -1568,7 +1568,15 @@ class Commands(object): # Default to putting the files where the module is if not outdir: outdir = self.path - for (csum, file) in self._read_sources(): + + for entry in self._read_sources(): + if len(entry) == 2: + csum, file = entry + hashtype = self.lookasidehash + + else: + hashtype, csum, file = entry + # See if we already have a valid copy downloaded outfile = os.path.join(outdir, file) if os.path.exists(outfile): diff --git a/src/pyrpkg/sources.py b/src/pyrpkg/sources.py index c4263ad..8d8b277 100644 --- a/src/pyrpkg/sources.py +++ b/src/pyrpkg/sources.py @@ -42,10 +42,11 @@ def _parse_line(line): stripped_line = line.strip() if not stripped_line: return [] - entries = stripped_line.split(' ', 1) - if len(entries) != 2: - raise ValueError("Malformed line: %r." % line) - return entries + entries = stripped_line.split(' ') + if len(entries) in (2, 3): + return entries + + raise ValueError("Malformed line: %r." % line)
def _format_line(entry): diff --git a/test/test_sources.py b/test/test_sources.py index 997ab83..71b49fb 100644 --- a/test/test_sources.py +++ b/test/test_sources.py @@ -14,7 +14,7 @@ class formatLineTestCase(unittest.TestCase): def test_wrong_number_of_fields(self): WRONG_ENTRIES = [ ('foo'), - ('foo', 'bar', 'foo'), + ('foo', 'bar', 'foo', 'bar'), ] for entry in WRONG_ENTRIES: self.assertRaises(ValueError, sources._format_line, entry)
From: Mathieu Bridon bochecha@daitauha.fr
It is added to each new line of the file. Existing lines are left untouched.
This is fine, because the reading of the file is capable (as per previous commit) to handle both type of lines indiscriminately. --- src/pyrpkg/__init__.py | 2 +- src/pyrpkg/sources.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/src/pyrpkg/__init__.py b/src/pyrpkg/__init__.py index f9606fe..38fe7e6 100644 --- a/src/pyrpkg/__init__.py +++ b/src/pyrpkg/__init__.py @@ -2271,7 +2271,7 @@ class Commands(object): self.log.info("Uploading: %s %s" % (file_hash, f)) file_basename = os.path.basename(f) if (file_hash, file_basename) not in sources: - sources.append((file_hash, file_basename)) + sources.append((self.lookasidehash, file_hash, file_basename))
# Add this file to .gitignore if it's not already there: if not gitignore.match(file_basename): diff --git a/src/pyrpkg/sources.py b/src/pyrpkg/sources.py index 8d8b277..c4c31be 100644 --- a/src/pyrpkg/sources.py +++ b/src/pyrpkg/sources.py @@ -50,7 +50,7 @@ def _parse_line(line):
def _format_line(entry): - if len(entry) != 0 and len(entry) != 2: + if len(entry) not in (0, 2, 3): raise ValueError("Incorrect number of fields for entry: %r." % (entry,)) return " ".join(entry)
From: Mathieu Bridon bochecha@daitauha.fr
This wasn't passing a tuple, but a string to the _format_line() function. --- test/test_sources.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/test/test_sources.py b/test/test_sources.py index 71b49fb..edff838 100644 --- a/test/test_sources.py +++ b/test/test_sources.py @@ -13,7 +13,7 @@ sys.path = old_path class formatLineTestCase(unittest.TestCase): def test_wrong_number_of_fields(self): WRONG_ENTRIES = [ - ('foo'), + ('foo',), ('foo', 'bar', 'foo', 'bar'), ] for entry in WRONG_ENTRIES:
From: Mathieu Bridon bochecha@daitauha.fr
This ensures that we can read/write lines in the new format, as well as mix lines in both format within a single sources file. --- test/test_sources.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-)
diff --git a/test/test_sources.py b/test/test_sources.py index edff838..c0722b1 100644 --- a/test/test_sources.py +++ b/test/test_sources.py @@ -25,6 +25,7 @@ class formatLineTestCase(unittest.TestCase): def test_correct_entry(self): CORRECT_ENTRIES = [ (['foo', 'bar'], ('foo bar')), + (['foo', 'bar', 'baz'], ('foo bar baz')), ] for entry, line in CORRECT_ENTRIES: self.assertEqual(line, @@ -53,7 +54,8 @@ class parseLineTestCase(unittest.TestCase): def test_correct_line(self): CORRECT_LINES = [ ('foo bar\n', ['foo', 'bar']), - ('foo bar\n', ['foo', ' bar']) + ('foo bar\n', ['foo', ' bar']), + ('foo bar baz\n', ['foo', ' bar', 'baz']) ] for line, entry in CORRECT_LINES: self.assertEqual(entry, sources._parse_line(line)) @@ -80,6 +82,10 @@ class ReaderTestCase(unittest.TestCase): ('foo bar\nfooo baaar\n', [['foo', 'bar'], ['fooo', 'baaar'], ]), + ('foo bar baz\n', [['foo', 'bar', 'baz']]), + ('foo bar\nfoo bar baz\n', [['foo', 'bar'], + ['foo', 'bar', 'baz'], + ]), ] for buffer, entries in CORRECT_SOURCES: fp = StringIO.StringIO(buffer) @@ -95,6 +101,9 @@ class WriterTestCase(unittest.TestCase): ([['foo', 'bar'], ['fooo', 'baaar'], ], 'foo bar\nfooo baaar\n'), + ([['foo', 'bar', 'baz']], 'foo bar baz\n'), + ([['foo', 'bar'], ['foo', 'bar', 'baz']], + 'foo bar\nfoo bar baz\n'), ] for entries, buffer in CORRECT_SOURCES: fp = StringIO.StringIO()
From: Mathieu Bridon bochecha@daitauha.fr
The current format for the 'sources file is as follows:
$hash $filename
We're eventually going to move to:
$hashtype $hash $filename
This commit just prepares us for that, allowing to parse both formats.
If the 'sources' file doesn't contain the $hashtype, it is assumed to be the one defined in the config file as lookasidehash. --- This is just a new version of the previous 0001 patch, replacing the earlier one.
It fixes an issue with the hashtype used by default when none was explicitly specified in the 'sources' file.
src/pyrpkg/__init__.py | 16 +++++++++++++++- src/pyrpkg/sources.py | 9 +++++---- test/test_sources.py | 2 +- 3 files changed, 21 insertions(+), 6 deletions(-)
diff --git a/src/pyrpkg/__init__.py b/src/pyrpkg/__init__.py index 5c4dbc5..3a1874f 100644 --- a/src/pyrpkg/__init__.py +++ b/src/pyrpkg/__init__.py @@ -1568,7 +1568,21 @@ class Commands(object): # Default to putting the files where the module is if not outdir: outdir = self.path - for (csum, file) in self._read_sources(): + + for entry in self._read_sources(): + if len(entry) == 2: + csum, file = entry + + # We have to hardcode md5 here, because this is the historical + # default. We can't rely on self.lookasidehash (from the config + # file), because it represents the **current** default, which + # will eventually change, which would break older 'sources' + # files. + hashtype = 'md5' + + else: + hashtype, csum, file = entry + # See if we already have a valid copy downloaded outfile = os.path.join(outdir, file) if os.path.exists(outfile): diff --git a/src/pyrpkg/sources.py b/src/pyrpkg/sources.py index c4263ad..8d8b277 100644 --- a/src/pyrpkg/sources.py +++ b/src/pyrpkg/sources.py @@ -42,10 +42,11 @@ def _parse_line(line): stripped_line = line.strip() if not stripped_line: return [] - entries = stripped_line.split(' ', 1) - if len(entries) != 2: - raise ValueError("Malformed line: %r." % line) - return entries + entries = stripped_line.split(' ') + if len(entries) in (2, 3): + return entries + + raise ValueError("Malformed line: %r." % line)
def _format_line(entry): diff --git a/test/test_sources.py b/test/test_sources.py index 997ab83..71b49fb 100644 --- a/test/test_sources.py +++ b/test/test_sources.py @@ -14,7 +14,7 @@ class formatLineTestCase(unittest.TestCase): def test_wrong_number_of_fields(self): WRONG_ENTRIES = [ ('foo'), - ('foo', 'bar', 'foo'), + ('foo', 'bar', 'foo', 'bar'), ] for entry in WRONG_ENTRIES: self.assertRaises(ValueError, sources._format_line, entry)
As discussed face to face and in the last releng meeting, this new format is not great, as it breaks things for people (or scripts) try to verify the hashes of the sources with things like:
$ md5sum -c sources
However, md5sum (and others like sha512sum) can check files written in the BSD-style format:
MD5 (afile) = ahash
This format serves our purpose (having the hashtype written on the line), and doesn't break for people using `md5sum -c sources`
Patches to move to that are coming soon, the current patch series can be dropped.
buildsys@lists.fedoraproject.org