java/buildconf/build-props.xml | 2
java/buildconf/build-utils.xml | 2
java/code/src/com/redhat/rhn/common/util/MD5Sum.java | 23 -----
java/code/src/com/redhat/rhn/common/util/test/MD5SumTest.java | 40 +++++++++-
java/code/src/com/redhat/rhn/testing/TestUtils.java | 27 ++++++
java/ivy.xml | 1
6 files changed, 71 insertions(+), 24 deletions(-)
New commits:
commit 1f617dc25a0bd3551fc112e366d9a24e6ee05345
Author: Mike McCune <mmccune(a)redhat.com>
Date: Wed Aug 12 11:40:02 2009 -0700
514305 - switching the algorithm for reading the file into memory.
New algorithm ditches the ByteArrayOutputStream which has its own
internal byte array buffer which means we are maintaining 2x
copies of the file in memory which doubles the heap required.
diff --git a/java/buildconf/build-props.xml b/java/buildconf/build-props.xml
index c23c3bc..8d6576c 100644
--- a/java/buildconf/build-props.xml
+++ b/java/buildconf/build-props.xml
@@ -52,7 +52,7 @@
<property name="common.jar.dependencies"
value="bcel cglib commons-beanutils commons-cli commons-codec
commons-collections commons-digester commons-discovery
- commons-el commons-lang commons-logging commons-validator hibernate3
+ commons-el commons-lang commons-logging commons-io commons-validator
hibernate3
mailapi smtp jdom jpam
jspapi log4j oro redstone-xmlrpc-client redstone-xmlrpc servletapi5 struts
bcprov bcpg jcommon stringtree-json postgresql-jdbc
diff --git a/java/buildconf/build-utils.xml b/java/buildconf/build-utils.xml
index e4e16a9..16be3d5 100644
--- a/java/buildconf/build-utils.xml
+++ b/java/buildconf/build-utils.xml
@@ -60,7 +60,7 @@ Running @{banner} ...
1.6.2, remove forkmode attribute -->
<junit forkmode="once" fork="yes"
printsummary="off" showoutput="yes"
haltonfailure="${halt-tests-on-failure}"
- failureproperty="junit_test_failure" >
+ failureproperty="junit_test_failure"
maxmemory="256m">
<!-- kind of hokey, but we need a property for the DataSourceParser
test to work. This keeps people from breaking until we move
the DataSource xml files into rhn-java. -->
diff --git a/java/code/src/com/redhat/rhn/common/util/MD5Sum.java
b/java/code/src/com/redhat/rhn/common/util/MD5Sum.java
index 5d5c4cd..01abe50 100644
--- a/java/code/src/com/redhat/rhn/common/util/MD5Sum.java
+++ b/java/code/src/com/redhat/rhn/common/util/MD5Sum.java
@@ -142,28 +142,11 @@ public class MD5Sum {
*/
public static String getFileMD5Sum(File f) throws IOException {
String sum = null;
- FileInputStream in = new FileInputStream(f.getAbsolutePath());
- byte[] b = new byte[1024];
- int num = 0;
- ByteArrayOutputStream out = new ByteArrayOutputStream();
-
- while ((num = in.read(b)) != -1) {
- out.write(b, 0, num);
-
- if (out.size() > SCOUR_MD5_BYTE_LIMIT) {
- sum = md5Sum(out.toByteArray(), SCOUR_MD5_BYTE_LIMIT);
- break;
- }
- }
-
- if (sum == null) {
- sum = md5Sum(out.toByteArray(), SCOUR_MD5_BYTE_LIMIT);
- }
-
- in.close();
- out.close();
+ byte[] barray = FileUtils.readByteArrayFromFile(f, 0, f.length());
+ sum = md5Sum(barray, SCOUR_MD5_BYTE_LIMIT);
return sum;
}
+
}
diff --git a/java/code/src/com/redhat/rhn/common/util/test/MD5SumTest.java
b/java/code/src/com/redhat/rhn/common/util/test/MD5SumTest.java
index 7e97efe..2e5bdfd 100644
--- a/java/code/src/com/redhat/rhn/common/util/test/MD5SumTest.java
+++ b/java/code/src/com/redhat/rhn/common/util/test/MD5SumTest.java
@@ -15,21 +15,57 @@
package com.redhat.rhn.common.util.test;
import com.redhat.rhn.common.util.MD5Sum;
-import com.redhat.rhn.testing.RhnBaseTestCase;
import com.redhat.rhn.testing.TestUtils;
import java.io.File;
+import java.util.Random;
+
+import junit.framework.TestCase;
/**
* @author mmccune
*
*/
-public class MD5SumTest extends RhnBaseTestCase {
+public class MD5SumTest extends TestCase {
public void testMD5Sum() throws Exception {
File testFile = new
File(TestUtils.findTestData("test.file").getFile());
String sum = MD5Sum.getFileMD5Sum(testFile);
assertEquals("ab0aa62f30d67085cd07ea9004a1437f", sum);
}
+
+ /**
+ * Note that this test creates a large 100MB file in /tmp
+ * and then does an md5sum on the file.
+ *
+ * With a previous implementation of MD5Sum.getFileMD5Sum() this would
+ * cause an OOME with a max-heap size in junit of 256m. This is configured in:
+ * spacewalk/buildconf/build-utils.xml:
+ *
+ * <junit forkmode="once" fork="yes"
printsummary="off" showoutput="yes"
+ * haltonfailure="${halt-tests-on-failure}"
+ * failureproperty="junit_test_failure"
maxmemory="256m">
+ *
+ * The new implementation uses less memory and thus passes the test below when ran
+ * from Junit CLI.
+ *
+ * @throws Exception
+ */
+ public void testOOMEMD5Sum() throws Exception {
+ File large = new File("/tmp/large-file.dat");
+ // Create a large 100mb file
+ large.createNewFile();
+ writeRandomLargeBytesToFile(large);
+ String sum = MD5Sum.getFileMD5Sum(large);
+ assertNotNull(sum);
+ large.delete();
+ }
+
+ private static void writeRandomLargeBytesToFile(File f) {
+ byte[] ba = new byte[101326592];
+ Random r = new Random();
+ r.nextBytes(ba);
+ TestUtils.writeByteArrayToFile(f, ba);
+ }
}
diff --git a/java/code/src/com/redhat/rhn/testing/TestUtils.java
b/java/code/src/com/redhat/rhn/testing/TestUtils.java
index b5df56d..a021c8c 100644
--- a/java/code/src/com/redhat/rhn/testing/TestUtils.java
+++ b/java/code/src/com/redhat/rhn/testing/TestUtils.java
@@ -43,6 +43,8 @@ import org.hibernate.Query;
import org.hibernate.Session;
import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
@@ -88,6 +90,31 @@ public class TestUtils {
}
/**
+ * Write a byte array to a file
+ * @param file to write bytearray to
+ * @param array to write out
+ */
+ public static void writeByteArrayToFile(File file, byte[] array) {
+ try {
+ if (file.exists()) {
+ file.delete();
+ }
+ file.createNewFile();
+ FileOutputStream fos = new FileOutputStream(file);
+ try {
+ fos.write(array);
+ }
+ finally {
+ fos.close();
+ }
+ }
+ catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+
+ /**
* Read whole stream into a string.
*
* @param stream the stream to consume
diff --git a/java/ivy.xml b/java/ivy.xml
index 955ea90..7ab8e43 100644
--- a/java/ivy.xml
+++ b/java/ivy.xml
@@ -22,6 +22,7 @@
<dependency org="redhat" name="commons-httpclient"
rev="3.0" />
<dependency org="redhat" name="commons-lang"
rev="2.1" />
<dependency org="redhat" name="commons-logging"
rev="1.0.4" />
+ <dependency org="redhat" name="commons-io"
rev="1.3.2" />
<dependency org="redhat" name="commons-validator"
rev="1.1.4" />
<dependency org="redhat" name="concurrent"
rev="1.3.2" />
<dependency org="redhat" name="dom4j"
rev="1.6.1" />
Show replies by date