java/buildconf/build-props.xml | 2
java/buildconf/build-utils.xml | 2
java/code/src/com/redhat/rhn/common/util/MD5Sum.java | 23 -----
java/code/src/com/redhat/rhn/common/util/test/MD5SumTest.java | 40 +++++++++-
java/code/src/com/redhat/rhn/testing/TestUtils.java | 27 ++++++
java/ivy.xml | 1
6 files changed, 71 insertions(+), 24 deletions(-)
New commits:
commit a77cbced5176518088c8090667138c5d74f982df
Author: Mike McCune <mmccune(a)redhat.com>
Date: Wed Aug 12 13:12:45 2009 -0700
514305 - switching the algorithm for reading the file into memory.
New algorithm ditches the ByteArrayOutputStream which has its own
internal byte array buffer which means we are maintaining 2x
copies of the file in memory which doubles the heap required.
diff --git a/java/buildconf/build-props.xml b/java/buildconf/build-props.xml
index 17c0bc2..2854c1e 100644
--- a/java/buildconf/build-props.xml
+++ b/java/buildconf/build-props.xml
@@ -52,7 +52,7 @@
<property name="common.jar.dependencies"
value="bcel cglib commons-beanutils commons-cli commons-codec
commons-collections commons-digester commons-discovery
- commons-el commons-lang commons-logging commons-validator hibernate3
+ commons-el commons-lang commons-logging commons-io commons-validator
hibernate3
mailapi smtp jdom jpam
jspapi log4j oro redstone-xmlrpc-client redstone-xmlrpc servletapi5 struts
bcprov bcpg jcommon stringtree-json
diff --git a/java/buildconf/build-utils.xml b/java/buildconf/build-utils.xml
index e4e16a9..16be3d5 100644
--- a/java/buildconf/build-utils.xml
+++ b/java/buildconf/build-utils.xml
@@ -60,7 +60,7 @@ Running @{banner} ...
1.6.2, remove forkmode attribute -->
<junit forkmode="once" fork="yes"
printsummary="off" showoutput="yes"
haltonfailure="${halt-tests-on-failure}"
- failureproperty="junit_test_failure" >
+ failureproperty="junit_test_failure"
maxmemory="256m">
<!-- kind of hokey, but we need a property for the DataSourceParser
test to work. This keeps people from breaking until we move
the DataSource xml files into rhn-java. -->
diff --git a/java/code/src/com/redhat/rhn/common/util/MD5Sum.java
b/java/code/src/com/redhat/rhn/common/util/MD5Sum.java
index 5d5c4cd..01abe50 100644
--- a/java/code/src/com/redhat/rhn/common/util/MD5Sum.java
+++ b/java/code/src/com/redhat/rhn/common/util/MD5Sum.java
@@ -142,28 +142,11 @@ public class MD5Sum {
*/
public static String getFileMD5Sum(File f) throws IOException {
String sum = null;
- FileInputStream in = new FileInputStream(f.getAbsolutePath());
- byte[] b = new byte[1024];
- int num = 0;
- ByteArrayOutputStream out = new ByteArrayOutputStream();
-
- while ((num = in.read(b)) != -1) {
- out.write(b, 0, num);
-
- if (out.size() > SCOUR_MD5_BYTE_LIMIT) {
- sum = md5Sum(out.toByteArray(), SCOUR_MD5_BYTE_LIMIT);
- break;
- }
- }
-
- if (sum == null) {
- sum = md5Sum(out.toByteArray(), SCOUR_MD5_BYTE_LIMIT);
- }
-
- in.close();
- out.close();
+ byte[] barray = FileUtils.readByteArrayFromFile(f, 0, f.length());
+ sum = md5Sum(barray, SCOUR_MD5_BYTE_LIMIT);
return sum;
}
+
}
diff --git a/java/code/src/com/redhat/rhn/common/util/test/MD5SumTest.java
b/java/code/src/com/redhat/rhn/common/util/test/MD5SumTest.java
index 7e97efe..2e5bdfd 100644
--- a/java/code/src/com/redhat/rhn/common/util/test/MD5SumTest.java
+++ b/java/code/src/com/redhat/rhn/common/util/test/MD5SumTest.java
@@ -15,21 +15,57 @@
package com.redhat.rhn.common.util.test;
import com.redhat.rhn.common.util.MD5Sum;
-import com.redhat.rhn.testing.RhnBaseTestCase;
import com.redhat.rhn.testing.TestUtils;
import java.io.File;
+import java.util.Random;
+
+import junit.framework.TestCase;
/**
* @author mmccune
*
*/
-public class MD5SumTest extends RhnBaseTestCase {
+public class MD5SumTest extends TestCase {
public void testMD5Sum() throws Exception {
File testFile = new
File(TestUtils.findTestData("test.file").getFile());
String sum = MD5Sum.getFileMD5Sum(testFile);
assertEquals("ab0aa62f30d67085cd07ea9004a1437f", sum);
}
+
+ /**
+ * Note that this test creates a large 100MB file in /tmp
+ * and then does an md5sum on the file.
+ *
+ * With a previous implementation of MD5Sum.getFileMD5Sum() this would
+ * cause an OOME with a max-heap size in junit of 256m. This is configured in:
+ * spacewalk/buildconf/build-utils.xml:
+ *
+ * <junit forkmode="once" fork="yes"
printsummary="off" showoutput="yes"
+ * haltonfailure="${halt-tests-on-failure}"
+ * failureproperty="junit_test_failure"
maxmemory="256m">
+ *
+ * The new implementation uses less memory and thus passes the test below when ran
+ * from Junit CLI.
+ *
+ * @throws Exception
+ */
+ public void testOOMEMD5Sum() throws Exception {
+ File large = new File("/tmp/large-file.dat");
+ // Create a large 100mb file
+ large.createNewFile();
+ writeRandomLargeBytesToFile(large);
+ String sum = MD5Sum.getFileMD5Sum(large);
+ assertNotNull(sum);
+ large.delete();
+ }
+
+ private static void writeRandomLargeBytesToFile(File f) {
+ byte[] ba = new byte[101326592];
+ Random r = new Random();
+ r.nextBytes(ba);
+ TestUtils.writeByteArrayToFile(f, ba);
+ }
}
diff --git a/java/code/src/com/redhat/rhn/testing/TestUtils.java
b/java/code/src/com/redhat/rhn/testing/TestUtils.java
index b5df56d..a021c8c 100644
--- a/java/code/src/com/redhat/rhn/testing/TestUtils.java
+++ b/java/code/src/com/redhat/rhn/testing/TestUtils.java
@@ -43,6 +43,8 @@ import org.hibernate.Query;
import org.hibernate.Session;
import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
@@ -88,6 +90,31 @@ public class TestUtils {
}
/**
+ * Write a byte array to a file
+ * @param file to write bytearray to
+ * @param array to write out
+ */
+ public static void writeByteArrayToFile(File file, byte[] array) {
+ try {
+ if (file.exists()) {
+ file.delete();
+ }
+ file.createNewFile();
+ FileOutputStream fos = new FileOutputStream(file);
+ try {
+ fos.write(array);
+ }
+ finally {
+ fos.close();
+ }
+ }
+ catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+
+ /**
* Read whole stream into a string.
*
* @param stream the stream to consume
diff --git a/java/ivy.xml b/java/ivy.xml
index f3c990e..1eb79b6 100644
--- a/java/ivy.xml
+++ b/java/ivy.xml
@@ -22,6 +22,7 @@
<dependency org="redhat" name="commons-httpclient"
rev="3.0" />
<dependency org="redhat" name="commons-lang"
rev="2.1" />
<dependency org="redhat" name="commons-logging"
rev="1.0.4" />
+ <dependency org="redhat" name="commons-io"
rev="1.3.2" />
<dependency org="redhat" name="commons-validator"
rev="1.1.4" />
<dependency org="redhat" name="concurrent"
rev="1.3.2" />
<dependency org="redhat" name="dom4j"
rev="1.6.1" />
Show replies by date