modules/core/domain/src/main/java/org/rhq/core/domain/content/PackageBits.java
| 7
modules/enterprise/server/jar/src/main/java/org/rhq/enterprise/server/content/ContentManagerBean.java
| 303 ++++++++--
modules/enterprise/server/jar/src/main/java/org/rhq/enterprise/server/content/ContentManagerLocal.java
| 11
modules/enterprise/server/jar/src/test/java/org/rhq/enterprise/server/content/test/ContentUIManagerBeanTest.java
| 97 +++
4 files changed, 358 insertions(+), 60 deletions(-)
New commits:
commit 6f4eb034b12085258f98c92713af8a499ea78261
Author: Simeon Pinder <spinder(a)redhat.com>
Date: Fri Jun 25 15:53:16 2010 -0400
unit test fix for blob.
diff --git
a/modules/enterprise/server/jar/src/test/java/org/rhq/enterprise/server/content/test/ContentUIManagerBeanTest.java
b/modules/enterprise/server/jar/src/test/java/org/rhq/enterprise/server/content/test/ContentUIManagerBeanTest.java
index 7fae6e0..5f73b89 100644
---
a/modules/enterprise/server/jar/src/test/java/org/rhq/enterprise/server/content/test/ContentUIManagerBeanTest.java
+++
b/modules/enterprise/server/jar/src/test/java/org/rhq/enterprise/server/content/test/ContentUIManagerBeanTest.java
@@ -284,7 +284,7 @@ public class ContentUIManagerBeanTest extends AbstractEJB3Test {
}
FileInputStream fos = new FileInputStream(originalBinary);
- contentManager.updateBlobStream(fos, packageBits);
+ contentManager.updateBlobStream(fos, packageBits, null);
packageBits = em.find(PackageBits.class, packageBits.getId());
// test that the bits are available and stored in the DB: Reading the Blob
commit 17b0f8bb1a60583a0d1ea24bd894cc9a3dbd5c42
Merge: 863201f... bfd6f7d...
Author: Simeon Pinder <spinder(a)redhat.com>
Date: Fri Jun 25 15:50:49 2010 -0400
Merge branch 'release-3.0.0' of ssh://spinder@git.fedorahosted.org/git/rhq/rhq
into release-3.0.0
commit 863201f8b91e01632ec7453940f4b08fd3d0dda7
Author: Simeon Pinder <spinder(a)redhat.com>
Date: Fri Jun 25 09:53:02 2010 -0400
BZ-597246: fixing errors, modified loadPackageBits to calculate digest and file length
as well.
diff --git
a/modules/enterprise/server/jar/src/main/java/org/rhq/enterprise/server/content/ContentManagerBean.java
b/modules/enterprise/server/jar/src/main/java/org/rhq/enterprise/server/content/ContentManagerBean.java
index b2cac05..6ee805e 100644
---
a/modules/enterprise/server/jar/src/main/java/org/rhq/enterprise/server/content/ContentManagerBean.java
+++
b/modules/enterprise/server/jar/src/main/java/org/rhq/enterprise/server/content/ContentManagerBean.java
@@ -18,6 +18,7 @@
*/
package org.rhq.enterprise.server.content;
+import java.io.BufferedInputStream;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.FileInputStream;
@@ -945,7 +946,7 @@ public class ContentManagerBean implements ContentManagerLocal,
ContentManagerRe
pkgName = packageVersion.getGeneralPackage().getName();
}
bits = loadPackageBits(bitStream, packageVersion.getId(), pkgName,
packageVersion.getVersion(),
- bits);
+ bits, null);
entityManager.merge(bits);
} finally {
@@ -1235,17 +1236,13 @@ public class ContentManagerBean implements ContentManagerLocal,
ContentManagerRe
PackageVersion newPackageVersion = new PackageVersion(existingPackage, version,
architecture);
newPackageVersion.setDisplayName(existingPackage.getName());
- PackageBits bits = loadPackageBits(packageBitStream, newPackageVersion.getId(),
packageName, version, null);
+ Map<String, String> contentDetails = new HashMap<String, String>();
+ PackageBits bits = loadPackageBits(packageBitStream, newPackageVersion.getId(),
packageName, version, null,
+ contentDetails);
newPackageVersion.setPackageBits(bits);
- //TODO: write down to file system to calculate length and calculate hash.
- newPackageVersion.setFileSize((long) bits.getBits().length);
- try {
- newPackageVersion.setSHA256(new
MessageDigestGenerator(MessageDigestGenerator.SHA_256)
- .calcDigestString(bits.getBits()));
- } catch (IOException e) {
- newPackageVersion.setSHA256(null);
- }
+
newPackageVersion.setFileSize(Long.valueOf(contentDetails.get(UPLOAD_FILE_SIZE)).longValue());
+ newPackageVersion.setSHA256(contentDetails.get(UPLOAD_SHA256));
newPackageVersion = persistOrMergePackageVersionSafely(newPackageVersion);
existingPackage.addVersion(newPackageVersion);
@@ -1559,7 +1556,7 @@ public class ContentManagerBean implements ContentManagerLocal,
ContentManagerRe
}
//get the data
- PackageBits bits = loadPackageBits(packageBitStream, packageVersion.getId(),
packageName, version, null);
+ PackageBits bits = loadPackageBits(packageBitStream, packageVersion.getId(),
packageName, version, null, null);
packageVersion.setPackageBits(bits);
@@ -1583,11 +1580,12 @@ public class ContentManagerBean implements ContentManagerLocal,
ContentManagerRe
*
* @param packageBitStream
* @param packageVersionId
+ * @param contentDetails
* @return PackageBits ref populated.
*/
@TransactionAttribute(TransactionAttributeType.REQUIRES_NEW)
private PackageBits loadPackageBits(InputStream packageBitStream, int
packageVersionId, String packageName,
- String packageVersion, PackageBits existingPkgBits) {
+ String packageVersion, PackageBits existingPkgBits, Map<String, String>
contentDetails) {
PackageBits bits = null;
//use or instantiate PackageBits instance.
@@ -1608,7 +1606,7 @@ public class ContentManagerBean implements ContentManagerLocal,
ContentManagerRe
bits = entityManager.find(PackageBits.class, bits.getId());
//write data from stream into db using Hibernate Blob mechanism
- updateBlobStream(packageBitStream, bits);
+ updateBlobStream(packageBitStream, bits, contentDetails);
return bits;
}
@@ -1617,8 +1615,9 @@ public class ContentManagerBean implements ContentManagerLocal,
ContentManagerRe
* Blob mechanism with PreparedStatements. As all content into Bits are not stored
as type OID, t
*
* @param stream
+ * @param contentDetails Map to store content details in used in PackageVersioning
*/
- public void updateBlobStream(InputStream stream, PackageBits bits) {
+ public void updateBlobStream(InputStream stream, PackageBits bits, Map<String,
String> contentDetails) {
//TODO: are there any db specific limits that we should check/verify here before
stuffing
// the contents of a stream into the db? Should we just let the db complain and
take care of
@@ -1632,7 +1631,7 @@ public class ContentManagerBean implements ContentManagerLocal,
ContentManagerRe
//persist to db if not already
if (bits.getId() <= 0) {
- entityManager.persist(this);
+ entityManager.persist(bits);
}
//locate the existing PackageBits instance
@@ -1647,7 +1646,6 @@ public class ContentManagerBean implements ContentManagerLocal,
ContentManagerRe
Connection conn = null;
PreparedStatement ps = null;
PreparedStatement ps2 = null;
- DataSource ds = null;
try {
conn = dataSource.getConnection();
@@ -1666,7 +1664,7 @@ public class ContentManagerBean implements ContentManagerLocal,
ContentManagerRe
Blob blb = rs.getBlob(1);
//copy the stream to the Blob
- long transferred = StreamUtil.copy(stream, blb.setBinaryStream(1),
false);
+ long transferred = copyAndDigest(stream, blb.setBinaryStream(1),
false, contentDetails);
stream.close();
//populate the prepared statement for update
@@ -1687,6 +1685,37 @@ public class ContentManagerBean implements ContentManagerLocal,
ContentManagerRe
log.error("An error occurred while updating Blob with stream for
PackageBits[" + bits.getId() + "], "
+ e.getMessage());
e.printStackTrace();
+ } finally {
+ if (ps != null) {
+ try {
+ ps.close();
+ } catch (Exception e) {
+ log.warn("Failed to close prepared statement for package bits
[" + bits.getId() + "]");
+ }
+ }
+
+ if (ps2 != null) {
+ try {
+ ps2.close();
+ } catch (Exception e) {
+ log.warn("Failed to close prepared statement for package bits
[" + bits.getId() + "]");
+ }
+ }
+
+ if (conn != null) {
+ try {
+ conn.close();
+ } catch (Exception e) {
+ log.warn("Failed to close connection for package bits [" +
bits.getId() + "]");
+ }
+ }
+ if (stream != null) {
+ try {
+ stream.close();
+ } catch (Exception e) {
+ log.warn("Failed to close stream to package bits located at
[" + +bits.getId() + "]");
+ }
+ }
}
entityManager.merge(bits);
@@ -1717,7 +1746,6 @@ public class ContentManagerBean implements ContentManagerLocal,
ContentManagerRe
ps.setInt(1, bits.getId());
ResultSet results = ps.executeQuery();
if (results.next()) {
- // results.next();
//retrieve the Blob
Blob blob = results.getBlob(1);
//now copy the contents to the stream passed in
@@ -1728,4 +1756,63 @@ public class ContentManagerBean implements ContentManagerLocal,
ContentManagerRe
ex.printStackTrace();
}
}
+
+ /** Functions same as StreamUtil.copy(), but calculates SHA hash and file size and
write it to
+ * the Map<String,String> passed in.
+ *
+ * @param input
+ * @param output
+ * @param closeStreams
+ * @param contentDetails
+ * @return
+ * @throws RuntimeException
+ */
+ private long copyAndDigest(InputStream input, OutputStream output, boolean
closeStreams,
+ Map<String, String> contentDetails) throws RuntimeException {
+ long numBytesCopied = 0;
+ int bufferSize = 32768;
+ MessageDigestGenerator digestGenerator = null;
+ if (contentDetails != null) {
+ digestGenerator = new
MessageDigestGenerator(MessageDigestGenerator.SHA_256);
+ }
+ try {
+ // make sure we buffer the input
+ input = new BufferedInputStream(input, bufferSize);
+
+ byte[] buffer = new byte[bufferSize];
+
+ for (int bytesRead = input.read(buffer); bytesRead != -1; bytesRead =
input.read(buffer)) {
+ output.write(buffer, 0, bytesRead);
+ numBytesCopied += bytesRead;
+ if (digestGenerator != null) {
+ digestGenerator.add(buffer, 0, bytesRead);
+ }
+ }
+
+ if (contentDetails != null) {//if we're calculating a digest as well
+ contentDetails.put(UPLOAD_FILE_SIZE, String.valueOf(numBytesCopied));
+ contentDetails.put(UPLOAD_SHA256, digestGenerator.getDigestString());
+ }
+ output.flush();
+ } catch (IOException ioe) {
+ throw new RuntimeException("Stream data cannot be copied", ioe);
+ } finally {
+ if (closeStreams) {
+ try {
+ output.close();
+ } catch (IOException ioe2) {
+ log.warn("Streams could not be closed", ioe2);
+ }
+
+ try {
+ input.close();
+ } catch (IOException ioe2) {
+ log.warn("Streams could not be closed", ioe2);
+ }
+ }
+ }
+
+ return numBytesCopied;
+ }
+
}
\ No newline at end of file
diff --git
a/modules/enterprise/server/jar/src/main/java/org/rhq/enterprise/server/content/ContentManagerLocal.java
b/modules/enterprise/server/jar/src/main/java/org/rhq/enterprise/server/content/ContentManagerLocal.java
index 65b0365..0b0e5a5 100644
---
a/modules/enterprise/server/jar/src/main/java/org/rhq/enterprise/server/content/ContentManagerLocal.java
+++
b/modules/enterprise/server/jar/src/main/java/org/rhq/enterprise/server/content/ContentManagerLocal.java
@@ -349,6 +349,6 @@ public interface ContentManagerLocal {
void writeBlobOutToStream(OutputStream stream, PackageBits bits, boolean
closeStreams);
- void updateBlobStream(InputStream stream, PackageBits bits);
+ void updateBlobStream(InputStream stream, PackageBits bits, Map<String, String>
contentDetails);
}
\ No newline at end of file
commit 8bbfe126b897a21135b21987aefcce1b45e76cb3
Author: Simeon Pinder <spinder(a)redhat.com>
Date: Thu Jun 24 14:33:18 2010 -0400
refactoring BZ-597246 a little.
diff --git
a/modules/core/domain/src/main/java/org/rhq/core/domain/content/PackageBits.java
b/modules/core/domain/src/main/java/org/rhq/core/domain/content/PackageBits.java
index 634ebeb..17ad568 100644
--- a/modules/core/domain/src/main/java/org/rhq/core/domain/content/PackageBits.java
+++ b/modules/core/domain/src/main/java/org/rhq/core/domain/content/PackageBits.java
@@ -95,14 +95,20 @@ public class PackageBits implements Serializable {
* return <code>null</code>. In this case, the content is probably stored
somewhere else on a local file system.
* When <code>null</code> is returned, it is assumed that who ever needs
the content can know where to find it based
* on the {@link PackageVersion} details.
+ *
+ * For large file contents, you should use ContentManager.updateBlobStream() to write
and
+ * ContentManager.writeBlobOutToStream() to read/direct file contents into as no
byte[] is used.
*/
@XmlTransient
public byte[] getBits() {
return bits;
}
+ /** For large file contents, you should use ContentManager.updateBlobStream() to
write and
+ * ContentManager.writeBlobOutToStream() to read/direct file contents into as no
byte[] is used.
+ *
+ */
public void setBits(byte[] bits) {
this.bits = bits;
}
-
}
\ No newline at end of file
diff --git
a/modules/enterprise/server/jar/src/main/java/org/rhq/enterprise/server/content/ContentManagerBean.java
b/modules/enterprise/server/jar/src/main/java/org/rhq/enterprise/server/content/ContentManagerBean.java
index 5bae392..b2cac05 100644
---
a/modules/enterprise/server/jar/src/main/java/org/rhq/enterprise/server/content/ContentManagerBean.java
+++
b/modules/enterprise/server/jar/src/main/java/org/rhq/enterprise/server/content/ContentManagerBean.java
@@ -939,7 +939,6 @@ public class ContentManagerBean implements ContentManagerLocal,
ContentManagerRe
try {
PackageBits bits = entityManager.find(PackageBits.class,
packageBits.getId());
- // bits.setBits(StreamUtil.slurp(bitStream));
String pkgName = "(set packageName)";
if ((packageVersion != null) &&
(packageVersion.getGeneralPackage() != null)) {
//update it to whatever package name is if we can get to it.
@@ -947,7 +946,6 @@ public class ContentManagerBean implements ContentManagerLocal,
ContentManagerRe
}
bits = loadPackageBits(bitStream, packageVersion.getId(), pkgName,
packageVersion.getVersion(),
bits);
- // bits.setBits(StreamUtil.slurp(bitStream));
entityManager.merge(bits);
} finally {
@@ -1237,31 +1235,14 @@ public class ContentManagerBean implements ContentManagerLocal,
ContentManagerRe
PackageVersion newPackageVersion = new PackageVersion(existingPackage, version,
architecture);
newPackageVersion.setDisplayName(existingPackage.getName());
- // PackageBits bits = loadPackageBits(packageBitStream);
PackageBits bits = loadPackageBits(packageBitStream, newPackageVersion.getId(),
packageName, version, null);
- // // TODO: THIS IS VERY BAD - MUST FIX - DO NOT SLURP THE ENTIRE FILE IN
MEMORY - USE JDBC STREAMING
- // // Write the content into the newly created package version. This may
eventually move, but for now we'll just
- // // use the byte array in the package version to store the bits.
- // byte[] packageBits;
- // try {
- // packageBits = StreamUtil.slurp(packageBitStream);
- // } catch (RuntimeException re) {
- // throw new RuntimeException("Error reading in the package
file", re);
- // }
- //
- // PackageBits bits = new PackageBits();
- // try {
- // bits.setBits(packageBits);
- // } catch (Exception e) {
- // log.error("Error saving the package.", e);
- // }
newPackageVersion.setPackageBits(bits);
+ //TODO: write down to file system to calculate length and calculate hash.
newPackageVersion.setFileSize((long) bits.getBits().length);
try {
newPackageVersion.setSHA256(new
MessageDigestGenerator(MessageDigestGenerator.SHA_256)
.calcDigestString(bits.getBits()));
- // .calcDigestString(packageBits));
} catch (IOException e) {
newPackageVersion.setSHA256(null);
}
@@ -1579,23 +1560,6 @@ public class ContentManagerBean implements ContentManagerLocal,
ContentManagerRe
//get the data
PackageBits bits = loadPackageBits(packageBitStream, packageVersion.getId(),
packageName, version, null);
- // // TODO: THIS IS VERY BAD - MUST FIX - DO NOT SLURP THE ENTIRE FILE IN
MEMORY - USE JDBC STREAMING
- // // Write the content into the newly created package version. This may
eventually move, but for now we'll just
- // // use the byte array in the package version to store the bits.
- // byte[] packageBits;
- // try {
- // packageBits = StreamUtil.slurp(packageBitStream);
- // } catch (RuntimeException re) {
- // throw new RuntimeException("Error reading in the package
file", re);
- // }
- //
- // PackageBits bits = new PackageBits();
- // // bits = new PackageBits();
- // try {
- // bits.setBits(packageBits);
- // } catch (Exception e) {
- // log.error("Error savinf the package.", e);
- // }
packageVersion.setPackageBits(bits);
@@ -1643,10 +1607,8 @@ public class ContentManagerBean implements ContentManagerLocal,
ContentManagerRe
bits = entityManager.find(PackageBits.class, bits.getId());
- Subject user = null;
-
//write data from stream into db using Hibernate Blob mechanism
- updateBlobStream(user, packageBitStream, bits);
+ updateBlobStream(packageBitStream, bits);
return bits;
}
@@ -1656,19 +1618,7 @@ public class ContentManagerBean implements ContentManagerLocal,
ContentManagerRe
*
* @param stream
*/
- public void updateBlobStream(Subject user, InputStream stream, PackageBits bits) {
-
- // int resourceId = -1;
- // if (bits != null) {
- // resourceId = bits.getId();
- // }
- // log.info("Updating data for PackageBits [" + bits + "]
with resource ID [" + resourceId + "]");
- //
- // // Check permissions first
- // if (!authorizationManager.hasResourcePermission(user,
Permission.MANAGE_CONTENT, resourceId)) {
- // throw new PermissionException("User [" + user.getName()
- // + "] does not have permission to update packageBits "
+ bits + " with resource ID [" + resourceId + "]");
- // }
+ public void updateBlobStream(InputStream stream, PackageBits bits) {
//TODO: are there any db specific limits that we should check/verify here before
stuffing
// the contents of a stream into the db? Should we just let the db complain and
take care of
@@ -1747,24 +1697,11 @@ public class ContentManagerBean implements ContentManagerLocal,
ContentManagerRe
*
* @param stream non null stream where contents to be written to.
*/
- public void writeBlobOutToStream(Subject user, OutputStream stream, PackageBits bits,
boolean closeStreams) {
-
- // int resourceId = -1;
- // if (bits != null) {
- // resourceId = bits.getId();
- // }
- // log.info("Reading data for PackageBits [" + bits + "]
with resource ID [" + resourceId + "]");
- //
- // // Check permissions first
- // if (!authorizationManager.hasResourcePermission(user,
Permission.MANAGE_CONTENT, resourceId)) {
- // throw new PermissionException("User [" + user.getName() +
"] does not have permission to read packageBits "
- // + bits + " with resource ID [" + resourceId +
"]");
- // }
+ public void writeBlobOutToStream(OutputStream stream, PackageBits bits, boolean
closeStreams) {
if (stream == null) {
return; // no locate to write to
}
- // if ((bits == null) || (bits.getId() <= 0) || bits.getBits() == null)
{
if ((bits == null) || (bits.getId() <= 0)) {
//then PackageBits instance passed in is insufficiently initialized.
log.warn("PackageBits insufficiently initialized. No data to write
out.");
diff --git
a/modules/enterprise/server/jar/src/main/java/org/rhq/enterprise/server/content/ContentManagerLocal.java
b/modules/enterprise/server/jar/src/main/java/org/rhq/enterprise/server/content/ContentManagerLocal.java
index 83c0037..65b0365 100644
---
a/modules/enterprise/server/jar/src/main/java/org/rhq/enterprise/server/content/ContentManagerLocal.java
+++
b/modules/enterprise/server/jar/src/main/java/org/rhq/enterprise/server/content/ContentManagerLocal.java
@@ -347,8 +347,8 @@ public interface ContentManagerLocal {
byte[] getPackageBytes(Subject user, int resourceId, int installedPackageId);
- void writeBlobOutToStream(Subject user, OutputStream stream, PackageBits bits,
boolean closeStreams);
+ void writeBlobOutToStream(OutputStream stream, PackageBits bits, boolean
closeStreams);
- void updateBlobStream(Subject user, InputStream stream, PackageBits bits);
+ void updateBlobStream(InputStream stream, PackageBits bits);
}
\ No newline at end of file
diff --git
a/modules/enterprise/server/jar/src/test/java/org/rhq/enterprise/server/content/test/ContentUIManagerBeanTest.java
b/modules/enterprise/server/jar/src/test/java/org/rhq/enterprise/server/content/test/ContentUIManagerBeanTest.java
index c8ac4e3..7fae6e0 100644
---
a/modules/enterprise/server/jar/src/test/java/org/rhq/enterprise/server/content/test/ContentUIManagerBeanTest.java
+++
b/modules/enterprise/server/jar/src/test/java/org/rhq/enterprise/server/content/test/ContentUIManagerBeanTest.java
@@ -29,7 +29,6 @@ import org.apache.commons.logging.LogFactory;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;
-import org.rhq.core.domain.auth.Subject;
import org.rhq.core.domain.content.Architecture;
import org.rhq.core.domain.content.Package;
import org.rhq.core.domain.content.PackageBits;
@@ -285,8 +284,7 @@ public class ContentUIManagerBeanTest extends AbstractEJB3Test {
}
FileInputStream fos = new FileInputStream(originalBinary);
- Subject empty = null;
- contentManager.updateBlobStream(empty, fos, packageBits);
+ contentManager.updateBlobStream(fos, packageBits);
packageBits = em.find(PackageBits.class, packageBits.getId());
// test that the bits are available and stored in the DB: Reading the Blob
@@ -298,7 +296,7 @@ public class ContentUIManagerBeanTest extends AbstractEJB3Test {
assert composite.isPackageBitsInDatabase();
FileOutputStream outputStream = new FileOutputStream(retrieved);
- contentManager.writeBlobOutToStream(empty, outputStream, packageBits,
false);
+ contentManager.writeBlobOutToStream(outputStream, packageBits, false);
//Check that db content equal to file system content
String originalDigest = new
MessageDigestGenerator(MessageDigestGenerator.SHA_256)
commit 408bc20049ff0f06738e70b2dbafdbbd83366fdb
Author: Simeon Pinder <spinder(a)redhat.com>
Date: Thu Jun 24 11:03:02 2010 -0400
BZ-597246: changed StreamUtil.slurp to use Hibernate Blob mechanism, and added unit
test for read/write Blob data.
diff --git
a/modules/core/domain/src/main/java/org/rhq/core/domain/content/PackageBits.java
b/modules/core/domain/src/main/java/org/rhq/core/domain/content/PackageBits.java
index 2bf059c..634ebeb 100644
--- a/modules/core/domain/src/main/java/org/rhq/core/domain/content/PackageBits.java
+++ b/modules/core/domain/src/main/java/org/rhq/core/domain/content/PackageBits.java
@@ -104,4 +104,5 @@ public class PackageBits implements Serializable {
public void setBits(byte[] bits) {
this.bits = bits;
}
+
}
\ No newline at end of file
diff --git
a/modules/enterprise/server/jar/src/main/java/org/rhq/enterprise/server/content/ContentManagerBean.java
b/modules/enterprise/server/jar/src/main/java/org/rhq/enterprise/server/content/ContentManagerBean.java
index 244478e..5bae392 100644
---
a/modules/enterprise/server/jar/src/main/java/org/rhq/enterprise/server/content/ContentManagerBean.java
+++
b/modules/enterprise/server/jar/src/main/java/org/rhq/enterprise/server/content/ContentManagerBean.java
@@ -24,8 +24,11 @@ import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
+import java.io.OutputStream;
+import java.sql.Blob;
import java.sql.Connection;
import java.sql.PreparedStatement;
+import java.sql.ResultSet;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
@@ -936,7 +939,15 @@ public class ContentManagerBean implements ContentManagerLocal,
ContentManagerRe
try {
PackageBits bits = entityManager.find(PackageBits.class,
packageBits.getId());
- bits.setBits(StreamUtil.slurp(bitStream));
+ // bits.setBits(StreamUtil.slurp(bitStream));
+ String pkgName = "(set packageName)";
+ if ((packageVersion != null) &&
(packageVersion.getGeneralPackage() != null)) {
+ //update it to whatever package name is if we can get to it.
+ pkgName = packageVersion.getGeneralPackage().getName();
+ }
+ bits = loadPackageBits(bitStream, packageVersion.getId(), pkgName,
packageVersion.getVersion(),
+ bits);
+ // bits.setBits(StreamUtil.slurp(bitStream));
entityManager.merge(bits);
} finally {
@@ -1227,28 +1238,30 @@ public class ContentManagerBean implements ContentManagerLocal,
ContentManagerRe
newPackageVersion.setDisplayName(existingPackage.getName());
// PackageBits bits = loadPackageBits(packageBitStream);
- // TODO: THIS IS VERY BAD - MUST FIX - DO NOT SLURP THE ENTIRE FILE IN MEMORY -
USE JDBC STREAMING
- // Write the content into the newly created package version. This may eventually
move, but for now we'll just
- // use the byte array in the package version to store the bits.
- byte[] packageBits;
- try {
- packageBits = StreamUtil.slurp(packageBitStream);
- } catch (RuntimeException re) {
- throw new RuntimeException("Error reading in the package file",
re);
- }
-
- PackageBits bits = new PackageBits();
- try {
- bits.setBits(packageBits);
- } catch (Exception e) {
- log.error("Error saving the package.", e);
- }
+ PackageBits bits = loadPackageBits(packageBitStream, newPackageVersion.getId(),
packageName, version, null);
+ // // TODO: THIS IS VERY BAD - MUST FIX - DO NOT SLURP THE ENTIRE FILE IN
MEMORY - USE JDBC STREAMING
+ // // Write the content into the newly created package version. This may
eventually move, but for now we'll just
+ // // use the byte array in the package version to store the bits.
+ // byte[] packageBits;
+ // try {
+ // packageBits = StreamUtil.slurp(packageBitStream);
+ // } catch (RuntimeException re) {
+ // throw new RuntimeException("Error reading in the package
file", re);
+ // }
+ //
+ // PackageBits bits = new PackageBits();
+ // try {
+ // bits.setBits(packageBits);
+ // } catch (Exception e) {
+ // log.error("Error saving the package.", e);
+ // }
newPackageVersion.setPackageBits(bits);
newPackageVersion.setFileSize((long) bits.getBits().length);
try {
newPackageVersion.setSHA256(new
MessageDigestGenerator(MessageDigestGenerator.SHA_256)
- .calcDigestString(packageBits));
+ .calcDigestString(bits.getBits()));
+ // .calcDigestString(packageBits));
} catch (IOException e) {
newPackageVersion.setSHA256(null);
}
@@ -1564,24 +1577,25 @@ public class ContentManagerBean implements ContentManagerLocal,
ContentManagerRe
entityManager.persist(packageVersion);
}
- //get the data and persist/merge packageVersion
- // PackageBits bits = loadPackageBits(packageBitStream);
- // TODO: THIS IS VERY BAD - MUST FIX - DO NOT SLURP THE ENTIRE FILE IN MEMORY -
USE JDBC STREAMING
- // Write the content into the newly created package version. This may eventually
move, but for now we'll just
- // use the byte array in the package version to store the bits.
- byte[] packageBits;
- try {
- packageBits = StreamUtil.slurp(packageBitStream);
- } catch (RuntimeException re) {
- throw new RuntimeException("Error reading in the package file",
re);
- }
-
- PackageBits bits = new PackageBits();
- try {
- bits.setBits(packageBits);
- } catch (Exception e) {
- log.error("Error savinf the package.", e);
- }
+ //get the data
+ PackageBits bits = loadPackageBits(packageBitStream, packageVersion.getId(),
packageName, version, null);
+ // // TODO: THIS IS VERY BAD - MUST FIX - DO NOT SLURP THE ENTIRE FILE IN
MEMORY - USE JDBC STREAMING
+ // // Write the content into the newly created package version. This may
eventually move, but for now we'll just
+ // // use the byte array in the package version to store the bits.
+ // byte[] packageBits;
+ // try {
+ // packageBits = StreamUtil.slurp(packageBitStream);
+ // } catch (RuntimeException re) {
+ // throw new RuntimeException("Error reading in the package
file", re);
+ // }
+ //
+ // PackageBits bits = new PackageBits();
+ // // bits = new PackageBits();
+ // try {
+ // bits.setBits(packageBits);
+ // } catch (Exception e) {
+ // log.error("Error savinf the package.", e);
+ // }
packageVersion.setPackageBits(bits);
@@ -1604,26 +1618,177 @@ public class ContentManagerBean implements ContentManagerLocal,
ContentManagerRe
/** Pulls in package bits from the stream. Currently inefficient.
*
* @param packageBitStream
+ * @param packageVersionId
* @return PackageBits ref populated.
*/
- private PackageBits loadPackageBits(InputStream packageBitStream) {
+ @TransactionAttribute(TransactionAttributeType.REQUIRES_NEW)
+ private PackageBits loadPackageBits(InputStream packageBitStream, int
packageVersionId, String packageName,
+ String packageVersion, PackageBits existingPkgBits) {
PackageBits bits = null;
- // TODO: THIS IS VERY BAD - MUST FIX - DO NOT SLURP THE ENTIRE FILE IN MEMORY -
USE JDBC STREAMING
- // Write the content into the newly created package version. This may eventually
move, but for now we'll just
- // use the byte array in the package version to store the bits.
- byte[] packageBits;
- try {
- packageBits = StreamUtil.slurp(packageBitStream);
- } catch (RuntimeException re) {
- throw new RuntimeException("Error reading in the package file",
re);
+
+ //use or instantiate PackageBits instance.
+ if (existingPkgBits == null) {
+ bits = new PackageBits();
+ entityManager.persist(bits);
+ } else {
+ bits = existingPkgBits;
}
- bits = new PackageBits();
+ //locate related packageVersion
+ PackageVersion pv = entityManager.find(PackageVersion.class, packageVersionId);
+
+ //associate the two
+ pv.setPackageBits(bits);
+ entityManager.flush(); // push the new package bits row to the DB
+
+ bits = entityManager.find(PackageBits.class, bits.getId());
+
+ Subject user = null;
+
+ //write data from stream into db using Hibernate Blob mechanism
+ updateBlobStream(user, packageBitStream, bits);
+
+ return bits;
+ }
+
+ /** Takes an input stream and copies it into the PackageBits table using Hibernate
+ * Blob mechanism with PreparedStatements. As all content into Bits are not stored
as type OID, t
+ *
+ * @param stream
+ */
+ public void updateBlobStream(Subject user, InputStream stream, PackageBits bits) {
+
+ // int resourceId = -1;
+ // if (bits != null) {
+ // resourceId = bits.getId();
+ // }
+ // log.info("Updating data for PackageBits [" + bits + "]
with resource ID [" + resourceId + "]");
+ //
+ // // Check permissions first
+ // if (!authorizationManager.hasResourcePermission(user,
Permission.MANAGE_CONTENT, resourceId)) {
+ // throw new PermissionException("User [" + user.getName()
+ // + "] does not have permission to update packageBits "
+ bits + " with resource ID [" + resourceId + "]");
+ // }
+
+ //TODO: are there any db specific limits that we should check/verify here before
stuffing
+ // the contents of a stream into the db? Should we just let the db complain and
take care of
+ // input validation?
+ if (stream == null) {
+ return; // no stream content to update.
+ }
+ if (bits == null) {
+ bits = new PackageBits();
+ }
+
+ //persist to db if not already
+ if (bits.getId() <= 0) {
+ entityManager.persist(this);
+ }
+
+ //locate the existing PackageBits instance
+ bits = entityManager.find(PackageBits.class, bits.getId());
+
+ ///initialize Bits field so non-empty.
+ bits.setBits(new String("a").getBytes());
+ entityManager.merge(bits);
+ entityManager.flush();
+
+ //Create prepared statements to work with Blobs and hibernate.
+ Connection conn = null;
+ PreparedStatement ps = null;
+ PreparedStatement ps2 = null;
+ DataSource ds = null;
try {
- bits.setBits(packageBits);
+
+ conn = dataSource.getConnection();
+
+ //we are loading the PackageBits saved in the previous step
+ //we need to lock the row which will be updated so we are using FOR UPDATE
+ ps = conn.prepareStatement("SELECT BITS FROM " +
PackageBits.TABLE_NAME + " WHERE ID = ? FOR UPDATE");
+ ps.setInt(1, bits.getId());
+ ResultSet rs = ps.executeQuery();
+ if (rs != null) {
+ while (rs.next()) {
+
+ //We can not create a blob directly because BlobImpl from Hibernate
is not acceptable
+ //for oracle and Connection.createBlob is not working on postgres.
+ //This blob will be not empty because we saved there a bytes from
String("a").
+ Blob blb = rs.getBlob(1);
+
+ //copy the stream to the Blob
+ long transferred = StreamUtil.copy(stream, blb.setBinaryStream(1),
false);
+ stream.close();
+
+ //populate the prepared statement for update
+ ps2 = conn.prepareStatement("UPDATE " +
PackageBits.TABLE_NAME + " SET bits = ? where id = ?");
+ ps2.setBlob(1, blb);
+ ps2.setInt(2, bits.getId());
+
+ //initiate the update.
+ if (ps2.execute()) {
+ throw new Exception("Unable to upload the package bits to
the DB:");
+ }
+ ps2.close();
+ }
+ }
+ ps.close();
+ conn.close();
} catch (Exception e) {
- log.error("Error saving the package.", e);
+ log.error("An error occurred while updating Blob with stream for
PackageBits[" + bits.getId() + "], "
+ + e.getMessage());
+ e.printStackTrace();
+ }
+
+ entityManager.merge(bits);
+ entityManager.flush();
+ }
+
+ /**Writes the contents of a the Blob out to the stream passed in.
+ *
+ * @param stream non null stream where contents to be written to.
+ */
+ public void writeBlobOutToStream(Subject user, OutputStream stream, PackageBits bits,
boolean closeStreams) {
+
+ // int resourceId = -1;
+ // if (bits != null) {
+ // resourceId = bits.getId();
+ // }
+ // log.info("Reading data for PackageBits [" + bits + "]
with resource ID [" + resourceId + "]");
+ //
+ // // Check permissions first
+ // if (!authorizationManager.hasResourcePermission(user,
Permission.MANAGE_CONTENT, resourceId)) {
+ // throw new PermissionException("User [" + user.getName() +
"] does not have permission to read packageBits "
+ // + bits + " with resource ID [" + resourceId +
"]");
+ // }
+
+ if (stream == null) {
+ return; // no locate to write to
+ }
+ // if ((bits == null) || (bits.getId() <= 0) || bits.getBits() == null)
{
+ if ((bits == null) || (bits.getId() <= 0)) {
+ //then PackageBits instance passed in is insufficiently initialized.
+ log.warn("PackageBits insufficiently initialized. No data to write
out.");
+ return;
+ }
+ try {
+ //open connection
+ Connection conn = dataSource.getConnection();
+
+ //prepared statement for retrieval of Blob.bits
+ PreparedStatement ps = conn
+ .prepareStatement("SELECT BITS FROM " + PackageBits.TABLE_NAME
+ " WHERE ID = ?");
+ ps.setInt(1, bits.getId());
+ ResultSet results = ps.executeQuery();
+ if (results.next()) {
+ // results.next();
+ //retrieve the Blob
+ Blob blob = results.getBlob(1);
+ //now copy the contents to the stream passed in
+ StreamUtil.copy(blob.getBinaryStream(), stream, closeStreams);
+ }
+ } catch (Exception ex) {
+ log.error("An error occurred while writing Blob contents out to stream
:" + ex.getMessage());
+ ex.printStackTrace();
}
- return bits;
}
-}
+}
\ No newline at end of file
diff --git
a/modules/enterprise/server/jar/src/main/java/org/rhq/enterprise/server/content/ContentManagerLocal.java
b/modules/enterprise/server/jar/src/main/java/org/rhq/enterprise/server/content/ContentManagerLocal.java
index fc128a4..83c0037 100644
---
a/modules/enterprise/server/jar/src/main/java/org/rhq/enterprise/server/content/ContentManagerLocal.java
+++
b/modules/enterprise/server/jar/src/main/java/org/rhq/enterprise/server/content/ContentManagerLocal.java
@@ -19,6 +19,7 @@
package org.rhq.enterprise.server.content;
import java.io.InputStream;
+import java.io.OutputStream;
import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -32,6 +33,7 @@ import org.rhq.core.domain.content.Architecture;
import org.rhq.core.domain.content.ContentServiceRequest;
import org.rhq.core.domain.content.InstalledPackage;
import org.rhq.core.domain.content.Package;
+import org.rhq.core.domain.content.PackageBits;
import org.rhq.core.domain.content.PackageDetailsKey;
import org.rhq.core.domain.content.PackageType;
import org.rhq.core.domain.content.PackageVersion;
@@ -211,9 +213,8 @@ public interface ContentManagerLocal {
PackageVersion createPackageVersion(String packageName, int packageTypeId, String
version, int architectureId,
InputStream packageBitStream);
- PackageVersion getUploadedPackageVersion(String packageName, int packageTypeId,
String version,
- int architectureId, InputStream packageBitStream, Map<String, String>
packageUploadDetails,
- int newResourceTypeId);
+ PackageVersion getUploadedPackageVersion(String packageName, int packageTypeId,
String version, int architectureId,
+ InputStream packageBitStream, Map<String, String> packageUploadDetails, int
newResourceTypeId);
/**
* Very simple method that persists the given package version within its own
transaction.
@@ -346,4 +347,8 @@ public interface ContentManagerLocal {
byte[] getPackageBytes(Subject user, int resourceId, int installedPackageId);
+ void writeBlobOutToStream(Subject user, OutputStream stream, PackageBits bits,
boolean closeStreams);
+
+ void updateBlobStream(Subject user, InputStream stream, PackageBits bits);
+
}
\ No newline at end of file
diff --git
a/modules/enterprise/server/jar/src/test/java/org/rhq/enterprise/server/content/test/ContentUIManagerBeanTest.java
b/modules/enterprise/server/jar/src/test/java/org/rhq/enterprise/server/content/test/ContentUIManagerBeanTest.java
index b83695f..c8ac4e3 100644
---
a/modules/enterprise/server/jar/src/test/java/org/rhq/enterprise/server/content/test/ContentUIManagerBeanTest.java
+++
b/modules/enterprise/server/jar/src/test/java/org/rhq/enterprise/server/content/test/ContentUIManagerBeanTest.java
@@ -18,6 +18,10 @@
*/
package org.rhq.enterprise.server.content.test;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+
import javax.persistence.EntityManager;
import org.apache.commons.logging.Log;
@@ -25,6 +29,7 @@ import org.apache.commons.logging.LogFactory;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;
+import org.rhq.core.domain.auth.Subject;
import org.rhq.core.domain.content.Architecture;
import org.rhq.core.domain.content.Package;
import org.rhq.core.domain.content.PackageBits;
@@ -32,7 +37,9 @@ import org.rhq.core.domain.content.PackageType;
import org.rhq.core.domain.content.PackageVersion;
import org.rhq.core.domain.content.composite.LoadedPackageBitsComposite;
import org.rhq.core.domain.resource.Resource;
+import org.rhq.core.util.MessageDigestGenerator;
import org.rhq.enterprise.server.auth.SubjectManagerLocal;
+import org.rhq.enterprise.server.content.ContentManagerLocal;
import org.rhq.enterprise.server.content.ContentUIManagerLocal;
import org.rhq.enterprise.server.test.AbstractEJB3Test;
import org.rhq.enterprise.server.util.LookupUtil;
@@ -52,6 +59,7 @@ public class ContentUIManagerBeanTest extends AbstractEJB3Test {
private ContentUIManagerLocal contentUIManager;
private SubjectManagerLocal subjectManager;
+ private ContentManagerLocal contentManager;
// Setup --------------------------------------------
@@ -59,6 +67,7 @@ public class ContentUIManagerBeanTest extends AbstractEJB3Test {
public void setupBeforeClass() throws Exception {
contentUIManager = LookupUtil.getContentUIManager();
subjectManager = LookupUtil.getSubjectManager();
+ contentManager = LookupUtil.getContentManager();
}
// Test Cases --------------------------------------------
@@ -214,4 +223,94 @@ public class ContentUIManagerBeanTest extends AbstractEJB3Test {
getTransactionManager().rollback();
}
}
+
+ @Test(enabled = ENABLE_TESTS)
+ public void testPackageBitsBlobStream() throws Throwable {
+ LoadedPackageBitsComposite composite;
+
+ getTransactionManager().begin();
+
+ try {
+ EntityManager em = getEntityManager();
+ Resource resource = SessionTestHelper.createNewResource(em,
"testPkgBitsLargeResource");
+ PackageType pkgType = new PackageType("testPkgBitsLargePT",
resource.getResourceType());
+ org.rhq.core.domain.content.Package pkg = new
Package("testPkgBitsLargeP", pkgType);
+ Architecture arch = new Architecture("testPkgLargeArch");
+ PackageVersion pkgVer = new PackageVersion(pkg, "1", arch);
+
+ em.persist(pkgType);
+ em.persist(pkg);
+ em.persist(arch);
+ em.persist(pkgVer);
+ em.flush();
+
+ // test that no bits are available right now
+ composite = contentUIManager.getLoadedPackageBitsComposite(pkgVer.getId());
+ assert composite != null;
+ assert composite.getPackageVersionId() == pkgVer.getId();
+ assert composite.getPackageBitsId() == null;
+ assert !composite.isPackageBitsAvailable();
+ assert !composite.isPackageBitsInDatabase();
+
+ // pretend we loaded the bits, but we stored them somewhere other then the
DB
+ PackageBits packageBits = new PackageBits();
+ em.persist(packageBits);
+ pkgVer.setPackageBits(packageBits);
+ pkgVer = em.merge(pkgVer);
+ em.flush();
+
+ // test that the bits are available, but are not stored in the DB
+ composite = contentUIManager.getLoadedPackageBitsComposite(pkgVer.getId());
+ assert composite != null;
+ assert composite.getPackageVersionId() == pkgVer.getId();
+ assert composite.getPackageBitsId() == packageBits.getId();
+ assert composite.isPackageBitsAvailable();
+ assert !composite.isPackageBitsInDatabase();
+
+ // let's make sure there really is no data in the DB
+ packageBits = em.find(PackageBits.class, packageBits.getId());
+ assert packageBits != null;
+ assert packageBits.getBits() == null;
+
+ // now lets store some bits in the DB using PreparedStatements and BLOB
mechanism
+ // to simulate large file transfers where streaming is used instead of
reading entire
+ // contents into memory every time.
+
+ //any jar should be fine. Use generated server jar
+ File originalBinary = new
File("./target/rhq-enterprise-server.jar");
+ //destination once pulled from db
+ File retrieved = new File("./target/pulled.jar");
+ if (retrieved.exists()) {
+ assertTrue("Unable to delete ./target/pulled.jar for test
cleanup.", retrieved.delete());
+ }
+ FileInputStream fos = new FileInputStream(originalBinary);
+
+ Subject empty = null;
+ contentManager.updateBlobStream(empty, fos, packageBits);
+ packageBits = em.find(PackageBits.class, packageBits.getId());
+
+ // test that the bits are available and stored in the DB: Reading the Blob
+ composite = contentUIManager.getLoadedPackageBitsComposite(pkgVer.getId());
+ assert composite != null;
+ assert composite.getPackageVersionId() == pkgVer.getId();
+ assert composite.getPackageBitsId() == packageBits.getId();
+ assert composite.isPackageBitsAvailable();
+ assert composite.isPackageBitsInDatabase();
+
+ FileOutputStream outputStream = new FileOutputStream(retrieved);
+ contentManager.writeBlobOutToStream(empty, outputStream, packageBits,
false);
+
+ //Check that db content equal to file system content
+ String originalDigest = new
MessageDigestGenerator(MessageDigestGenerator.SHA_256)
+ .calcDigestString(originalBinary);
+ String newDigest = new
MessageDigestGenerator(MessageDigestGenerator.SHA_256).calcDigestString(retrieved);
+ assertEquals("Uploaded and retrieved digests differ:",
originalDigest, newDigest);
+
+ } catch (Throwable t) {
+ t.printStackTrace();
+ throw t;
+ } finally {
+ getTransactionManager().rollback();
+ }
+ }
}
\ No newline at end of file