modules/core/domain/src/main/java/org/rhq/core/domain/resource/Agent.java
| 48 ++++++----
modules/enterprise/server/jar/src/main/java/org/rhq/enterprise/server/core/AgentManagerBean.java
| 27 ++---
modules/enterprise/server/jar/src/main/java/org/rhq/enterprise/server/core/AgentManagerLocal.java
| 47 +++++----
modules/enterprise/server/jar/src/main/java/org/rhq/enterprise/server/measurement/AvailabilityManagerBean.java
| 8 -
4 files changed, 75 insertions(+), 55 deletions(-)
New commits:
commit d01bb14a36a1121d377dc2445d1456c611c37007
Author: Jay Shaughnessy <jshaughn(a)redhat.com>
Date: Mon Dec 9 14:13:52 2013 -0500
[998945] Too many statement timeout exceptions in server log
[999165] Frequent statement timeouts while updating availabilities
I did a visual inspection of code looking for things that may
contribute to the slow updates and/or timeouts reported in these BZs. In
general the updates for availReports (infrequent but can spike in
certain scenarios, 1 per avail report) and availPing
(frequent, 1 per minute per agent) should be very fast and not in
themselves be an issue. Slowness in these queries is likely caused by the
Agent table being locked by some other longer running transaction.
Everything looked pretty clean with one exception. When backfilling an agent
due to an agent shutdown or suspect job detection, it looked like we
may have locked th agent table for the duration of the backfilling, which
can be a fairly large operation for a big agent inventory. I'm not sure
this is the problem, but this commit should allevaite the Agent table
locking while the backfilling is performed.
Also:
- move some inline JPA updates on the Agent to be NamedQuery in the Agent
entity. No need to build those every time we update an agent ping time.
diff --git a/modules/core/domain/src/main/java/org/rhq/core/domain/resource/Agent.java
b/modules/core/domain/src/main/java/org/rhq/core/domain/resource/Agent.java
index 788e189..186c968 100644
--- a/modules/core/domain/src/main/java/org/rhq/core/domain/resource/Agent.java
+++ b/modules/core/domain/src/main/java/org/rhq/core/domain/resource/Agent.java
@@ -75,7 +75,7 @@ import org.rhq.core.domain.cloud.Server;
+ "SELECT a.id " //
+ " FROM Agent a " //
+ " WHERE a.server.name = :serverName " //
- + " AND a.status <> 0 "), //
+ + " AND a.status <> 0 "), //
@NamedQuery(name = Agent.QUERY_FIND_ALL_WITH_STATUS, query = "" //
+ "SELECT a.id " //
+ " FROM Agent a " //
@@ -83,7 +83,7 @@ import org.rhq.core.domain.cloud.Server;
@NamedQuery(name = Agent.QUERY_UPDATE_CLEAR_STATUS_BY_IDS, query = "" //
+ "UPDATE Agent a " //
+ " SET a.status = 0 " //
- + " WHERE a.id IN ( :agentIds ) "), //
+ + " WHERE a.id IN ( :agentIds ) "), //
@NamedQuery(name = Agent.QUERY_FIND_BY_AFFINITY_GROUP, query = "" //
+ "SELECT a " //
+ " FROM Agent a " //
@@ -137,7 +137,16 @@ import org.rhq.core.domain.cloud.Server;
@NamedQuery(name = Agent.QUERY_UPDATE_STATUS_FOR_ALL, query = "" //
+ " UPDATE Agent a " //
+ " SET a.status = -1 " // negative numbers so that bitmask strategy
does not conflict with this one
- + " WHERE a.status = 0 ") //
+ + " WHERE a.status = 0 "), //
+ @NamedQuery(name = Agent.QUERY_UPDATE_LAST_AVAIL_REPORT, query = "" //
+ + " UPDATE Agent a " //
+ + " SET lastAvailabilityReport = :reportTime, backFilled = FALSE "
//
+ + " WHERE id = :agentId "), //
+ @NamedQuery(name = Agent.QUERY_UPDATE_LAST_AVAIL_PING, query = "" //
+ + " UPDATE Agent a " //
+ + " SET lastAvailabilityPing = :now, backFilled = FALSE " //
+ + " WHERE name = :agentName ") //
+
})
@SequenceGenerator(allocationSize = org.rhq.core.domain.util.Constants.ALLOCATION_SIZE,
name = "RHQ_AGENT_ID_SEQ", sequenceName = "RHQ_AGENT_ID_SEQ")
@Table(name = "RHQ_AGENT")
@@ -172,7 +181,10 @@ public class Agent implements Serializable {
public static final String QUERY_UPDATE_STATUS_BY_MEASUREMENT_BASELINE =
"Agent.updateStatusByMeasurementBasleine";
public static final String QUERY_UPDATE_STATUS_BY_AGENT =
"Agent.updateStatusByAgent";
public static final String QUERY_UPDATE_STATUS_FOR_ALL =
"Agent.updateStatusForAll";
-
+
+ public static final String QUERY_UPDATE_LAST_AVAIL_REPORT =
"Agent.updateLastAvailReport";
+ public static final String QUERY_UPDATE_LAST_AVAIL_PING =
"Agent.updateLastAvailPing";
+
// this value is set, when authorized user wants to reset the token
public static final String SECURITY_TOKEN_RESET = "@#$reset$#@";
@@ -237,7 +249,8 @@ public class Agent implements Serializable {
* @param remoteEndpoint
* @param agentToken
*/
- public Agent(@NotNull String name, String address, int port, String remoteEndpoint,
String agentToken) {
+ public Agent(@NotNull
+ String name, String address, int port, String remoteEndpoint, String agentToken) {
this.name = name;
this.address = address;
this.port = port;
@@ -269,7 +282,8 @@ public class Agent implements Serializable {
return this.name;
}
- public void setName(@NotNull String name) {
+ public void setName(@NotNull
+ String name) {
this.name = name;
}
@@ -284,7 +298,8 @@ public class Agent implements Serializable {
return this.address;
}
- public void setAddress(@NotNull String address) {
+ public void setAddress(@NotNull
+ String address) {
this.address = address;
}
@@ -313,7 +328,8 @@ public class Agent implements Serializable {
return agentToken;
}
- public void setAgentToken(@NotNull String agentToken) {
+ public void setAgentToken(@NotNull
+ String agentToken) {
this.agentToken = agentToken;
}
@@ -392,7 +408,7 @@ public class Agent implements Serializable {
/**
* Returns the {@link AffinityGroup} this agent currently belongs to.
- *
+ *
* @return the {@link AffinityGroup} this agent currently belongs to
*/
public AffinityGroup getAffinityGroup() {
@@ -401,7 +417,7 @@ public class Agent implements Serializable {
/**
* Sets the {@link AffinityGroup} this agent should belong to.
- *
+ *
* @param affinityGroup the {@link AffinityGroup} this agent should belong to
*/
public void setAffinityGroup(AffinityGroup affinityGroup) {
@@ -410,7 +426,7 @@ public class Agent implements Serializable {
/**
* Returns the {@link Server} this agent is currently communicating to.
- *
+ *
* @return the {@link Server} this agent is currently communicating to
*/
public Server getServer() {
@@ -418,8 +434,8 @@ public class Agent implements Serializable {
}
/**
- * Sets the {@link Server} this agent should communicate with.
- *
+ * Sets the {@link Server} this agent should communicate with.
+ *
* @param server the {@link Server} this agent should communicate with
*/
public void setServer(Server server) {
@@ -429,7 +445,7 @@ public class Agent implements Serializable {
/**
* Returns 0 if this agent is current. Otherwise, returns a mask of {@link
Agent.Status}
* elements corresponding to the updates that have occurred that are related to this
agent.
- *
+ *
* @return 0 if this agent is current. Otherwise, returns a mask of {@link
Agent.Status}
* elements corresponding to the updates that have occurred that are related to this
agent.
*/
@@ -440,8 +456,8 @@ public class Agent implements Serializable {
/**
* If this status was non-zero, some scheduled job would have had to come along to
perform
* some work on behalf of this agent. After that work is complete, the status can be
reset
- * (set to 0) signifying that no further work needs to be done on this agent (as long
as the
- * status remains 0).
+ * (set to 0) signifying that no further work needs to be done on this agent (as long
as the
+ * status remains 0).
*/
public void clearStatus() {
status = 0;
diff --git
a/modules/enterprise/server/jar/src/main/java/org/rhq/enterprise/server/core/AgentManagerBean.java
b/modules/enterprise/server/jar/src/main/java/org/rhq/enterprise/server/core/AgentManagerBean.java
index 1c23836..2d4d58d 100644
---
a/modules/enterprise/server/jar/src/main/java/org/rhq/enterprise/server/core/AgentManagerBean.java
+++
b/modules/enterprise/server/jar/src/main/java/org/rhq/enterprise/server/core/AgentManagerBean.java
@@ -242,8 +242,7 @@ public class AgentManagerBean implements AgentManagerLocal {
long maximumQuietTimeAllowed = 300000L;
try {
- String prop = systemManager.getUnmaskedSystemSettings(true).get(
- SystemSetting.AGENT_MAX_QUIET_TIME_ALLOWED);
+ String prop =
systemManager.getUnmaskedSystemSettings(true).get(SystemSetting.AGENT_MAX_QUIET_TIME_ALLOWED);
if (prop != null) {
maximumQuietTimeAllowed = Long.parseLong(prop);
}
@@ -286,7 +285,7 @@ public class AgentManagerBean implements AgentManagerLocal {
// about a downed agent once, at the time it is first backfilled.
if (!record.isBackFilled()) {
LOG.info("Have not heard from agent [" + record.getAgentName()
+ "] since ["
- + new Date(record.getLastAvailabilityPing()) + "]. Will be
backfilled since we suspect it is down");
+ + new Date(record.getLastAvailabilityPing()) + "]. Will be
backfilled since we suspect it is down");
agentManager.backfillAgentInNewTransaction(subjectManager.getOverlord(),
record.getAgentName(),
record.getAgentId());
@@ -304,20 +303,27 @@ public class AgentManagerBean implements AgentManagerLocal {
// problems
AvailabilityReportSerializer.getSingleton().lock(agentName);
try {
- // This marks the Agent as backfilled=true
- agentManager.setAgentBackfilled(agentId, true);
-
// This marks the top-level platform DOWN since we have not heard from it and
all
// child resources UNKNOWN since they are not reporting and may very well be
up
availabilityManager.updateAgentResourceAvailabilities(agentId,
AvailabilityType.DOWN,
AvailabilityType.UNKNOWN);
+ // This marks the Agent as backfilled=true. Although doing this in its own
transaction means the
+ // agent will be reflected as backfilled before the resource availability
changes are committed, that
+ // should not matter, as it affects only availability reporting which 1) is
locked out (see a few line above)
+ // and 2) at worst would only request a full avail report anyway, when
detecting a backfilled agent.
+ // What this buys is it shortens the locking on the agent table, because
backfilling a large inventory
+ // can take time. Performing this after the backfill to minimize the window
between the commits as
+ // much as possible.
+ agentManager.setAgentBackfilledInNewTransaction(agentId, true);
+
} finally {
AvailabilityReportSerializer.getSingleton().unlock(agentName);
}
}
- public void setAgentBackfilled(int agentId, boolean backfilled) {
+ @TransactionAttribute(TransactionAttributeType.REQUIRES_NEW)
+ public void setAgentBackfilledInNewTransaction(int agentId, boolean backfilled) {
Query query = entityManager.createNamedQuery(Agent.QUERY_SET_AGENT_BACKFILLED);
query.setParameter("agentId", agentId);
query.setParameter("backfilled", backfilled);
@@ -662,12 +668,7 @@ public class AgentManagerBean implements AgentManagerLocal {
* since we already know we have to update the agent row with the last avail ping
time, might as well
* set the backfilled to false here (as opposed to called
agentManager.setBackfilled(agentId, false)
*/
- String updateStatement = "" //
- + "UPDATE Agent " //
- + " SET lastAvailabilityPing = :now, backFilled = FALSE " //
- + " WHERE name = :agentName ";
-
- Query query = entityManager.createQuery(updateStatement);
+ Query query =
entityManager.createNamedQuery(Agent.QUERY_UPDATE_LAST_AVAIL_PING);
query.setParameter("now", now);
query.setParameter("agentName", agentName);
diff --git
a/modules/enterprise/server/jar/src/main/java/org/rhq/enterprise/server/core/AgentManagerLocal.java
b/modules/enterprise/server/jar/src/main/java/org/rhq/enterprise/server/core/AgentManagerLocal.java
index 7e2c3b9..4e07a08 100644
---
a/modules/enterprise/server/jar/src/main/java/org/rhq/enterprise/server/core/AgentManagerLocal.java
+++
b/modules/enterprise/server/jar/src/main/java/org/rhq/enterprise/server/core/AgentManagerLocal.java
@@ -44,11 +44,11 @@ import org.rhq.enterprise.server.agentclient.AgentClient;
@Local
public interface AgentManagerLocal {
- /**
+ /**
* Call this method to set the agent DOWN and mark it 'backfilled'. Also,
sets all of its monitored resources
* to an UNKNOWN avail state since the agent is no longer reporting availability.
Done in its own transaction to
* avoid large transactions if many agents are simultaneously backfilled.
- *
+ *
* @param subject
* @param agentName
* @param agentId
@@ -113,15 +113,17 @@ public interface AgentManagerLocal {
* @return list of all known agents in inventory
* @deprecated Use <code>findAgentsByCriteria()</code> instead
*/
+ @Deprecated
List<Agent> getAllAgents();
/**
* Returns a collection of paged agents, filtered by Server (if non-null).
*
- * @param serverId the server to filter the agent list by. pass null to view
unfiltered results.
+ * @param serverId the server to filter the agent list by. pass null to view
unfiltered results.
* @return list of all known agents in inventory
* @deprecated Use <code>findAgentsByCriteria()</code> instead
*/
+ @Deprecated
PageList<Agent> getAgentsByServer(Subject subject, Integer serverId,
PageControl pageControl);
/**
@@ -136,7 +138,7 @@ public interface AgentManagerLocal {
* name exists, <code>null</code> is returned.
* This method is very efficient if you want to find a single agent by its name.
* If you need to get more than one agent, you could use
<code>findAgentsByCriteria</code>.
- *
+ *
* @param agentName
*
* @return the agent whose name matches the given name; <code>null</code>
if there is no agent with the given name
@@ -184,7 +186,7 @@ public interface AgentManagerLocal {
/**
* Given a resource ID, this will return the agent responsible for servicing that
resource.
- * @param subject
+ * @param subject
*
* @param resourceId
*
@@ -246,9 +248,9 @@ public interface AgentManagerLocal {
/**
* Determines if the given agent version is supported by this server. In other words,
this will
* return <code>true</code> if this server can talk to any agent of the
given version.
- *
+ *
* @param agentVersion the version of the agent to verify
- *
+ *
* @return <code>true</code> if this server can support an agent with the
given version; if the server
* knows it cannot communicate successfully with an agent of that version,
<code>false</code>
* will be returned
@@ -259,9 +261,9 @@ public interface AgentManagerLocal {
* Returns the path on the server's file system where the agent update version
file is found.
* The agent update version file contains information about the agent update binary,
such
* as what version it is.
- *
+ *
* @return agent update version file location
- *
+ *
* @throws Exception if the file could not be created or found
*/
File getAgentUpdateVersionFile() throws Exception;
@@ -271,9 +273,9 @@ public interface AgentManagerLocal {
* of some name/value pairs.
* The agent update version file contains information about the agent update binary,
such
* as what version it is.
- *
+ *
* @return version properties found in the agent update version file.
- *
+ *
* @throws Exception if cannot read the agent update version file
*/
Properties getAgentUpdateVersionFileContent() throws Exception;
@@ -281,9 +283,9 @@ public interface AgentManagerLocal {
/**
* Returns the path on the server's file system where the agent update binary is
found.
* This is the actual agent distribution that can be installed on the agent
machines.
- *
+ *
* @return agent update binary location
- *
+ *
* @throws Exception if the binary file does not exist
*/
File getAgentUpdateBinaryFile() throws Exception;
@@ -292,13 +294,18 @@ public interface AgentManagerLocal {
* DO NOT USE THIS. You should be using one of the getAgentUpdateXXX methods directly
rather
* than looking in the download directory. Not all agent update files are located in
this download
* directory anymore. This API will be removed from the public API in the near
future.
- *
+ *
* @deprecated
*/
@Deprecated
File getAgentDownloadDir() throws Exception;
- void setAgentBackfilled(int agentId, boolean backfilled);
+ /**
+ * Do this in its own transaction to minimize locking on the agent table.
+ * @param agentId
+ * @param backfilled
+ */
+ void setAgentBackfilledInNewTransaction(int agentId, boolean backfilled);
/**
* Returns <code>true</code> if the agent is "suspect" and has
been backfilled. A "suspect agent" means one that the
@@ -313,7 +320,7 @@ public interface AgentManagerLocal {
/**
* Returns <code>true</code> indicating successful ping of agent. Exposed
so server could
- * initiate N requests so gwt clients wont face Single Origin Policy issues.
+ * initiate N requests so gwt clients wont face Single Origin Policy issues.
*
* @param agentId the id of the agent
*
@@ -323,17 +330,17 @@ public interface AgentManagerLocal {
/**
* Process a ping request from an agent, performing any requested actions and
returning any requested data.
- *
+ *
* @param request
* @return The updated request object.
*/
public PingRequest handlePingRequest(PingRequest request);
-
+
/**
* Fetches the agents based on provided criteria.
- *
+ *
* Subject needs MANAGE_SETTINGS and MANAGE_INVENTORY permissions.
- *
+ *
* @param subject caller
* @param criteria the criteria
* @return list of agents
diff --git
a/modules/enterprise/server/jar/src/main/java/org/rhq/enterprise/server/measurement/AvailabilityManagerBean.java
b/modules/enterprise/server/jar/src/main/java/org/rhq/enterprise/server/measurement/AvailabilityManagerBean.java
index 34922fe..629069f 100644
---
a/modules/enterprise/server/jar/src/main/java/org/rhq/enterprise/server/measurement/AvailabilityManagerBean.java
+++
b/modules/enterprise/server/jar/src/main/java/org/rhq/enterprise/server/measurement/AvailabilityManagerBean.java
@@ -54,6 +54,7 @@ import org.rhq.core.domain.discovery.AvailabilityReport.Datum;
import org.rhq.core.domain.measurement.Availability;
import org.rhq.core.domain.measurement.AvailabilityType;
import org.rhq.core.domain.measurement.ResourceAvailability;
+import org.rhq.core.domain.resource.Agent;
import org.rhq.core.domain.resource.InventoryStatus;
import org.rhq.core.domain.resource.Resource;
import org.rhq.core.domain.resource.composite.ResourceIdWithAvailabilityComposite;
@@ -1073,12 +1074,7 @@ public class AvailabilityManagerBean implements
AvailabilityManagerLocal, Availa
* since we already know we have to update the agent row with the last avail
report time, might as well
* set the backfilled to false here (as opposed to called
agentManager.setBackfilled(agentId, false)
*/
- String updateStatement = "" //
- + "UPDATE Agent " //
- + " SET lastAvailabilityReport = :reportTime, backFilled = FALSE
" //
- + " WHERE id = :agentId ";
-
- Query query = entityManager.createQuery(updateStatement);
+ Query query =
entityManager.createNamedQuery(Agent.QUERY_UPDATE_LAST_AVAIL_REPORT);
query.setParameter("reportTime", System.currentTimeMillis());
query.setParameter("agentId", agentId);