Hi, I hope someone can help me.
I have an application that is due to go live in less than 1 month and it's failing when stress testing using Loadrunner.
The application is using a Servlet Filter to keep the session open for the presentation layer. I am also using the ThreadLocal pattern to keep the sessions/transactions threadsafe.
I am using Oracle's OC4J 10.1.3 server and am making use of the servers JTA Transaction manager.
The Datasource has max connections set to 5 and I'm failing with 20 concurrent users.
I would really appreciated any help here, I'm stumped and don't know where to go next.
Hibernate version: 3.1.3
Mapping documents: Hibernate.cfg.xml
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE hibernate-configuration PUBLIC
"-//Hibernate/Hibernate Configuration DTD 3.0//EN"
"http://hibernate.sourceforge.net/hibernate-configuration-3.0.dtd">
<hibernate-configuration>
<session-factory>
<property name="hibernate.connection.datasource">jdbc/dpcs</property>
<property name="hibernate.current_session_context_class">thread</property>
<property name="hibernate.transaction.factory_class">org.hibernate.transaction.JTATransactionFactory</property>
<property name="hibernate.transaction.manager_lookup_class">org.hibernate.transaction.OC4JTransactionManagerLookup</property>
<property name="hibernate.dialect">org.hibernate.dialect.OracleDialect</property>
<property name="hibernate.query.factory_class">org.hibernate.hql.classic.ClassicQueryTranslatorFactory</property>
<property name="hibernate.cglib.use_reflection_optimizer">false</property>
<property name="show_sql">true</property>
<property name="hibernate.format_sql">true</property>
<property name="hbm2ddl.auto">none</property>
<property name="cache.use_second_level_cache">false</property>
<mapping resource="ie/gov/agriculture/dpcs/cert/Certificate.hbm.xml"/>
<mapping resource="ie/gov/agriculture/dpcs/cert/CertificateDisclaimer.hbm.xml"/>
<mapping resource="ie/gov/agriculture/dpcs/cert/CertificateForeignStatementBody.hbm.xml"/>
<mapping resource="ie/gov/agriculture/dpcs/cert/CertificateStatement.hbm.xml"/>
<mapping resource="ie/gov/agriculture/dpcs/cert/CertificateStatementBody.hbm.xml"/>
<mapping resource="ie/gov/agriculture/dpcs/cert/CertificateStatementTitle.hbm.xml"/>
<mapping resource="ie/gov/agriculture/dpcs/cert/CertificateTitle.hbm.xml"/>
<mapping resource="ie/gov/agriculture/dpcs/cert/CertLanguage.hbm.xml"/>
<mapping resource="ie/gov/agriculture/dpcs/cert/CertTitle.hbm.xml"/>
<mapping resource="ie/gov/agriculture/dpcs/cert/CertType.hbm.xml"/>
<mapping resource="ie/gov/agriculture/dpcs/cert/CertTypeStatementTitle.hbm.xml"/>
<mapping resource="ie/gov/agriculture/dpcs/cert/CertTypeStatement.hbm.xml"/>
<mapping resource="ie/gov/agriculture/dpcs/cert/ConsignmentDetails.hbm.xml"/>
<mapping resource="ie/gov/agriculture/dpcs/cert/Country.hbm.xml"/>
<mapping resource="ie/gov/agriculture/dpcs/cert/CountryGroup.hbm.xml"/>
<mapping resource="ie/gov/agriculture/dpcs/cert/Disclaimer.hbm.xml"/>
<mapping resource="ie/gov/agriculture/dpcs/cert/Filter.hbm.xml"/>
<mapping resource="ie/gov/agriculture/dpcs/cert/ForeignDisclaimer.hbm.xml"/>
<mapping resource="ie/gov/agriculture/dpcs/cert/ForeignStatementBody.hbm.xml"/>
<mapping resource="ie/gov/agriculture/dpcs/cert/Manufacturer.hbm.xml"/>
<mapping resource="ie/gov/agriculture/dpcs/cert/Note.hbm.xml"/>
<mapping resource="ie/gov/agriculture/dpcs/cert/Packer.hbm.xml"/>
<mapping resource="ie/gov/agriculture/dpcs/cert/Product.hbm.xml"/>
<mapping resource="ie/gov/agriculture/dpcs/cert/ProductGroup.hbm.xml"/>
<mapping resource="ie/gov/agriculture/dpcs/cert/Signature.hbm.xml"/>
<mapping resource="ie/gov/agriculture/dpcs/cert/SignatureType.hbm.xml"/>
<mapping resource="ie/gov/agriculture/dpcs/cert/Statement.hbm.xml"/>
<mapping resource="ie/gov/agriculture/dpcs/cert/StatementBody.hbm.xml"/>
<mapping resource="ie/gov/agriculture/dpcs/cert/StatementTitle.hbm.xml"/>
<mapping resource="ie/gov/agriculture/dpcs/cert/UnitOfMeasure.hbm.xml"/>
<mapping resource="ie/gov/agriculture/dpcs/cert/VetDpiNames.hbm.xml"/>
</session-factory>
</hibernate-configuration>
Code between sessionFactory.openSession() and session.close():
HibernateUtil.class
Code:
package ie.gov.agriculture.dpcs.hibernate;
import java.io.FileInputStream;
import java.util.Properties;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.hibernate.HibernateException;
import org.hibernate.Session;
import org.hibernate.SessionFactory;
import org.hibernate.Transaction;
import org.hibernate.cfg.Configuration;
import org.hibernate.exception.ConstraintViolationException;
public class HibernateUtil {
private static final SessionFactory sessionFactory;
private static final ThreadLocal threadSession = new ThreadLocal();
private static final ThreadLocal threadTransaction = new ThreadLocal();
private static Log log = LogFactory.getLog(HibernateUtil.class);
static {
try {
// Create the SessionFactory from hibernate.cfg.xml
sessionFactory = new Configuration().configure().buildSessionFactory();
log.debug("Created Hibernate Session Factory");
} catch (Throwable ex) {
// Make sure you log the exception, as it might be swallowed
log.debug("Initial SessionFactory creation failed." + ex);
throw new ExceptionInInitializerError(ex);
}
}
/**
*
* @return
*/
public static SessionFactory getSessionFactory() {
return sessionFactory;
}
/**
* Create a new Session if no session exists
* and add it to the current thread.
*
* @return the current session
*/
public static Session getSession() {
Session s = (Session)threadSession.get();
// Open a new Session, it this thread has none yet
try {
if(s== null) {
log.debug("SessionFactory is closed = " + sessionFactory.isClosed());
s = sessionFactory.openSession();
log.debug("Session is connected = " + s.isConnected()+ " And is open = " + s.isOpen());
log.debug("HibernateUtil::Opening new Hibernate Session");
threadSession.set(s);
log.debug("HibernateUtil::Added Session to current Thread");
}
} catch (HibernateException e) {
log.debug("HibernateException Getting Session::" + e.getMessage());
e.printStackTrace();
}
return s;
}
/**
* Remove the session from the current thread
* and if it is open, close it.
*
*/
public static void closeSession() {
try {
Session s = (Session)threadSession.get();
threadSession.set(null);
log.debug("HibernateUtil::Closing Hibernate Session");
if( s != null && s.isOpen()) {
s.close();
}
} catch (HibernateException e) {
log.debug("HibernateException Closing Session::" + e.getMessage());
e.printStackTrace();
}
}
/**
* If no transaction exists, create a new one and add it to
* this thread
*
*/
public static void beginTransaction() {
Transaction tx = (Transaction)threadTransaction.get();
try{
if(tx == null) {
log.debug("HibernateUtil::Starting new Hibernate Transaction");
tx = getSession().beginTransaction();
threadTransaction.set(tx);
log.debug("HibernateUtil::Added Hibernate Transaction to local thread:: tx is Active= " + tx.isActive());
}
} catch (HibernateException e) {
log.debug("HibernateException Beginning Transaction::" + e.getMessage());
e.printStackTrace();
}
}
/**
* Commit the Transaction if it has not been comitted or
* rolled back already.
*
*/
public static void commitTransaction() {
Transaction tx = (Transaction)threadTransaction.get();
try {
threadTransaction.set(null);
if(tx != null && !tx.wasCommitted()
&& !tx.wasRolledBack()) {
log.debug("HibernateUtil::Commiting Hibernate Transaction");
tx.commit();
}
} catch (ConstraintViolationException e) {
log.debug("HibernateException Rolling Back Transaction::" + e.getMessage());
rollbackTransaction();
throw e;
} catch (HibernateException e) {
log.debug("HibernateException Rolling Back Transaction::" + e.getMessage());
rollbackTransaction();
e.printStackTrace();
}
}
public static void attemptCommitTransaction() {
Transaction tx = (Transaction)threadTransaction.get();
threadTransaction.set(null);
if(tx != null && !tx.wasCommitted()
&& !tx.wasRolledBack()) {
log.debug("HibernateUtil::Attempting to Commit Hibernate Transaction");
tx.commit();
}
}
/**
* Roll back transaction if it has not been comitted
* or rolled back already
*
*/
public static void rollbackTransaction() {
Transaction tx = (Transaction)threadTransaction.get();
log.debug("HibernateUtil::About to Rollback Hibernate Transaction:: tx is Active= " + tx.isActive());
try {
threadTransaction.set(null);
if(tx != null && !tx.wasCommitted()
&& !tx.wasRolledBack()) {
log.debug("HibernateUtil::Rollingback Hibernate Transaction");
tx.rollback();
}
} catch (HibernateException e) {
log.debug("HibernateException Rolling Back Transaction::" + e.getMessage());
e.printStackTrace();
} finally {
closeSession();
}
}
}
Servlet Filter Code:
public class HibernateFilter implements Filter {
private static Log log = LogFactory.getLog(HibernateFilter.class);
public void destroy() {
// TODO Auto-generated method stub
}
public void doFilter(ServletRequest request, ServletResponse response,
FilterChain filterChain) throws IOException, ServletException {
try{
// We are using Tiles and so have to explicitly put our action errors
// into the session instead of the request. We remove any errors here
// from a previous request.
HttpServletRequest req = (HttpServletRequest)request;
req.getSession().removeAttribute(Globals.ERROR_KEY);
filterChain.doFilter(request,response);
HibernateUtil.commitTransaction();
// catch all exception for any uncaught
} catch (Exception e) {
log.debug("HibernateFilter::" + e.getMessage());
System.out.println("HibernateFilter::" + e.getMessage());
e.printStackTrace();
} finally {
HibernateUtil.closeSession();
}
}
public void init(FilterConfig arg0) throws ServletException {
// TODO Auto-generated method stub
}
Full stack trace of any exception that occurs:
2007-11-23 15:01:48.839 WARNING J2EE JTA-00109 Transaction has been marked for rollback: Timed out
[-] 2007-11-23 15:01:48,840 ERROR JDBCExceptionReporter::logExceptions - javax.resource.ResourceException: RollbackException: Transaction has been marked for rollback: Timed out
[-] 2007-11-23 15:01:48,842 ERROR CreateCertStep6::executeAction - HibernateException::could not initialize a collection: [ie.gov.agriculture.dpcs.cert.Disclaimer.foreignDisclaimers#1]
org.hibernate.exception.GenericJDBCException: could not initialize a collection: [ie.gov.agriculture.dpcs.cert.Disclaimer.foreignDisclaimers#1]
at org.hibernate.exception.SQLStateConverter.handledNonSpecificException(SQLStateConverter.java:103)
at org.hibernate.exception.SQLStateConverter.convert(SQLStateConverter.java:91)
at org.hibernate.exception.JDBCExceptionHelper.convert(JDBCExceptionHelper.java:43)
at org.hibernate.loader.Loader.loadCollection(Loader.java:1926)
at org.hibernate.loader.collection.CollectionLoader.initialize(CollectionLoader.java:36)
at org.hibernate.persister.collection.AbstractCollectionPersister.initialize(AbstractCollectionPersister.java:520)
at org.hibernate.event.def.DefaultInitializeCollectionEventListener.onInitializeCollection(DefaultInitializeCollectionEventListener.java:60)
at org.hibernate.impl.SessionImpl.initializeCollection(SessionImpl.java:1676)
at org.hibernate.collection.AbstractPersistentCollection.initialize(AbstractPersistentCollection.java:344)
at org.hibernate.collection.AbstractPersistentCollection.read(AbstractPersistentCollection.java:86)
at org.hibernate.collection.PersistentSet.iterator(PersistentSet.java:138)
at ie.gov.agriculture.dpcs.actions.exporter.CreateCertStep6.executeAction(CreateCertStep6.java:103)
at ie.gov.agriculture.dpcs.actions.BaseAction.execute(BaseAction.java:84)
at org.apache.struts.action.RequestProcessor.processActionPerform(RequestProcessor.java:431)
at org.apache.struts.action.RequestProcessor.process(RequestProcessor.java:236)
at org.apache.struts.action.ActionServlet.process(ActionServlet.java:1196)
at org.apache.struts.action.ActionServlet.doPost(ActionServlet.java:432)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:763)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:856)
at com.evermind.server.http.ResourceFilterChain.doFilter(ResourceFilterChain.java:64)
at ie.gov.agriculture.dpcs.hibernate.HibernateFilter.doFilter(HibernateFilter.java:31)
at com.evermind.server.http.ServletRequestDispatcher.invoke(ServletRequestDispatcher.java:622)
at com.evermind.server.http.ServletRequestDispatcher.forwardInternal(ServletRequestDispatcher.java:369)
at com.evermind.server.http.HttpRequestHandler.doProcessRequest(HttpRequestHandler.java:865)
at com.evermind.server.http.HttpRequestHandler.processRequest(HttpRequestHandler.java:447)
at com.evermind.server.http.HttpRequestHandler.serveOneRequest(HttpRequestHandler.java:215)
at com.evermind.server.http.HttpRequestHandler.run(HttpRequestHandler.java:117)
at com.evermind.server.http.HttpRequestHandler.run(HttpRequestHandler.java:110)
at oracle.oc4j.network.ServerSocketReadHandler$SafeRunnable.run(ServerSocketReadHandler.java:260)
at com.evermind.util.ReleasableResourcePooledExecutor$MyWorker.run(ReleasableResourcePooledExecutor.java:303)
at java.lang.Thread.run(Thread.java:595)
Caused by: java.sql.SQLException: javax.resource.ResourceException: RollbackException: Transaction has been marked for rollback: Timed out
at oracle.oc4j.sql.spi.ManagedConnectionImpl.setupTransaction(ManagedConnectionImpl.java:850)
at oracle.oc4j.sql.spi.ConnectionHandle.oc4j_intercept(ConnectionHandle.java:305)
at oracle_jdbc_driver_LogicalConnection_Proxy.prepareStatement()
at org.hibernate.jdbc.AbstractBatcher.getPreparedStatement(AbstractBatcher.java:442)
at org.hibernate.jdbc.AbstractBatcher.getPreparedStatement(AbstractBatcher.java:368)
at org.hibernate.jdbc.AbstractBatcher.prepareQueryStatement(AbstractBatcher.java:105)
at org.hibernate.loader.Loader.prepareQueryStatement(Loader.java:1561)
at org.hibernate.loader.Loader.doQuery(Loader.java:661)
at org.hibernate.loader.Loader.doQueryAndInitializeNonLazyCollections(Loader.java:224)
at org.hibernate.loader.Loader.loadCollection(Loader.java:1919)
... 27 more
Caused by: javax.resource.ResourceException: RollbackException: Transaction has been marked for rollback: Timed out
at com.evermind.server.connector.ConnectionContext.setupForJTATransaction(ConnectionContext.java:342)
at com.evermind.server.connector.ConnectionContext.setupForTransaction(ConnectionContext.java:279)
at com.evermind.server.connector.ConnectionContext.setupForTransaction(ConnectionContext.java:269)
at com.evermind.server.connector.ApplicationConnectionManager.lazyEnlist(ApplicationConnectionManager.java:1983)
at oracle.j2ee.connector.OracleConnectionManager.lazyEnlist(OracleConnectionManager.java:285)
at oracle.oc4j.sql.spi.ManagedConnectionFactoryImpl.enlist(ManagedConnectionFactoryImpl.java:526)
at oracle.oc4j.sql.spi.ManagedConnectionImpl.setupTransaction(ManagedConnectionImpl.java:848)
... 36 more
[-] 2007-11-23 15:02:18,842 ERROR JDBCExceptionReporter::logExceptions - Unable to get a physical connection from the database...there are no connections available.
[-] 2007-11-23 15:02:18,844 ERROR CreateCertStep5::executeAction - HibernateException::Cannot open connection
org.hibernate.exception.GenericJDBCException: Cannot open connection
at org.hibernate.exception.SQLStateConverter.handledNonSpecificException(SQLStateConverter.java:103)
at org.hibernate.exception.SQLStateConverter.convert(SQLStateConverter.java:91)
at org.hibernate.exception.JDBCExceptionHelper.convert(JDBCExceptionHelper.java:43)
at org.hibernate.exception.JDBCExceptionHelper.convert(JDBCExceptionHelper.java:29)
at org.hibernate.jdbc.ConnectionManager.openConnection(ConnectionManager.java:420)
at org.hibernate.jdbc.ConnectionManager.getConnection(ConnectionManager.java:144)
at org.hibernate.jdbc.AbstractBatcher.prepareQueryStatement(AbstractBatcher.java:105)
at org.hibernate.loader.Loader.prepareQueryStatement(Loader.java:1561)
at org.hibernate.loader.Loader.doQuery(Loader.java:661)
at org.hibernate.loader.Loader.doQueryAndInitializeNonLazyCollections(Loader.java:224)
at org.hibernate.loader.Loader.loadEntity(Loader.java:1785)
at org.hibernate.loader.entity.AbstractEntityLoader.load(AbstractEntityLoader.java:47)
at org.hibernate.loader.entity.AbstractEntityLoader.load(AbstractEntityLoader.java:41)
at org.hibernate.persister.entity.AbstractEntityPersister.load(AbstractEntityPersister.java:2730)
at org.hibernate.event.def.DefaultLoadEventListener.loadFromDatasource(DefaultLoadEventListener.java:365)
at org.hibernate.event.def.DefaultLoadEventListener.doLoad(DefaultLoadEventListener.java:346)
at org.hibernate.event.def.DefaultLoadEventListener.load(DefaultLoadEventListener.java:123)
at org.hibernate.event.def.DefaultLoadEventListener.proxyOrLoad(DefaultLoadEventListener.java:177)
at org.hibernate.event.def.DefaultLoadEventListener.onLoad(DefaultLoadEventListener.java:87)
at org.hibernate.impl.SessionImpl.fireLoad(SessionImpl.java:862)
at org.hibernate.impl.SessionImpl.get(SessionImpl.java:799)
at org.hibernate.impl.SessionImpl.get(SessionImpl.java:792)
at ie.gov.agriculture.dpcs.actions.exporter.CreateCertStep5.executeAction(CreateCertStep5.java:84)
at ie.gov.agriculture.dpcs.actions.BaseAction.execute(BaseAction.java:84)
at org.apache.struts.action.RequestProcessor.processActionPerform(RequestProcessor.java:431)
at org.apache.struts.action.RequestProcessor.process(RequestProcessor.java:236)
at org.apache.struts.action.ActionServlet.process(ActionServlet.java:1196)
at org.apache.struts.action.ActionServlet.doPost(ActionServlet.java:432)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:763)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:856)
at com.evermind.server.http.ResourceFilterChain.doFilter(ResourceFilterChain.java:64)
at ie.gov.agriculture.dpcs.hibernate.HibernateFilter.doFilter(HibernateFilter.java:31)
at com.evermind.server.http.ServletRequestDispatcher.invoke(ServletRequestDispatcher.java:622)
at com.evermind.server.http.ServletRequestDispatcher.forwardInternal(ServletRequestDispatcher.java:369)
at com.evermind.server.http.HttpRequestHandler.doProcessRequest(HttpRequestHandler.java:865)
at com.evermind.server.http.HttpRequestHandler.processRequest(HttpRequestHandler.java:447)
at com.evermind.server.http.HttpRequestHandler.serveOneRequest(HttpRequestHandler.java:215)
at com.evermind.server.http.HttpRequestHandler.run(HttpRequestHandler.java:117)
at com.evermind.server.http.HttpRequestHandler.run(HttpRequestHandler.java:110)
at oracle.oc4j.network.ServerSocketReadHandler$SafeRunnable.run(ServerSocketReadHandler.java:260)
at com.evermind.util.ReleasableResourcePooledExecutor$MyWorker.run(ReleasableResourcePooledExecutor.java:303)
at java.lang.Thread.run(Thread.java:595)
Caused by: java.sql.SQLException: Unable to get a physical connection from the database...there are no connections available.
at oracle.oc4j.sql.DataSourceConnectionPoolDataSource.getPooledConnection(DataSourceConnectionPoolDataSource.java:64)
at oracle.oc4j.sql.xa.EmulatedXADataSource.getXAConnection(EmulatedXADataSource.java:92)
at oracle.oc4j.sql.spi.ManagedConnectionFactoryImpl.createXAConnection(ManagedConnectionFactoryImpl.java:208)
at oracle.oc4j.sql.spi.ManagedConnectionFactoryImpl.createManagedConnection(ManagedConnectionFactoryImpl.java:167)
at com.evermind.server.connector.ApplicationConnectionManager.createManagedConnection(ApplicationConnectionManager.java:1377)
at oracle.j2ee.connector.ConnectionPoolImpl.createManagedConnectionFromFactory(ConnectionPoolImpl.java:327)
at oracle.j2ee.connector.ConnectionPoolImpl.access$800(ConnectionPoolImpl.java:98)
at oracle.j2ee.connector.ConnectionPoolImpl$NonePoolingScheme.getManagedConnection(ConnectionPoolImpl.java:1211)
at oracle.j2ee.connector.ConnectionPoolImpl.getManagedConnection(ConnectionPoolImpl.java:785)
at com.evermind.server.connector.ApplicationConnectionManager.getConnectionFromPool(ApplicationConnectionManager.java:1575)
at com.evermind.server.connector.ApplicationConnectionManager.acquireConnectionContext(ApplicationConnectionManager.java:1520)
at com.evermind.server.connector.ApplicationConnectionManager.allocateConnection(ApplicationConnectionManager.java:1465)
at oracle.j2ee.connector.OracleConnectionManager.unprivileged_allocateConnection(OracleConnectionManager.java:238)
at oracle.j2ee.connector.OracleConnectionManager.allocateConnection(OracleConnectionManager.java:192)
at oracle.oc4j.sql.ManagedDataSource.getConnection(ManagedDataSource.java:197)
at oracle.oc4j.sql.ManagedDataSource.getConnection(ManagedDataSource.java:142)
at oracle.oc4j.sql.ManagedDataSource.getConnection(ManagedDataSource.java:127)
at org.hibernate.connection.DatasourceConnectionProvider.getConnection(DatasourceConnectionProvider.java:69)
at org.hibernate.jdbc.ConnectionManager.openConnection(ConnectionManager.java:417)
... 37 more
Name and version of the database you are using: Oracle 8Code: