Hibernate search : Indexing a 1 million rows table.

romaincherchi · **Joined:** Tue Dec 23, 2008 10:42 am **Posts:** 7

Hi guys,

I'm trying to index a large table (> 1 million rows) using Hibernate search version:3.0.0 GA.

I read the instructions in the manual (Chapter 6. Manual indexing) :

Code:

fullTextSession.setFlushMode(FlushMode.MANUAL);
fullTextSession.setCacheMode(CacheMode.IGNORE);
transaction = fullTextSession.beginTransaction();
//Scrollable results will avoid loading too many objects in memory
ScrollableResults results = fullTextSession.createCriteria( Email.class ).scroll( ScrollMode.FORWARD_ONLY );
int index = 0;
while( results.next() ) {
    index++;
    fullTextSession.index( results.get(0) ); //index each element
    if (index % batchSize == 0) s.clear(); //clear every batchSize since the queue is processed
}
transaction.commit();

but I can't figure out how I can use it with JPA. I have a FullTextEntityManager (no fullTextSession). I'm using JBoss Seam 2.1.1.GA.

Thanks for you help.

sanne.grinovero · **Posted:** Wed Dec 24, 2008 4:10 pm

Hi,
you can get always get the session from the EntityManager, however you'll need some casting:

Code:

Session session = (Session) em.getDelegate();

Using Seam you can actually just inject it:

Code:

@In FullTextSession session;

were the variable "session" needs to match your configuration from components.xml ; If I recall correctly it is named "hibernateSession" since Seam 2.1

By the way, you mention Search version 3.0 GA which is quite old now: I would suggest to upgrade to the latest one especially because of indexing performance improvements.

romaincherchi · **Joined:** Tue Dec 23, 2008 10:42 am **Posts:** 7

You solved the problem :-)

bemanuel.pe · **Joined:** Fri Jul 17, 2009 2:26 pm **Posts:** 1

romaincherchi · **Joined:** Tue Dec 23, 2008 10:42 am **Posts:** 7

Here is my IndexerBean if it can help :

Code:

package com.easycity.ejb.util;

import java.util.Date;

import javax.ejb.Remove;
import javax.ejb.Stateful;
import javax.persistence.EntityManager;
import javax.persistence.PersistenceContext;
import javax.persistence.PersistenceContextType;

import org.hibernate.CacheMode;
import org.hibernate.ScrollMode;
import org.hibernate.criterion.CriteriaSpecification;
import org.hibernate.search.FullTextSession;
import org.jboss.seam.ScopeType;
import org.jboss.seam.annotations.Create;
import org.jboss.seam.annotations.Destroy;
import org.jboss.seam.annotations.Logger;
import org.jboss.seam.annotations.Name;
import org.jboss.seam.annotations.Scope;
import org.jboss.seam.annotations.Startup;
import org.jboss.seam.log.Log;

import com.easycity.entity.community.Mb;
import com.easycity.entity.i18n.City;
import com.easycity.entity.core.business.LocalBusinessOwner;
import com.easycity.entity.core.poi.POI;
import com.easycity.entity.stats.StatsSearch;
import com.easycity.entity.utils.Message;

/**
 * Re index the needed entities
 * 
 * @author Emmanuel Bernard, Romain Cherchi
 */
@Name("indexer")
@Stateful
@Scope(ScopeType.APPLICATION)
@Startup
public class IndexerBean implements IndexerLocal {
   private Date lastIndexingTime;

   @PersistenceContext(type=PersistenceContextType.EXTENDED)
   private EntityManager em;
   
   @Logger
   private Log log;

   private final int BATCH_SIZE = 1000;
   private final int FETCH_SIZE = 100;

   public Date getLastIndexingTime() {
      return lastIndexingTime;
   }

   @Create
   public void index() {
      PropertiesManagerBean propertiesManager = new PropertiesManagerBean();

      if (propertiesManager.getLuceneCreateIndexes()) {
         indexPOI();

         indexLargeClass(Mb.class);
         
         indexLargeClass(LocalBusinessOwner.class);
         
         indexLargeClass(Message.class);
         
         indexLargeClass(City.class);

         indexLargeClass(StatsSearch.class);
         
         lastIndexingTime = new Date();

         System.out.println("Last indexing time : " + lastIndexingTime.toString());
      } else {
         System.out.println("Lucene index not recreated (on purpose).");
      }
   }
   
   private void indexPOI() {
      FullTextSession fullTextSession = getFullTextSession();
      
      /*
      fullTextSession.setFlushMode(org.hibernate.FlushMode.MANUAL);
      fullTextSession.setCacheMode(CacheMode.IGNORE);
      */
      
      org.hibernate.Transaction transaction = fullTextSession
            .beginTransaction();
      /*
      // Scrollable results will avoid loading too many objects in memory
      org.hibernate.ScrollableResults results = fullTextSession
      .createCriteria(POI.class).setFetchSize(BATCH_SIZE)
      .setFetchMode("detailsSet", org.hibernate.FetchMode.JOIN)
      .setFetchMode("reviewSet", org.hibernate.FetchMode.JOIN)
      .scroll(org.hibernate.ScrollMode.FORWARD_ONLY);
      */
       org.hibernate.Criteria query = fullTextSession
            .createCriteria(POI.class)
            .setFetchMode("detailsSet", org.hibernate.FetchMode.JOIN)
            .setResultTransformer(CriteriaSpecification.DISTINCT_ROOT_ENTITY)
            .setCacheMode(CacheMode.IGNORE)
            .setFetchSize(FETCH_SIZE)
            .setFlushMode(org.hibernate.FlushMode.MANUAL);
       
       query.setFetchMode("reviewSet", org.hibernate.FetchMode.JOIN)
         .setResultTransformer(CriteriaSpecification.DISTINCT_ROOT_ENTITY)
         .setCacheMode(CacheMode.IGNORE)
         .setFetchSize(FETCH_SIZE)
         .setFlushMode(org.hibernate.FlushMode.MANUAL);
       
       query.setFetchMode("businessOwner", org.hibernate.FetchMode.JOIN)
         .setResultTransformer(CriteriaSpecification.DISTINCT_ROOT_ENTITY)
         .setCacheMode(CacheMode.IGNORE)
         .setFetchSize(FETCH_SIZE)
         .setFlushMode(org.hibernate.FlushMode.MANUAL);
       
       query.setFetchMode("mainCategory", org.hibernate.FetchMode.JOIN)
         .setResultTransformer(CriteriaSpecification.DISTINCT_ROOT_ENTITY)
         .setCacheMode(CacheMode.IGNORE)
         .setFetchSize(FETCH_SIZE)
         .setFlushMode(org.hibernate.FlushMode.MANUAL);
       
       query.setFetchMode("mainCategory.detailsSet", org.hibernate.FetchMode.JOIN)
         .setResultTransformer(CriteriaSpecification.DISTINCT_ROOT_ENTITY)
         .setCacheMode(CacheMode.IGNORE)
         .setFetchSize(FETCH_SIZE)
         .setFlushMode(org.hibernate.FlushMode.MANUAL);
       
      org.hibernate.ScrollableResults results = query.scroll(ScrollMode.FORWARD_ONLY);
       
      int index = 0;
      while (results.next()) {
         index++;
         fullTextSession.index(results.get(0)); // index each element
         if (index % BATCH_SIZE == 0) {
            System.out.println("...indexing POI : " + index);
            fullTextSession.clear(); // clear every batchSize since the
            // queue is processed
         }
      }
      transaction.commit();

      try {
         if (results != null) {
            results.close();
         }
      } catch (Exception ex) {
         ex.printStackTrace();
      }
   }
   
   private FullTextSession getFullTextSession() {
      return (FullTextSession) em.getDelegate();
   }

   /*
   @SuppressWarnings("unchecked")
   private void indexAllSmallClasses(Class... entityTypes) {
      FullTextSession fullTextSession = getFullTextSession();
      for (Class entityType : entityTypes) {
         System.out.println("...indexing " + entityType.getSimpleName());
         for (Object obj : fullTextSession.createCriteria(entityType).list()) {
            fullTextSession.index(obj);
         }
      }
   }
   */

   @SuppressWarnings("unchecked")
   private void indexLargeClass(Class entityType) {
      FullTextSession fullTextSession = getFullTextSession();
      
      fullTextSession.setFlushMode(org.hibernate.FlushMode.MANUAL);    // Disable flush operations
      fullTextSession.setCacheMode(CacheMode.IGNORE);               // Disable 2nd-level cache operations

      org.hibernate.Transaction transaction = fullTextSession
            .beginTransaction();
      // Scrollable results will avoid loading too many objects in memory

      org.hibernate.ScrollableResults results = fullTextSession
            .createCriteria(entityType).setFetchSize(BATCH_SIZE).scroll(
                  org.hibernate.ScrollMode.FORWARD_ONLY);       // Ensure forward only result set
      int index = 0;
      while (results.next()) {
         index++;
         fullTextSession.index(results.get(0)); // index entities
         if (index % BATCH_SIZE == 0) {
            System.out.println("...indexing " + entityType.getSimpleName()
                  + ": " + index);
            fullTextSession.clear(); // Clear the session releasing memory
         }
      }
      transaction.commit(); // Apply the remaining index changes

      try {
         if (results != null)
            results.close();
      } catch (Exception ex) {
         ex.printStackTrace();
      }
   }

   @Remove
   @Destroy
   public void stop() {
   }

}

Regards,

Romain