Java Mailing List Archive

http://www.gg3721.com/

Home » Hibernate Commits List »

[hibernate-commits] Hibernate SVN: r14817 - in search/trunk/src:
 java/org/hibernate/search/backend/impl and 5 other directories.

hibernate-commits

2008-06-26


Author Login • Post Reply
Author: sannegrinovero
Date: 2008-06-26 20:25:45 -0400 (Thu, 26 Jun 2008)
New Revision: 14817

Modified:
 search/trunk/src/java/org/hibernate/search/backend/Workspace.java
 search/trunk/src/java/org/hibernate/search/backend/impl/BatchedQueueingProcessor.java
 search/trunk/src/java/org/hibernate/search/engine/SearchFactoryImplementor.java
 search/trunk/src/java/org/hibernate/search/impl/FullTextSessionImpl.java
 search/trunk/src/java/org/hibernate/search/impl/SearchFactoryImpl.java
 search/trunk/src/java/org/hibernate/search/reader/SharedReaderProvider.java
 search/trunk/src/java/org/hibernate/search/reader/SharingBufferReaderProvider.java
 search/trunk/src/java/org/hibernate/search/store/DirectoryProviderFactory.java
 search/trunk/src/java/org/hibernate/search/store/FSMasterDirectoryProvider.java
 search/trunk/src/test/org/hibernate/search/test/reader/performance/ReaderPerformance.java
Log:
minor DirectoryProviders's Locks reorganization (and relative API)

Modified: search/trunk/src/java/org/hibernate/search/backend/Workspace.java
===================================================================
--- search/trunk/src/java/org/hibernate/search/backend/Workspace.java  2008-06-26 22:15:36 UTC (rev 14816)
+++ search/trunk/src/java/org/hibernate/search/backend/Workspace.java  2008-06-27 00:25:45 UTC (rev 14817)
@@(protected) @@
import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
-import java.util.concurrent.locks.ReentrantLock;
+import java.util.concurrent.locks.Lock;

import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.SimpleAnalyzer;
@@(protected) @@
 private class DPWorkspace {
   
   private final DirectoryProvider directoryProvider;
-    private final ReentrantLock lock;
+    private final Lock lock;
   
   private IndexReader reader;
   private IndexWriter writer;
@@(protected) @@
   
   DPWorkspace(DirectoryProvider dp) {
     this.directoryProvider = dp;
-      this.lock = searchFactoryImplementor.getLockableDirectoryProviders().get( dp );
+      this.lock = searchFactoryImplementor.getDirectoryProviderLock( dp );
   }
   
   public boolean needsOptimization() {

Modified: search/trunk/src/java/org/hibernate/search/backend/impl/BatchedQueueingProcessor.java
===================================================================
--- search/trunk/src/java/org/hibernate/search/backend/impl/BatchedQueueingProcessor.java  2008-06-26 22:15:36 UTC (rev 14816)
+++ search/trunk/src/java/org/hibernate/search/backend/impl/BatchedQueueingProcessor.java  2008-06-27 00:25:45 UTC (rev 14817)
@@(protected) @@

 private static final Logger log = LoggerFactory.getLogger( BatchedQueueingProcessor.class );

-  private boolean sync;
-  private int batchSize;
-  private ExecutorService executorService;
-  private BackendQueueProcessorFactory backendQueueProcessorFactory;
-  private SearchFactoryImplementor searchFactoryImplementor;
+  private final boolean sync;
+  private final int batchSize;
+  private final ExecutorService executorService;
+  private final BackendQueueProcessorFactory backendQueueProcessorFactory;
+  private final SearchFactoryImplementor searchFactoryImplementor;

-  public BatchedQueueingProcessor(SearchFactoryImplementor searchFactoryImplementor,
-                  Properties properties) {
+  public BatchedQueueingProcessor(SearchFactoryImplementor searchFactoryImplementor, Properties properties) {
   this.searchFactoryImplementor = searchFactoryImplementor;
   //default to sync if none defined
   this.sync = !"async".equalsIgnoreCase( properties.getProperty( Environment.WORKER_EXECUTION ) );
@@(protected) @@
         new ThreadPoolExecutor.CallerRunsPolicy()
     );
   }
+    else {
+      executorService = null;
+    }
   String backend = properties.getProperty( Environment.WORKER_BACKEND );
   if ( StringHelper.isEmpty( backend ) || "lucene".equalsIgnoreCase( backend ) ) {
     backendQueueProcessorFactory = new LuceneBackendQueueProcessorFactory();

Modified: search/trunk/src/java/org/hibernate/search/engine/SearchFactoryImplementor.java
===================================================================
--- search/trunk/src/java/org/hibernate/search/engine/SearchFactoryImplementor.java  2008-06-26 22:15:36 UTC (rev 14816)
+++ search/trunk/src/java/org/hibernate/search/engine/SearchFactoryImplementor.java  2008-06-27 00:25:45 UTC (rev 14817)
@@(protected) @@

import java.util.Map;
import java.util.Set;
-import java.util.concurrent.locks.ReentrantLock;
+import java.util.concurrent.locks.Lock;

import org.hibernate.search.SearchFactory;
import org.hibernate.search.backend.BackendQueueProcessorFactory;
@@(protected) @@

 Map<Class, DocumentBuilder<Object>> getDocumentBuilders();

-  Map<DirectoryProvider, ReentrantLock> getLockableDirectoryProviders();
-
 Worker getWorker();

 void addOptimizerStrategy(DirectoryProvider<?> provider, OptimizerStrategy optimizerStrategy);
@@(protected) @@
 void addClassToDirectoryProvider(Class clazz, DirectoryProvider<?> directoryProvider);

 Set<Class> getClassesInDirectoryProvider(DirectoryProvider<?> directoryProvider);
+
+  Set<DirectoryProvider> getDirectoryProviders();
+
+  Lock getDirectoryProviderLock(DirectoryProvider dp);
+
+  void addDirectoryProvider(DirectoryProvider<?> provider);
+  
}

Modified: search/trunk/src/java/org/hibernate/search/impl/FullTextSessionImpl.java
===================================================================
--- search/trunk/src/java/org/hibernate/search/impl/FullTextSessionImpl.java  2008-06-26 22:15:36 UTC (rev 14816)
+++ search/trunk/src/java/org/hibernate/search/impl/FullTextSessionImpl.java  2008-06-27 00:25:45 UTC (rev 14817)
@@(protected) @@
import org.hibernate.search.FullTextQuery;
import org.hibernate.search.FullTextSession;
import org.hibernate.search.SearchFactory;
-import org.hibernate.search.SearchException;
import org.hibernate.search.backend.Work;
import org.hibernate.search.backend.WorkType;
import org.hibernate.search.engine.DocumentBuilder;
@@(protected) @@
   //need to add elements in a queue kept at the Session level
   //the queue will be processed by a Lucene(Auto)FlushEventListener
   //note that we could keep this queue somewhere in the event listener in the mean time but that requires
-    // a synchronized hashmap holding this queue on a per session basis plus some session house keeping (yuk)
-    //an other solution would be to subclass SessionImpl instead of having this LuceneSession delecation model
-    // this is an open discussion
+    //a synchronized hashmap holding this queue on a per session basis plus some session house keeping (yuk)
+    //another solution would be to subclass SessionImpl instead of having this LuceneSession delegation model
+    //this is an open discussion
 }

 public SearchFactory getSearchFactory() {

Modified: search/trunk/src/java/org/hibernate/search/impl/SearchFactoryImpl.java
===================================================================
--- search/trunk/src/java/org/hibernate/search/impl/SearchFactoryImpl.java  2008-06-26 22:15:36 UTC (rev 14816)
+++ search/trunk/src/java/org/hibernate/search/impl/SearchFactoryImpl.java  2008-06-27 00:25:45 UTC (rev 14817)
@@(protected) @@
import java.util.Properties;
import java.util.Set;
import java.util.WeakHashMap;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;

import org.apache.lucene.analysis.Analyzer;
@@(protected) @@
   Version.touch();
 }

-  private static final Logger log = LoggerFactory.getLogger( SearchFactoryImpl.class );
+  private final Logger log = LoggerFactory.getLogger( SearchFactoryImpl.class );

 private final Map<Class, DocumentBuilder<Object>> documentBuilders = new HashMap<Class, DocumentBuilder<Object>>();
 //keep track of the index modifiers per DirectoryProvider since multiple entity can use the same directory provider
-  //TODO move the ReentrantLock into DirectoryProviderData.lock, add a getDPLock(DP) and add a Set<DP> getDirectoryProviders() method.
-  private final Map<DirectoryProvider, ReentrantLock> lockableDirectoryProviders =
-      new HashMap<DirectoryProvider, ReentrantLock>();
-  private final Map<DirectoryProvider, DirectoryProviderData> dirProviderData =
-      new HashMap<DirectoryProvider, DirectoryProviderData>();
-  private Worker worker;
-  private ReaderProvider readerProvider;
+  private final Map<DirectoryProvider, DirectoryProviderData> dirProviderData = new HashMap<DirectoryProvider, DirectoryProviderData>();
+  private final Worker worker;
+  private final ReaderProvider readerProvider;
 private BackendQueueProcessorFactory backendQueueProcessorFactory;
 private final Map<String, FilterDef> filterDefinitions = new HashMap<String, FilterDef>();
-  private FilterCachingStrategy filterCachingStrategy;
+  private final FilterCachingStrategy filterCachingStrategy;
 private Map<String, Analyzer> analyzers;
-  private boolean stopped = false;
+  private final AtomicBoolean stopped = new AtomicBoolean( false );

 /**
  * Each directory provider (index) can have its own performance settings.
  */
 private Map<DirectoryProvider, LuceneIndexingParameters> dirProviderIndexingParams =
   new HashMap<DirectoryProvider, LuceneIndexingParameters>();
-  private String indexingStrategy;
+  private final String indexingStrategy;


 public BackendQueueProcessorFactory getBackendQueueProcessorFactory() {
@@(protected) @@
 public SearchFactoryImpl(Configuration cfg) {
   //yuk
   ReflectionManager reflectionManager = getReflectionManager( cfg );
-    setIndexingStrategy(cfg); //need to be done before the document builds
-    InitContext context = new InitContext(cfg);
-    initDocumentBuilders(cfg, reflectionManager, context );
+    this.indexingStrategy = defineIndexingStrategy( cfg ); //need to be done before the document builds
+    initDocumentBuilders( cfg, reflectionManager );

   Set<Class> indexedClasses = documentBuilders.keySet();
   for (DocumentBuilder builder : documentBuilders.values()) {
     builder.postInitialize( indexedClasses );
   }
-    worker = WorkerFactory.createWorker( cfg, this );
-    readerProvider = ReaderProviderFactory.createReaderProvider( cfg, this );
-    buildFilterCachingStrategy( cfg.getProperties() );
+    this.worker = WorkerFactory.createWorker( cfg, this );
+    this.readerProvider = ReaderProviderFactory.createReaderProvider( cfg, this );
+    this.filterCachingStrategy = buildFilterCachingStrategy( cfg.getProperties() );
 }

-  private void setIndexingStrategy(Configuration cfg) {
-    indexingStrategy = cfg.getProperties().getProperty( Environment.INDEXING_STRATEGY, "event" );
+  private static String defineIndexingStrategy(Configuration cfg) {
+    String indexingStrategy = cfg.getProperties().getProperty( Environment.INDEXING_STRATEGY, "event" );
   if ( ! ("event".equals( indexingStrategy ) || "manual".equals( indexingStrategy ) ) ) {
-      throw new SearchException(Environment.INDEXING_STRATEGY + " unknown: " + indexingStrategy);
+      throw new SearchException( Environment.INDEXING_STRATEGY + " unknown: " + indexingStrategy );
   }
+    return indexingStrategy;
 }

 public String getIndexingStrategy() {
@@(protected) @@
 }

 public void close() {
-    if (!stopped) {
-      stopped = true;
+    if ( stopped.compareAndSet( false, true) ) {
     try {
       worker.close();
     }
@@(protected) @@
       log.error( "Worker raises an exception on close()", e );
     }
     //TODO move to DirectoryProviderFactory for cleaner
-      for (DirectoryProvider dp : lockableDirectoryProviders.keySet() ) {
+      for (DirectoryProvider dp : getDirectoryProviders() ) {
       try {
         dp.stop();
       }
@@(protected) @@
   SearchFactoryImpl searchFactory = contextMap.get( cfg );
   if ( searchFactory == null ) {
     searchFactory = new SearchFactoryImpl( cfg );
-
     contextMap.put( cfg, searchFactory );
   }
   return searchFactory;
@@(protected) @@
   return documentBuilders;
 }

-  public Map<DirectoryProvider, ReentrantLock> getLockableDirectoryProviders() {
-    return lockableDirectoryProviders;
+  public Set<DirectoryProvider> getDirectoryProviders() {
+    return this.dirProviderData.keySet();
 }

 public Worker getWorker() {
@@(protected) @@
   ReflectionManager reflectionManager;
   try {
     //TODO introduce a ReflectionManagerHolder interface to avoid reflection
-      //I want to avoid hard link between HAN and Validator for usch a simple need
+      //I want to avoid hard link between HAN and Validator for such a simple need
     //reuse the existing reflectionManager one when possible
     reflectionManager =
         (ReflectionManager) cfg.getClass().getMethod( "getReflectionManager" ).invoke( cfg );
@@(protected) @@
   return analyzer;
 }

-  private void initDocumentBuilders(Configuration cfg, ReflectionManager reflectionManager, InitContext context) {
+  private void initDocumentBuilders(Configuration cfg, ReflectionManager reflectionManager) {
+    InitContext context = new InitContext( cfg );
   Iterator iter = cfg.getClassMappings();
   DirectoryProviderFactory factory = new DirectoryProviderFactory();

-    while (iter.hasNext()) {
+    while ( iter.hasNext() ) {
     PersistentClass clazz = (PersistentClass) iter.next();
     Class<?> mappedClass = clazz.getMappedClass();
     if (mappedClass != null) {
@@(protected) @@
   factory.startDirectoryProviders();
 }

-  private void buildFilterCachingStrategy(Properties properties) {
+  private static FilterCachingStrategy buildFilterCachingStrategy(Properties properties) {
+    FilterCachingStrategy filterCachingStrategy;
   String impl = properties.getProperty( Environment.FILTER_CACHING_STRATEGY );
   if ( StringHelper.isEmpty( impl ) || "mru".equalsIgnoreCase( impl ) ) {
     filterCachingStrategy = new MRUFilterCachingStrategy();
@@(protected) @@
     }
   }
   filterCachingStrategy.initialize( properties );
+    return filterCachingStrategy;
 }

 public FilterCachingStrategy getFilterCachingStrategy() {
@@(protected) @@
 }

 private static class DirectoryProviderData {
+    public final Lock dirLock = new ReentrantLock();
   public OptimizerStrategy optimizerStrategy;
   public Set<Class> classes = new HashSet<Class>(2);
 }
+
+  public Lock getDirectoryProviderLock(DirectoryProvider dp) {
+    return this.dirProviderData.get( dp ).dirLock;
+  }
+
+  public void addDirectoryProvider(DirectoryProvider<?> provider) {
+    this.dirProviderData.put( provider, new DirectoryProviderData() );
+  }
+  
}

Modified: search/trunk/src/java/org/hibernate/search/reader/SharedReaderProvider.java
===================================================================
--- search/trunk/src/java/org/hibernate/search/reader/SharedReaderProvider.java  2008-06-26 22:15:36 UTC (rev 14816)
+++ search/trunk/src/java/org/hibernate/search/reader/SharedReaderProvider.java  2008-06-27 00:25:45 UTC (rev 14817)
@@(protected) @@
 }

 public void initialize(Properties props, SearchFactoryImplementor searchFactoryImplementor) {
-    Set<DirectoryProvider> providers = searchFactoryImplementor.getLockableDirectoryProviders().keySet();
+    Set<DirectoryProvider> providers = searchFactoryImplementor.getDirectoryProviders();
   perDirectoryProviderManipulationLocks = new HashMap<DirectoryProvider, Lock>( providers.size() );
   for (DirectoryProvider dp : providers) {
     perDirectoryProviderManipulationLocks.put( dp, new ReentrantLock() );

Modified: search/trunk/src/java/org/hibernate/search/reader/SharingBufferReaderProvider.java
===================================================================
--- search/trunk/src/java/org/hibernate/search/reader/SharingBufferReaderProvider.java  2008-06-26 22:15:36 UTC (rev 14816)
+++ search/trunk/src/java/org/hibernate/search/reader/SharingBufferReaderProvider.java  2008-06-27 00:25:45 UTC (rev 14817)
@@(protected) @@

 public void initialize(Properties props, SearchFactoryImplementor searchFactoryImplementor) {
   Map<DirectoryProvider,PerDirectoryLatestReader> map = new HashMap<DirectoryProvider,PerDirectoryLatestReader>();
-    Set<DirectoryProvider> providers = searchFactoryImplementor.getLockableDirectoryProviders().keySet();
+    Set<DirectoryProvider> providers = searchFactoryImplementor.getDirectoryProviders();
   for ( DirectoryProvider provider : providers ) {
     try {
       map.put( provider, new PerDirectoryLatestReader( provider ) );

Modified: search/trunk/src/java/org/hibernate/search/store/DirectoryProviderFactory.java
===================================================================
--- search/trunk/src/java/org/hibernate/search/store/DirectoryProviderFactory.java  2008-06-26 22:15:36 UTC (rev 14816)
+++ search/trunk/src/java/org/hibernate/search/store/DirectoryProviderFactory.java  2008-06-27 00:25:45 UTC (rev 14817)
@@(protected) @@
public class DirectoryProviderFactory {
 
 private final List<DirectoryProvider<?>> providers = new ArrayList<DirectoryProvider<?>>();
-  private static String DEFAULT_DIRECTORY_PROVIDER = FSDirectoryProvider.class.getName();
+  private static final String DEFAULT_DIRECTORY_PROVIDER = FSDirectoryProvider.class.getName();
 
 private static final String SHARDING_STRATEGY = "sharding_strategy";
 private static final String NBR_OF_SHARDS = SHARDING_STRATEGY + ".nbr_of_shards";
@@(protected) @@
     configureIndexingParameters( searchFactoryImplementor, indexProps, provider );
     providers.add( provider );
     searchFactoryImplementor.addClassToDirectoryProvider(entity, provider);
-      if ( !searchFactoryImplementor.getLockableDirectoryProviders().containsKey( provider ) ) {
-        searchFactoryImplementor.getLockableDirectoryProviders().put( provider, new ReentrantLock() );
+      if ( ! searchFactoryImplementor.getDirectoryProviders().contains( provider ) ) {
+        searchFactoryImplementor.addDirectoryProvider( provider );
     }
     return provider;
   }

Modified: search/trunk/src/java/org/hibernate/search/store/FSMasterDirectoryProvider.java
===================================================================
--- search/trunk/src/java/org/hibernate/search/store/FSMasterDirectoryProvider.java  2008-06-26 22:15:36 UTC (rev 14816)
+++ search/trunk/src/java/org/hibernate/search/store/FSMasterDirectoryProvider.java  2008-06-27 00:25:45 UTC (rev 14817)
@@(protected) @@
     long start = System.currentTimeMillis();
     inProgress = true;
     if ( directoryProviderLock == null ) {
-        directoryProviderLock = searchFactory.getLockableDirectoryProviders().get( directoryProvider );
+        directoryProviderLock = searchFactory.getDirectoryProviderLock( directoryProvider );
       directoryProvider = null;
       searchFactory = null; //get rid of any useless link (help hot redeployment?)
     }

Modified: search/trunk/src/test/org/hibernate/search/test/reader/performance/ReaderPerformance.java
===================================================================
--- search/trunk/src/test/org/hibernate/search/test/reader/performance/ReaderPerformance.java  2008-06-26 22:15:36 UTC (rev 14816)
+++ search/trunk/src/test/org/hibernate/search/test/reader/performance/ReaderPerformance.java  2008-06-27 00:25:45 UTC (rev 14817)
@@(protected) @@
 private static final File baseIndexDir = new File( new File( "." ), "indextemp" );
 
 //more iterations for more reliable measures:
-  private static final int TOTAL_WORK_BATCHES = 1000;
+  private static final int TOTAL_WORK_BATCHES = 10;
 //the next 3 define the kind of workload mix to test on:
 private static final int SEARCHERS_PER_BATCH = 10;
 private static final int UPDATES_PER_BATCH = 2;
 private static final int INSERTIONS_PER_BATCH = 1;

-  private static final int WORKER_THREADS = 30;
+  private static final int WORKER_THREADS = 20;
+
+  private static final int WARMUP_CYCLES = 6;
 
 protected void setUp() throws Exception {
   baseIndexDir.mkdir();
@@(protected) @@
     FileHelper.delete( file );
   }
   super.setUp();
-    //enable this line:
-//    buildBigIndex();
 }
 
 public void testFakeTest(){
@@(protected) @@
 }

 private void buildBigIndex() throws InterruptedException, CorruptIndexException, LockObtainFailedException, IOException {
+    System.out.println( "Going to create fake index..." );
   FSDirectory directory = FSDirectory.getDirectory(new File(baseIndexDir, Detective.class.getCanonicalName()));
   IndexWriter iw = new IndexWriter( directory, new SimpleAnalyzer(), true );
   IndexFillRunnable filler = new IndexFillRunnable( iw );
   ThreadPoolExecutor executor = (ThreadPoolExecutor) Executors.newFixedThreadPool( WORKER_THREADS );
-    for (int batch=0; batch<=100000; batch++){
+    for (int batch=0; batch<=5000000; batch++){
     executor.execute( filler );
   }
   executor.shutdown();
   executor.awaitTermination( 600, TimeUnit.SECONDS );
+    iw.optimize();
   iw.close();
   System.out.println( "Index created." );
 }
@@(protected) @@

 protected void tearDown() throws Exception {
   super.tearDown();
-//    FileHelper.delete( baseIndexDir );
+    FileHelper.delete( baseIndexDir );
 }
 
 protected void configure(org.hibernate.cfg.Configuration cfg) {
@@(protected) @@
 protected abstract String getReaderStrategyName();
 
 //this test is disabled as it is very slow (and someone should read the output)
-  public final void disabled_testPerformance() throws InterruptedException{
+  public final void disabled_testPerformance() throws InterruptedException, CorruptIndexException, LockObtainFailedException, IOException {
+    buildBigIndex();
+    for (int i=0; i<WARMUP_CYCLES; i++) {
+      timeMs();
+    }
+  }
+  
+  private final void timeMs() throws InterruptedException {
   ThreadPoolExecutor executor = (ThreadPoolExecutor) Executors.newFixedThreadPool( WORKER_THREADS );
   CountDownLatch startSignal = new CountDownLatch(1);
   InsertActivity insertionTask = new InsertActivity( getSessions(), startSignal );

_______________________________________________
hibernate-commits mailing list
hibernate-commits@(protected)
https://lists.jboss.org/mailman/listinfo/hibernate-commits
©2008 gg3721.com - Jax Systems, LLC, U.S.A.