Skip to content

Commit 7b0bc2d

Browse files
ligangtyligangty
authored
Merge pull request #1475 from ligangty/master
Revert "Async the group de-index during upload in content-index service"
2 parents 0a08e56 + 06b5fc7 commit 7b0bc2d

File tree

6 files changed

+70
-297
lines changed

6 files changed

+70
-297
lines changed

‎addons/content-index/src/main/java/org/commonjava/indy/content/index/ContentIndexCacheProducer.java

+1-11
Original file line numberDiff line numberDiff line change
@@ -15,10 +15,8 @@
1515
*/
1616
package org.commonjava.indy.content.index;
1717

18-
import org.commonjava.indy.content.index.deindex.DeIndexHandlingCache;
19-
import org.commonjava.indy.content.index.deindex.DeIndexInfo;
18+
import org.commonjava.indy.model.core.StoreKey;
2019
import org.commonjava.indy.subsys.infinispan.BasicCacheHandle;
21-
import org.commonjava.indy.subsys.infinispan.CacheHandle;
2220
import org.commonjava.indy.subsys.infinispan.CacheProducer;
2321

2422
import javax.enterprise.context.ApplicationScoped;
@@ -83,12 +81,4 @@ public BasicCacheHandle<IndexedStorePath, IndexedStorePath> contentIndexCacheCfg
8381
{
8482
return cacheProducer.getBasicCache( "content-index" );
8583
}
86-
87-
@DeIndexHandlingCache
88-
@Produces
89-
@ApplicationScoped
90-
public CacheHandle<Long, DeIndexInfo> produceDeIndexHandlingCache()
91-
{
92-
return cacheProducer.getCache( "content-de-index" );
93-
}
9484
}

‎addons/content-index/src/main/java/org/commonjava/indy/content/index/IndexingContentManagerDecorator.java

+69-47
Original file line numberDiff line numberDiff line change
@@ -18,20 +18,17 @@
1818
import org.commonjava.indy.IndyWorkflowException;
1919
import org.commonjava.indy.content.ContentManager;
2020
import org.commonjava.indy.content.index.conf.ContentIndexConfig;
21-
import org.commonjava.indy.content.index.deindex.DeIndexHandlingCache;
22-
import org.commonjava.indy.content.index.deindex.DeIndexHandlingCacheListener;
23-
import org.commonjava.indy.content.index.deindex.DeIndexInfo;
2421
import org.commonjava.indy.core.content.PathMaskChecker;
2522
import org.commonjava.indy.data.IndyDataException;
2623
import org.commonjava.indy.data.StoreDataManager;
2724
import org.commonjava.indy.measure.annotation.Measure;
25+
import org.commonjava.indy.measure.annotation.MetricNamed;
2826
import org.commonjava.indy.model.core.ArtifactStore;
2927
import org.commonjava.indy.model.core.Group;
3028
import org.commonjava.indy.model.core.HostedRepository;
3129
import org.commonjava.indy.model.core.StoreKey;
3230
import org.commonjava.indy.model.core.StoreType;
3331
import org.commonjava.indy.model.galley.KeyedLocation;
34-
import org.commonjava.indy.subsys.infinispan.CacheHandle;
3532
import org.commonjava.indy.util.LocationUtils;
3633
import org.commonjava.maven.galley.event.EventMetadata;
3734
import org.commonjava.maven.galley.model.ConcreteResource;
@@ -43,7 +40,6 @@
4340
import org.slf4j.Logger;
4441
import org.slf4j.LoggerFactory;
4542

46-
import javax.annotation.PostConstruct;
4743
import javax.decorator.Decorator;
4844
import javax.decorator.Delegate;
4945
import javax.enterprise.inject.Any;
@@ -56,10 +52,9 @@
5652
import java.util.Objects;
5753
import java.util.Set;
5854

59-
import static org.commonjava.indy.content.index.deindex.DeIndexInfo.TYPE_MULTI;
60-
import static org.commonjava.indy.content.index.deindex.DeIndexInfo.TYPE_SINGLE;
6155
import static org.commonjava.indy.core.content.group.GroupMergeHelper.GROUP_METADATA_EXISTS;
6256
import static org.commonjava.indy.core.content.group.GroupMergeHelper.GROUP_METADATA_GENERATED;
57+
import static org.commonjava.indy.measure.annotation.MetricNamed.DEFAULT;
6358

6459
/**
6560
* Decorator for ContentManager which uses Infinispan to index content to avoid having to iterate all members of large
@@ -93,14 +88,6 @@ public abstract class IndexingContentManagerDecorator
9388
@Inject
9489
private ContentIndexConfig indexCfg;
9590

96-
@Inject
97-
@DeIndexHandlingCache
98-
private CacheHandle<Long, DeIndexInfo> deIndexQueue;
99-
100-
@Inject
101-
private DeIndexHandlingCacheListener deIndexHandler;
102-
103-
10491
protected IndexingContentManagerDecorator()
10592
{
10693
}
@@ -125,27 +112,6 @@ protected IndexingContentManagerDecorator( final ContentManager delegate, final
125112
this.indexCfg = indexCfg;
126113
}
127114

128-
protected IndexingContentManagerDecorator( final ContentManager delegate, final StoreDataManager storeDataManager,
129-
final SpecialPathManager specialPathManager,
130-
final ContentIndexManager indexManager, final NotFoundCache nfc,
131-
final ContentIndexConfig indexCfg,
132-
final CacheHandle<Long, DeIndexInfo> deIndexQueue,
133-
final DeIndexHandlingCacheListener deIndexHandler )
134-
{
135-
this( delegate, storeDataManager, specialPathManager, indexManager, nfc, indexCfg );
136-
this.deIndexQueue = deIndexQueue;
137-
this.deIndexHandler = deIndexHandler;
138-
}
139-
140-
@PostConstruct
141-
public void init()
142-
{
143-
deIndexQueue.executeCache( c -> {
144-
c.addListener( deIndexHandler );
145-
return null;
146-
} );
147-
}
148-
149115
@Override
150116
public Transfer retrieveFirst( final List<? extends ArtifactStore> stores, final String path )
151117
throws IndyWorkflowException
@@ -234,7 +200,7 @@ public Transfer retrieve( final ArtifactStore store, final String path, final Ev
234200
else if ( isAuthoritativelyMissing( store ) )
235201
{
236202
logger.debug(
237-
"Not found indexed transfer: {} and authoritative index switched on. Considering not found and return null.", transfer );
203+
"Not found indexed transfer: {} and authoritative index switched on. Considering not found and return null." );
238204
return null;
239205
}
240206

@@ -492,7 +458,7 @@ public Transfer getTransfer( final ArtifactStore store, final String path, final
492458
else if ( isAuthoritativelyMissing( store ) )
493459
{
494460
logger.info(
495-
"Not found indexed transfer: {} and authoritative index switched on. Considering not found and return null.", transfer );
461+
"Not found indexed transfer: {} and authoritative index switched on. Considering not found and return null." );
496462
return null;
497463
}
498464

@@ -544,7 +510,7 @@ else if ( isAuthoritativelyMissing( store ) )
544510
return transfer;
545511
}
546512

547-
@Measure
513+
@Measure( timers = @MetricNamed( DEFAULT ), exceptions = @MetricNamed( DEFAULT ) )
548514
@Deprecated
549515
public Transfer getIndexedMemberTransfer( final Group group, final String path, TransferOperation op,
550516
ContentManagementFunction func, final EventMetadata metadata )
@@ -636,7 +602,7 @@ public Transfer getTransfer( final StoreKey storeKey, final String path, final T
636602

637603
if ( isAuthoritativelyMissing( store ) )
638604
{
639-
logger.debug( "Not found indexed transfer: {} and authoritative index switched on. Return null.", transfer );
605+
logger.debug( "Not found indexed transfer: {} and authoritative index switched on. Return null." );
640606
return null;
641607
}
642608

@@ -730,11 +696,41 @@ public Transfer store( final ArtifactStore store, final String path, final Input
730696
{
731697
Logger logger = LoggerFactory.getLogger( getClass() );
732698
logger.trace( "Storing: {} in: {} from indexing level", path, store.getKey() );
733-
final Transfer transfer = delegate.store( store, path, stream, op, eventMetadata );
734-
//NOSSUP-87 & NOS-2233: As the getGroupsAffectedBy is a time-consumed ops which is killing the
735-
//upload performance, and seems the de-index ops for the uploaded path is not a big factor to
736-
//affect following operations, we should think about to async it here.
737-
deIndexQueue.put( System.currentTimeMillis(), new DeIndexInfo( transfer, store, TYPE_SINGLE ) );
699+
Transfer transfer = delegate.store( store, path, stream, op, eventMetadata );
700+
if ( transfer != null )
701+
{
702+
if ( indexCfg.isEnabled() )
703+
{
704+
logger.trace( "Indexing: {} in: {}", transfer, store.getKey() );
705+
indexManager.indexTransferIn( transfer, store.getKey() );
706+
}
707+
708+
if ( store instanceof Group )
709+
{
710+
nfc.clearMissing( new ConcreteResource( LocationUtils.toLocation( store ), path ) );
711+
}
712+
// We should deIndex the path for all parent groups because the new content of the path
713+
// may change the content index sequence based on the constituents sequence in parent groups
714+
if ( store.getType() == StoreType.hosted )
715+
{
716+
try
717+
{
718+
Set<Group> groups = storeDataManager.query().getGroupsAffectedBy( store.getKey() );
719+
if ( groups != null && !groups.isEmpty() && indexCfg.isEnabled() )
720+
{
721+
groups.forEach( g -> indexManager.deIndexStorePath( g.getKey(), path ) );
722+
}
723+
}
724+
catch ( IndyDataException e )
725+
{
726+
throw new IndyWorkflowException(
727+
"Failed to get groups which contains: %s for NFC handling. Reason: %s", e, store.getKey(),
728+
e.getMessage() );
729+
}
730+
731+
}
732+
}
733+
// nfcClearByContaining( store, path );
738734

739735
return transfer;
740736
}
@@ -751,8 +747,34 @@ public Transfer store( final List<? extends ArtifactStore> stores, final StoreKe
751747
final InputStream stream, final TransferOperation op, final EventMetadata eventMetadata )
752748
throws IndyWorkflowException
753749
{
754-
final Transfer transfer = delegate.store( stores, topKey, path, stream, op, eventMetadata );
755-
deIndexQueue.put( System.currentTimeMillis(), new DeIndexInfo( transfer, null, topKey, TYPE_MULTI ) );
750+
Transfer transfer = delegate.store( stores, topKey, path, stream, op, eventMetadata );
751+
if ( transfer != null )
752+
{
753+
if ( indexCfg.isEnabled() )
754+
{
755+
indexManager.indexTransferIn( transfer, topKey );
756+
}
757+
758+
try
759+
{
760+
ArtifactStore topStore = storeDataManager.getArtifactStore( topKey );
761+
nfc.clearMissing( new ConcreteResource( LocationUtils.toLocation( topStore ), path ) );
762+
763+
if ( indexCfg.isEnabled() )
764+
{
765+
// We should deIndex the path for all parent groups because the new content of the path
766+
// may change the content index sequence based on the constituents sequence in parent groups
767+
indexManager.deIndexStorePath( topKey, path );
768+
}
769+
}
770+
catch ( IndyDataException e )
771+
{
772+
Logger logger = LoggerFactory.getLogger( getClass() );
773+
logger.error( String.format( "Failed to retrieve top store: %s for NFC management. Reason: %s",
774+
topKey, e.getMessage()), e );
775+
}
776+
}
777+
756778
return transfer;
757779
}
758780

‎addons/content-index/src/main/java/org/commonjava/indy/content/index/deindex/DeIndexHandlingCache.java

-19
This file was deleted.

0 commit comments

Comments
 (0)