Skip to content

Commit 492c5ea

Browse files
ligangtyligangty
authored
Merge pull request #1480 from jdcasey/master
Implement affected-by reverse cache
2 parents 44a4eda + 9c26897 commit 492c5ea

File tree

12 files changed

+291
-74
lines changed

12 files changed

+291
-74
lines changed

‎addons/pkg-npm/jaxrs/src/main/java/org/commonjava/indy/pkg/npm/jaxrs/NPMContentAccessHandler.java

+4-1
Original file line numberDiff line numberDiff line change
@@ -82,6 +82,9 @@ public class NPMContentAccessHandler
8282
@Inject
8383
private ResponseHelper responseHelper;
8484

85+
@Inject
86+
private PackageMetadataMerger packageMetadataMerger;
87+
8588
@Override
8689
public Response doCreate( String packageType, String type, String name, String path, HttpServletRequest request,
8790
EventMetadata eventMetadata, Supplier<URI> uriBuilder )
@@ -127,7 +130,7 @@ public Response doCreate( String packageType, String type, String name, String p
127130
// then store the transfer, delete unuseful temp and meta transfers.
128131
if ( temp != null && temp.exists() )
129132
{
130-
stream = new PackageMetadataMerger().merge( temp, tomerge );
133+
stream = packageMetadataMerger.merge( temp, tomerge );
131134
Transfer merged = contentController.store( sk, path, stream, eventMetadata );
132135

133136
// for npm group, will not replace with the new http meta when re-upload,

‎api/src/main/java/org/commonjava/indy/data/StoreDataManager.java

+5
Original file line numberDiff line numberDiff line change
@@ -17,11 +17,13 @@
1717

1818
import org.commonjava.indy.audit.ChangeSummary;
1919
import org.commonjava.indy.model.core.ArtifactStore;
20+
import org.commonjava.indy.model.core.Group;
2021
import org.commonjava.indy.model.core.HostedRepository;
2122
import org.commonjava.indy.model.core.StoreKey;
2223
import org.commonjava.indy.model.core.StoreType;
2324
import org.commonjava.maven.galley.event.EventMetadata;
2425

26+
import java.util.Collection;
2527
import java.util.Map;
2628
import java.util.Set;
2729
import java.util.stream.Stream;
@@ -138,4 +140,7 @@ void reload()
138140
Set<StoreKey> getStoreKeysByPkg( String pkg );
139141

140142
Set<StoreKey> getStoreKeysByPkgAndType( final String pkg, final StoreType type );
143+
144+
Set<Group> affectedBy( Collection<StoreKey> keys )
145+
throws IndyDataException;
141146
}

‎db/common/src/main/java/org/commonjava/indy/db/common/AbstractStoreDataManager.java

+89
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@
2828
import org.commonjava.indy.data.StoreValidator;
2929
import org.commonjava.indy.measure.annotation.Measure;
3030
import org.commonjava.indy.model.core.ArtifactStore;
31+
import org.commonjava.indy.model.core.Group;
3132
import org.commonjava.indy.model.core.HostedRepository;
3233
import org.commonjava.indy.model.core.StoreKey;
3334
import org.commonjava.indy.model.core.StoreType;
@@ -37,7 +38,11 @@
3738
import org.slf4j.LoggerFactory;
3839

3940
import javax.inject.Inject;
41+
import java.util.ArrayList;
42+
import java.util.Collection;
4043
import java.util.Collections;
44+
import java.util.HashSet;
45+
import java.util.List;
4146
import java.util.Map;
4247
import java.util.Set;
4348
import java.util.concurrent.atomic.AtomicReference;
@@ -47,6 +52,8 @@
4752
import java.util.stream.Collectors;
4853
import java.util.stream.Stream;
4954

55+
import static org.commonjava.indy.db.common.StoreUpdateAction.DELETE;
56+
import static org.commonjava.indy.db.common.StoreUpdateAction.STORE;
5057
import static org.commonjava.indy.model.core.StoreType.hosted;
5158

5259
public abstract class AbstractStoreDataManager
@@ -146,6 +153,11 @@ else if ( !store.isDisabled() && original.isDisabled() )
146153
}
147154
}
148155
}
156+
// Hosted or Remote update does not change affectedBy relationships
157+
if ( store instanceof Group )
158+
{
159+
refreshAffectedBy( store, original, STORE );
160+
}
149161
}
150162

151163
protected void preDelete( final ArtifactStore store, final ChangeSummary summary, final boolean fireEvents,
@@ -169,6 +181,14 @@ protected void postDelete( final ArtifactStore store, final ChangeSummary summar
169181
{
170182
dispatcher.deleted( eventMetadata, store );
171183
}
184+
185+
refreshAffectedBy( store, null, DELETE );
186+
}
187+
188+
protected void refreshAffectedBy( final ArtifactStore store, final ArtifactStore original, StoreUpdateAction action )
189+
throws IndyDataException
190+
{
191+
//do nothing by default
172192
}
173193

174194
protected abstract ArtifactStore removeArtifactStoreInternal( StoreKey key );
@@ -422,4 +442,73 @@ public Set<StoreKey> getStoreKeysByPkgAndType( final String pkg, final StoreType
422442
return streamArtifactStoreKeys().filter( key -> key.getPackageType().equals( pkg ) && key.getType() == type )
423443
.collect( Collectors.toSet() );
424444
}
445+
446+
public Set<Group> affectedBy( final Collection<StoreKey> keys )
447+
throws IndyDataException
448+
{
449+
return affectedByFromStores( keys );
450+
}
451+
452+
protected Set<Group> affectedByFromStores( final Collection<StoreKey> keys )
453+
throws IndyDataException
454+
{
455+
Logger logger = LoggerFactory.getLogger( getClass() );
456+
logger.debug( "Getting groups affected by: {}", keys );
457+
458+
List<StoreKey> toProcess = new ArrayList<>( new HashSet<>( keys ) );
459+
460+
Set<Group> groups = new HashSet<>();
461+
if ( toProcess.isEmpty() )
462+
{
463+
return groups;
464+
}
465+
466+
Set<StoreKey> processed = new HashSet<>();
467+
final String packageType = toProcess.get( 0 ).getPackageType();
468+
469+
Set<ArtifactStore> all = this.getAllArtifactStores().stream().filter( st -> {
470+
if ( packageType != null && !st.getPackageType().equals( packageType ) )
471+
{
472+
return false;
473+
}
474+
475+
if ( st.getType() != StoreType.group )
476+
{
477+
return false;
478+
}
479+
480+
return true;
481+
} ).collect( Collectors.toSet() );
482+
483+
while ( !toProcess.isEmpty() )
484+
{
485+
// as long as we have another key to process, pop it off the list (remove it) and process it.
486+
StoreKey next = toProcess.remove( 0 );
487+
if ( processed.contains( next ) )
488+
{
489+
// if we've already handled this group (via another branch in the group membership tree, etc. then don't bother.
490+
continue;
491+
}
492+
493+
// use this to avoid reprocessing groups we've already encountered.
494+
processed.add( next );
495+
496+
for ( ArtifactStore store : all )
497+
{
498+
if ( ( store instanceof Group ) && !processed.contains( store.getKey() ) )
499+
{
500+
Group g = (Group) store;
501+
if ( g.getConstituents() != null && g.getConstituents().contains( next ) )
502+
{
503+
groups.add( g );
504+
505+
// add this group as another one to process for groups that contain it...and recurse upwards
506+
toProcess.add( g.getKey() );
507+
}
508+
}
509+
}
510+
}
511+
512+
return groups;
513+
}
425514
}

‎db/common/src/main/java/org/commonjava/indy/db/common/DefaultArtifactStoreQuery.java

+5-70
Original file line numberDiff line numberDiff line change
@@ -404,65 +404,7 @@ public Set<Group> getGroupsAffectedBy( StoreKey... keys )
404404
public Set<Group> getGroupsAffectedBy( Collection<StoreKey> keys )
405405
throws IndyDataException
406406
{
407-
Logger logger = LoggerFactory.getLogger( getClass() );
408-
logger.debug( "Getting groups affected by: {}", keys );
409-
410-
List<StoreKey> toProcess = new ArrayList<>( new HashSet<>( keys ) );
411-
412-
Set<Group> groups = new HashSet<>();
413-
if ( toProcess.isEmpty() )
414-
{
415-
return groups;
416-
}
417-
418-
Set<StoreKey> processed = new HashSet<>();
419-
420-
Set<StoreKey> all = dataManager.getStoreKeysByPkgAndType( toProcess.get( 0 ).getPackageType(), group );
421-
422-
logger.debug( "There are {} groups need to loop checking for affected by", all.size() );
423-
424-
Set<ArtifactStore> allStores = all.stream().map( k-> {
425-
try
426-
{
427-
return dataManager.getArtifactStore( k );
428-
}
429-
catch ( IndyDataException e )
430-
{
431-
logger.error( "Error to get store {}", k );
432-
return null;
433-
}
434-
} ).filter( Objects::nonNull ).collect( Collectors.toSet());
435-
436-
while ( !toProcess.isEmpty() )
437-
{
438-
// as long as we have another key to process, pop it off the list (remove it) and process it.
439-
StoreKey next = toProcess.remove( 0 );
440-
if ( processed.contains( next ) )
441-
{
442-
// if we've already handled this group (via another branch in the group membership tree, etc. then don't bother.
443-
continue;
444-
}
445-
446-
// use this to avoid reprocessing groups we've already encountered.
447-
processed.add( next );
448-
449-
for ( ArtifactStore store : allStores )
450-
{
451-
if ( ( store instanceof Group ) && !processed.contains( store.getKey() ) )
452-
{
453-
Group g = (Group) store;
454-
if ( g.getConstituents() != null && g.getConstituents().contains( next ) )
455-
{
456-
groups.add( g );
457-
458-
// add this group as another one to process for groups that contain it...and recurse upwards
459-
toProcess.add( g.getKey() );
460-
}
461-
}
462-
}
463-
}
464-
465-
return groups;
407+
return dataManager.affectedBy( keys );
466408
}
467409

468410
public Stream<StoreKey> keyStream()
@@ -585,18 +527,10 @@ private List<ArtifactStore> getGroupOrdering( final String groupName,
585527
}
586528

587529
final List<ArtifactStore> result = new ArrayList<>();
588-
recurseGroup( master, result, new HashSet<>(), includeGroups, recurseGroups );
589530

590-
return result;
591-
}
592-
593-
private void recurseGroup( final Group master,
594-
final List<ArtifactStore> result, final Set<StoreKey> seen, final boolean includeGroups,
595-
final boolean recurseGroups )
596-
throws IndyDataException
597-
{
598531
AtomicReference<IndyDataException> errorRef = new AtomicReference<>();
599532
LinkedList<Group> toCheck = new LinkedList<>();
533+
Set<StoreKey> seen = new HashSet<>();
600534
toCheck.add( master );
601535

602536
while ( !toCheck.isEmpty() )
@@ -605,7 +539,7 @@ private void recurseGroup( final Group master,
605539

606540
if ( next == null || next.isDisabled() && Boolean.TRUE.equals( enabled ) )
607541
{
608-
return;
542+
continue;
609543
}
610544

611545
List<StoreKey> members = new ArrayList<>( next.getConstituents() );
@@ -614,7 +548,6 @@ private void recurseGroup( final Group master,
614548
result.add( next );
615549
}
616550

617-
// TODO: Need to refactor away from actual recursion.
618551
members.forEach( ( key ) ->
619552
{
620553
if ( !seen.contains( key ) )
@@ -651,6 +584,8 @@ private void recurseGroup( final Group master,
651584
throw error;
652585
}
653586
}
587+
588+
return result;
654589
}
655590

656591
}

‎db/common/src/main/java/org/commonjava/indy/db/common/StoreUpdateAction.java

+6
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
package org.commonjava.indy.db.common;
2+
3+
public enum StoreUpdateAction
4+
{
5+
STORE, DELETE;
6+
}

‎db/infinispan/src/main/java/org/commonjava/indy/infinispan/data/AffectedByStoreCache.java

+16
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
package org.commonjava.indy.infinispan.data;
2+
3+
import javax.inject.Qualifier;
4+
import java.lang.annotation.Documented;
5+
import java.lang.annotation.ElementType;
6+
import java.lang.annotation.Retention;
7+
import java.lang.annotation.RetentionPolicy;
8+
import java.lang.annotation.Target;
9+
10+
@Qualifier
11+
@Target( { ElementType.FIELD, ElementType.PARAMETER, ElementType.METHOD})
12+
@Retention( RetentionPolicy.RUNTIME)
13+
@Documented
14+
public @interface AffectedByStoreCache
15+
{
16+
}

0 commit comments

Comments
 (0)