Skip to content

Commit adcf210

Browse files
jasonk000jasonk000
committed
push in netflix services
1 parent c9b57ed commit adcf210

10 files changed

+628
-1
lines changed

Diff for: ‎server/server.gradle

+28
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
plugins {
1414
id 'application'
1515
id "org.springframework.boot" version "3.1.4"
16+
id "com.netflix.nebula.jakartaee-migration"
1617
}
1718

1819
apply from: "$rootDir/gradle/java.gradle"
@@ -22,6 +23,20 @@ springBoot {
2223
applicationName = 'jifa'
2324
}
2425

26+
jakartaeeMigration {
27+
includeTransform('com.netflix.identity:identity-context-spring')
28+
includeTransform('com.netflix.identity:identity-context-core')
29+
includeTransform('com.netflix.identity:identity-context-api')
30+
includeTransform('com.netflix.identity:identity-context-servlet')
31+
includeTransform('com.netflix.identity:identity-context-servlet')
32+
includeTransform('com.netflix.archaius:archaius2-api')
33+
includeTransform('com.netflix.archaius:archaius2-core')
34+
includeTransform('com.netflix.archaius:archaius2-guice')
35+
includeTransform('com.netflix.archaius:archaius2-persisted2')
36+
includeTransform('com.netflix.spring:spring-boot-netflix-configuration')
37+
migrate()
38+
}
39+
2540
dependencies {
2641
implementation project(':common')
2742
implementation project(':analysis')
@@ -33,7 +48,19 @@ dependencies {
3348
implementation 'org.springframework.boot:spring-boot-starter-web'
3449
implementation 'org.springframework.boot:spring-boot-starter-websocket'
3550
implementation 'org.springframework.boot:spring-boot-starter-validation'
51+
implementation 'org.springframework.boot:spring-boot'
3652
implementation 'org.springframework.boot:spring-boot-starter-security'
53+
54+
implementation 'com.netflix.spring:spring-boot-netflix-sso:3.1.19'
55+
implementation 'com.netflix.spring:spring-boot-netflix-sso-authn-embedded:3.1.19'
56+
implementation 'com.netflix.spring:spring-boot-netflix-starter-configuration:3.1.19'
57+
implementation 'com.netflix.spring:spring-boot-netflix-starter-security:3.1.19'
58+
implementation 'com.netflix.spring:spring-boot-netflix-starter-security-authn-embedded:3.1.19'
59+
60+
implementation 'netflix:gandalf-authz-client:latest.release'
61+
implementation 'com.netflix.nflxe2etokens:nflx-e2etokens-validation:latest.release'
62+
implementation 'com.netflix.identity:identity-context-vertx4:1.8.0'
63+
3764
implementation 'org.springframework.boot:spring-boot-starter-oauth2-client'
3865
implementation 'org.springframework.boot:spring-boot-starter-oauth2-resource-server'
3966
implementation 'org.springframework.boot:spring-boot-starter-data-jpa'
@@ -144,6 +171,7 @@ bootJar {
144171
attributes('Implementation-Title': 'Eclipse Jifa')
145172
}
146173
archiveFileName = 'jifa.jar'
174+
duplicatesStrategy = DuplicatesStrategy.EXCLUDE
147175
}
148176

149177
bootDistTar {

Diff for: ‎server/src/main/java/org/eclipse/jifa/server/service/impl/netflix/FileAccessService.java

+8
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
package org.eclipse.jifa.server.service.impl.netflix;
2+
3+
import org.eclipse.jifa.server.domain.entity.shared.file.BaseFileEntity;
4+
5+
public interface FileAccessService {
6+
7+
public void checkAuthorityForCurrentUser(BaseFileEntity file);
8+
}

Diff for: ‎server/src/main/java/org/eclipse/jifa/server/service/impl/netflix/InstanceCommand.java

+49
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
package org.eclipse.jifa.server.service.impl.netflix;
2+
3+
import java.util.regex.Matcher;
4+
import java.util.regex.Pattern;
5+
6+
public class InstanceCommand {
7+
final String instanceId;
8+
final String commandId;
9+
final long pid;
10+
11+
final static Pattern VALID_REGEX = Pattern.compile("[0-9a-z-]+");
12+
13+
public InstanceCommand(final String instanceId, final String commandId, final long pid) {
14+
this.instanceId = instanceId;
15+
this.commandId = commandId;
16+
this.pid = pid;
17+
}
18+
19+
public static InstanceCommand parseParam(String param) {
20+
if (param == null) {
21+
return null;
22+
}
23+
24+
final String[] parts = param.split("!");
25+
26+
if (parts.length != 4 || !parts[0].equals("s3")) {
27+
return null;
28+
}
29+
30+
// validate the data; three requirements:
31+
// 1) instance id should be only alpha and hyphen
32+
// 2) command id is a uuid
33+
Matcher instanceIdMatcher = VALID_REGEX.matcher(parts[1]);
34+
Matcher commandIdMatcher = VALID_REGEX.matcher(parts[2]);
35+
if (!instanceIdMatcher.matches() || !commandIdMatcher.matches()) {
36+
return null;
37+
}
38+
39+
// 3) pid should be parsable as a number (only)
40+
final long pid;
41+
try {
42+
pid = Long.valueOf(parts[3]);
43+
} catch (NumberFormatException e) {
44+
return null;
45+
}
46+
47+
return new InstanceCommand(parts[1], parts[2], pid);
48+
}
49+
}

Diff for: ‎server/src/main/java/org/eclipse/jifa/server/service/impl/netflix/NetflixGandalfUserAccessService.java

+131
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,131 @@
1+
package org.eclipse.jifa.server.service.impl.netflix;
2+
3+
import static org.eclipse.jifa.common.enums.CommonErrorCode.ILLEGAL_ARGUMENT;
4+
import static org.eclipse.jifa.server.enums.ServerErrorCode.ACCESS_DENIED;
5+
import static org.eclipse.jifa.server.enums.ServerErrorCode.FILE_NOT_FOUND;
6+
import static org.eclipse.jifa.server.enums.ServerErrorCode.UNAVAILABLE;
7+
8+
import java.io.IOException;
9+
import java.io.InputStreamReader;
10+
import java.net.URL;
11+
12+
import javax.net.ssl.HttpsURLConnection;
13+
14+
import org.eclipse.jifa.common.domain.exception.ErrorCode;
15+
import org.eclipse.jifa.common.util.Validate;
16+
import org.eclipse.jifa.server.domain.entity.shared.file.BaseFileEntity;
17+
import org.eclipse.jifa.server.domain.entity.shared.user.UserEntity;
18+
import org.eclipse.jifa.server.enums.ServerErrorCode;
19+
import org.eclipse.jifa.server.service.UserService;
20+
import org.springframework.beans.factory.annotation.Autowired;
21+
import org.springframework.context.annotation.Primary;
22+
import org.springframework.stereotype.Component;
23+
24+
import com.google.gson.Gson;
25+
import com.google.gson.JsonObject;
26+
import com.google.gson.annotations.SerializedName;
27+
import com.netflix.gandalf.agent.AuthorizationClient;
28+
import com.netflix.gandalf.agent.GandalfException;
29+
import com.netflix.gandalf.agent.protogen.AuthorizationResponse;
30+
import com.netflix.metatron.ipc.security.MetatronSslContext;
31+
32+
import lombok.extern.slf4j.Slf4j;
33+
34+
@Primary
35+
@Component
36+
@Slf4j
37+
public class NetflixGandalfUserAccessService implements FileAccessService {
38+
39+
private static final String flamecommanderApi = "https://flamecommander.cluster.us-east-1.test.cloud.netflix.net:7004/";
40+
41+
AuthorizationClient authorizationClient;
42+
43+
@Autowired
44+
UserService userService;
45+
46+
public NetflixGandalfUserAccessService() {
47+
this.authorizationClient = new AuthorizationClient();
48+
this.authorizationClient.status();
49+
}
50+
51+
@Override
52+
public void checkAuthorityForCurrentUser(BaseFileEntity file)
53+
{
54+
UserEntity user = userService.getCurrentUser();
55+
56+
log.info("Checking authority for file '{}' and user '{}'",
57+
file.getUniqueName(),
58+
user.getName());
59+
60+
InstanceCommand ic = InstanceCommand.parseParam(file.getUniqueName());
61+
Validate.notNull(ic, ILLEGAL_ARGUMENT, "Not an s3! formed pathname: " + file.getUniqueName());
62+
63+
// look up historic information in flamecommander database for instance
64+
final FcEvent fcEvent;
65+
try {
66+
fcEvent = loadFlamecommanderEvent(ic);
67+
} catch (IOException e) {
68+
log.error("flamecommander api error, sending error", e);
69+
Validate.error(FILE_NOT_FOUND, "flamecommander api error: " + e.getMessage());
70+
return;
71+
}
72+
73+
Validate.notNull(fcEvent, FILE_NOT_FOUND,
74+
String.format("No event found in flamecommander for instanceId=%s, commandId=%s",
75+
ic.instanceId, ic.commandId));
76+
77+
Validate.notNull(user.getName(), ACCESS_DENIED, "No user identity found");
78+
79+
log.debug("found identity, verifying resources against gandalf");
80+
JsonObject subject = createGandalfSubject(user.getName());
81+
JsonObject resource = createGandalfResource();
82+
try
83+
{
84+
AuthorizationResponse authorizationResponse = authorizationClient.isAuthorized(fcEvent.application,
85+
fcEvent.stack, fcEvent.accountId, fcEvent.region, subject, "SSH", resource, null);
86+
log.debug("gandalf response: {}", authorizationResponse);
87+
Validate.isTrue(authorizationResponse.getAllowed(), ACCESS_DENIED, "Access denied by gandalf");
88+
}
89+
catch (GandalfException e)
90+
{
91+
Validate.error(UNAVAILABLE, "gandalf error: " + e.getMessage());
92+
}
93+
94+
}
95+
96+
static class FcEvent {
97+
@SerializedName("account_id")
98+
String accountId;
99+
String application;
100+
String stack;
101+
String region;
102+
}
103+
104+
String getFcProfileLookupUrl(InstanceCommand ic) {
105+
return flamecommanderApi + "api/jifa/fc-event/" + ic.instanceId + "/" + ic.commandId;
106+
}
107+
108+
FcEvent loadFlamecommanderEvent(InstanceCommand ic) throws IOException {
109+
final String eventUrl = getFcProfileLookupUrl(ic);
110+
HttpsURLConnection connection = (HttpsURLConnection) new URL(eventUrl).openConnection();
111+
connection.setSSLSocketFactory(MetatronSslContext.forClient("flamecommander").getSocketFactory());
112+
connection.setHostnameVerifier((hostname, session) -> true);
113+
FcEvent[] events = new Gson().fromJson(new InputStreamReader(connection.getInputStream()), FcEvent[].class);
114+
return (events.length == 1) ? events[0] : null;
115+
}
116+
117+
JsonObject createGandalfSubject(String email) {
118+
final JsonObject user = new JsonObject();
119+
user.addProperty("email", email);
120+
final JsonObject subject = new JsonObject();
121+
subject.add("user", user);
122+
return subject;
123+
}
124+
125+
JsonObject createGandalfResource() {
126+
final JsonObject resource = new JsonObject();
127+
resource.addProperty("username", "nfsuper");
128+
return resource;
129+
}
130+
131+
}

Diff for: ‎server/src/main/java/org/eclipse/jifa/server/service/impl/netflix/PathLookupService.java

+10
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
package org.eclipse.jifa.server.service.impl.netflix;
2+
3+
import java.nio.file.Path;
4+
5+
import org.eclipse.jifa.server.enums.FileType;
6+
7+
public interface PathLookupService
8+
{
9+
Path lookupFile(Path basePath, FileType type, String name);
10+
}

Diff for: ‎server/src/main/java/org/eclipse/jifa/server/service/impl/netflix/PathLookupServiceImpl.java

+29
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
package org.eclipse.jifa.server.service.impl.netflix;
2+
3+
import java.nio.file.Path;
4+
import java.nio.file.Paths;
5+
6+
import org.eclipse.jifa.server.enums.FileType;
7+
import org.springframework.stereotype.Component;
8+
9+
import lombok.extern.slf4j.Slf4j;
10+
11+
@Component
12+
@Slf4j
13+
public class PathLookupServiceImpl implements PathLookupService {
14+
@Override
15+
public Path lookupFile(Path basePath, FileType type, String name)
16+
{
17+
if (name.startsWith("s3!") && type == FileType.HEAP_DUMP) {
18+
InstanceCommand ic = InstanceCommand.parseParam(name);
19+
String path = String.format("%s/heapdump/%s/%s/%s", "/efs/ssm-outputs", ic.instanceId, ic.commandId, ic.pid);
20+
Path result = Paths.get(path).resolve("heapdump.hprof");
21+
log.info("Path lookup (Netflix) for {} of type {} resolved to {}", name, type, result);
22+
return result;
23+
}
24+
25+
Path result = basePath.resolve(type.getStorageDirectoryName()).resolve(name).resolve(name);
26+
log.info("Path lookup (default) for {} of type {} resolved to {}", name, type, result);
27+
return result;
28+
}
29+
}

Diff for: ‎server/src/main/java/org/eclipse/jifa/server/service/impl/netflix/ReadAheadService.java

+105
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,105 @@
1+
package org.eclipse.jifa.server.service.impl.netflix;
2+
3+
import java.io.FileInputStream;
4+
import java.io.IOException;
5+
import java.nio.file.Path;
6+
import java.time.Instant;
7+
import java.util.concurrent.atomic.AtomicInteger;
8+
9+
import org.springframework.stereotype.Component;
10+
11+
import java.util.concurrent.ExecutorService;
12+
import java.util.concurrent.Executors;
13+
import java.util.concurrent.ConcurrentHashMap;
14+
import java.util.concurrent.ConcurrentMap;
15+
import lombok.extern.slf4j.Slf4j;
16+
17+
@Component
18+
@Slf4j
19+
public class ReadAheadService {
20+
final static int CHUNK_SIZE = 8*1024*1024;
21+
final static int READ_THREADS = 16;
22+
23+
final static int MAX_AGE_SECONDS = 300;
24+
final static AtomicInteger threadCount = new AtomicInteger(0);
25+
final ExecutorService readers = Executors.newFixedThreadPool(READ_THREADS, r -> new Thread(r, "ReadAheader-" + threadCount.getAndIncrement()));
26+
27+
final ConcurrentMap<String, Instant> readAheadRequests = new ConcurrentHashMap<>();
28+
29+
static final String[] INDEXES = new String[]{
30+
// this assumes the files are all based on heapdump.hprof
31+
32+
// any order
33+
"heapdump.i2sv2.index", // retained size cache (histogram & leak suspects)
34+
"heapdump.domOut.index", // dominator tree outbound
35+
"heapdump.o2ret.index", // retained size for object
36+
"heapdump.o2hprof.index", // file pointer
37+
"heapdump.o2c.index", // class id for object
38+
"heapdump.a2s.index", // array to size
39+
"heapdump.inbound.index", // inbound pointers for an object
40+
"heapdump.outbound.index", // outbound pointers for object
41+
"heapdump.domIn.index", // dominator tree inbound
42+
43+
// last
44+
// "heapdump.idx.index",
45+
// "heapdump.index",
46+
47+
// of course, also the hprof
48+
// "heapdump.hprof"
49+
};
50+
51+
// the readahead request is likely to come through concurrently, so maybe
52+
// it will occur again before completing
53+
// note that because we use standard CompletableFuture it will run on fj pool
54+
55+
public ReadAheadService() {
56+
log.info("init(): setting up readahead, read_threads = {}, chunk_size = {}, max_age_seconds = {}",
57+
READ_THREADS, CHUNK_SIZE, MAX_AGE_SECONDS);
58+
}
59+
60+
public synchronized void issueForPath(Path hprofFile) {
61+
Path path = hprofFile.getParent();
62+
for (String index : INDEXES) {
63+
issueReadFile(path.resolve(index));
64+
}
65+
}
66+
67+
void issueReadFile(final Path file) {
68+
String filename = file.toString();
69+
Instant existingRequest = readAheadRequests.get(filename);
70+
if (needsLoad(existingRequest)) {
71+
readAheadRequests.put(filename, Instant.now());
72+
log.info("issueReadFile(): issuing for {}", filename);
73+
readers.submit(() -> {
74+
readAheadRequests.put(filename, Instant.now());
75+
76+
log.info("issueReadFile(): issued for {}", filename);
77+
try {
78+
byte[] chunk = new byte[CHUNK_SIZE];
79+
FileInputStream fis = new FileInputStream(filename);
80+
while(fis.read(chunk) > 0) {
81+
// read again
82+
}
83+
log.info("issueReadFile(): completed for {}", filename);
84+
} catch (IOException e) {
85+
log.info("issueReadFile(): failed for {}, {}", filename, e);
86+
}
87+
88+
readAheadRequests.put(filename, Instant.now());
89+
});
90+
}
91+
}
92+
93+
boolean needsLoad(Instant instant) {
94+
if (instant == null) {
95+
return true;
96+
}
97+
98+
Instant OLDEST_OK_AGE = Instant.now().minusSeconds(MAX_AGE_SECONDS);
99+
if (instant.isBefore(OLDEST_OK_AGE)) {
100+
return true;
101+
}
102+
103+
return false;
104+
}
105+
}

0 commit comments

Comments
 (0)