Skip to content

Commit 1b890cd

Browse files
maeldonnmaeldonn
committed
Manage CRR endpoints without explicit port
Issue: BB-746
1 parent 7c513ca commit 1b890cd

File tree

4 files changed

+296
-51
lines changed

4 files changed

+296
-51
lines changed

‎extensions/replication/queueProcessor/QueueProcessor.js‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -259,7 +259,7 @@ class QueueProcessor extends EventEmitter {
259259

260260
if (Array.isArray(this.destConfig.replicationEndpoint.servers)) {
261261
this.destHosts =
262-
new RoundRobin(this.destConfig.replicationEndpoint.servers, { defaultPort: 80 });
262+
new RoundRobin(this.destConfig.replicationEndpoint.servers);
263263
if (this.destConfig.replicationEndpoint.echo) {
264264
this._setupEcho();
265265
}

‎lib/Config.js‎

Lines changed: 13 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -90,14 +90,19 @@ class Config extends EventEmitter {
9090
throw new Error('dataMoverTopic is required when lifecycle transitions is supported');
9191
}
9292

93-
if (parsedConfig.extensions && parsedConfig.extensions.replication
94-
&& parsedConfig.extensions.replication.destination
95-
&& parsedConfig.extensions.replication.destination.bootstrapList) {
96-
this.bootstrapList = parsedConfig.extensions.replication
97-
.destination.bootstrapList;
98-
} else {
99-
this.bootstrapList = [];
100-
}
93+
const destination = parsedConfig.extensions?.replication?.destination;
94+
this.bootstrapList = destination?.bootstrapList?.map(endpoint => {
95+
if (!endpoint.servers) {
96+
return endpoint;
97+
}
98+
99+
const transport = destination.sites?.[endpoint.site]?.transport || destination.transport || 'http';
100+
const defaultPort = transport === 'https' ? 443 : 80;
101+
const servers = endpoint.servers.map(server => server.includes(':') ? server : `${server}:${defaultPort}`);
102+
103+
return { ...endpoint, servers };
104+
}) ?? [];
105+
101106
const shouldRestrictToSite = process.env.BOOTSTRAP_SITE_NAME;
102107
if (shouldRestrictToSite) {
103108
this.bootstrapList = this.bootstrapList.filter(item => item.site === shouldRestrictToSite);

‎tests/unit/conf/Config.js‎

Lines changed: 78 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -91,63 +91,99 @@ describe('Site name', () => {
9191
});
9292
});
9393

94+
9495
describe('Config', () => {
9596
describe('getReplicationSiteDestConfig', () => {
9697
let ogConfigFileEnv;
98+
9799
before(() => {
98100
ogConfigFileEnv = process.env.BACKBEAT_CONFIG_FILE;
99101
});
100-
afterEach(() => {
101-
sinon.restore();
102-
});
102+
103+
afterEach(() => sinon.restore());
104+
103105
after(() => {
104106
if (ogConfigFileEnv) {
105107
process.env.BACKBEAT_CONFIG_FILE = ogConfigFileEnv;
106108
}
107109
});
108-
it('should return replication site destination config', () => {
109-
process.env.BACKBEAT_CONFIG_FILE = `${__dirname}/configs/replicationMultiDestConfig.json`;
110-
const conf = new Config();
111-
const destConfig = conf.getReplicationSiteDestConfig('aws3');
112-
assert.deepStrictEqual(destConfig, {
113-
transport: 'https',
114-
auth: {
115-
type: 'service',
116-
account: 'service-replication-3',
117-
},
118-
replicationEndpoint: {
119-
site: 'aws3',
120-
type: 'aws_s3',
121-
},
110+
111+
describe('bootstrapList server normalization', () => {
112+
let conf;
113+
114+
before(() => {
115+
process.env.BACKBEAT_CONFIG_FILE = `${__dirname}/configs/replicationServersConfig.json`;
116+
conf = new Config();
117+
});
118+
119+
it('should normalize server entries with default port 443 for https transport', () => {
120+
const entry = conf.bootstrapList.find(e => e.site === 'https-site');
121+
assert.deepStrictEqual(entry.servers, ['s3.example.com:443']);
122+
});
123+
124+
it('should normalize server entries with default port 80 for http transport', () => {
125+
const entry = conf.bootstrapList.find(e => e.site === 'http-site');
126+
assert.deepStrictEqual(entry.servers, ['s3.example.com:80']);
127+
});
128+
129+
it('should preserve explicit port in server entries', () => {
130+
const entry = conf.bootstrapList.find(e => e.site === 'explicit-port-site');
131+
assert.deepStrictEqual(entry.servers, ['s3.example.com:8443']);
132+
});
133+
134+
it('should not modify endpoint without servers array', () => {
135+
const entry = conf.bootstrapList.find(e => e.site === 'aws-site');
136+
assert.strictEqual(entry.servers, undefined);
137+
assert.strictEqual(entry.type, 'aws_s3');
122138
});
123139
});
124140

125-
it('should return default replication destination config when site one is not available', () => {
126-
process.env.BACKBEAT_CONFIG_FILE = `${__dirname}/configs/replicationMultiDestConfig.json`;
127-
const conf = new Config();
128-
sinon.stub(conf.extensions.replication, 'destination').value({
129-
transport: 'https',
130-
auth: {
131-
type: 'service',
132-
account: 'service-replication',
133-
},
134-
bootstrapList: [
135-
{ site: 'aws1', type: 'aws_s3' },
136-
{ site: 'aws2', type: 'aws_s3' },
137-
{ site: 'aws3', type: 'aws_s3' }
138-
]
141+
142+
describe('getReplicationSiteDestConfig', () => {
143+
it('should return replication site destination config', () => {
144+
process.env.BACKBEAT_CONFIG_FILE = `${__dirname}/configs/replicationMultiDestConfig.json`;
145+
const conf = new Config();
146+
const destConfig = conf.getReplicationSiteDestConfig('aws3');
147+
assert.deepStrictEqual(destConfig, {
148+
transport: 'https',
149+
auth: {
150+
type: 'service',
151+
account: 'service-replication-3',
152+
},
153+
replicationEndpoint: {
154+
site: 'aws3',
155+
type: 'aws_s3',
156+
},
157+
});
139158
});
140-
const destConfig = conf.getReplicationSiteDestConfig('aws3');
141-
assert.deepStrictEqual(destConfig, {
142-
transport: 'https',
143-
auth: {
144-
type: 'service',
145-
account: 'service-replication',
146-
},
147-
replicationEndpoint: {
148-
site: 'aws3',
149-
type: 'aws_s3',
150-
},
159+
160+
it('should return default replication destination config when site one is not available', () => {
161+
process.env.BACKBEAT_CONFIG_FILE = `${__dirname}/configs/replicationMultiDestConfig.json`;
162+
const conf = new Config();
163+
sinon.stub(conf.extensions.replication, 'destination').value({
164+
transport: 'https',
165+
auth: {
166+
type: 'service',
167+
account: 'service-replication',
168+
},
169+
bootstrapList: [
170+
{ site: 'aws1', type: 'aws_s3' },
171+
{ site: 'aws2', type: 'aws_s3' },
172+
{ site: 'aws3', type: 'aws_s3' }
173+
]
174+
});
175+
const destConfig = conf.getReplicationSiteDestConfig('aws3');
176+
assert.deepStrictEqual(destConfig, {
177+
transport: 'https',
178+
auth: {
179+
type: 'service',
180+
account: 'service-replication',
181+
},
182+
replicationEndpoint: {
183+
site: 'aws3',
184+
type: 'aws_s3',
185+
},
186+
});
151187
});
152188
});
153189
});

‎tests/unit/conf/configs/replicationServersConfig.json‎

Lines changed: 204 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,204 @@
1+
{
2+
"zookeeper": {
3+
"connectionString": "127.0.0.1:2181/backbeat",
4+
"autoCreateNamespace": false
5+
},
6+
"kafka": {
7+
"hosts": "127.0.0.1:9092",
8+
"backlogMetrics": {
9+
"zkPath": "/backbeat/run/kafka-backlog-metrics",
10+
"intervalS": 60
11+
},
12+
"maxRequestSize": 5000020
13+
},
14+
"s3": {
15+
"host": "127.0.0.1",
16+
"port": 8000
17+
},
18+
"vaultAdmin": {
19+
"host": "127.0.0.1",
20+
"port": 8500
21+
},
22+
"replicationGroupId": "RG001 ",
23+
"queuePopulator": {
24+
"cronRule": "*/5 * * * * *",
25+
"batchMaxRead": 10000,
26+
"batchTimeoutMs": 9000,
27+
"zookeeperPath": "/queue-populator",
28+
"logSource": "mongo",
29+
"bucketd": {
30+
"host": "127.0.0.1",
31+
"port": 9000
32+
},
33+
"dmd": {
34+
"host": "127.0.0.1",
35+
"port": 9990
36+
},
37+
"mongo": {
38+
"replicaSetHosts":
39+
"localhost:27017,localhost:27018,localhost:27019",
40+
"writeConcern": "majority",
41+
"replicaSet": "rs0",
42+
"readPreference": "primary",
43+
"database": "metadata"
44+
},
45+
"kafka": {
46+
"topic": "backbeat-oplog",
47+
"consumerGroupId": "backbeat-qp-oplog-group"
48+
},
49+
"probeServer": {
50+
"bindAddress": "0.0.0.0",
51+
"port": 4042
52+
}
53+
},
54+
"extensions": {
55+
"replication": {
56+
"source": {
57+
"transport": "http",
58+
"s3": {
59+
"host": "127.0.0.1",
60+
"port": 8000
61+
},
62+
"auth": {
63+
"type": "service",
64+
"account": "service-replication",
65+
"vault": {
66+
"host": "127.0.0.1",
67+
"port": 8500,
68+
"adminPort": 8600
69+
}
70+
}
71+
},
72+
"destination": {
73+
"transport": "http",
74+
"sites": {
75+
"https-site": {
76+
"transport": "https"
77+
},
78+
"http-site": {
79+
"transport": "http"
80+
}
81+
},
82+
"bootstrapList": [
83+
{ "site": "https-site", "servers": ["s3.example.com"] },
84+
{ "site": "http-site", "servers": ["s3.example.com"] },
85+
{ "site": "explicit-port-site", "servers": ["s3.example.com:8443"] },
86+
{ "site": "aws-site", "type": "aws_s3" }
87+
],
88+
"auth": {
89+
"type": "service",
90+
"account": "service-replication"
91+
}
92+
},
93+
"topic": "backbeat-replication",
94+
"dataMoverTopic": "backbeat-data-mover",
95+
"replicationStatusTopic": "backbeat-replication-status",
96+
"replicationFailedTopic": "backbeat-replication-failed",
97+
"monitorReplicationFailures": true,
98+
"monitorReplicationFailureExpiryTimeS": 86400,
99+
"replayTopics": [
100+
{
101+
"topicName": "backbeat-replication-replay-0",
102+
"retries": 5
103+
}
104+
],
105+
"queueProcessor": {
106+
"groupId": "backbeat-replication-group",
107+
"retry": {
108+
"aws_s3": {
109+
"maxRetries": 5,
110+
"timeoutS": 900,
111+
"backoff": {
112+
"min": 60000,
113+
"max": 900000,
114+
"jitter": 0.1,
115+
"factor": 1.5
116+
}
117+
},
118+
"azure": {
119+
"maxRetries": 5,
120+
"timeoutS": 900,
121+
"backoff": {
122+
"min": 60000,
123+
"max": 900000,
124+
"jitter": 0.1,
125+
"factor": 1.5
126+
}
127+
},
128+
"gcp": {
129+
"maxRetries": 5,
130+
"timeoutS": 900,
131+
"backoff": {
132+
"min": 60000,
133+
"max": 900000,
134+
"jitter": 0.1,
135+
"factor": 1.5
136+
}
137+
},
138+
"scality": {
139+
"maxRetries": 5,
140+
"timeoutS": 300,
141+
"backoff": {
142+
"min": 1000,
143+
"max": 300000,
144+
"jitter": 0.1,
145+
"factor": 1.5
146+
}
147+
}
148+
},
149+
"concurrency": 10,
150+
"mpuPartsConcurrency": 10,
151+
"probeServer": {
152+
"bindAddress": "localhost",
153+
"port": 4043
154+
}
155+
},
156+
"replicationStatusProcessor": {
157+
"groupId": "backbeat-replication-group",
158+
"retry": {
159+
"maxRetries": 5,
160+
"timeoutS": 300,
161+
"backoff": {
162+
"min": 1000,
163+
"max": 300000,
164+
"jitter": 0.1,
165+
"factor": 1.5
166+
}
167+
},
168+
"concurrency": 10,
169+
"probeServer": {
170+
"bindAddress": "localhost",
171+
"port": 4045
172+
}
173+
},
174+
"objectSizeMetrics": [
175+
66560,
176+
8388608,
177+
68157440
178+
]
179+
}
180+
},
181+
"log": {
182+
"logLevel": "info",
183+
"dumpLevel": "error"
184+
},
185+
"metrics": {
186+
"topic": "backbeat-metrics"
187+
},
188+
"server": {
189+
"healthChecks": {
190+
"allowFrom": ["127.0.0.1/8", "::1"]
191+
},
192+
"host": "127.0.0.1",
193+
"port": 8900
194+
},
195+
"redis": {
196+
"host": "localhost",
197+
"port": 6379
198+
},
199+
"certFilePaths": {
200+
"key": "",
201+
"cert": "",
202+
"ca": ""
203+
}
204+
}

0 commit comments

Comments
 (0)