Skip to content

Commit 486e7c8

Browse files
ntk148vntk148v
committed
test(e2e): add ulimit nofiles testcase
Signed-off-by: Kien Nguyen Tuan <[email protected]>
1 parent 8da48e2 commit 486e7c8

8 files changed

+101
-76
lines changed

‎Dockerfile.e2e-tests

+1-1
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ FROM golang:1.24.0
1212
LABEL maintainer="The Thanos Authors"
1313

1414
COPY --from=builder $GOBIN/thanos /bin/thanos
15-
COPY docker-entrypoint.sh /docker-entrypoint.sh
15+
COPY docker-entrypoint-test.sh /docker-entrypoint.sh
1616

1717
ENV GORACE="halt_on_error=1"
1818

‎docker-entrypoint-test.sh

+13
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
#!/bin/sh
2+
# NOTE(kiennt26): This is a test entrypoint script for the Thanos container.
3+
# It is used to test the container's behavior when the THANOS_ULIMIT_NOFILES
4+
# environment variable is set.
5+
set -e
6+
7+
if [ -n "${THANOS_ULIMIT_NOFILES:-}" ]; then
8+
echo "Setting file description limit to $THANOS_ULIMIT_NOFILES"
9+
ulimit -n "$THANOS_ULIMIT_NOFILES"
10+
ulimit -Hn "$THANOS_ULIMIT_NOFILES"
11+
fi
12+
13+
exec /bin/thanos "$@"

‎test/e2e/compact_test.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -459,7 +459,7 @@ func testCompactWithStoreGateway(t *testing.T, penaltyDedup bool) {
459459
}
460460

461461
// Crank down the deletion mark delay since deduplication can miss blocks in the presence of replica labels it doesn't know about.
462-
str := e2ethanos.NewStoreGW(e, "1", bktConfig, "", "", []string{"--ignore-deletion-marks-delay=2s"})
462+
str := e2ethanos.NewStoreGWBuilder(e, "1").Init(bktConfig, "", "", []string{"--ignore-deletion-marks-delay=2s"})
463463
testutil.Ok(t, e2e.StartAndWaitReady(str))
464464
testutil.Ok(t, str.WaitSumMetrics(e2emon.Equals(float64(len(rawBlockIDs)+8)), "thanos_blocks_meta_synced"))
465465
testutil.Ok(t, str.WaitSumMetrics(e2emon.Equals(0), "thanos_blocks_meta_sync_failures_total"))

‎test/e2e/distributed_query_test.go

+2-6
Original file line numberDiff line numberDiff line change
@@ -145,9 +145,7 @@ func TestDistributedEngineWithOverlappingIntervalsEnabled(t *testing.T) {
145145
)
146146
testutil.Ok(t, err)
147147
testutil.Ok(t, objstore.UploadDir(ctx, l, bkt1, path.Join(dir1, blockID2.String()), blockID2.String()))
148-
store1 := e2ethanos.NewStoreGW(
149-
e,
150-
"s1",
148+
store1 := e2ethanos.NewStoreGWBuilder(e, "s1").Init(
151149
client.BucketConfig{
152150
Type: client.S3,
153151
Config: e2ethanos.NewS3Config(bucket1, minio1.InternalEndpoint("http"), minio1.InternalDir()),
@@ -238,9 +236,7 @@ func TestDistributedEngineWithoutOverlappingIntervals(t *testing.T) {
238236
)
239237
testutil.Ok(t, err)
240238
testutil.Ok(t, objstore.UploadDir(ctx, l, bkt1, path.Join(dir1, blockID2.String()), blockID2.String()))
241-
store1 := e2ethanos.NewStoreGW(
242-
e,
243-
"s1",
239+
store1 := e2ethanos.NewStoreGWBuilder(e, "s1").Init(
244240
client.BucketConfig{
245241
Type: client.S3,
246242
Config: e2ethanos.NewS3Config(bucket1, minio1.InternalEndpoint("http"), minio1.InternalDir()),

‎test/e2e/e2ethanos/services.go

+28-8
Original file line numberDiff line numberDiff line change
@@ -934,32 +934,51 @@ receivers:
934934
})), "http")
935935
}
936936

937-
func NewStoreGW(e e2e.Environment, name string, bucketConfig client.BucketConfig, cacheConfig, indexCacheConfig string, extArgs []string, relabelConfig ...relabel.Config) *e2eobs.Observable {
937+
type StoreGWBuilder struct {
938+
e2e.Linkable
939+
f e2e.FutureRunnable
940+
name string
941+
envVars map[string]string
942+
}
943+
944+
// NewStoreGWBuilder is a Store Gateway future that allows extra configuration before initialization.
945+
func NewStoreGWBuilder(e e2e.Environment, name string) *StoreGWBuilder {
938946
f := e.Runnable(fmt.Sprintf("store-gw-%v", name)).
939947
WithPorts(map[string]int{"http": 8080, "grpc": 9091}).
940948
Future()
949+
return &StoreGWBuilder{
950+
Linkable: f,
951+
f: f,
952+
name: name,
953+
}
954+
}
941955

942-
if err := os.MkdirAll(f.Dir(), 0750); err != nil {
943-
return &e2eobs.Observable{Runnable: e2e.NewFailedRunnable(name, errors.Wrap(err, "create store dir"))}
956+
func (s *StoreGWBuilder) WithEnvVars(envVars map[string]string) *StoreGWBuilder {
957+
s.envVars = envVars
958+
return s
959+
}
960+
func (s *StoreGWBuilder) Init(bucketConfig client.BucketConfig, cacheConfig, indexCacheConfig string, extArgs []string, relabelConfig ...relabel.Config) *e2eobs.Observable {
961+
if err := os.MkdirAll(s.f.Dir(), 0750); err != nil {
962+
return &e2eobs.Observable{Runnable: e2e.NewFailedRunnable(s.name, errors.Wrap(err, "create store dir"))}
944963
}
945964

946965
bktConfigBytes, err := yaml.Marshal(bucketConfig)
947966
if err != nil {
948-
return &e2eobs.Observable{Runnable: e2e.NewFailedRunnable(name, errors.Wrapf(err, "generate store config file: %v", bucketConfig))}
967+
return &e2eobs.Observable{Runnable: e2e.NewFailedRunnable(s.name, errors.Wrapf(err, "generate store config file: %v", bucketConfig))}
949968
}
950969

951970
relabelConfigBytes, err := yaml.Marshal(relabelConfig)
952971
if err != nil {
953-
return &e2eobs.Observable{Runnable: e2e.NewFailedRunnable(name, errors.Wrapf(err, "generate store relabel file: %v", relabelConfig))}
972+
return &e2eobs.Observable{Runnable: e2e.NewFailedRunnable(s.name, errors.Wrapf(err, "generate store relabel file: %v", relabelConfig))}
954973
}
955974

956975
args := append(e2e.BuildArgs(map[string]string{
957-
"--debug.name": fmt.Sprintf("store-gw-%v", name),
976+
"--debug.name": fmt.Sprintf("store-gw-%v", s.name),
958977
"--grpc-address": ":9091",
959978
"--grpc-grace-period": "0s",
960979
"--http-address": ":8080",
961980
"--log.level": infoLogLevel,
962-
"--data-dir": f.InternalDir(),
981+
"--data-dir": s.f.InternalDir(),
963982
"--objstore.config": string(bktConfigBytes),
964983
// Accelerated sync time for quicker test (3m by default).
965984
"--sync-block-duration": "3s",
@@ -977,10 +996,11 @@ func NewStoreGW(e e2e.Environment, name string, bucketConfig client.BucketConfig
977996
args = append(args, "--index-cache.config", indexCacheConfig)
978997
}
979998

980-
return e2eobs.AsObservable(f.Init(wrapWithDefaults(e2e.StartOptions{
999+
return e2eobs.AsObservable(s.f.Init(wrapWithDefaults(e2e.StartOptions{
9811000
Image: DefaultImage(),
9821001
Command: e2e.NewCommand("store", args...),
9831002
Readiness: e2e.NewHTTPReadinessProbe("http", "/-/ready", 200, 200),
1003+
EnvVars: s.envVars,
9841004
})), "http")
9851005
}
9861006

‎test/e2e/info_api_test.go

+1-3
Original file line numberDiff line numberDiff line change
@@ -40,9 +40,7 @@ func TestInfo(t *testing.T) {
4040
const bucket = "info-api-test"
4141
m := e2edb.NewMinio(e, "thanos-minio", bucket, e2edb.WithMinioTLS())
4242
testutil.Ok(t, e2e.StartAndWaitReady(m))
43-
store := e2ethanos.NewStoreGW(
44-
e,
45-
"1",
43+
store := e2ethanos.NewStoreGWBuilder(e, "1").Init(
4644
client.BucketConfig{
4745
Type: client.S3,
4846
Config: e2ethanos.NewS3Config(bucket, m.InternalEndpoint("http"), m.InternalDir()),

‎test/e2e/query_test.go

+3-9
Original file line numberDiff line numberDiff line change
@@ -660,9 +660,7 @@ func TestQueryStoreMetrics(t *testing.T) {
660660
}
661661
}
662662

663-
storeGW := e2ethanos.NewStoreGW(
664-
e,
665-
"s1",
663+
storeGW := e2ethanos.NewStoreGWBuilder(e, "s1").Init(
666664
client.BucketConfig{
667665
Type: client.S3,
668666
Config: e2ethanos.NewS3Config(bucket, minio.InternalEndpoint("http"), minio.InternalDir()),
@@ -811,9 +809,7 @@ func TestQueryStoreDedup(t *testing.T) {
811809
bkt, err := s3.NewBucketWithConfig(l, e2ethanos.NewS3Config(bucket, minio.Endpoint("http"), minio.Dir()), "test", nil)
812810
testutil.Ok(t, err)
813811

814-
storeGW := e2ethanos.NewStoreGW(
815-
e,
816-
"s1",
812+
storeGW := e2ethanos.NewStoreGWBuilder(e, "s1").Init(
817813
client.BucketConfig{
818814
Type: client.S3,
819815
Config: e2ethanos.NewS3Config(bucket, minio.InternalEndpoint("http"), minio.InternalDir()),
@@ -2097,9 +2093,7 @@ func TestQueryTenancyEnforcement(t *testing.T) {
20972093
testutil.Ok(t, objstore.UploadDir(ctx, l, bkt, path.Join(dir, blockID2.String()), blockID2.String()))
20982094
testutil.Ok(t, objstore.UploadDir(ctx, l, bkt, path.Join(dir, blockID3.String()), blockID3.String()))
20992095

2100-
storeGW := e2ethanos.NewStoreGW(
2101-
e,
2102-
"s1",
2096+
storeGW := e2ethanos.NewStoreGWBuilder(e, "s1").Init(
21032097
client.BucketConfig{
21042098
Type: client.S3,
21052099
Config: e2ethanos.NewS3Config(bucket, minio.InternalEndpoint("http"), minio.InternalDir()),

‎test/e2e/store_gateway_test.go

+52-48
Original file line numberDiff line numberDiff line change
@@ -73,22 +73,19 @@ metafile_exists_ttl: 0s
7373
metafile_doesnt_exist_ttl: 0s
7474
metafile_content_ttl: 0s`, memcached.InternalEndpoint("memcached"))
7575

76-
s1 := e2ethanos.NewStoreGW(
77-
e,
78-
"1",
79-
client.BucketConfig{
80-
Type: client.S3,
81-
Config: e2ethanos.NewS3Config(bucket, m.InternalEndpoint("http"), m.InternalDir()),
82-
},
76+
s1 := e2ethanos.NewStoreGWBuilder(e, "1").Init(client.BucketConfig{
77+
Type: client.S3,
78+
Config: e2ethanos.NewS3Config(bucket, m.InternalEndpoint("http"), m.InternalDir()),
79+
},
8380
memcachedConfig,
8481
"",
8582
nil,
8683
relabel.Config{
8784
Action: relabel.Drop,
8885
Regex: relabel.MustNewRegexp("value2"),
8986
SourceLabels: model.LabelNames{"ext1"},
90-
},
91-
)
87+
})
88+
9289
testutil.Ok(t, e2e.StartAndWaitReady(s1))
9390
// Ensure bucket UI.
9491
ensureGETStatusCode(t, http.StatusOK, "http://"+path.Join(s1.Endpoint("http"), "loaded"))
@@ -409,9 +406,7 @@ func TestStoreGatewayNoCacheFile(t *testing.T) {
409406
m := e2edb.NewMinio(e, "thanos-minio", bucket, e2edb.WithMinioTLS())
410407
testutil.Ok(t, e2e.StartAndWaitReady(m))
411408

412-
s1 := e2ethanos.NewStoreGW(
413-
e,
414-
"1",
409+
s1 := e2ethanos.NewStoreGWBuilder(e, "1").Init(
415410
client.BucketConfig{
416411
Type: client.S3,
417412
Config: e2ethanos.NewS3Config(bucket, m.InternalEndpoint("http"), m.InternalDir()),
@@ -641,9 +636,7 @@ config:
641636
addresses: [%s]
642637
blocks_iter_ttl: 0s`, memcached.InternalEndpoint("memcached"))
643638

644-
s1 := e2ethanos.NewStoreGW(
645-
e,
646-
"1",
639+
s1 := e2ethanos.NewStoreGWBuilder(e, "1").Init(
647640
client.BucketConfig{
648641
Type: client.S3,
649642
Config: e2ethanos.NewS3Config(bucket, m.InternalEndpoint("http"), m.InternalDir()),
@@ -750,9 +743,7 @@ metafile_exists_ttl: 0s
750743
metafile_doesnt_exist_ttl: 0s
751744
metafile_content_ttl: 0s`
752745

753-
store1 := e2ethanos.NewStoreGW(
754-
e,
755-
"1",
746+
store1 := e2ethanos.NewStoreGWBuilder(e, "1").Init(
756747
client.BucketConfig{
757748
Type: client.S3,
758749
Config: e2ethanos.NewS3Config(bucket, m.InternalEndpoint("http"), m.InternalDir()),
@@ -761,9 +752,7 @@ metafile_content_ttl: 0s`
761752
"",
762753
nil,
763754
)
764-
store2 := e2ethanos.NewStoreGW(
765-
e,
766-
"2",
755+
store2 := e2ethanos.NewStoreGWBuilder(e, "2").Init(
767756
client.BucketConfig{
768757
Type: client.S3,
769758
Config: e2ethanos.NewS3Config(bucket, m.InternalEndpoint("http"), m.InternalDir()),
@@ -772,9 +761,7 @@ metafile_content_ttl: 0s`
772761
"",
773762
nil,
774763
)
775-
store3 := e2ethanos.NewStoreGW(
776-
e,
777-
"3",
764+
store3 := e2ethanos.NewStoreGWBuilder(e, "3").Init(
778765
client.BucketConfig{
779766
Type: client.S3,
780767
Config: e2ethanos.NewS3Config(bucket, m.InternalEndpoint("http"), m.InternalDir()),
@@ -869,9 +856,7 @@ config:
869856
m := e2edb.NewMinio(e, "thanos-minio", bucket, e2edb.WithMinioTLS())
870857
testutil.Ok(t, e2e.StartAndWaitReady(m))
871858

872-
store1 := e2ethanos.NewStoreGW(
873-
e,
874-
"1",
859+
store1 := e2ethanos.NewStoreGWBuilder(e, "1").Init(
875860
client.BucketConfig{
876861
Type: client.S3,
877862
Config: e2ethanos.NewS3Config(bucket, m.InternalEndpoint("http"), m.InternalDir()),
@@ -881,9 +866,7 @@ config:
881866
[]string{"--store.grpc.downloaded-bytes-limit=1B"},
882867
)
883868

884-
store2 := e2ethanos.NewStoreGW(
885-
e,
886-
"2",
869+
store2 := e2ethanos.NewStoreGWBuilder(e, "2").Init(
887870
client.BucketConfig{
888871
Type: client.S3,
889872
Config: e2ethanos.NewS3Config(bucket, m.InternalEndpoint("http"), m.InternalDir()),
@@ -892,9 +875,7 @@ config:
892875
"",
893876
[]string{"--store.grpc.downloaded-bytes-limit=100B"},
894877
)
895-
store3 := e2ethanos.NewStoreGW(
896-
e,
897-
"3",
878+
store3 := e2ethanos.NewStoreGWBuilder(e, "3").Init(
898879
client.BucketConfig{
899880
Type: client.S3,
900881
Config: e2ethanos.NewS3Config(bucket, m.InternalEndpoint("http"), m.InternalDir()),
@@ -1037,9 +1018,7 @@ config:
10371018
dns_provider_update_interval: 1s
10381019
auto_discovery: false`, memcached.InternalEndpoint("memcached"))
10391020

1040-
s1 := e2ethanos.NewStoreGW(
1041-
e,
1042-
"1",
1021+
s1 := e2ethanos.NewStoreGWBuilder(e, "1").Init(
10431022
client.BucketConfig{
10441023
Type: client.S3,
10451024
Config: e2ethanos.NewS3Config(bucket, m.InternalEndpoint("http"), m.InternalDir()),
@@ -1133,9 +1112,7 @@ func TestStoreGatewayLazyExpandedPostingsEnabled(t *testing.T) {
11331112
testutil.Ok(t, e2e.StartAndWaitReady(m))
11341113

11351114
// Create 2 store gateways, one with lazy expanded postings enabled and another one disabled.
1136-
s1 := e2ethanos.NewStoreGW(
1137-
e,
1138-
"1",
1115+
s1 := e2ethanos.NewStoreGWBuilder(e, "1").Init(
11391116
client.BucketConfig{
11401117
Type: client.S3,
11411118
Config: e2ethanos.NewS3Config(bucket, m.InternalEndpoint("http"), m.InternalDir()),
@@ -1144,9 +1121,7 @@ func TestStoreGatewayLazyExpandedPostingsEnabled(t *testing.T) {
11441121
"",
11451122
[]string{"--store.enable-lazy-expanded-postings"},
11461123
)
1147-
s2 := e2ethanos.NewStoreGW(
1148-
e,
1149-
"2",
1124+
s2 := e2ethanos.NewStoreGWBuilder(e, "2").Init(
11501125
client.BucketConfig{
11511126
Type: client.S3,
11521127
Config: e2ethanos.NewS3Config(bucket, m.InternalEndpoint("http"), m.InternalDir()),
@@ -1290,9 +1265,7 @@ func TestStoreGatewayLazyExpandedPostingsPromQLSmithFuzz(t *testing.T) {
12901265
testutil.Ok(t, e2e.StartAndWaitReady(m))
12911266

12921267
// Create 2 store gateways, one with lazy expanded postings enabled and another one disabled.
1293-
s1 := e2ethanos.NewStoreGW(
1294-
e,
1295-
"1",
1268+
s1 := e2ethanos.NewStoreGWBuilder(e, "1").Init(
12961269
client.BucketConfig{
12971270
Type: client.S3,
12981271
Config: e2ethanos.NewS3Config(bucket, m.InternalEndpoint("http"), m.InternalDir()),
@@ -1301,9 +1274,7 @@ func TestStoreGatewayLazyExpandedPostingsPromQLSmithFuzz(t *testing.T) {
13011274
"",
13021275
[]string{"--store.enable-lazy-expanded-postings"},
13031276
)
1304-
s2 := e2ethanos.NewStoreGW(
1305-
e,
1306-
"2",
1277+
s2 := e2ethanos.NewStoreGWBuilder(e, "2").Init(
13071278
client.BucketConfig{
13081279
Type: client.S3,
13091280
Config: e2ethanos.NewS3Config(bucket, m.InternalEndpoint("http"), m.InternalDir()),
@@ -1437,3 +1408,36 @@ func TestStoreGatewayLazyExpandedPostingsPromQLSmithFuzz(t *testing.T) {
14371408
require.Failf(t, "finished store gateway lazy expanded posting fuzzing tests", "%d test cases failed", failures)
14381409
}
14391410
}
1411+
1412+
func TestStoreGatewayUlimitNoFiles(t *testing.T) {
1413+
t.Parallel()
1414+
1415+
e, err := e2e.NewDockerEnvironment("store-nofiles")
1416+
testutil.Ok(t, err)
1417+
t.Cleanup(e2ethanos.CleanScenario(t, e))
1418+
1419+
const bucket = "store-gateway-nofiles-test"
1420+
m := e2edb.NewMinio(e, "thanos-minio", bucket, e2edb.WithMinioTLS())
1421+
testutil.Ok(t, e2e.StartAndWaitReady(m))
1422+
1423+
// Set THANOS_ULIMIT_NOFILES to a very small value to trigger ulimit error.
1424+
env := map[string]string{
1425+
"THANOS_ULIMIT_NOFILES": "1",
1426+
}
1427+
1428+
s1 := e2ethanos.NewStoreGWBuilder(e, "32").
1429+
WithEnvVars(env).
1430+
Init(
1431+
client.BucketConfig{
1432+
Type: client.S3,
1433+
Config: e2ethanos.NewS3Config(bucket, m.InternalEndpoint("http"), m.InternalDir()),
1434+
},
1435+
"",
1436+
"",
1437+
nil,
1438+
)
1439+
err = e2e.StartAndWaitReady(s1)
1440+
// We expect the process to fail to start due to the low ulimit.
1441+
// error while loading shared libraries: libc.so.6: cannot open shared object file: Error 24
1442+
testutil.NotOk(t, err)
1443+
}

0 commit comments

Comments
 (0)