Skip to content

Enable setting different timeouts for web vs workers #19326

@miketheman

Description

@miketheman

As noted in #19325 (comment)

A request.opensearch request method is set up during request instantiation, and it picks up a configuration that is hard-coded to the same value for both web and worker instances.

def opensearch(request):
client = request.registry["opensearch.client"]
doc_types = request.registry.get("search.doc_types", set())
index_name = request.registry["opensearch.index"]
index = get_index(
index_name,
doc_types,
using=client,
shards=request.registry.get("opensearch.shards", 1),
replicas=request.registry.get("opensearch.replicas", 0),
)
return index.search()
def includeme(config):
ratelimit_string = config.registry.settings.get("warehouse.search.ratelimit_string")
config.register_service_factory(
RateLimit(ratelimit_string), IRateLimiter, name="search"
)
p = parse_url(config.registry.settings["opensearch.url"])
assert p.path, "The URL for the OpenSearch instance must include the index name."
qs = urllib.parse.parse_qs(p.query)
kwargs = {
"hosts": [urllib.parse.urlunparse((p.scheme, p.netloc) + ("",) * 4)],
"verify_certs": True,
"ca_certs": certifi.where(),
"timeout": 1,
"retry_on_timeout": True,
"serializer": opensearchpy.serializer.serializer,
"max_retries": 1,
}
aws_auth = bool(qs.get("aws_auth", False))
if aws_auth:
aws_region = qs.get("region", ["us-east-1"])[0]
kwargs["connection_class"] = opensearchpy.RequestsHttpConnection
kwargs["http_auth"] = requests_aws4auth.AWS4Auth(
config.registry.settings["aws.key_id"],
config.registry.settings["aws.secret_key"],
aws_region,
"es",
)
config.registry["opensearch.client"] = opensearchpy.OpenSearch(**kwargs)
config.registry["opensearch.index"] = p.path.strip("/")
config.registry["opensearch.shards"] = int(qs.get("shards", ["1"])[0])
config.registry["opensearch.replicas"] = int(qs.get("replicas", ["0"])[0])
config.add_request_method(opensearch, name="opensearch", reify=True)
from warehouse.search.tasks import reindex
config.add_periodic_task(crontab(minute=0, hour=6), reindex)
config.register_service_factory(SearchService.create_service, iface=ISearchService)

The desired outcome is to be able to express different timeout settings for web vs worker instances so that we can continue to have a "snappy" timeout for frontend and have longer timeouts for background worker processes that are less latency-sensitive.

Metadata

Metadata

Assignees

No one assigned

    Labels

    searchOpensearch, search filters, and so on

    Projects

    No projects

    Milestone

    No milestone

    Relationships

    None yet

    Development

    No branches or pull requests

    Issue actions