-
Notifications
You must be signed in to change notification settings - Fork 1.1k
Open
Labels
searchOpensearch, search filters, and so onOpensearch, search filters, and so on
Description
As noted in #19325 (comment)
A request.opensearch request method is set up during request instantiation, and it picks up a configuration that is hard-coded to the same value for both web and worker instances.
warehouse/warehouse/search/__init__.py
Lines 68 to 120 in e1b0ab3
| def opensearch(request): | |
| client = request.registry["opensearch.client"] | |
| doc_types = request.registry.get("search.doc_types", set()) | |
| index_name = request.registry["opensearch.index"] | |
| index = get_index( | |
| index_name, | |
| doc_types, | |
| using=client, | |
| shards=request.registry.get("opensearch.shards", 1), | |
| replicas=request.registry.get("opensearch.replicas", 0), | |
| ) | |
| return index.search() | |
| def includeme(config): | |
| ratelimit_string = config.registry.settings.get("warehouse.search.ratelimit_string") | |
| config.register_service_factory( | |
| RateLimit(ratelimit_string), IRateLimiter, name="search" | |
| ) | |
| p = parse_url(config.registry.settings["opensearch.url"]) | |
| assert p.path, "The URL for the OpenSearch instance must include the index name." | |
| qs = urllib.parse.parse_qs(p.query) | |
| kwargs = { | |
| "hosts": [urllib.parse.urlunparse((p.scheme, p.netloc) + ("",) * 4)], | |
| "verify_certs": True, | |
| "ca_certs": certifi.where(), | |
| "timeout": 1, | |
| "retry_on_timeout": True, | |
| "serializer": opensearchpy.serializer.serializer, | |
| "max_retries": 1, | |
| } | |
| aws_auth = bool(qs.get("aws_auth", False)) | |
| if aws_auth: | |
| aws_region = qs.get("region", ["us-east-1"])[0] | |
| kwargs["connection_class"] = opensearchpy.RequestsHttpConnection | |
| kwargs["http_auth"] = requests_aws4auth.AWS4Auth( | |
| config.registry.settings["aws.key_id"], | |
| config.registry.settings["aws.secret_key"], | |
| aws_region, | |
| "es", | |
| ) | |
| config.registry["opensearch.client"] = opensearchpy.OpenSearch(**kwargs) | |
| config.registry["opensearch.index"] = p.path.strip("/") | |
| config.registry["opensearch.shards"] = int(qs.get("shards", ["1"])[0]) | |
| config.registry["opensearch.replicas"] = int(qs.get("replicas", ["0"])[0]) | |
| config.add_request_method(opensearch, name="opensearch", reify=True) | |
| from warehouse.search.tasks import reindex | |
| config.add_periodic_task(crontab(minute=0, hour=6), reindex) | |
| config.register_service_factory(SearchService.create_service, iface=ISearchService) |
The desired outcome is to be able to express different timeout settings for web vs worker instances so that we can continue to have a "snappy" timeout for frontend and have longer timeouts for background worker processes that are less latency-sensitive.
Reactions are currently unavailable
Metadata
Metadata
Assignees
Labels
searchOpensearch, search filters, and so onOpensearch, search filters, and so on