|
55 | 55 | from .errors import ( |
56 | 56 | BaseOTACacheError, |
57 | 57 | CacheCommitFailed, |
58 | | - CacheProviderNotReady, |
59 | | - ReaderPoolBusy, |
60 | 58 | ) |
61 | 59 | from .external_cache import mount_external_cache, umount_external_cache |
62 | 60 | from .lru_cache_helper import LRUCacheHelper |
@@ -695,50 +693,39 @@ async def retrieve_file( |
695 | 693 | raw_url, headers=headers_from_client |
696 | 694 | ) |
697 | 695 |
|
698 | | - try: |
699 | | - # NOTE(20241202): behavior changed: even if _cache_enabled is False, if external_cache is configured |
700 | | - # and loaded, still try to use external cache source. |
701 | | - if self._external_cache_data_dir and ( |
702 | | - _res := await self._retrieve_file_by_external_cache( |
703 | | - cache_policy, self._external_cache_data_dir |
704 | | - ) |
705 | | - ): |
706 | | - self._metrics_data.cache_external_hits += 1 |
707 | | - return _res |
708 | | - |
709 | | - # NOTE(20251212): use local cache first before going to NFS cache |
710 | | - if not cache_policy.retry_caching and ( |
711 | | - _res := await self._retrieve_file_by_cache_lookup( |
712 | | - raw_url=raw_url, cache_policy=cache_policy |
713 | | - ) |
714 | | - ): |
715 | | - self._metrics_data.cache_local_hits += 1 |
716 | | - return _res |
| 696 | + # NOTE(20241202): behavior changed: even if _cache_enabled is False, if external_cache is configured |
| 697 | + # and loaded, still try to use external cache source. |
| 698 | + if self._external_cache_data_dir and ( |
| 699 | + _res := await self._retrieve_file_by_external_cache( |
| 700 | + cache_policy, self._external_cache_data_dir |
| 701 | + ) |
| 702 | + ): |
| 703 | + self._metrics_data.cache_external_hits += 1 |
| 704 | + return _res |
717 | 705 |
|
718 | | - if self._external_nfs_cache_data_dir and ( |
719 | | - _res := await self._retrieve_file_by_external_cache( |
720 | | - cache_policy, self._external_nfs_cache_data_dir |
721 | | - ) |
722 | | - ): |
723 | | - self._metrics_data.cache_external_nfs_hits += 1 |
724 | | - return _res |
725 | | - |
726 | | - if _res := await self._retrieve_file_by_new_caching( |
727 | | - raw_url=raw_url, |
728 | | - cache_policy=cache_policy, |
729 | | - headers_from_client=headers_from_client, |
730 | | - ): |
731 | | - return _res |
732 | | - except (ReaderPoolBusy, CacheProviderNotReady): |
733 | | - raise # common errors, will indicate server_app to return 503 |
734 | | - except Exception as e: |
735 | | - # NOTE(20260403): sometime the caching might fail due to cache bucket too |
736 | | - # shallow, LRU crushing the cache files too fast under multi-client |
737 | | - # condition. |
738 | | - burst_suppressed_logger_exc.exception( |
739 | | - f"internal errors when processing {raw_url}: {e!r}\n" |
740 | | - "fallback to direct downloading ..." |
| 706 | + # NOTE(20251212): use local cache first before going to NFS cache |
| 707 | + if not cache_policy.retry_caching and ( |
| 708 | + _res := await self._retrieve_file_by_cache_lookup( |
| 709 | + raw_url=raw_url, cache_policy=cache_policy |
741 | 710 | ) |
| 711 | + ): |
| 712 | + self._metrics_data.cache_local_hits += 1 |
| 713 | + return _res |
| 714 | + |
| 715 | + if self._external_nfs_cache_data_dir and ( |
| 716 | + _res := await self._retrieve_file_by_external_cache( |
| 717 | + cache_policy, self._external_nfs_cache_data_dir |
| 718 | + ) |
| 719 | + ): |
| 720 | + self._metrics_data.cache_external_nfs_hits += 1 |
| 721 | + return _res |
| 722 | + |
| 723 | + if _res := await self._retrieve_file_by_new_caching( |
| 724 | + raw_url=raw_url, |
| 725 | + cache_policy=cache_policy, |
| 726 | + headers_from_client=headers_from_client, |
| 727 | + ): |
| 728 | + return _res |
742 | 729 |
|
743 | 730 | # as last resort, finally try to handle the request by directly downloading |
744 | 731 | return await self._retrieve_file_by_downloading( |
|
0 commit comments