HTTPError('403 Client Error: Forbidden for url

[Potracheno]

Seems otodom change link and if try to run scrape_data (http://localhost:8000/scrape_data)

pgadmin | postfix/postlog: starting the Postfix mail system
pgadmin | [2025-05-16 17:38:49 +0000] [1] [INFO] Starting gunicorn 23.0.0
pgadmin | [2025-05-16 17:38:49 +0000] [1] [INFO] Listening at: http://[::]:80 (1)
pgadmin | [2025-05-16 17:38:49 +0000] [1] [INFO] Using worker: gthread
pgadmin | [2025-05-16 17:38:49 +0000] [119] [INFO] Booting worker with pid: 119
fastapi | INFO: 172.31.0.1:47624 - "GET / HTTP/1.1" 404 Not Found
fastapi | INFO: 172.31.0.1:47632 - "GET /favicon.ico HTTP/1.1" 404 Not Found
fastapi | INFO:root:Running web page scraper ...
fastapi | INFO: 172.31.0.1:55576 - "GET /scrape_data HTTP/1.1" 500 Internal Server Error
fastapi | --- Logging error ---
fastapi | Traceback (most recent call last):
fastapi | File "/otodom.py", line 55, in perform_request
fastapi | response.raise_for_status()
fastapi | File "/usr/local/lib/python3.8/site-packages/requests/models.py", line 1021, in raise_for_status
fastapi | raise HTTPError(http_error_msg, response=self)
fastapi | requests.exceptions.HTTPError: 403 Client Error: Forbidden for url: https://www.otodom.pl/pl/wyniki/sprzedaz/mieszkanie?&page=1&limit=36&ownerTypeSingleSelect=ALL&priceMin=300000&priceMax=1000000&by=DEFAULT&direction=DESC&viewType=listing&locations=%5Bmalopolskie/wielicki/wieliczka%5D
fastapi |
fastapi | During handling of the above exception, another exception occurred:
fastapi |
fastapi | Traceback (most recent call last):
fastapi | File "/usr/local/lib/python3.8/logging/init.py", line 1081, in emit
fastapi | msg = self.format(record)
fastapi | File "/usr/local/lib/python3.8/logging/init.py", line 925, in format
fastapi | return fmt.format(record)
fastapi | File "/usr/local/lib/python3.8/logging/init.py", line 664, in format
fastapi | record.message = record.getMessage()
fastapi | File "/usr/local/lib/python3.8/logging/init.py", line 369, in getMessage
fastapi | msg = msg % self.args
fastapi | TypeError: not all arguments converted during string formatting
fastapi | Call stack:
fastapi | File "/usr/local/bin/uvicorn", line 8, in
fastapi | sys.exit(main())
fastapi | File "/usr/local/lib/python3.8/site-packages/click/core.py", line 1161, in call
fastapi | return self.main(*args, **kwargs)
fastapi | File "/usr/local/lib/python3.8/site-packages/click/core.py", line 1082, in main
fastapi | rv = self.invoke(ctx)
fastapi | File "/usr/local/lib/python3.8/site-packages/click/core.py", line 1443, in invoke
fastapi | return ctx.invoke(self.callback, **ctx.params)
fastapi | File "/usr/local/lib/python3.8/site-packages/click/core.py", line 788, in invoke
fastapi | return __callback(*args, **kwargs)
fastapi | File "/usr/local/lib/python3.8/site-packages/uvicorn/main.py", line 416, in main
fastapi | run(
fastapi | File "/usr/local/lib/python3.8/site-packages/uvicorn/main.py", line 587, in run
fastapi | server.run()
fastapi | File "/usr/local/lib/python3.8/site-packages/uvicorn/server.py", line 61, in run
fastapi | return asyncio.run(self.serve(sockets=sockets))
fastapi | File "/usr/local/lib/python3.8/asyncio/runners.py", line 43, in run
fastapi | return loop.run_until_complete(main)
fastapi | File "/usr/local/lib/python3.8/asyncio/base_events.py", line 603, in run_until_complete
fastapi | self.run_forever()
fastapi | File "/usr/local/lib/python3.8/asyncio/base_events.py", line 570, in run_forever
fastapi | self._run_once()
fastapi | File "/usr/local/lib/python3.8/asyncio/base_events.py", line 1859, in _run_once
fastapi | handle._run()
fastapi | File "/usr/local/lib/python3.8/asyncio/events.py", line 81, in _run
fastapi | self._context.run(self._callback, *self._args)
fastapi | File "/usr/local/lib/python3.8/site-packages/uvicorn/protocols/http/h11_impl.py", line 408, in run_asgi
fastapi | result = await app( # type: ignore[func-returns-value]
fastapi | File "/usr/local/lib/python3.8/site-packages/uvicorn/middleware/proxy_headers.py", line 84, in call
fastapi | return await self.app(scope, receive, send)
fastapi | File "/usr/local/lib/python3.8/site-packages/fastapi/applications.py", line 289, in call
fastapi | await super().call(scope, receive, send)
fastapi | File "/usr/local/lib/python3.8/site-packages/starlette/applications.py", line 122, in call
fastapi | await self.middleware_stack(scope, receive, send)
fastapi | File "/usr/local/lib/python3.8/site-packages/starlette/middleware/errors.py", line 162, in call
fastapi | await self.app(scope, receive, _send)
fastapi | File "/usr/local/lib/python3.8/site-packages/starlette/middleware/exceptions.py", line 68, in call
fastapi | await self.app(scope, receive, sender)
fastapi | File "/usr/local/lib/python3.8/site-packages/fastapi/middleware/asyncexitstack.py", line 17, in call
fastapi | await self.app(scope, receive, send)
fastapi | File "/usr/local/lib/python3.8/site-packages/starlette/routing.py", line 718, in call
fastapi | await route.handle(scope, receive, send)
fastapi | File "/usr/local/lib/python3.8/site-packages/starlette/routing.py", line 276, in handle
fastapi | await self.app(scope, receive, send)
fastapi | File "/usr/local/lib/python3.8/site-packages/starlette/routing.py", line 66, in app
fastapi | response = await func(request)
fastapi | File "/usr/local/lib/python3.8/site-packages/fastapi/routing.py", line 273, in app
fastapi | raw_response = await run_endpoint_function(
fastapi | File "/usr/local/lib/python3.8/site-packages/fastapi/routing.py", line 190, in run_endpoint_function
fastapi | return await dependant.call(**values)
fastapi | File "/main.py", line 64, in insert_data
fastapi | data = ps.run()
fastapi | File "/otodom.py", line 176, in run
fastapi | self.get_data()
fastapi | File "/otodom.py", line 69, in get_data
fastapi | response_data = self.perform_request(self.url_builder(page=1))
fastapi | File "/otodom.py", line 58, in perform_request
fastapi | logging.error("Http Error:", err)
fastapi | Message: 'Http Error:'
fastapi | Arguments: (HTTPError('403 Client Error: Forbidden for url: https://www.otodom.pl/pl/wyniki/sprzedaz/mieszkanie?&page=1&limit=36&ownerTypeSingleSelect=ALL&priceMin=300000&priceMax=1000000&by=DEFAULT&direction=DESC&viewType=listing&locations=%5Bmalopolskie/wielicki/wieliczka%5D'),)
fastapi | ERROR: Exception in ASGI application
fastapi | Traceback (most recent call last):
fastapi | File "/usr/local/lib/python3.8/site-packages/uvicorn/protocols/http/h11_impl.py", line 408, in run_asgi
fastapi | result = await app( # type: ignore[func-returns-value]
fastapi | File "/usr/local/lib/python3.8/site-packages/uvicorn/middleware/proxy_headers.py", line 84, in call
fastapi | return await self.app(scope, receive, send)
fastapi | File "/usr/local/lib/python3.8/site-packages/fastapi/applications.py", line 289, in call
fastapi | await super().call(scope, receive, send)
fastapi | File "/usr/local/lib/python3.8/site-packages/starlette/applications.py", line 122, in call
fastapi | await self.middleware_stack(scope, receive, send)
fastapi | File "/usr/local/lib/python3.8/site-packages/starlette/middleware/errors.py", line 184, in call
fastapi | raise exc
fastapi | File "/usr/local/lib/python3.8/site-packages/starlette/middleware/errors.py", line 162, in call
fastapi | await self.app(scope, receive, _send)
fastapi | File "/usr/local/lib/python3.8/site-packages/starlette/middleware/exceptions.py", line 79, in call
fastapi | raise exc
fastapi | File "/usr/local/lib/python3.8/site-packages/starlette/middleware/exceptions.py", line 68, in call
fastapi | await self.app(scope, receive, sender)
fastapi | File "/usr/local/lib/python3.8/site-packages/fastapi/middleware/asyncexitstack.py", line 20, in call
fastapi | raise e
fastapi | File "/usr/local/lib/python3.8/site-packages/fastapi/middleware/asyncexitstack.py", line 17, in call
fastapi | await self.app(scope, receive, send)
fastapi | File "/usr/local/lib/python3.8/site-packages/starlette/routing.py", line 718, in call
fastapi | await route.handle(scope, receive, send)
fastapi | File "/usr/local/lib/python3.8/site-packages/starlette/routing.py", line 276, in handle
fastapi | await self.app(scope, receive, send)
fastapi | File "/usr/local/lib/python3.8/site-packages/starlette/routing.py", line 66, in app
fastapi | response = await func(request)
fastapi | File "/usr/local/lib/python3.8/site-packages/fastapi/routing.py", line 273, in app
fastapi | raw_response = await run_endpoint_function(
fastapi | File "/usr/local/lib/python3.8/site-packages/fastapi/routing.py", line 190, in run_endpoint_function
fastapi | return await dependant.call(**values)
fastapi | File "/main.py", line 64, in insert_data
fastapi | data = ps.run()
fastapi | File "/otodom.py", line 176, in run
fastapi | self.get_data()
fastapi | File "/otodom.py", line 73, in get_data
fastapi | self._offers_raw_data = json.loads(soup_list.text)
fastapi | AttributeError: 'NoneType' object has no attribute 'text'