diff --git a/Dockerfile b/Dockerfile index 38583fe5a..18e11d856 100644 --- a/Dockerfile +++ b/Dockerfile @@ -25,29 +25,9 @@ EXPOSE 8080 WORKDIR /app RUN chmod +x /app/icloud /app/icloudpd -# Use a shell script to allow command selection -COPY < icloudpd [options]" - echo " or: docker run icloud [options]" - exit 1 - ;; -esac -EOF - +# Use a shell script to allow command selection and environment variable conversion +COPY entrypoint-wrapper.sh /app/entrypoint.sh RUN chmod +x /app/entrypoint.sh -# Default entrypoint allows command selection +# Default entrypoint allows command selection and env var conversion ENTRYPOINT ["/app/entrypoint.sh"] diff --git a/Dockerfile.build-simple b/Dockerfile.build-simple new file mode 100644 index 000000000..ef7804e72 --- /dev/null +++ b/Dockerfile.build-simple @@ -0,0 +1,25 @@ +# Simplified build Dockerfile that doesn't require BuildKit +FROM python:3.13-alpine3.19 +WORKDIR /app +RUN apk update && \ + apk add git curl binutils gcc libc-dev libffi-dev zlib-dev openssl-dev tzdata bash patchelf python3-dev musl-dev pkgconfig cargo +COPY LICENSE.md . +COPY README_PYPI.md . +COPY requirements-pip.txt . +COPY scripts scripts/ +COPY binary_dist binary_dist/ +COPY pyproject.toml . +COPY src src/ +RUN python3 -m venv .venv && \ + . .venv/bin/activate && \ + python3 -m pip install --disable-pip-version-check -r requirements-pip.txt && \ + pip3 install --disable-pip-version-check . --group dev --group devlinux +RUN . .venv/bin/activate && \ + scripts/build_bin1 icloudpd && \ + scripts/build_bin1 icloud +# Copy binaries to a known location +RUN mkdir -p /output && \ + cp /app/dist/icloudpd-*-linux-musl-amd64 /output/icloudpd && \ + cp /app/dist/icloud-*-linux-musl-amd64 /output/icloud && \ + chmod +x /output/icloudpd /output/icloud + diff --git a/Dockerfile.github b/Dockerfile.github new file mode 100644 index 000000000..879730f41 --- /dev/null +++ b/Dockerfile.github @@ -0,0 +1,40 @@ +# Dockerfile para construir desde GitHub +# Clona el repositorio y construye los binarios +FROM python:3.13-alpine3.19 AS builder +WORKDIR /app + +# Instalar git y dependencias de compilación +RUN apk update && \ + apk add git curl binutils gcc libc-dev libffi-dev zlib-dev openssl-dev tzdata bash patchelf python3-dev musl-dev pkgconfig cargo + +# Clonar el repositorio desde GitHub +# Usa tu fork o el repositorio oficial según prefieras +ARG GIT_REPO=https://github.com/jibanez-staticduo/icloud_photos_downloader.git +ARG GIT_BRANCH=master +RUN git clone --depth 1 --branch ${GIT_BRANCH} ${GIT_REPO} /app + +# Construir los binarios +RUN python3 -m venv .venv && \ + . .venv/bin/activate && \ + python3 -m pip install --disable-pip-version-check -r requirements-pip.txt && \ + pip3 install --disable-pip-version-check . --group dev --group devlinux +RUN . .venv/bin/activate && \ + scripts/build_bin1 icloudpd && \ + scripts/build_bin1 icloud + +# Imagen final +FROM alpine:3.18 +ENV MUSL_LOCPATH="/usr/share/i18n/locales/musl" +RUN apk update && apk add --no-cache tzdata musl-locales musl-locales-lang +WORKDIR /app +COPY --from=builder /app/dist/icloud icloud +COPY --from=builder /app/dist/icloudpd icloudpd +RUN chmod +x /app/icloud /app/icloudpd + +# Copiar el entrypoint wrapper desde el repositorio clonado +COPY --from=builder /app/entrypoint-wrapper.sh /app/entrypoint.sh +RUN chmod +x /app/entrypoint.sh +ENV TZ=UTC +EXPOSE 8080 +ENTRYPOINT ["/app/entrypoint.sh"] + diff --git a/Dockerfile.local b/Dockerfile.local new file mode 100644 index 000000000..aabdcf69f --- /dev/null +++ b/Dockerfile.local @@ -0,0 +1,35 @@ +# Dockerfile para construir desde el código fuente local +# Construye los binarios y luego crea la imagen final en un solo paso +FROM python:3.13-alpine3.19 AS builder +WORKDIR /app +RUN apk update && \ + apk add git curl binutils gcc libc-dev libffi-dev zlib-dev openssl-dev tzdata bash patchelf python3-dev musl-dev pkgconfig cargo +COPY LICENSE.md . +COPY README_PYPI.md . +COPY requirements-pip.txt . +COPY scripts scripts/ +COPY binary_dist binary_dist/ +COPY pyproject.toml . +COPY src src/ +RUN python3 -m venv .venv && \ + . .venv/bin/activate && \ + python3 -m pip install --disable-pip-version-check -r requirements-pip.txt && \ + pip3 install --disable-pip-version-check . --group dev --group devlinux +RUN . .venv/bin/activate && \ + scripts/build_bin1 icloudpd && \ + scripts/build_bin1 icloud + +# Imagen final +FROM alpine:3.18 +ENV MUSL_LOCPATH="/usr/share/i18n/locales/musl" +RUN apk update && apk add --no-cache tzdata musl-locales musl-locales-lang +WORKDIR /app +COPY --from=builder /app/dist/icloud icloud +COPY --from=builder /app/dist/icloudpd icloudpd +RUN chmod +x /app/icloud /app/icloudpd +COPY entrypoint-wrapper.sh /app/entrypoint.sh +RUN chmod +x /app/entrypoint.sh +ENV TZ=UTC +EXPOSE 8080 +ENTRYPOINT ["/app/entrypoint.sh"] + diff --git a/LICENSE.md b/LICENSE.md index cf8c8ceab..7e913fd1e 100644 --- a/LICENSE.md +++ b/LICENSE.md @@ -1,6 +1,7 @@ The MIT License (MIT) Copyright (c) 2016 Nathan Broadbent +Copyright (c) 2016-2025 The iCloud Photo Downloader Authors Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -19,3 +20,12 @@ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +--- + +This is a fork of the original iCloud Photos Downloader project: +https://github.com/icloud-photos-downloader/icloud_photos_downloader + +This fork includes additional features such as Telegram bot integration for +remote control and authentication. All modifications are also licensed under +the MIT License. diff --git a/README.md b/README.md index 74c717877..1182671d8 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,15 @@ -# iCloud Photos Downloader [![Quality Checks](https://github.com/icloud-photos-downloader/icloud_photos_downloader/workflows/Quality%20Checks/badge.svg)](https://github.com/icloud-photos-downloader/icloud_photos_downloader/actions/workflows/quality-checks.yml) [![Build and Package](https://github.com/icloud-photos-downloader/icloud_photos_downloader/workflows/Produce%20Artifacts/badge.svg)](https://github.com/icloud-photos-downloader/icloud_photos_downloader/actions/workflows/produce-artifacts.yml) [![MIT License](https://img.shields.io/badge/license-MIT-blue.svg)](LICENSE) +# iCloud Photos Downloader [![MIT License](https://img.shields.io/badge/license-MIT-blue.svg)](LICENSE.md) + +> **Note:** This is a fork of the original [iCloud Photos Downloader](https://github.com/icloud-photos-downloader/icloud_photos_downloader) project with additional features including Telegram bot integration for remote control and authentication. - A command-line tool to download all your iCloud photos. - Works on Linux, Windows, and macOS; laptop, desktop, and NAS - Available as an executable for direct downloading and through package managers/ecosystems ([Docker](https://icloud-photos-downloader.github.io/icloud_photos_downloader/install.html#docker), [PyPI](https://icloud-photos-downloader.github.io/icloud_photos_downloader/install.html#pypi), [AUR](https://icloud-photos-downloader.github.io/icloud_photos_downloader/install.html#aur), [npm](https://icloud-photos-downloader.github.io/icloud_photos_downloader/install.html#npm)) -- Developed and maintained by volunteers (we are always looking for [help](CONTRIBUTING.md)). +- **Additional features in this fork:** + - Telegram bot integration for remote control (`/sync`, `/syncall`, `/stop`, `/status`, `/auth` commands) + - Telegram-based MFA authentication (no SSH required for cookie renewal) + - Automatic authentication expiration detection and notifications +- Based on the original project developed and maintained by volunteers (we are always looking for [help](CONTRIBUTING.md)). See [Documentation](https://icloud-photos-downloader.github.io/icloud_photos_downloader/) for more details. Also, check [Issues](https://github.com/icloud-photos-downloader/icloud_photos_downloader/issues) diff --git a/entrypoint-wrapper.sh b/entrypoint-wrapper.sh new file mode 100644 index 000000000..dde3e46ee --- /dev/null +++ b/entrypoint-wrapper.sh @@ -0,0 +1,133 @@ +#!/bin/sh +# Wrapper script to convert environment variables to icloudpd command line arguments + +ARGS="" + +# Username (required) +if [ -n "$apple_id" ]; then + ARGS="$ARGS --username $apple_id" +elif [ -n "$APPLE_ID" ]; then + ARGS="$ARGS --username $APPLE_ID" +fi + +# Directory (required) +if [ -n "$download_path" ]; then + ARGS="$ARGS --directory $download_path" +elif [ -n "$DOWNLOAD_PATH" ]; then + ARGS="$ARGS --directory $DOWNLOAD_PATH" +fi + +# Watch interval +if [ -n "$synchronisation_interval" ]; then + ARGS="$ARGS --watch-with-interval $synchronisation_interval" +elif [ -n "$SYNCHRONISATION_INTERVAL" ]; then + ARGS="$ARGS --watch-with-interval $SYNCHRONISATION_INTERVAL" +fi + +# Folder structure +if [ -n "$folder_structure" ]; then + ARGS="$ARGS --folder-structure \"$folder_structure\"" +elif [ -n "$FOLDER_STRUCTURE" ]; then + ARGS="$ARGS --folder-structure \"$FOLDER_STRUCTURE\"" +fi + +# Cookie directory (default to /config if mounted) +if [ -n "$cookie_directory" ]; then + ARGS="$ARGS --cookie-directory $cookie_directory" +elif [ -n "$COOKIE_DIRECTORY" ]; then + ARGS="$ARGS --cookie-directory $COOKIE_DIRECTORY" +elif [ -d "/config" ]; then + ARGS="$ARGS --cookie-directory /config" +fi + +# Boolean flags +# Nota: --skip-check y --delete-empty-directories no existen en el repositorio oficial +# --skip-album tampoco existe, pero podemos usar --album para especificar qué descargar + +[ "$auto_delete" = "true" ] && ARGS="$ARGS --auto-delete" || true +[ "$AUTO_DELETE" = "true" ] && ARGS="$ARGS --auto-delete" || true + +# convert_heic_to_jpeg no está disponible en el repositorio oficial + +[ "$skip_videos" = "true" ] && ARGS="$ARGS --skip-videos" || true +[ "$SKIP_VIDEOS" = "true" ] && ARGS="$ARGS --skip-videos" || true + +[ "$skip_live_photos" = "true" ] && ARGS="$ARGS --skip-live-photos" || true +[ "$SKIP_LIVE_PHOTOS" = "true" ] && ARGS="$ARGS --skip-live-photos" || true + +[ "$set_exif_datetime" = "true" ] && ARGS="$ARGS --set-exif-datetime" || true +[ "$SET_EXIF_DATETIME" = "true" ] && ARGS="$ARGS --set-exif-datetime" || true + +[ "$keep_unicode" = "true" ] && ARGS="$ARGS --keep-unicode-in-filenames" || true +[ "$KEEP_UNICODE" = "true" ] && ARGS="$ARGS --keep-unicode-in-filenames" || true + +# Album (para especificar qué descargar, no para saltar) +if [ -n "$photo_album" ]; then + ARGS="$ARGS --album $photo_album" +elif [ -n "$PHOTO_ALBUM" ]; then + ARGS="$ARGS --album $PHOTO_ALBUM" +fi + +# Photo size +if [ -n "$photo_size" ]; then + ARGS="$ARGS --size $photo_size" +elif [ -n "$PHOTO_SIZE" ]; then + ARGS="$ARGS --size $PHOTO_SIZE" +fi + +# Live photo size +if [ -n "$live_photo_size" ]; then + ARGS="$ARGS --live-photo-size $live_photo_size" +elif [ -n "$LIVE_PHOTO_SIZE" ]; then + ARGS="$ARGS --live-photo-size $LIVE_PHOTO_SIZE" +fi + +# Recent photos +if [ -n "$recent_only" ] && [ "$recent_only" != "0" ]; then + ARGS="$ARGS --recent $recent_only" +elif [ -n "$RECENT_ONLY" ] && [ "$RECENT_ONLY" != "0" ]; then + ARGS="$ARGS --recent $RECENT_ONLY" +fi + +# Library +if [ -n "$photo_library" ]; then + ARGS="$ARGS --library $photo_library" +elif [ -n "$PHOTO_LIBRARY" ]; then + ARGS="$ARGS --library $PHOTO_LIBRARY" +fi + +# Log level +if [ -n "$debug_logging" ] && [ "$debug_logging" = "true" ]; then + ARGS="$ARGS --log-level debug" +elif [ -n "$DEBUG_LOGGING" ] && [ "$DEBUG_LOGGING" = "true" ]; then + ARGS="$ARGS --log-level debug" +fi + +# Password (if provided) +if [ -n "$password" ]; then + ARGS="$ARGS --password \"$password\"" +elif [ -n "$PASSWORD" ]; then + ARGS="$ARGS --password \"$PASSWORD\"" +fi + +# Auth only mode +[ "$auth_only" = "true" ] && ARGS="$ARGS --auth-only" || true +[ "$AUTH_ONLY" = "true" ] && ARGS="$ARGS --auth-only" || true + +# If no arguments were provided, show help +if [ -z "$ARGS" ] && [ "$1" != "icloudpd" ] && [ "$1" != "icloud" ]; then + exec /app/icloudpd --help + exit 0 +fi + +# If first argument is 'icloud' or 'icloudpd', use it, otherwise default to icloudpd +if [ "$1" = "icloud" ] || [ "$1" = "icloudpd" ]; then + CMD="$1" + shift + # Combine remaining args with our generated args + exec /app/$CMD $ARGS "$@" +else + # Default to icloudpd with our generated args + exec /app/icloudpd $ARGS "$@" +fi + diff --git a/src/icloudpd/authentication.py b/src/icloudpd/authentication.py index d6a7d1568..034cafd81 100644 --- a/src/icloudpd/authentication.py +++ b/src/icloudpd/authentication.py @@ -106,6 +106,8 @@ def password_provider(username: str, valid_password: List[str]) -> str | None: notificator() if mfa_provider == MFAProvider.WEBUI: request_2fa_web(icloud, logger, status_exchange) + elif mfa_provider == MFAProvider.TELEGRAM: + request_2fa_telegram(icloud, logger, status_exchange) else: request_2fa(icloud, logger) @@ -281,3 +283,63 @@ def request_2fa_web( ) else: raise PyiCloudFailedMFAException("Failed to change status") + + +def request_2fa_telegram( + icloud: PyiCloudService, logger: logging.Logger, status_exchange: StatusExchange +) -> None: + """Request two-factor authentication through Telegram.""" + if not status_exchange.replace_status(Status.NO_INPUT_NEEDED, Status.NEED_MFA): + raise PyiCloudFailedMFAException( + f"Expected NO_INPUT_NEEDED, but got {status_exchange.get_status()}" + ) + + # Get telegram bot from status_exchange if available + telegram_bot = status_exchange.get_telegram_bot() + if telegram_bot: + username = status_exchange.get_current_user() or "user" + telegram_bot.request_auth_code(username) + else: + logger.warning("Telegram bot not available, falling back to console") + # Fallback to console if Telegram bot not available + request_2fa(icloud, logger) + return + + # wait for input + while True: + status = status_exchange.get_status() + if status == Status.NEED_MFA: + time.sleep(1) + continue + else: + pass + + if status_exchange.replace_status(Status.SUPPLIED_MFA, Status.CHECKING_MFA): + code = status_exchange.get_payload() + if not code: + raise PyiCloudFailedMFAException( + "Internal error: did not get code for SUPPLIED_MFA status" + ) + + if not icloud.validate_2fa_code(code): + if status_exchange.set_error("Failed to verify two-factor authentication code"): + # Reset waiting flag and request code again + if telegram_bot: + telegram_bot.request_auth_code(username) + continue + else: + raise PyiCloudFailedMFAException("Failed to change status of invalid code") + else: + status_exchange.replace_status(Status.CHECKING_MFA, Status.NO_INPUT_NEEDED) # done + if telegram_bot: + telegram_bot.send_message("✅ Authentication completed successfully") + logger.info( + "Great, you're all set up. The script can now be run without " + "user interaction until 2FA expires.\n" + "You can set up email notifications for when " + "the two-factor authentication expires.\n" + "(Use --help to view information about SMTP options.)" + ) + break + else: + raise PyiCloudFailedMFAException("Failed to change status") diff --git a/src/icloudpd/base.py b/src/icloudpd/base.py index 2c9fe24dc..b16e3c014 100644 --- a/src/icloudpd/base.py +++ b/src/icloudpd/base.py @@ -23,6 +23,7 @@ Iterable, List, Mapping, + Optional, Sequence, Tuple, ) @@ -45,6 +46,7 @@ from icloudpd.counter import Counter from icloudpd.email_notifications import send_2sa_notification from icloudpd.filename_policies import build_filename_with_policies, create_filename_builder +from icloudpd.file_cache import FileCache from icloudpd.log_level import LogLevel from icloudpd.mfa_provider import MFAProvider from icloudpd.password_provider import PasswordProvider @@ -56,6 +58,7 @@ from pyicloud_ipd.asset_version import add_suffix_to_filename, calculate_version_filename from pyicloud_ipd.base import PyiCloudService from pyicloud_ipd.exceptions import ( + PyiCloud2SARequiredException, PyiCloudAPIResponseException, PyiCloudConnectionErrorException, PyiCloudFailedLoginException, @@ -247,18 +250,55 @@ def run_with_configs(global_config: GlobalConfig, user_configs: Sequence[UserCon provider == PasswordProvider.WEBUI for provider in global_config.password_providers ) + # Start Telegram bot if configured (before web server to pass it if needed) + telegram_bot = None + if global_config.telegram_polling and global_config.telegram_token and global_config.telegram_chat_id: + from icloudpd.telegram_bot import TelegramBot + telegram_bot = TelegramBot( + logger, + global_config.telegram_token, + global_config.telegram_chat_id, + shared_status_exchange, + global_config.telegram_polling_interval, + global_config.telegram_webhook_url, + ) + telegram_bot.start_polling() + # Store telegram bot reference in status_exchange for auth requests + shared_status_exchange.set_telegram_bot(telegram_bot) + # Start web server ONCE if needed, outside all loops - if needs_web_server: - logger.info("Starting web server for WebUI authentication...") - server_thread = Thread(target=serve_app, daemon=True, args=[logger, shared_status_exchange]) + # Pass telegram_bot if available for webhook support + # Use webhook port if Telegram webhook is configured, otherwise default port 8080 + webhook_port = ( + global_config.telegram_webhook_port + if telegram_bot and telegram_bot.webhook_url + else 8080 + ) + if needs_web_server or telegram_bot: + if needs_web_server: + logger.info(f"Starting web server for WebUI authentication on port {webhook_port}...") + if telegram_bot and telegram_bot.webhook_url: + logger.info(f"Starting web server for Telegram webhooks on port {webhook_port}...") + server_thread = Thread( + target=serve_app, + daemon=True, + args=[logger, shared_status_exchange, telegram_bot, webhook_port], + ) server_thread.start() # Check if we're in watch mode watch_interval = global_config.watch_with_interval + + # Set watch_interval in progress for status messages (even before first sync) + if watch_interval: + shared_status_exchange.get_progress().watch_interval = watch_interval + # Set initial sync time to now so we can calculate time until first sync + import time + shared_status_exchange.get_progress().last_sync_time = time.time() if not watch_interval: # No watch mode - process each user once and exit - return _process_all_users_once(global_config, user_configs, logger, shared_status_exchange) + return _process_all_users_once(global_config, user_configs, logger, shared_status_exchange, telegram_bot) else: # Watch mode - infinite loop processing all users, then wait skip_bar = not os.environ.get("FORCE_TQDM") and ( @@ -268,9 +308,14 @@ def run_with_configs(global_config: GlobalConfig, user_configs: Sequence[UserCon ) while True: + # Check if resume was requested before processing (might have been set while waiting) + if shared_status_exchange.get_progress().resume: + logger.info("Sync requested, starting immediately...") + shared_status_exchange.get_progress().resume = False # Clear resume flag + # Process all user configs in this iteration result = _process_all_users_once( - global_config, user_configs, logger, shared_status_exchange + global_config, user_configs, logger, shared_status_exchange, telegram_bot ) # If any critical operation (auth-only, list commands) succeeded, exit @@ -304,9 +349,16 @@ def run_with_configs(global_config: GlobalConfig, user_configs: Sequence[UserCon # Update shared status exchange with wait progress shared_status_exchange.get_progress().waiting = watch_interval - counter if shared_status_exchange.get_progress().resume: - shared_status_exchange.get_progress().reset() + logger.info("Sync requested, breaking wait loop...") + shared_status_exchange.get_progress().resume = False # Clear resume flag + shared_status_exchange.get_progress().waiting = 0 # Clear waiting break time.sleep(1) + + # Check if resume was requested (might have been set while processing) + if shared_status_exchange.get_progress().resume: + logger.info("Sync requested, starting immediately...") + shared_status_exchange.get_progress().resume = False # Clear resume flag def _process_all_users_once( @@ -314,6 +366,7 @@ def _process_all_users_once( user_configs: Sequence[UserConfig], logger: logging.Logger, shared_status_exchange: StatusExchange, + telegram_bot: Optional[Any] = None, ) -> int: """Process all user configs once (used by both single run and watch mode)""" @@ -396,8 +449,22 @@ def password_provider(_username: str) -> str | None: filename_builder, ) - downloader = ( - partial( + # Initialize file cache only for sync date tracking (not for file caching) + file_cache: FileCache | None = None + if user_config.directory is not None: + # Create cache database path in the cookie directory or config directory + # We only use it for storing last sync date, not for file caching + cache_dir = user_config.cookie_directory or os.path.dirname(user_config.directory) + cache_db_path = os.path.join(cache_dir, "file_cache.db") + file_cache = FileCache(cache_db_path, logger) + + # Reset force_full_sync flag after checking + status_exchange.set_force_full_sync(False) + + # Create downloader partial - file_cache and use_cache will be passed later + # We need to create a wrapper that captures file_cache and use_cache + if user_config.directory is not None: + download_builder_partial = partial( download_builder, logger, user_config.folder_structure, @@ -415,9 +482,13 @@ def password_provider(_username: str) -> str | None: filename_builder, user_config.align_raw, ) - if user_config.directory is not None - else (lambda _s, _c, _p: False) - ) + # Create a wrapper (file_cache no longer used for file caching, only for sync date) + # total_photos and start_time will be set later when we know the actual count + def downloader_wrapper(icloud: PyiCloudService, counter: Counter, photo: PhotoAsset, total_photos: int | None = None, start_time: float | None = None) -> bool: + return download_builder_partial(icloud, counter, photo, file_cache=None, total_photos=total_photos, start_time=start_time) + downloader = downloader_wrapper + else: + downloader = lambda _s, _c, _p: False notificator = partial( notificator_builder, @@ -445,6 +516,9 @@ def password_provider(_username: str) -> str | None: downloader, notificator, lp_filename_generator, + file_cache, # Only used for sync date tracking, not for file caching + filename_builder, + telegram_bot, ) # If any user config fails and we're not in watch mode, return the error code @@ -580,6 +654,9 @@ def download_builder( icloud: PyiCloudService, counter: Counter, photo: PhotoAsset, + file_cache: FileCache | None = None, # Not used anymore, kept for compatibility + total_photos: int | None = None, # Total photos to process (for counter display) + start_time: float | None = None, # Start time for rate calculation ) -> bool: """function for actually downloading the photos""" @@ -662,6 +739,7 @@ def download_builder( download_path = local_download_path(filename, download_dir) original_download_path = None + # Check if file exists using os.path.isfile() (no cache) file_exists = os.path.isfile(download_path) if not file_exists and download_size == AssetVersionSize.ORIGINAL: # Deprecation - We used to download files like IMG_1234-original.jpg, @@ -671,6 +749,7 @@ def download_builder( file_exists = os.path.isfile(original_download_path) if file_exists: + # Only do additional checks for deduplication if file_match_policy == FileMatchPolicy.NAME_SIZE_DEDUP_WITH_SUFFIX: # for later: this crashes if download-size medium is specified file_size = os.stat(original_download_path or download_path).st_size @@ -678,10 +757,18 @@ def download_builder( if file_size != photo_size: download_path = (f"-{photo_size}.").join(download_path.rsplit(".", 1)) logger.debug("%s deduplicated", truncate_middle(download_path, 96)) + # Check again with os.path.isfile() (not using cache here) file_exists = os.path.isfile(download_path) if file_exists: counter.increment() - logger.debug("%s already exists", truncate_middle(download_path, 96)) + # Only show "already exists" when using is_file (not cache) + current_count = counter.value() + counter_text = "" + if total_photos is not None and start_time is not None: + elapsed = time.time() - start_time + rate = current_count / elapsed if elapsed > 0 else 0.0 + counter_text = f" ({current_count}/{total_photos}) (Rate: {rate:.2f} items/s)" + logger.debug("%s already exists%s", truncate_middle(download_path, 96), counter_text) if not file_exists: counter.reset() @@ -754,7 +841,16 @@ def download_builder( pass lp_download_path = os.path.join(download_dir, lp_filename) - lp_file_exists = os.path.isfile(lp_download_path) + # Use cache if available and enabled, otherwise use os.path.isfile() + # verify_disk=True only when use_cache=False (e.g., /syncall command) + if file_cache is not None and use_cache: + lp_file_exists = file_cache.file_exists(lp_download_path, verify_disk=False) + # If file not in cache but exists on disk, add it to cache silently + if not lp_file_exists and os.path.isfile(lp_download_path): + file_cache.add_file(lp_download_path) + lp_file_exists = True + else: + lp_file_exists = os.path.isfile(lp_download_path) if only_print_filenames: if not lp_file_exists: @@ -775,6 +871,7 @@ def download_builder( print(lp_download_path) else: if lp_file_exists: + # Only do additional checks for deduplication if file_match_policy == FileMatchPolicy.NAME_SIZE_DEDUP_WITH_SUFFIX: lp_file_size = os.stat(lp_download_path).st_size lp_photo_size = version.size @@ -783,9 +880,17 @@ def download_builder( lp_download_path.rsplit(".", 1) ) logger.debug("%s deduplicated", truncate_middle(lp_download_path, 96)) + # Check again with os.path.isfile() lp_file_exists = os.path.isfile(lp_download_path) if lp_file_exists: - logger.debug("%s already exists", truncate_middle(lp_download_path, 96)) + # For live photos, use the same counter and rate info + current_count = counter.value() + counter_text = "" + if total_photos is not None and start_time is not None: + elapsed = time.time() - start_time + rate = current_count / elapsed if elapsed > 0 else 0.0 + counter_text = f" ({current_count}/{total_photos}) (Rate: {rate:.2f} items/s)" + logger.debug("%s already exists%s", truncate_middle(lp_download_path, 96), counter_text) if not lp_file_exists: truncated_path = truncate_middle(lp_download_path, 96) logger.debug("Downloading %s...", truncated_path) @@ -884,6 +989,10 @@ def core_single_run( downloader: Callable[[PyiCloudService, Counter, PhotoAsset], bool], notificator: Callable[[], None], lp_filename_generator: Callable[[str], str], + file_cache: FileCache | None = None, + use_cache: bool = True, + filename_builder: Callable[[PhotoAsset], str] | None = None, + telegram_bot: Optional[Any] = None, ) -> int: """Download all iCloud photos to a local directory for a single execution (no watch loop)""" @@ -899,6 +1008,25 @@ def append_response(captured: List[Mapping[str, Any]], response: Mapping[str, An captured.append(response) try: + # Check if authentication was requested via Telegram /auth command + # If so, delete cookies to force re-authentication + if telegram_bot and hasattr(telegram_bot, '_auth_requested') and telegram_bot._auth_requested: + telegram_bot._auth_requested = False # Reset flag + cookie_dir = user_config.cookie_directory or os.path.expanduser("~/.pyicloud") + # Delete cookie and session files to force re-authentication + # Cookie file name is based on username (alphanumeric characters only) + import re + cookie_filename = "".join([c for c in user_config.username if re.match(r'\w', c)]) + cookie_path = os.path.join(cookie_dir, cookie_filename) + session_path = cookie_path + ".session" + for file_path in [cookie_path, session_path]: + if os.path.exists(file_path): + try: + os.remove(file_path) + logger.info(f"Deleted {file_path} to force re-authentication") + except OSError as e: + logger.warning(f"Could not delete {file_path}: {e}") + icloud = authenticator( logger, global_config.domain, @@ -960,6 +1088,14 @@ def append_response(captured: List[Mapping[str, Any]], response: Mapping[str, An pass directory = os.path.normpath(user_config.directory) + + # Create filename_builder if not provided (for backward compatibility) + if filename_builder is None: + from icloudpd.filename_policies import build_filename_cleaner, create_filename_builder + filename_cleaner = build_filename_cleaner(user_config.keep_unicode_in_filenames) + filename_builder = create_filename_builder( + user_config.file_match_policy, filename_cleaner + ) if user_config.skip_photos or user_config.skip_videos: photo_video_phrase = "photos" if user_config.skip_videos else "videos" @@ -987,11 +1123,77 @@ def sum_(inp: Iterable[int]) -> int: return sum(inp) photos_count: int | None = compose(sum_, album_lengths)(albums) + total_photos_in_icloud = photos_count if photos_count is not None else 0 + logger.info(f"Found {total_photos_in_icloud} total photos in iCloud") + for photo_album in albums: + # OPTIMIZATION: Increase page_size to reduce number of HTTP requests + # Default is 100, which means 200 records per request (page_size * 2) + # Increasing to 500 means 1000 records per request, reducing HTTP calls by 5x + if hasattr(photo_album, 'page_size'): + original_page_size = photo_album.page_size + photo_album.page_size = 500 # Increase from 100 to 500 (1000 records per request) + logger.debug(f"Increased page_size from {original_page_size} to {photo_album.page_size} for faster loading") + + # OPTIMIZATION: Filter by addedDate if not doing full sync and we have a last sync date + # This dramatically reduces the number of photos to process + # Use 1 day margin to account for timing differences (photo added to device vs uploaded to iCloud) + if file_cache is not None and not status_exchange.get_force_full_sync(): + last_sync_timestamp = file_cache.get_last_sync_date() + if last_sync_timestamp: + # Subtract 1 day (86400 seconds) as margin for timing differences + margin_seconds = 86400 # 1 day + last_sync_with_margin = last_sync_timestamp - margin_seconds + + # Add filter for addedDate >= (last_sync_date - 1 day) + # Convert timestamp (seconds) to milliseconds (what iCloud uses) + added_date_ms = int(last_sync_with_margin * 1000) + + # Create or extend query_filter + added_date_filter = { + "fieldName": "addedDate", + "fieldValue": {"type": "INT64", "value": added_date_ms}, + "comparator": "GREATER_THAN_OR_EQUALS", + } + + if photo_album.query_filter is None: + photo_album.query_filter = [added_date_filter] + else: + # Add to existing filters + photo_album.query_filter = list(photo_album.query_filter) + [added_date_filter] + + last_sync_readable = datetime.datetime.fromtimestamp(last_sync_timestamp).strftime("%Y-%m-%d %H:%M:%S") + margin_readable = datetime.datetime.fromtimestamp(last_sync_with_margin).strftime("%Y-%m-%d %H:%M:%S") + logger.info(f"🔄 INCREMENTAL SYNC: Filtering photos added since {margin_readable} (last sync: {last_sync_readable}, 1 day margin)") + logger.info(f" This will only process NEW photos, not all {total_photos_in_icloud} photos") + else: + logger.info("🔄 FULL SYNC: No previous sync date found, processing all photos (first sync)") + photos_enumerator: Iterable[PhotoAsset] = photo_album + + # Note: photos_count is calculated before filter, so it shows total + # The actual number of photos returned by Apple after filter will be less + # We'll update photos_to_download as we process them + photos_to_download = photos_count if photos_count is not None else 0 + + progress = status_exchange.get_progress() + progress.total_photos_in_icloud = total_photos_in_icloud + progress.photos_to_download = photos_to_download + + # Send initial message if manual sync + if telegram_bot and status_exchange.get_manual_sync(): + telegram_bot.send_sync_start_message(photos_to_download, total_photos_in_icloud) + # Mark that we should send progress updates + progress.last_progress_message_time = time.time() + # Reset manual sync flag after sending initial message + status_exchange.set_manual_sync(False) # Optional: Only download the x most recent photos. if user_config.recent is not None: + # Adjust photos_to_download if using recent limit + photos_to_download = min(photos_to_download, user_config.recent) + progress = status_exchange.get_progress() + progress.photos_to_download = photos_to_download photos_count = user_config.recent photos_top: Iterable[PhotoAsset] = itertools.islice( photos_enumerator, user_config.recent @@ -1001,6 +1203,9 @@ def sum_(inp: Iterable[int]) -> int: if user_config.until_found is not None: photos_count = None + # Can't know photos_to_download in advance with until_found + progress = status_exchange.get_progress() + progress.photos_to_download = 0 # Will be updated dynamically # ensure photos iterator doesn't have a known length # photos_enumerator = (p for p in photos_enumerator) @@ -1059,15 +1264,27 @@ def should_break(counter: Counter) -> bool: and counter.value() >= user_config.until_found ) - status_exchange.get_progress().photos_count = ( - 0 if photos_count is None else photos_count - ) + # For cache mode, we don't know exact count upfront, so start with 0 + # It will be updated as we process photos + progress = status_exchange.get_progress() + if file_cache is not None and use_cache: + # Start with 0, will be updated dynamically + progress.photos_count = 0 + else: + progress.photos_count = photos_to_download photos_counter = 0 + photos_downloaded = 0 # Track photos actually downloaded in this iteration + # Initialize photos_checked for status tracking + progress.photos_checked = 0 now = datetime.datetime.now(get_localzone()) - # photos_iterator = iter(photos_enumerator) - - download_photo = partial(downloader, icloud) + last_progress_message_time = time.time() # Track last progress message time + loop_start_time = time.time() # Track start time for rate calculation + # Store processing start time in progress for status messages + progress.processing_start_time = loop_start_time + # Create download_photo with total_photos and start_time for counter display + # Use lambda to pass keyword arguments correctly + download_photo = lambda counter, photo: downloader(icloud, counter, photo, total_photos=photos_to_download, start_time=loop_start_time) for item in photos_bar: try: @@ -1080,10 +1297,33 @@ def should_break(counter: Counter) -> bool: # item = next(photos_iterator) should_delete = False + # Update photos_checked for status tracking (every photo we process) + progress = status_exchange.get_progress() + progress.photos_checked += 1 + passer_result = passer(item) download_result = passer_result and download_photo( consecutive_files_found, item ) + + # Count photos that were actually downloaded + if download_result: + photos_downloaded += 1 + photos_counter += 1 + status_exchange.get_progress().photos_counter = photos_counter + + # Send progress update every minute if manual sync was triggered + current_time = time.time() + if telegram_bot and progress.last_progress_message_time > 0: + # Manual sync was triggered, send updates every minute + if (current_time - last_progress_message_time) >= 60: + telegram_bot.send_progress_update() + last_progress_message_time = current_time + progress.last_progress_message_time = current_time + elif passer_result: + # Photo passed filters but was already downloaded + photos_counter += 1 + status_exchange.get_progress().photos_counter = photos_counter if download_result and user_config.delete_after_download: should_delete = True @@ -1142,8 +1382,6 @@ def should_break(counter: Counter) -> bool: # retrier(delete_local, error_handler) photo_album.increment_offset(-1) - photos_counter += 1 - status_exchange.get_progress().photos_counter = photos_counter if status_exchange.get_progress().cancel: break @@ -1171,7 +1409,33 @@ def should_break(counter: Counter) -> bool: message = f"All {photo_video_phrase} have been downloaded" logger.info(message) status_exchange.get_progress().photos_last_message = message - status_exchange.get_progress().reset() + + # Send final message via Telegram + # Only send if manual sync was triggered (last_progress_message_time > 0) + # OR if it's a periodic sync (always send final message for periodic) + progress = status_exchange.get_progress() + if telegram_bot: + watch_interval = global_config.watch_with_interval or 0 + # Send final message if manual sync OR if periodic (last_progress_message_time == 0 means periodic) + if progress.last_progress_message_time > 0: + # Manual sync - send final message + telegram_bot.send_sync_complete_message(photos_downloaded, watch_interval) + elif watch_interval > 0: + # Periodic sync - send final message (but no initial or progress messages) + telegram_bot.send_sync_complete_message(photos_downloaded, watch_interval) + + # Update last sync time and watch interval before reset + progress = status_exchange.get_progress() + current_time = time.time() + if global_config.watch_with_interval: + progress.watch_interval = global_config.watch_with_interval + progress.last_sync_time = current_time + # Always save last sync date to cache for incremental syncs (even without watch mode) + # This allows subsequent runs to only process new photos + if file_cache is not None: + file_cache.set_last_sync_date(current_time) + logger.info(f"Saved last sync date: {datetime.datetime.fromtimestamp(current_time).strftime('%Y-%m-%d %H:%M:%S')} (next sync will only process new photos)") + progress.reset() if user_config.auto_delete: autodelete_photos( @@ -1200,8 +1464,20 @@ def should_break(counter: Counter) -> bool: if global_config.mfa_provider == MFAProvider.WEBUI: update_auth_error_in_webui(status_exchange, str(error)) continue + elif global_config.mfa_provider == MFAProvider.TELEGRAM and telegram_bot: + # Error already handled in request_2fa_telegram, just continue + continue else: return 1 + except PyiCloud2SARequiredException as error: + logger.info(str(error)) + dump_responses(logger.debug, captured_responses) + # Notify via Telegram if available + if telegram_bot: + username = user_config.username + telegram_bot.notify_auth_required(username) + # Continue to retry authentication (will detect requires_2fa and use Telegram if configured) + continue except ( PyiCloudServiceNotActivatedException, PyiCloudServiceUnavailableException, @@ -1210,6 +1486,15 @@ def should_break(counter: Counter) -> bool: ) as error: logger.info(error) dump_responses(logger.debug, captured_responses) + # Check if it's an authentication error (421, 450, 500) + if isinstance(error, PyiCloudAPIResponseException) and str(error.code) in ["421", "450", "500"]: + # Authentication required - notify via Telegram if available + if telegram_bot: + username = user_config.username + telegram_bot.notify_auth_required(username) + # Continue to retry authentication (will detect requires_2fa and use Telegram if configured) + if global_config.mfa_provider == MFAProvider.TELEGRAM and telegram_bot: + continue # webui will display error and wait for password again if ( PasswordProvider.WEBUI in global_config.password_providers diff --git a/src/icloudpd/cli.py b/src/icloudpd/cli.py index 9c33ac1af..7b3e0aba4 100644 --- a/src/icloudpd/cli.py +++ b/src/icloudpd/cli.py @@ -281,6 +281,7 @@ def parse_mfa_provider(provider: str) -> MFAProvider: provider_map = { "console": MFAProvider.CONSOLE, "webui": MFAProvider.WEBUI, + "telegram": MFAProvider.TELEGRAM, } normalized_provider = lower(provider) @@ -342,6 +343,42 @@ def add_global_options(parser: argparse.ArgumentParser) -> argparse.ArgumentPars type=int, default=None, ) + cloned.add_argument( + "--telegram-token", + help="Telegram bot token for notifications and remote control", + default=None, + type=str, + ) + cloned.add_argument( + "--telegram-chat-id", + help="Telegram chat ID for notifications and remote control", + default=None, + type=str, + ) + cloned.add_argument( + "--telegram-polling", + action="store_true", + help="Enable Telegram bot polling for remote control commands", + default=False, + ) + cloned.add_argument( + "--telegram-polling-interval", + help="Interval in seconds for Telegram bot polling (default: 30 seconds)", + default=30, + type=int, + ) + cloned.add_argument( + "--telegram-webhook-url", + help="Webhook URL for Telegram bot (push notifications). If not set, uses polling.", + default=None, + type=str, + ) + cloned.add_argument( + "--telegram-webhook-port", + help="Port for Telegram webhook server (default: 48080)", + default=48080, + type=int, + ) cloned.add_argument( "--password-provider", dest="password_providers", @@ -354,7 +391,7 @@ def add_global_options(parser: argparse.ArgumentParser) -> argparse.ArgumentPars cloned.add_argument( "--mfa-provider", help="Specify where to get the MFA code from", - choices=["console", "webui"], + choices=["console", "webui", "telegram"], default="console", type=lower, ) @@ -528,6 +565,12 @@ def parse(args: Sequence[str]) -> Tuple[GlobalConfig, Sequence[UserConfig]]: ) ), mfa_provider=MFAProvider(global_ns.mfa_provider), + telegram_token=global_ns.telegram_token, + telegram_chat_id=global_ns.telegram_chat_id, + telegram_polling=global_ns.telegram_polling, + telegram_polling_interval=global_ns.telegram_polling_interval, + telegram_webhook_url=global_ns.telegram_webhook_url, + telegram_webhook_port=global_ns.telegram_webhook_port, ), user_nses, ) diff --git a/src/icloudpd/config.py b/src/icloudpd/config.py index 019843864..1933407a4 100644 --- a/src/icloudpd/config.py +++ b/src/icloudpd/config.py @@ -71,3 +71,9 @@ class GlobalConfig: watch_with_interval: int | None password_providers: Sequence[PasswordProvider] mfa_provider: MFAProvider + telegram_token: str | None = None + telegram_chat_id: str | None = None + telegram_polling: bool = False + telegram_polling_interval: int = 30 + telegram_webhook_url: str | None = None + telegram_webhook_port: int = 48080 \ No newline at end of file diff --git a/src/icloudpd/download.py b/src/icloudpd/download.py index 78da134b5..33a975f54 100644 --- a/src/icloudpd/download.py +++ b/src/icloudpd/download.py @@ -160,6 +160,8 @@ def download_media( except PyiCloudAPIResponseException as ex: if "Invalid global session" in str(ex): logger.error("Session error, re-authenticating...") + # Note: re-authentication will be handled by the main loop + # which will detect requires_2fa and trigger MFA flow if retries > 0: # If the first re-authentication attempt failed, # start waiting a few seconds before retrying in case diff --git a/src/icloudpd/file_cache.py b/src/icloudpd/file_cache.py new file mode 100644 index 000000000..8469eeb85 --- /dev/null +++ b/src/icloudpd/file_cache.py @@ -0,0 +1,258 @@ +"""File cache module to track which files exist on disk for faster synchronization""" + +import logging +import os +import sqlite3 +import threading +import time +from datetime import datetime, timedelta +from pathlib import Path +from typing import Optional + +from icloudpd.config import UserConfig + + +class FileCache: + """SQLite-based cache to track files on disk""" + + def __init__(self, cache_db_path: str, logger: logging.Logger) -> None: + self.cache_db_path = cache_db_path + self.logger = logger + self.lock = threading.Lock() + self._ensure_cache_db() + + def _ensure_cache_db(self) -> None: + """Create cache database if it doesn't exist""" + with self.lock: + conn = sqlite3.connect(self.cache_db_path, timeout=30.0) + try: + cursor = conn.cursor() + cursor.execute( + """ + CREATE TABLE IF NOT EXISTS file_cache ( + file_path TEXT PRIMARY KEY, + file_size INTEGER, + mtime REAL, + last_verified REAL + ) + """ + ) + cursor.execute( + """ + CREATE INDEX IF NOT EXISTS idx_file_path ON file_cache(file_path) + """ + ) + conn.commit() + finally: + conn.close() + + def file_exists(self, file_path: str, verify_disk: bool = False) -> bool: + """Check if file exists in cache + + Args: + file_path: Path to check + verify_disk: If True, verify file exists on disk even if in cache. + If False (default), trust cache completely (faster). + Use True only for /syncall command. + """ + with self.lock: + conn = sqlite3.connect(self.cache_db_path, timeout=30.0) + try: + cursor = conn.cursor() + cursor.execute( + "SELECT file_path FROM file_cache WHERE file_path = ?", + (file_path,), + ) + result = cursor.fetchone() + if result: + # File is in cache + if verify_disk: + # Only verify on disk if explicitly requested (e.g., /syncall) + if os.path.isfile(file_path): + # Update last_verified timestamp + cursor.execute( + "UPDATE file_cache SET last_verified = ? WHERE file_path = ?", + (time.time(), file_path), + ) + conn.commit() + return True + else: + # File was deleted, remove from cache + cursor.execute("DELETE FROM file_cache WHERE file_path = ?", (file_path,)) + conn.commit() + return False + else: + # Trust cache completely (much faster) - cache is rebuilt every 24h + return True + return False + finally: + conn.close() + + def add_file(self, file_path: str, file_size: Optional[int] = None) -> None: + """Add file to cache""" + try: + if file_size is None and os.path.isfile(file_path): + file_size = os.path.getsize(file_path) + mtime = os.path.getmtime(file_path) if os.path.isfile(file_path) else time.time() + + with self.lock: + conn = sqlite3.connect(self.cache_db_path, timeout=30.0) + try: + cursor = conn.cursor() + cursor.execute( + """ + INSERT OR REPLACE INTO file_cache (file_path, file_size, mtime, last_verified) + VALUES (?, ?, ?, ?) + """, + (file_path, file_size, mtime, time.time()), + ) + conn.commit() + finally: + conn.close() + except Exception as e: + self.logger.debug(f"Error adding file to cache: {e}") + + def remove_file(self, file_path: str) -> None: + """Remove file from cache""" + with self.lock: + conn = sqlite3.connect(self.cache_db_path, timeout=30.0) + try: + cursor = conn.cursor() + cursor.execute("DELETE FROM file_cache WHERE file_path = ?", (file_path,)) + conn.commit() + finally: + conn.close() + + def rebuild_cache(self, directory: str, user_config: UserConfig) -> None: + """Rebuild cache by scanning disk directory""" + self.logger.info("Rebuilding file cache by scanning disk...") + start_time = time.time() + file_count = 0 + + directory_path = Path(directory) + if not directory_path.exists(): + self.logger.warning(f"Directory {directory} does not exist, skipping cache rebuild") + return + + # Clear existing cache for this directory + with self.lock: + conn = sqlite3.connect(self.cache_db_path, timeout=30.0) + try: + cursor = conn.cursor() + # Only remove files from this directory + cursor.execute( + "DELETE FROM file_cache WHERE file_path LIKE ?", + (f"{directory}%",), + ) + conn.commit() + finally: + conn.close() + + # Scan directory and add files to cache + try: + for root, dirs, files in os.walk(directory): + for file in files: + file_path = os.path.join(root, file) + try: + if os.path.isfile(file_path): + self.add_file(file_path) + file_count += 1 + if file_count % 1000 == 0: + self.logger.debug(f"Scanned {file_count} files...") + except Exception as e: + self.logger.debug(f"Error scanning file {file_path}: {e}") + + elapsed = time.time() - start_time + self.logger.info( + f"Cache rebuild completed: {file_count} files scanned in {elapsed:.2f} seconds" + ) + except Exception as e: + self.logger.error(f"Error rebuilding cache: {e}") + + def should_rebuild_cache(self, rebuild_interval_hours: int = 24) -> bool: + """Check if cache should be rebuilt based on last rebuild time""" + cache_dir = os.path.dirname(self.cache_db_path) + rebuild_flag_file = os.path.join(cache_dir, ".cache_last_rebuild") + + if not os.path.exists(rebuild_flag_file): + return True + + try: + last_rebuild = os.path.getmtime(rebuild_flag_file) + hours_since_rebuild = (time.time() - last_rebuild) / 3600 + return hours_since_rebuild >= rebuild_interval_hours + except Exception: + return True + + def mark_cache_rebuilt(self) -> None: + """Mark cache as recently rebuilt""" + cache_dir = os.path.dirname(self.cache_db_path) + rebuild_flag_file = os.path.join(cache_dir, ".cache_last_rebuild") + try: + Path(rebuild_flag_file).touch() + except Exception as e: + self.logger.debug(f"Error marking cache as rebuilt: {e}") + + def get_cache_stats(self) -> dict: + """Get cache statistics""" + with self.lock: + conn = sqlite3.connect(self.cache_db_path, timeout=30.0) + try: + cursor = conn.cursor() + cursor.execute("SELECT COUNT(*) FROM file_cache") + total_files = cursor.fetchone()[0] + + cursor.execute( + "SELECT COUNT(*) FROM file_cache WHERE last_verified > ?", + (time.time() - 86400,), # Last 24 hours + ) + recently_verified = cursor.fetchone()[0] + + return { + "total_files": total_files, + "recently_verified": recently_verified, + } + finally: + conn.close() + + def get_last_sync_date(self) -> Optional[float]: + """Get the last sync date (timestamp) from cache metadata""" + cache_dir = os.path.dirname(self.cache_db_path) + sync_date_file = os.path.join(cache_dir, ".last_sync_date") + + if not os.path.exists(sync_date_file): + return None + + try: + with open(sync_date_file, 'r') as f: + timestamp = float(f.read().strip()) + return timestamp + except (ValueError, IOError): + return None + + def set_last_sync_date(self, timestamp: float) -> None: + """Set the last sync date (timestamp) in cache metadata""" + cache_dir = os.path.dirname(self.cache_db_path) + sync_date_file = os.path.join(cache_dir, ".last_sync_date") + + try: + with open(sync_date_file, 'w') as f: + f.write(str(timestamp)) + except IOError as e: + self.logger.debug(f"Error saving last sync date: {e}") + + def get_all_cached_paths(self) -> set[str]: + """Get all file paths from cache as a set for fast lookup + + Returns: + Set of all file paths in cache + """ + with self.lock: + conn = sqlite3.connect(self.cache_db_path, timeout=30.0) + try: + cursor = conn.cursor() + cursor.execute("SELECT file_path FROM file_cache") + paths = {row[0] for row in cursor.fetchall()} + return paths + finally: + conn.close() diff --git a/src/icloudpd/mfa_provider.py b/src/icloudpd/mfa_provider.py index 500654f02..cd3da2ed3 100644 --- a/src/icloudpd/mfa_provider.py +++ b/src/icloudpd/mfa_provider.py @@ -4,6 +4,7 @@ class MFAProvider(Enum): CONSOLE = "console" WEBUI = "webui" + TELEGRAM = "telegram" def __str__(self) -> str: return self.name diff --git a/src/icloudpd/progress.py b/src/icloudpd/progress.py index dfafd397b..18d993c0c 100644 --- a/src/icloudpd/progress.py +++ b/src/icloudpd/progress.py @@ -11,6 +11,13 @@ def __init__(self) -> None: self.resume = False self.cancel = False self._waiting = 0 + self.total_photos_in_icloud = 0 # Total photos in iCloud + self.photos_to_download = 0 # Photos not in cache that need download + self.last_progress_message_time = 0.0 # Timestamp of last progress message + self.watch_interval = 0 # Watch interval in seconds (0 = no watch mode) + self.last_sync_time = 0.0 # Timestamp of last sync completion + self.photos_checked = 0 # Photos checked during filtering (for progress tracking) + self.processing_start_time = 0.0 # Timestamp when processing started (for rate calculation) @property def waiting(self) -> int: @@ -46,10 +53,18 @@ def photos_counter(self, photos_counter: int) -> None: self.photos_percent = 0 def reset(self) -> None: + # Save resume flag before resetting (it should persist across resets) + resume_flag = self.resume self._photos_count = 0 self._photos_counter = 0 self.photos_percent = 0 self._waiting = 0 self.waiting_readable = "" - self.resume = False self.cancel = False + self.total_photos_in_icloud = 0 + self.photos_to_download = 0 + self.last_progress_message_time = 0.0 + self.photos_checked = 0 + # Don't reset processing_start_time - it should persist until processing completes + # Restore resume flag and don't reset watch_interval and last_sync_time - they persist across resets + self.resume = resume_flag diff --git a/src/icloudpd/server/__init__.py b/src/icloudpd/server/__init__.py index 650ee1e5e..c713e5ebb 100644 --- a/src/icloudpd/server/__init__.py +++ b/src/icloudpd/server/__init__.py @@ -8,7 +8,7 @@ from icloudpd.status import Status, StatusExchange -def serve_app(logger: Logger, _status_exchange: StatusExchange) -> None: +def serve_app(logger: Logger, _status_exchange: StatusExchange, telegram_bot=None, port: int = 8080) -> None: app = Flask(__name__) app.logger = logger # for running in pyinstaller @@ -88,5 +88,19 @@ def cancel() -> Response | str: _status_exchange.get_progress().cancel = True return make_response("Ok", 200) - logger.debug("Starting web server...") - return waitress.serve(app) + # Telegram webhook endpoint (for push notifications) + @app.route("/telegram/webhook", methods=["POST"]) + def telegram_webhook() -> Response: + if telegram_bot: + try: + update = request.get_json() + if update: + telegram_bot.process_update(update) + return make_response("Ok", 200) + except Exception as e: + logger.error(f"Error processing Telegram webhook: {e}") + return make_response("Error", 500) + return make_response("Telegram bot not configured", 404) + + logger.debug(f"Starting web server on port {port}...") + return waitress.serve(app, host="0.0.0.0", port=port) diff --git a/src/icloudpd/status.py b/src/icloudpd/status.py index 860ef848f..b0d620817 100644 --- a/src/icloudpd/status.py +++ b/src/icloudpd/status.py @@ -29,6 +29,9 @@ def __init__(self) -> None: self._user_configs: Sequence[UserConfig] = [] self._current_user: str | None = None self._progress = Progress() + self._force_full_sync = False + self._manual_sync = False # True if sync was triggered manually via Telegram + self._telegram_bot = None # Reference to Telegram bot for auth requests def get_status(self) -> Status: with self.lock: @@ -121,3 +124,29 @@ def get_current_user(self) -> str | None: def clear_current_user(self) -> None: with self.lock: self._current_user = None + + def set_force_full_sync(self, force: bool) -> None: + with self.lock: + self._force_full_sync = force + + def get_force_full_sync(self) -> bool: + with self.lock: + return self._force_full_sync + + def set_manual_sync(self, manual: bool) -> None: + with self.lock: + self._manual_sync = manual + + def get_manual_sync(self) -> bool: + with self.lock: + return self._manual_sync + + def set_telegram_bot(self, telegram_bot) -> None: + """Set Telegram bot reference for authentication requests""" + with self.lock: + self._telegram_bot = telegram_bot + + def get_telegram_bot(self): + """Get Telegram bot reference""" + with self.lock: + return self._telegram_bot diff --git a/src/icloudpd/telegram_bot.py b/src/icloudpd/telegram_bot.py new file mode 100644 index 000000000..35616dbd7 --- /dev/null +++ b/src/icloudpd/telegram_bot.py @@ -0,0 +1,375 @@ +"""Telegram bot integration for icloudpd to handle sync commands""" + +import datetime +import logging +import os +import threading +import time +from typing import Optional + +import requests + +from icloudpd.status import StatusExchange, Status + + +class TelegramBot: + """Telegram bot to handle commands and trigger sync""" + + def __init__( + self, + logger: logging.Logger, + token: str, + chat_id: str, + status_exchange: StatusExchange, + polling_interval: int = 5, + webhook_url: str | None = None, + ) -> None: + self.logger = logger + self.token = token + self.chat_id = chat_id + self.status_exchange = status_exchange + self.polling_interval = polling_interval + self.webhook_url = webhook_url + self.last_update_id = 0 + self.base_url = f"https://api.telegram.org/bot{self.token}" + self.thread = threading.Thread(target=self._poll_updates, daemon=True) + self._waiting_for_auth_code = False # Track if we're waiting for 6-digit code + self._auth_requested = False # Track if /auth command was sent + + def send_message(self, text: str) -> bool: + """Send a message to the configured chat""" + try: + url = f"{self.base_url}/sendMessage" + data = {"chat_id": self.chat_id, "text": text} + response = requests.post(url, data=data, timeout=10) + response.raise_for_status() + return True + except Exception as e: + self.logger.error(f"Error sending Telegram message: {e}") + return False + + def process_message(self, message: dict) -> None: + """Process incoming Telegram message""" + try: + text = message.get("text", "").strip() + if not text: + return + + # Process /sync command + if text == "/sync": + self.logger.info("Telegram /sync command received, triggering sync (with cache)...") + progress = self.status_exchange.get_progress() + progress.resume = True + progress.cancel = False + self.status_exchange.set_force_full_sync(False) + self.status_exchange.set_manual_sync(True) # Mark as manual sync + # Don't send message here - will be sent when sync starts with photo counts + # Process /syncall command (full sync without cache) + elif text == "/syncall": + self.logger.info("Telegram /syncall command received, triggering full sync (no cache)...") + progress = self.status_exchange.get_progress() + progress.resume = True + progress.cancel = False + self.status_exchange.set_force_full_sync(True) + self.status_exchange.set_manual_sync(True) # Mark as manual sync + # Don't send message here - will be sent when sync starts with photo counts + # Process /stop command + elif text == "/stop": + self.logger.info("Telegram /stop command received, stopping current sync...") + progress = self.status_exchange.get_progress() + progress.cancel = True + self.send_message("⏹️ Current synchronization stopped") + # Process /status command + elif text == "/status": + self.logger.info("Telegram /status command received") + status_message = self._get_status_message() + self.logger.debug(f"Status message: {status_message}") + if self.send_message(status_message): + self.logger.debug("Status message sent successfully") + else: + self.logger.error("Failed to send status message") + # Process /auth command - initiate authentication + elif text == "/auth": + self.logger.info("Telegram /auth command received, initiating authentication...") + self._initiate_auth() + # Legacy support: "Staticduo" command (case insensitive) + elif text.lower() == "staticduo": + self.logger.info("Telegram Staticduo command received, triggering sync...") + self.status_exchange.get_progress().resume = True + self.send_message("✅ Synchronization started") + # Check if message is a 6-digit code (for MFA) + elif self._is_six_digit_code(text) and self._waiting_for_auth_code: + self.logger.info(f"Telegram 6-digit code received: {text}") + self._handle_auth_code(text) + # If waiting for auth code but received something else, remind user + elif self._waiting_for_auth_code: + self.send_message("❌ Please send a 6-digit code for authentication.") + except Exception as e: + self.logger.error(f"Error processing Telegram message: {e}") + + def _get_status_message(self) -> str: + """Get current status message""" + import time + + progress = self.status_exchange.get_progress() + current_user = self.status_exchange.get_current_user() + + # Determine if downloading or idle + # If there's a current user, we're processing (either downloading or filtering) + is_processing = current_user is not None + # Check if processing has actually started (processing_start_time > 0 means loop has started) + has_started_processing = progress.processing_start_time > 0 + # Check if we're actually downloading (photos_count > 0 and counter < count) + # If photos_checked > 0, we're processing photos (either filtering or downloading) + # We're filtering if photos_checked > 0 (we're processing photos) + # is_downloading should only be true if we're actually downloading NEW photos + # For now, we'll show filtering progress whenever photos_checked > 0 + # and only show downloading if photos_counter is significantly less than photos_checked + # (meaning we're downloading new photos, not just processing existing ones) + is_filtering = (has_started_processing and + progress.photos_checked > 0 and + progress.total_photos_in_icloud > 0) + # Only show downloading if we have photos_count set and counter is progressing + # But prioritize filtering if photos_checked is much larger than photos_counter + is_downloading = (has_started_processing and + progress.photos_count > 0 and + progress.photos_counter > 0 and + progress.photos_counter < progress.photos_count and + progress.photos_counter >= progress.photos_checked * 0.9) # Only if counter is close to checked (downloading new photos) + is_waiting = progress.waiting > 0 + + if is_processing or is_filtering or is_downloading: + # Show filtering progress if we're checking photos + # Always show filtering if photos_checked > 0 (we're processing photos) + # Only show downloading if we're actually downloading new photos (not just processing existing ones) + if is_filtering: + status_text = "🔄 Filtering photos" + user_text = f"\n👤 User: {current_user}" if current_user else "" + if progress.photos_checked > 0: + percent = round(100 * progress.photos_checked / progress.total_photos_in_icloud) + # Calculate rate if processing has started + rate_text = "" + if progress.processing_start_time > 0: + elapsed = time.time() - progress.processing_start_time + rate = progress.photos_checked / elapsed if elapsed > 0 else 0.0 + rate_text = f" (Rate: {rate:.2f} items/s)" + progress_text = f"\n📊 Checked: {progress.photos_checked}/{progress.total_photos_in_icloud} ({percent}%){rate_text}" + else: + # Processing just started, show initial message + progress_text = f"\n📊 Starting: 0/{progress.total_photos_in_icloud} (0%)" + if progress.photos_to_download > 0: + progress_text += f"\n📥 {progress.photos_to_download} photos to download" + return f"{status_text}{user_text}{progress_text}" + elif is_downloading: + # Actually downloading photos + status_text = "🔄 Downloading" + user_text = f"\n👤 User: {current_user}" if current_user else "" + # Calculate rate if processing has started + rate_text = "" + if progress.processing_start_time > 0: + elapsed = time.time() - progress.processing_start_time + rate = progress.photos_counter / elapsed if elapsed > 0 else 0.0 + rate_text = f" (Rate: {rate:.2f} items/s)" + progress_text = ( + f"\n📊 Progress: {progress.photos_counter}/{progress.photos_count} photos " + f"({progress.photos_percent}%){rate_text}" + ) + if progress.photos_last_message: + progress_text += f"\n📝 {progress.photos_last_message}" + return f"{status_text}{user_text}{progress_text}" + else: + # Processing but not filtering or downloading yet (authenticating, etc.) + # If processing_start_time is set but photos_checked is 0, loop just started + status_text = "🔄 Processing" + user_text = f"\n👤 User: {current_user}" if current_user else "" + if has_started_processing and progress.total_photos_in_icloud > 0: + # Loop has started, show initial progress + progress_text = f"\n📊 Starting processing of {progress.total_photos_in_icloud} photos..." + return f"{status_text}{user_text}{progress_text}" + else: + return f"{status_text}{user_text}\n⏳ Preparing synchronization..." + elif is_waiting: + # Idle (waiting) status - in wait loop + status_text = "⏸️ Idle (Waiting)" + user_text = f"\n👤 User: {current_user}" if current_user else "" + waiting_text = f"\n⏱️ Next synchronization in: {progress.waiting_readable}" + return f"{status_text}{user_text}{waiting_text}" + else: + # Completely idle - calculate time until next sync + status_text = "✅ Idle" + user_text = f"\n👤 User: {current_user}" if current_user else "" + + # Calculate time until next sync if watch mode is active + if progress.watch_interval > 0: + current_time = time.time() + if progress.last_sync_time > 0: + # Calculate based on last sync time + elapsed = current_time - progress.last_sync_time + remaining = progress.watch_interval - elapsed + else: + # First sync hasn't happened yet, use full interval + remaining = progress.watch_interval + + if remaining > 0: + waiting_readable = str(datetime.timedelta(seconds=int(remaining))) + waiting_text = f"\n⏱️ Next synchronization in: {waiting_readable}" + return f"{status_text}{user_text}{waiting_text}" + else: + # Should have started already, but show anyway + waiting_text = "\n⏱️ Next synchronization: Imminent" + return f"{status_text}{user_text}{waiting_text}" + else: + # No watch mode + return f"{status_text}{user_text}\n💤 No activity at this time" + + def set_webhook(self, webhook_url: str) -> bool: + """Set webhook URL for Telegram bot (push notifications instead of polling)""" + try: + url = f"{self.base_url}/setWebhook" + data = {"url": webhook_url} + response = requests.post(url, data=data, timeout=10) + response.raise_for_status() + result = response.json() + if result.get("ok"): + self.logger.info(f"Webhook configured successfully: {webhook_url}") + return True + else: + self.logger.error(f"Failed to set webhook: {result.get('description')}") + return False + except Exception as e: + self.logger.error(f"Error setting webhook: {e}") + return False + + def delete_webhook(self) -> bool: + """Delete webhook and fall back to polling""" + try: + url = f"{self.base_url}/deleteWebhook" + response = requests.post(url, timeout=10) + response.raise_for_status() + result = response.json() + if result.get("ok"): + self.logger.info("Webhook deleted, falling back to polling") + return True + return False + except Exception as e: + self.logger.error(f"Error deleting webhook: {e}") + return False + + def start_polling(self) -> None: + """Start the Telegram bot (webhook if available, otherwise polling)""" + if self.webhook_url: + if self.set_webhook(self.webhook_url): + self.logger.info("Telegram bot using webhooks (push notifications)") + return + else: + self.logger.warning("Failed to set webhook, falling back to polling") + + self.logger.info(f"Starting Telegram bot polling (interval: {self.polling_interval}s)...") + self.thread.start() + + def _poll_updates(self) -> None: + """Main polling loop for Telegram bot (uses long polling)""" + while True: + try: + updates = self._get_updates() + for update in updates: + self.process_update(update) + # No sleep needed - long polling already waits for timeout_seconds + # If we got updates, process immediately; if not, wait for next poll + except Exception as e: + self.logger.error(f"Error polling Telegram updates: {e}") + # On error, wait a bit before retrying + time.sleep(1) + + def _get_updates(self) -> list[dict]: + """Get updates from Telegram using long polling""" + url = f"{self.base_url}/getUpdates" + # Use long polling with shorter timeout (1-2 seconds) for faster response + # Telegram allows timeout up to 30 seconds, but shorter is better for responsiveness + # We use min(2, polling_interval) to balance between responsiveness and API calls + timeout_seconds = min(2, max(1, self.polling_interval // 3)) + params = {"timeout": timeout_seconds, "offset": self.last_update_id + 1} + response = requests.get(url, params=params, timeout=timeout_seconds + 5) + response.raise_for_status() + updates = response.json().get("result", []) + if updates: + self.last_update_id = updates[-1]["update_id"] + return updates + + def process_update(self, update: dict) -> None: + """Process a Telegram update""" + message = update.get("message") + if message and str(message.get("chat", {}).get("id")) == self.chat_id: + self.process_message(message) + else: + self.logger.debug(f"Ignoring message from unknown chat or without message: {update}") + + def send_progress_update(self) -> None: + """Send progress update message (called periodically during download)""" + progress = self.status_exchange.get_progress() + if progress.photos_to_download > 0: + message = f"📥 {progress.photos_counter}/{progress.photos_to_download}" + self.send_message(message) + + def send_sync_start_message(self, photos_to_download: int, total_photos: int) -> None: + """Send sync start message with photo counts""" + message = f"Downloading: {photos_to_download} of {total_photos} total" + self.send_message(message) + + def send_sync_complete_message(self, photos_downloaded: int, next_sync_seconds: int) -> None: + """Send sync complete message""" + next_sync_readable = str(datetime.timedelta(seconds=next_sync_seconds)) + message = f"Downloaded {photos_downloaded} photos. Next sync in {next_sync_readable}." + self.send_message(message) + + def _is_six_digit_code(self, text: str) -> bool: + """Check if text is a 6-digit code""" + return len(text) == 6 and text.isdigit() + + def _initiate_auth(self) -> None: + """Initiate authentication process""" + # Force authentication by clearing cookies or triggering re-auth + # The authentication will be handled in the main loop when it detects requires_2fa + self.send_message("🔐 Starting authentication process...\n\nAuthentication will be attempted on the next synchronization. If a 6-digit code is required, I will ask for it here.") + # Set resume flag to trigger sync, which will attempt authentication + progress = self.status_exchange.get_progress() + progress.resume = True + progress.cancel = False + # Set flag to force authentication (will be handled in base.py) + self._auth_requested = True + self._waiting_for_auth_code = False # Reset flag, will be set when MFA is required + + def _handle_auth_code(self, code: str) -> None: + """Handle 6-digit authentication code received via Telegram""" + if self.status_exchange.get_status() == Status.NEED_MFA: + if self.status_exchange.set_payload(code): + self._waiting_for_auth_code = False + self.send_message("✅ Code received, verifying...") + self.logger.info(f"Authentication code provided via Telegram: {code}") + else: + self.send_message("❌ Error: Could not process the code. Please try again.") + self.logger.error("Failed to set authentication code payload") + else: + self.send_message("❌ Error: No authentication code is expected at this time.") + self.logger.warning(f"Received auth code but status is {self.status_exchange.get_status()}, not NEED_MFA") + self._waiting_for_auth_code = False + + def request_auth_code(self, username: str) -> None: + """Request authentication code via Telegram""" + self._waiting_for_auth_code = True + message = ( + f"🔐 Authentication required for {username}\n\n" + f"Please send the 6-digit code that appears on your iPhone/iPad/Mac." + ) + self.send_message(message) + self.logger.info(f"Requested authentication code via Telegram for {username}") + + def notify_auth_required(self, username: str) -> None: + """Notify that authentication is required (cookie expired or about to expire)""" + message = ( + f"⚠️ Authentication required for {username}\n\n" + f"The authentication cookie has expired or is about to expire.\n" + f"Use the /auth command to renew authentication." + ) + self.send_message(message) + self.logger.info(f"Sent authentication required notification via Telegram for {username}")