diff --git a/README.md b/README.md index e4ca357..c903caa 100644 --- a/README.md +++ b/README.md @@ -8,13 +8,27 @@ --- -> [!WARNING] -> Danzo has seen a significant refactor that changed how commands were called in previous versions. This was done to support an easier CLI interface and make it easier to add additional downloaders in the future. - ## Quickstart This section gives a quick peek at the capabilities and the extremely simple command structure. For detailed descriptions, see [Usage](#usage). +The primary downloaders and their supported aliases are as follows: + +| Command | Aliases (Shorthands) | Description | +| --- | --- | --- | +| `http` | - | Multi-chunked or linear downloads for general HTTP(S) sources | +| `batch` | - | Multi-threaded, multi-downloader operation given a yaml config | +| `live-stream` | `hls`, `m3u8`, `livestream`, `stream` | Download a live stream format video (playlist.m3u8 files) with multi-threading | +| `youtube` | `yt` | Download YouTube videos using `yt-dlp` | +| `youtube-music` | `ytm`, `yt-music` | Download audio from YouTube with iTunes/Deezer metadata using `yt-dlp` and `ffmpeg` | +| `git-clone` | `gitclone`, `gitc`, `git`, `clone` | Clone a git repository with SSH/token authentication | +| `github-release` | `ghrelease`, `ghr` | Download a platform-correct release asset for a GitHub repo | +| `google-drive` | `gdrive`, `gd`, `drive` | Download file/folder from Google drive with API key or OAuth flow authentication | +| `s3` | - | Multi-threaded download for object, directory, or full AWS S3 bucket | +| `clean` | - | Clear local cache for interrupted/incomplete downloads | + +Following are examples to get started with various flags: + - HTTP(S) downloads ```bash danzo http https://example.com/internet-file.zip -o local.zip # (in lieu of `wget`) @@ -30,18 +44,18 @@ This section gives a quick peek at the capabilities and the extremely simple com danzo yt "https://www.youtube.com/watch?v=dQw4w9WgXcQ" # (default is <=1080p, <=60fps quality) danzo yt "https://www.youtube.com/watch?v=dQw4w9WgXcQ" --format 1080p # (download in 1080p) # allows customization with `best60`, `decent`, `1080p60`, and more (see Usage) - danzo ytmusic "https://www.youtube.com/watch?v=dQw4w9WgXcQ" # (download standard `.m4a` audio) - danzo ytmusic "https://youtu.be/JJpFTUP6fIo" --apple 1800533191 # (add music metadata from itunes) - danzo ytmusic "https://youtu.be/JJpFTUP6fIo" --deezer 3271607031 # (add music metadata from deezer) + danzo ytm "https://www.youtube.com/watch?v=dQw4w9WgXcQ" # (download standard `.m4a` audio) + danzo ytm "https://youtu.be/JJpFTUP6fIo" --apple 1800533191 # (add music metadata from itunes) + danzo ytm "https://youtu.be/JJpFTUP6fIo" --deezer 3271607031 # (add music metadata from deezer) ``` - Download file from Google Drive ```bash - danzo gdrive "https://drive.google.com/file/d/abc123/view" --api-key your_key # (static Key only for publicly shared files) - danzo gdrive "https://drive.google.com/file/d/abc123/view" --creds service-acc-key.json # (OAuth device code flow for private files) + danzo gd "https://drive.google.com/file/d/abc123/view" --api-key your_key # (static Key only for publicly shared files) + danzo gd "https://drive.google.com/file/d/abc123/view" --creds service-acc-key.json # (OAuth device code flow for private files) ``` - Download streamed output from an m3u8-manifest ```bash - danzo m3u8 "https://example.com/manifest.m3u8" -o video.mp4 + danzo hls "https://example.com/manifest.m3u8" -o video.mp4 ``` - Download an S3 object or folder ```bash @@ -50,15 +64,15 @@ This section gives a quick peek at the capabilities and the extremely simple com ``` - Download GitHub release asset ```bash - danzo ghrelease "username/repo" # (auto-selects release according to OS and arch) - danzo ghrelease "username/repo" --manual # (choose interactively) + danzo ghr "username/repo" # (auto-selects release according to OS and arch) + danzo ghr "username/repo" --manual # (choose asset interactively) ``` - Clone a git repository ```bash - danzo gitclone "gitlab.com/username/repo" # (supports `github.com/`, `bitbucket.org/`, and `git.com/`) - danzo gitclone "github.com/username/repo" --depth 1 # (clone with --depth=1) - danzo gitclone "github.com/tanq16/private" --token $(cat /secrets/ghtoken) # (use a PAT; auto-manages for different providers) - danzo gitclone "github.com/tanq16/private" --ssh "/secrets/gh-ssh.key" # (use an SSH key to authenticate) + danzo git "gitlab.com/username/repo" # (supports `github.com/`, `bitbucket.org/`, and `git.com/`) + danzo git "github.com/username/repo" --depth 1 # (clone with --depth=1) + danzo git "github.com/tanq16/private" --token $(cat /secrets/ghtoken) # (use a PAT; auto-manages for different providers) + danzo git "github.com/tanq16/private" --ssh "/secrets/gh-ssh.key" # (use an SSH key to authenticate) ``` ## Installation @@ -107,7 +121,7 @@ Danzo supports these global options: --header, -H Custom headers (repeatable) --workers, -w Number of parallel workers (default: 1) --connections, -c Connections per download (default: 8) ---log Enable debug logging (WORK IN PROGRESS) +--debug Enable debug logging at info or debug level (default: disabled, i.e., uses TUI) ``` Using a download directly won't always yield the best result, so to optimize according to file types, use multiple threads (read through the next couple sections to learn more). @@ -124,14 +138,16 @@ Follow these links to quickly jump to the relevant provider: ### HTTP(S) Downloads +
+Unfold to read + The output filename will be inferred from the URL and Danzo will use 8 connection threads and 1 worker by default. You can also specify an output filename manually like: ```bash danzo http https://example.com/largefile.zip -o ./path/to/file.zip ``` -> [!NOTE] -> The value for `-c` can be arbitrary. Danzo creates chunks equal to number of connections requested. Once all chunks are downloaded, they are combined into a single file. If the decided number of chunks are too small, Danzo falls back to a single threaded download for that file. +> ✎ The value for `-c` can be arbitrary. Danzo creates chunks equal to number of connections requested. Once all chunks are downloaded, they are combined into a single file. If the decided number of chunks are too small, Danzo falls back to a single threaded download for that file. You can customize the number of connections to use like so: @@ -139,8 +155,7 @@ You can customize the number of connections to use like so: danzo "https://example.com/largefile.zip" -c 16 ``` -> [!WARNING] -> You should be careful of the disk IO as well. Multi-connection download takes disk IO, which can add to overall time before the file is ready. +> ⚠ You should be careful of the disk IO as well. Multi-connection download takes disk IO, which can add to overall time before the file is ready. > > For example, a 1 GB file takes 54 seconds when using 50 connections vs. 62 seconds when using 64 connections. This is because combining 64 files takes longer than combining 50 files. > @@ -182,8 +197,7 @@ Single-connection downloads store a `OUTPUTPATH.part` file in the current workin These partial downloads on disk are useful when a download event is interrupted or failed. In that case, the temporary files are used to resume the download. -> [!WARNING] -> A resume operation is triggered automatically when the same output path is encountered. However, the feature will only work correctly if the number of connections are exactly the same. Otherwise, the resulting assembled file may contain faulty bytes. +> ⚠ A resume operation is triggered automatically when the same output path is encountered. However, the feature will only work correctly if the number of connections are exactly the same. Otherwise, the resulting assembled file may contain faulty bytes. To clear the temporary (partially downloaded) files, use the command with the `clean` flag: @@ -193,11 +207,15 @@ danzo clean "./path/for/download.zip" danzo clean ``` -> [!TIP] -> Failed chunks are automatically retried up to 5 times before failing the entire file. Additionally, Danzo automatically runs a clean for a download event once it is successful. +> ✦ Failed chunks are automatically retried up to 5 times before failing the entire file. Additionally, Danzo automatically runs a clean for a download event once it is successful. + +
### Google Drive Downloads +
+Unfold to read + Downloading a file from a Drive URL requires authentication, which Danzo supports in 2 ways: - `API Key`: @@ -216,8 +234,7 @@ Downloading a file from a Drive URL requires authentication, which Danzo support - Danzo will exchange this for an authentication token and save it to `.danzo-token.json`. - If you re-attempt the use of these credentials, Danzo will reuse the token from current directory if it exists, refresh it if possible, and fallback to reauthentication. -> [!TIP] -> The API Key method only works on files that are either publicly shared or shared with your user. It cannot be used to download private files that you own. So for your own files, use the OAuth device code method. +> ✦ The API Key method only works on files that are either publicly shared or shared with your user. It cannot be used to download private files that you own. So for your own files, use the OAuth device code method. Danzo can be used in this manner to download Google Drive files: @@ -231,14 +248,17 @@ OR danzo gdrive "https://drive.google.com/file/d/1w.....HK/view?usp=drive_link" --creds ~/secrets/gdrive-oauth.key ``` -> [!WARNING] -> Danzo does not perform multi-connection download for Google Drive files; instead it uses the simple download method. For Google Drive specifically, this does not present a significant loss in bandwidth. This is done because Google can throttle multiple connections after a while. +> ⚠︎ Danzo does not perform multi-connection download for Google Drive files; instead it uses the simple download method. For Google Drive specifically, this does not present a significant loss in bandwidth. This is done because Google can throttle multiple connections after a while. + +> ✎ Users who have never logged into GCP may be required to create a new GCP Project. This is normal and doesn't cost anything. -> [!NOTE] -> Users who have never logged into GCP may be required to create a new GCP Project. This is normal and doesn't cost anything. +
### YouTube Downloads +
+Unfold to read + Danzo supports downloading videos and audio from YouTube by using [yt-dlp](https://github.com/yt-dlp/yt-dlp) as a dependency. Some files and merge operations may also require `ffmpeg` and `ffprobe`. If not present, Danzo will make a temporary download of the appropriate `yt-dlp` binary. However, it is recommended to have `yt-dlp`, `ffmpeg`, and `ffprobe` pre-installed. To download a YouTube video: @@ -248,8 +268,7 @@ To download a YouTube video: danzo yt "https://www.youtube.com/watch?v=dQw4w9WgXcQ" ``` -> [!NOTE] -> In an effort to create a successful and simple integration, Danzo lets `yt-dlp` dictate the file extension for a given output. As such, the `-o` flag will not have an effect on the extension. Audio downloads will always have a `.m4a` download, while a video may have `.mp4`, or `.webm`. +> ✎ In an effort to create a successful and simple integration, Danzo lets `yt-dlp` dictate the file extension for a given output. As such, the `-o` flag will not have an effect on the extension. Audio downloads will always have a `.m4a` download, while a video may have `.mp4`, or `.webm`. A download type can be appended to the URL to control Danzo's behavior. These defaults were chosen based on heuristics and observed popularity. @@ -267,8 +286,7 @@ danzo yt "https://www.youtube.com/watch?v=dQw4w9WgXcQ" --format "decent" danzo yt "https://www.youtube.com/watch?v=dQw4w9WgXcQ" --format "audio" ``` -> [!NOTE] -> YouTube downloads require `yt-dlp` to be installed on your system. If it's not found, Danzo will automatically download and use a compatible version. Additionally, since the STDOUT and STDERR are directly streamed from `yt-dlp` to `danzo`, YouTube videos are not tracked for progress the way HTTP downloads are. When downloading a single YouTube URL, the output from `yt-dlp` will be streamed to the user's STDOUT. But if the URL is part of a batch file, then the output is hidden and the progress appears stalled until finished. +> ✎ YouTube downloads require `yt-dlp` to be installed on your system. If it's not found, Danzo will automatically download and use a compatible version. Additionally, since the STDOUT and STDERR are directly streamed from `yt-dlp` to `danzo`, YouTube videos are not tracked for progress the way HTTP downloads are. When downloading a single YouTube URL, the output from `yt-dlp` will be streamed to the user's STDOUT. But if the URL is part of a batch file, then the output is hidden and the progress appears stalled until finished. Danzo also supports downloading music from YouTube and automatically add metadata from the Deezer or the iTunes API, when the appropriate ID is provided. Example: @@ -277,14 +295,18 @@ danzo ytmusic "https://youtu.be/JJpFTUP6fIo" --apple "1800533191" danzo ytmusic "https://youtu.be/JJpFTUP6fIo" --deezer "3271607031" ``` +
+ ### M3U8 Stream Downloads +
+Unfold to read + Danzo supports downloading streamed content from M3U8 manifests. This is commonly used for video streaming services, live broadcasts, and VOD content. Danzo downloads the M3U8 manifest, parses the playlist (supports both master and media playlists), downloads all segments, and merges them into a single file. -> [!NOTE] -> Danzo requires `ffmpeg` to be installed for merging the segments. +> ✎ Danzo requires `ffmpeg` to be installed for merging the segments. ```bash danzo m3u8 "https://example.com/path/to/playlist.m3u8" -o video.mp4 @@ -293,8 +315,13 @@ danzo m3u8 "https://example.com/path/to/playlist.m3u8" -o video.mp4 danzo m3u8 "https://example.com/video/master.m3u8" ``` +
+ ### AWS S3 Downloads +
+Unfold to read + There are 2 ways of downloading objects from S3: - Public Buckets: These are often directly exposed as HTTP(S) sites or pre-signed URLs. Either of the two can be sufficiently handled by the HTTP(S) downloaders. @@ -315,14 +342,17 @@ danzo s3 "mybucket/some/directory/" AWS session profiles are used to allow for flexibility and ease of access. As a result, specifying the flag (`--profile`) allows using a profile of the user's choice. Additionally, when not set, Danzo uses the `default` profile. -> [!WARNING] -> For successful authentication, Danzo needs to use a profile that is configured for the same region as the S3 bucket. +> ⚠︎ For successful authentication, Danzo needs to use a profile that is configured for the same region as the S3 bucket. -> [!NOTE] -> For S3 downloads, the `connections` flag determines how many objects will be downloaded in parallel if downloading a folder. +> ✎ For S3 downloads, the `connections` flag determines how many objects will be downloaded in parallel if downloading a folder. + +
### GitHub Release Downloads +
+Unfold to read + It is often a task to download GitHub project releases because it requires figuring out the exact name of the asset file based on the OS and architecture of the machine. Danzo simplifies this process and only requires you to provide the owner and the project name. It uses that to automatically identify the correct latest release for its host's architecture and OS. ```bash @@ -338,14 +368,18 @@ If the user selection process needs to be manually kicked off, use Danzo like so danzo ghrelease "owner/repo" --manual ``` +
+ ### Git Repository Cloning +
+Unfold to read + Danzo can clone repositores sourced by various providers. While this is not particularly an expensive operation to run using just `git clone`, it serves to provide ease of setup when setting up a remote server with a large number of files as downloads and clones. As such, given a situation where a server needs to be prepared for operation by cloning a set of 8 repositories, 5 different tool assets, and an S3 folder; it would be slow to write a script incorporating several tools to get the environment ready. Danzo would be the perfect fir for such a scenario due to its batch-download capability via a YAML configuration. It is primarily for this purpose that an operation as simple and atomic as `git clone` was replicated in Danzo. -> [!WARNING] -> While Danzo as a tool is focused on conducting very fast downloads, it is important to note that in some cases where a git repository may be more than 1.5-2 GB in size, Danzo may experience easily noticeable slowdowns compared to plain old `git clone`. This is expected and usually, it's recommended to enforce depth (continue reading) when cloning repositories that large. +> ⚠︎ While Danzo as a tool is focused on conducting very fast downloads, it is important to note that in some cases where a git repository may be more than 1.5-2 GB in size, Danzo may experience easily noticeable slowdowns compared to plain old `git clone`. This is expected and usually, it's recommended to enforce depth (continue reading) when cloning repositories that large. Danzo supports the use of Personal Access Tokens as well as SSH keys when cloning repositories. The syntax has been simplified to refer to repositories with one of the following: @@ -375,8 +409,9 @@ danzo gitclone "github.com/tanq16/private" --token $(cat /secrets/ghtoken) danzo gitclone github.com/tanq16/private --ssh "/secrets/gh-ssh.key" ``` -> [!NOTE] -> Repository cloning is another download provider that does not use `-c` or number of connections. Number of workers, `-w`, is still applicable as usual in batch (YAML config) mode. +> ✎ Repository cloning is another download provider that does not use `-c` or number of connections. Number of workers, `-w`, is still applicable as usual in batch (YAML config) mode. + +
## Contributing @@ -398,8 +433,4 @@ Danzo draws inspiration from the following projects: - [ytmdl](https://github.com/deepjyoti30/ytmdl) - [aria2](https://github.com/aria2/aria2) -The following contributors helped improve Danzo: - -- [Whispard](https://github.com/Whispard) - [PR #8](https://github.com/Tanq16/danzo/pull/8) (support for custom HTTP headers) - Lastly, Danzo uses several Go packages referenced within `go.mod` that allow Danzo to be amazing. diff --git a/cmd/batch.go b/cmd/batch.go index 727f5ed..24070cb 100644 --- a/cmd/batch.go +++ b/cmd/batch.go @@ -6,6 +6,7 @@ import ( "strings" "github.com/goccy/go-yaml" + "github.com/rs/zerolog/log" "github.com/spf13/cobra" "github.com/tanq16/danzo/internal/scheduler" "github.com/tanq16/danzo/internal/utils" @@ -26,21 +27,26 @@ func newBatchCmd() *cobra.Command { Run: func(cmd *cobra.Command, args []string) { yamlFile := args[0] data, err := os.ReadFile(yamlFile) + log.Debug().Str("op", "cmd/batch").Msgf("Reading YAML file: %s", yamlFile) if err != nil { + log.Error().Str("op", "cmd/batch").Msgf("Error reading YAML file: %v", err) fmt.Fprintf(os.Stderr, "Error reading YAML file: %v\n", err) os.Exit(1) } var batchFile BatchFile if err := yaml.Unmarshal(data, &batchFile); err != nil { + log.Error().Str("op", "cmd/batch").Msgf("Error parsing YAML file: %v", err) fmt.Fprintf(os.Stderr, "Error parsing YAML file: %v\n", err) os.Exit(1) } jobs := buildJobsFromBatch(batchFile) if len(jobs) == 0 { + log.Error().Str("op", "cmd/batch").Msgf("No valid jobs found in the batch file") fmt.Fprintf(os.Stderr, "No valid jobs found in the batch file\n") os.Exit(1) } - scheduler.Run(jobs, workers, fileLog) + log.Debug().Str("op", "cmd/batch").Msgf("Starting scheduler with %d jobs", len(jobs)) + scheduler.Run(jobs, workers) }, } return cmd @@ -51,11 +57,13 @@ func buildJobsFromBatch(batchFile BatchFile) []utils.DanzoJob { for jobType, entries := range batchFile { normalizedType := normalizeJobType(jobType) if normalizedType == "" { + log.Warn().Str("op", "cmd/batch").Msgf("Unknown job type '%s', skipping...", jobType) fmt.Fprintf(os.Stderr, "Warning: Unknown job type '%s', skipping...\n", jobType) continue } for _, entry := range entries { if entry.Link == "" { + log.Warn().Str("op", "cmd/batch").Msgf("Empty link found in %s section, skipping...", jobType) fmt.Fprintf(os.Stderr, "Warning: Empty link found in %s section, skipping...\n", jobType) continue } @@ -67,14 +75,14 @@ func buildJobsFromBatch(batchFile BatchFile) []utils.DanzoJob { Metadata: make(map[string]any), } switch normalizedType { - case "http", "gdrive", "ghrelease", "m3u8": + case "http", "google-drive", "github-release", "live-stream": job.Connections = connections job.ProgressType = "progress" case "s3": job.Connections = connections job.ProgressType = "progress" job.Metadata["profile"] = "default" - case "youtube", "ytmusic", "gitclone": + case "youtube", "youtube-music", "git-clone": job.ProgressType = "stream" default: job.ProgressType = "progress" @@ -83,31 +91,36 @@ func buildJobsFromBatch(batchFile BatchFile) []utils.DanzoJob { jobs = append(jobs, job) } } + log.Debug().Str("op", "cmd/batch").Msgf("Built %d jobs from batch file", len(jobs)) return jobs } func normalizeJobType(jobType string) string { typeMap := map[string]string{ - "http": "http", - "https": "http", - "s3": "s3", - "gdrive": "gdrive", - "googledrive": "gdrive", - "google-drive": "gdrive", - "gitclone": "gitclone", - "git-clone": "gitclone", - "git": "gitclone", - "ghrelease": "ghrelease", - "gh-release": "ghrelease", - "github": "ghrelease", - "github-release": "ghrelease", - "m3u8": "m3u8", - "hls": "m3u8", - "youtube": "youtube", - "yt": "youtube", - "ytmusic": "ytmusic", - "youtube-music": "ytmusic", - "yt-music": "ytmusic", + "http": "http", + "https": "http", + "s3": "s3", + "gdrive": "google-drive", + "googledrive": "google-drive", + "google-drive": "google-drive", + "gitclone": "git-clone", + "git-clone": "git-clone", + "git": "git-clone", + "ghr": "github-release", + "ghrelease": "github-release", + "gh-release": "github-release", + "github": "github-release", + "github-release": "github-release", + "m3u8": "live-stream", + "hls": "live-stream", + "http-livestream": "live-stream", + "live-stream": "live-stream", + "youtube": "youtube", + "yt": "youtube", + "ytm": "yt-music", + "ytmusic": "yt-music", + "youtube-music": "yt-music", + "yt-music": "yt-music", } normalized := "" for key, value := range typeMap { @@ -125,9 +138,9 @@ func addJobTypeSpecificMetadata(job *utils.DanzoJob, jobType string) { if _, ok := job.Metadata["format"]; !ok { job.Metadata["format"] = "decent" } - case "ghrelease": + case "github-release": job.Metadata["manual"] = false - case "gitclone": + case "git-clone": if _, ok := job.Metadata["depth"]; !ok { job.Metadata["depth"] = 0 } diff --git a/cmd/clean.go b/cmd/clean.go index 69493f5..5a27c43 100644 --- a/cmd/clean.go +++ b/cmd/clean.go @@ -3,6 +3,7 @@ package cmd import ( "path/filepath" + "github.com/rs/zerolog/log" "github.com/spf13/cobra" "github.com/tanq16/danzo/internal/utils" ) @@ -14,8 +15,10 @@ func newCleanCmd() *cobra.Command { Args: cobra.MaximumNArgs(1), Run: func(cmd *cobra.Command, args []string) { if len(args) == 0 { + log.Debug().Str("op", "cmd/clean").Msgf("Cleaning local files in current directory") utils.CleanLocal() } else { + log.Debug().Str("op", "cmd/clean").Msgf("Cleaning local files in %s", filepath.Dir(args[0])) utils.CleanFunction(filepath.Dir(args[0])) } }, diff --git a/cmd/gitclone.go b/cmd/git-clone.go similarity index 71% rename from cmd/gitclone.go rename to cmd/git-clone.go index 7344165..550ea59 100644 --- a/cmd/gitclone.go +++ b/cmd/git-clone.go @@ -1,6 +1,7 @@ package cmd import ( + "github.com/rs/zerolog/log" "github.com/spf13/cobra" "github.com/tanq16/danzo/internal/scheduler" "github.com/tanq16/danzo/internal/utils" @@ -13,12 +14,13 @@ func newGitCloneCmd() *cobra.Command { var sshKey string cmd := &cobra.Command{ - Use: "gitclone [REPO_URL] [--output OUTPUT_PATH] [--depth DEPTH] [--token GIT_TOKEN] [--ssh SSH_KEY_PATH]", - Short: "Clone a Git repository", - Args: cobra.ExactArgs(1), + Use: "git-clone [REPO_URL] [--output OUTPUT_PATH] [--depth DEPTH] [--token GIT_TOKEN] [--ssh SSH_KEY_PATH]", + Short: "Clone a Git repository", + Aliases: []string{"gitclone", "gitc", "git", "clone"}, + Args: cobra.ExactArgs(1), Run: func(cmd *cobra.Command, args []string) { job := utils.DanzoJob{ - JobType: "gitclone", + JobType: "git-clone", URL: args[0], OutputPath: outputPath, ProgressType: "stream", @@ -35,7 +37,8 @@ func newGitCloneCmd() *cobra.Command { job.Metadata["sshKey"] = sshKey } jobs := []utils.DanzoJob{job} - scheduler.Run(jobs, workers, fileLog) + log.Debug().Str("op", "cmd/git-clone").Msgf("Starting scheduler with %d jobs", len(jobs)) + scheduler.Run(jobs, workers) }, } diff --git a/cmd/ghrelease.go b/cmd/github-release.go similarity index 65% rename from cmd/ghrelease.go rename to cmd/github-release.go index c2b9304..cd76649 100644 --- a/cmd/ghrelease.go +++ b/cmd/github-release.go @@ -1,6 +1,7 @@ package cmd import ( + "github.com/rs/zerolog/log" "github.com/spf13/cobra" "github.com/tanq16/danzo/internal/scheduler" "github.com/tanq16/danzo/internal/utils" @@ -11,12 +12,13 @@ func newGHReleaseCmd() *cobra.Command { var manual bool cmd := &cobra.Command{ - Use: "ghrelease [USER/REPO or URL] [--output OUTPUT_PATH] [--manual]", - Short: "Download a release asset for a GitHub repository", - Args: cobra.ExactArgs(1), + Use: "github-release [USER/REPO or URL] [--output OUTPUT_PATH] [--manual]", + Short: "Download a release asset for a GitHub repository", + Aliases: []string{"ghrelease", "ghr"}, + Args: cobra.ExactArgs(1), Run: func(cmd *cobra.Command, args []string) { job := utils.DanzoJob{ - JobType: "ghrelease", + JobType: "github-release", URL: args[0], OutputPath: outputPath, Connections: connections, @@ -26,7 +28,8 @@ func newGHReleaseCmd() *cobra.Command { } job.Metadata["manual"] = manual jobs := []utils.DanzoJob{job} - scheduler.Run(jobs, workers, fileLog) + log.Debug().Str("op", "cmd/github-release").Msgf("Starting scheduler with %d jobs", len(jobs)) + scheduler.Run(jobs, workers) }, } diff --git a/cmd/gdrive.go b/cmd/google-drive.go similarity index 70% rename from cmd/gdrive.go rename to cmd/google-drive.go index 5803518..fd4e6dd 100644 --- a/cmd/gdrive.go +++ b/cmd/google-drive.go @@ -1,6 +1,7 @@ package cmd import ( + "github.com/rs/zerolog/log" "github.com/spf13/cobra" "github.com/tanq16/danzo/internal/scheduler" "github.com/tanq16/danzo/internal/utils" @@ -12,12 +13,13 @@ func newGDriveCmd() *cobra.Command { var credentialsFile string cmd := &cobra.Command{ - Use: "gdrive [URL] [--output OUTPUT_PATH] [--api-key YOUR_KEY] [--creds creds.json]", - Short: "Download files or folders from Google Drive", - Args: cobra.ExactArgs(1), + Use: "google-drive [URL] [--output OUTPUT_PATH] [--api-key YOUR_KEY] [--creds creds.json]", + Short: "Download files or folders from Google Drive", + Aliases: []string{"gdrive", "gd", "drive"}, + Args: cobra.ExactArgs(1), Run: func(cmd *cobra.Command, args []string) { job := utils.DanzoJob{ - JobType: "gdrive", + JobType: "google-drive", URL: args[0], OutputPath: outputPath, Connections: connections, @@ -32,7 +34,8 @@ func newGDriveCmd() *cobra.Command { job.Metadata["credentialsFile"] = credentialsFile } jobs := []utils.DanzoJob{job} - scheduler.Run(jobs, workers, fileLog) + log.Debug().Str("op", "cmd/google-drive").Msgf("Starting scheduler with %d jobs", len(jobs)) + scheduler.Run(jobs, workers) }, } diff --git a/cmd/http.go b/cmd/http.go index 778e5d0..9b4ab0a 100644 --- a/cmd/http.go +++ b/cmd/http.go @@ -1,6 +1,7 @@ package cmd import ( + "github.com/rs/zerolog/log" "github.com/spf13/cobra" "github.com/tanq16/danzo/internal/scheduler" "github.com/tanq16/danzo/internal/utils" @@ -25,7 +26,8 @@ func newHTTPCmd() *cobra.Command { Metadata: make(map[string]any), } jobs := []utils.DanzoJob{job} - scheduler.Run(jobs, workers, fileLog) + log.Debug().Str("op", "cmd/http").Msgf("Starting scheduler with %d jobs", len(jobs)) + scheduler.Run(jobs, workers) }, } diff --git a/cmd/m3u8.go b/cmd/live-stream.go similarity index 64% rename from cmd/m3u8.go rename to cmd/live-stream.go index 100e921..dcc52d5 100644 --- a/cmd/m3u8.go +++ b/cmd/live-stream.go @@ -1,6 +1,7 @@ package cmd import ( + "github.com/rs/zerolog/log" "github.com/spf13/cobra" "github.com/tanq16/danzo/internal/scheduler" "github.com/tanq16/danzo/internal/utils" @@ -10,12 +11,13 @@ func newM3U8Cmd() *cobra.Command { var outputPath string cmd := &cobra.Command{ - Use: "m3u8 [URL] [--output OUTPUT_PATH]", - Short: "Download HLS/M3U8 streams", - Args: cobra.ExactArgs(1), + Use: "live-stream [URL] [--output OUTPUT_PATH]", + Short: "Download HLS/M3U8 live streams", + Aliases: []string{"hls", "m3u8", "livestream", "stream"}, + Args: cobra.ExactArgs(1), Run: func(cmd *cobra.Command, args []string) { job := utils.DanzoJob{ - JobType: "m3u8", + JobType: "live-stream", URL: args[0], OutputPath: outputPath, Connections: connections, @@ -24,7 +26,8 @@ func newM3U8Cmd() *cobra.Command { Metadata: make(map[string]any), } jobs := []utils.DanzoJob{job} - scheduler.Run(jobs, workers, fileLog) + log.Debug().Str("op", "cmd/live-stream").Msgf("Starting scheduler with %d jobs", len(jobs)) + scheduler.Run(jobs, workers) }, } diff --git a/cmd/root.go b/cmd/root.go index e95f88a..8e360c1 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -3,7 +3,10 @@ package cmd import ( "fmt" "os" + "time" + "github.com/rs/zerolog" + "github.com/rs/zerolog/log" "github.com/spf13/cobra" "github.com/tanq16/danzo/internal/utils" ) @@ -19,15 +22,12 @@ var ( headers []string workers int connections int - fileLog bool + debugFlag string ) // Global HTTP client config that will be passed to subcommands var globalHTTPConfig utils.HTTPClientConfig -// Registry for all subcommands -var commandRegistry = make(map[string]*cobra.Command) - var rootCmd = &cobra.Command{ Use: "danzo", Short: "Danzo is a swiss-army knife CLI download manager", @@ -45,6 +45,31 @@ var rootCmd = &cobra.Command{ }, } +func setupLogs() { + zerolog.TimeFieldFormat = zerolog.TimeFormatUnix + output := zerolog.ConsoleWriter{ + Out: os.Stdout, + TimeFormat: time.DateTime, + NoColor: false, // Enable color output + } + log.Logger = zerolog.New(output).With().Timestamp().Logger() + zerolog.SetGlobalLevel(zerolog.Disabled) + switch debugFlag { + case "debug": + zerolog.SetGlobalLevel(zerolog.DebugLevel) + utils.GlobalDebugFlag = true + case "info": + zerolog.SetGlobalLevel(zerolog.InfoLevel) + utils.GlobalDebugFlag = true + case "disabled": + zerolog.SetGlobalLevel(zerolog.Disabled) + utils.GlobalDebugFlag = false + default: + zerolog.SetGlobalLevel(zerolog.Disabled) + utils.GlobalDebugFlag = false + } +} + func Execute() { if err := rootCmd.Execute(); err != nil { fmt.Fprintln(os.Stderr, err) @@ -54,6 +79,8 @@ func Execute() { func init() { rootCmd.SetHelpCommand(&cobra.Command{Hidden: true}) + rootCmd.PersistentFlags().StringVar(&debugFlag, "debug", "disabled", "Enable logging for debug, info or disabled (TUI mode) level") + cobra.OnInitialize(setupLogs) // Global flags rootCmd.PersistentFlags().StringVarP(&proxyURL, "proxy", "p", "", "HTTP/HTTPS proxy URL") @@ -63,27 +90,20 @@ func init() { rootCmd.PersistentFlags().StringArrayVarP(&headers, "header", "H", []string{}, "Custom headers") rootCmd.PersistentFlags().IntVarP(&workers, "workers", "w", 1, "Number of parallel workers") rootCmd.PersistentFlags().IntVarP(&connections, "connections", "c", 8, "Number of connections per download") - rootCmd.PersistentFlags().BoolVar(&fileLog, "log", false, "Enable debug logging") registerCommands() fmt.Println() } -func RegisterCommand(name string, cmd *cobra.Command) { - commandRegistry[name] = cmd - rootCmd.AddCommand(cmd) -} - func registerCommands() { - RegisterCommand("clean", newCleanCmd()) - - RegisterCommand("http", newHTTPCmd()) - RegisterCommand("m3u8", newM3U8Cmd()) - RegisterCommand("s3", newS3Cmd()) - RegisterCommand("gitclone", newGitCloneCmd()) - RegisterCommand("ghrelease", newGHReleaseCmd()) - RegisterCommand("gdrive", newGDriveCmd()) - RegisterCommand("youtube", newYouTubeCmd()) - RegisterCommand("ytmusic", newYTMusicCmd()) - RegisterCommand("batch", newBatchCmd()) + rootCmd.AddCommand(newCleanCmd()) + rootCmd.AddCommand(newHTTPCmd()) + rootCmd.AddCommand(newM3U8Cmd()) + rootCmd.AddCommand(newS3Cmd()) + rootCmd.AddCommand(newGitCloneCmd()) + rootCmd.AddCommand(newGHReleaseCmd()) + rootCmd.AddCommand(newGDriveCmd()) + rootCmd.AddCommand(newYouTubeCmd()) + rootCmd.AddCommand(newYTMusicCmd()) + rootCmd.AddCommand(newBatchCmd()) } diff --git a/cmd/s3.go b/cmd/s3.go index 8686bde..7027d2a 100644 --- a/cmd/s3.go +++ b/cmd/s3.go @@ -1,6 +1,7 @@ package cmd import ( + "github.com/rs/zerolog/log" "github.com/spf13/cobra" "github.com/tanq16/danzo/internal/scheduler" "github.com/tanq16/danzo/internal/utils" @@ -26,7 +27,8 @@ func newS3Cmd() *cobra.Command { } job.Metadata["profile"] = profile jobs := []utils.DanzoJob{job} - scheduler.Run(jobs, workers, fileLog) + log.Debug().Str("op", "cmd/s3").Msgf("Starting scheduler with %d jobs", len(jobs)) + scheduler.Run(jobs, workers) }, } diff --git a/cmd/youtube-music.go b/cmd/youtube-music.go index 93084ad..d74775f 100644 --- a/cmd/youtube-music.go +++ b/cmd/youtube-music.go @@ -1,6 +1,7 @@ package cmd import ( + "github.com/rs/zerolog/log" "github.com/spf13/cobra" "github.com/tanq16/danzo/internal/scheduler" "github.com/tanq16/danzo/internal/utils" @@ -12,12 +13,13 @@ func newYTMusicCmd() *cobra.Command { var appleID string cmd := &cobra.Command{ - Use: "ytmusic [URL] [--output OUTPUT_PATH] [--deezer DEEZER_ID] [--apple APPLE_ID]", - Short: "Download YouTube music with metadata", - Args: cobra.ExactArgs(1), + Use: "youtube-music [URL] [--output OUTPUT_PATH] [--deezer DEEZER_ID] [--apple APPLE_ID]", + Short: "Download YouTube music with metadata", + Aliases: []string{"ytm", "yt-music"}, + Args: cobra.ExactArgs(1), Run: func(cmd *cobra.Command, args []string) { job := utils.DanzoJob{ - JobType: "ytmusic", + JobType: "youtube-music", URL: args[0], OutputPath: outputPath, ProgressType: "stream", @@ -32,7 +34,8 @@ func newYTMusicCmd() *cobra.Command { job.Metadata["musicID"] = appleID } jobs := []utils.DanzoJob{job} - scheduler.Run(jobs, workers, fileLog) + log.Debug().Str("op", "cmd/youtube-music").Msgf("Starting scheduler with %d jobs", len(jobs)) + scheduler.Run(jobs, workers) }, } diff --git a/cmd/youtube.go b/cmd/youtube.go index 112682a..ae8dbdf 100644 --- a/cmd/youtube.go +++ b/cmd/youtube.go @@ -1,6 +1,7 @@ package cmd import ( + "github.com/rs/zerolog/log" "github.com/spf13/cobra" "github.com/tanq16/danzo/internal/scheduler" "github.com/tanq16/danzo/internal/utils" @@ -11,9 +12,10 @@ func newYouTubeCmd() *cobra.Command { var format string cmd := &cobra.Command{ - Use: "yt [URL] [--output OUTPUT_PATH] [--format FORMAT]", - Short: "Download YouTube videos", - Args: cobra.ExactArgs(1), + Use: "youtube [URL] [--output OUTPUT_PATH] [--format FORMAT]", + Short: "Download YouTube videos", + Aliases: []string{"yt"}, + Args: cobra.ExactArgs(1), Run: func(cmd *cobra.Command, args []string) { job := utils.DanzoJob{ JobType: "youtube", @@ -27,7 +29,8 @@ func newYouTubeCmd() *cobra.Command { job.Metadata["format"] = format } jobs := []utils.DanzoJob{job} - scheduler.Run(jobs, workers, fileLog) + log.Debug().Str("op", "cmd/youtube").Msgf("Starting scheduler with %d jobs", len(jobs)) + scheduler.Run(jobs, workers) }, } diff --git a/go.mod b/go.mod index b092f95..fb8c957 100644 --- a/go.mod +++ b/go.mod @@ -10,6 +10,7 @@ require ( github.com/go-git/go-git/v5 v5.14.0 github.com/goccy/go-yaml v1.17.1 github.com/google/uuid v1.6.0 + github.com/rs/zerolog v1.34.0 github.com/spf13/cobra v1.9.1 golang.org/x/oauth2 v0.28.0 golang.org/x/term v0.29.0 @@ -50,6 +51,7 @@ require ( github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect github.com/kevinburke/ssh_config v1.2.0 // indirect github.com/lucasb-eyer/go-colorful v1.2.0 // indirect + github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/mattn/go-runewidth v0.0.16 // indirect github.com/muesli/termenv v0.16.0 // indirect diff --git a/go.sum b/go.sum index e33855a..cdb180b 100644 --- a/go.sum +++ b/go.sum @@ -61,6 +61,7 @@ github.com/charmbracelet/x/term v0.2.1 h1:AQeHeLZ1OqSXhrAWpYUtZyX1T3zVxfpZuEQMIQ github.com/charmbracelet/x/term v0.2.1/go.mod h1:oQ4enTYFV7QN4m0i9mzHrViD7TQKvNEEkHUMCmsxdUg= github.com/cloudflare/circl v1.6.0 h1:cr5JKic4HI+LkINy2lg3W2jF8sHCVTBncJr5gIIq7qk= github.com/cloudflare/circl v1.6.0/go.mod h1:uddAzsPgqdMAYatqJ0lsjX1oECcQLIlRpzZh3pJrofs= +github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= github.com/cyphar/filepath-securejoin v0.4.1 h1:JyxxyPEaktOD+GAnqIqTf9A8tHyAG22rowi7HkoSU1s= github.com/cyphar/filepath-securejoin v0.4.1/go.mod h1:Sdj7gXlvMcPZsbhwhQ33GguGLDGQL7h7bg04C/+u9jI= @@ -83,6 +84,7 @@ github.com/go-git/go-git/v5 v5.14.0 h1:/MD3lCrGjCen5WfEAzKg00MJJffKhC8gzS80ycmCi github.com/go-git/go-git/v5 v5.14.0/go.mod h1:Z5Xhoia5PcWA3NF8vRLURn9E5FRhSl7dGj9ItW3Wk5k= github.com/goccy/go-yaml v1.17.1 h1:LI34wktB2xEE3ONG/2Ar54+/HJVBriAGJ55PHls4YuY= github.com/goccy/go-yaml v1.17.1/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA= +github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 h1:f+oWsMOmNPc8JmEHVZIycC7hBoQxHH9pNKQORJNozsQ= github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8/go.mod h1:wcDNUvekVysuuOpQKo3191zZyTpiI6se1N1ULghS0sw= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= @@ -104,6 +106,10 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69Aj6K7nkY= github.com/lucasb-eyer/go-colorful v1.2.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= +github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= +github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc= @@ -123,6 +129,9 @@ github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= +github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= +github.com/rs/zerolog v1.34.0 h1:k43nTLIwcTVQAncfCw4KZ2VY6ukYoZaBPNOE8txlOeY= +github.com/rs/zerolog v1.34.0/go.mod h1:bJsvje4Z08ROH4Nhs5iH600c3IkWhwp44iRc54W6wYQ= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN3Uc8sB6B/s6Z4t2xvBgU1htSHuq8= github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= @@ -158,7 +167,9 @@ golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik= golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= diff --git a/internal/downloaders/gitclone/auth.go b/internal/downloaders/git-clone/auth.go similarity index 83% rename from internal/downloaders/gitclone/auth.go rename to internal/downloaders/git-clone/auth.go index 41e8e8c..82d7f75 100644 --- a/internal/downloaders/gitclone/auth.go +++ b/internal/downloaders/git-clone/auth.go @@ -8,6 +8,7 @@ import ( "github.com/go-git/go-git/v5/plumbing/transport" "github.com/go-git/go-git/v5/plumbing/transport/http" "github.com/go-git/go-git/v5/plumbing/transport/ssh" + "github.com/rs/zerolog/log" ) func getAuthMethod(repoURL string, metadata map[string]any) (transport.AuthMethod, error) { @@ -15,6 +16,7 @@ func getAuthMethod(repoURL string, metadata map[string]any) (transport.AuthMetho token := "" if ok { token = tokenStr.(string) + log.Debug().Str("op", "git-clone/auth").Msg("token found") } if token != "" { if strings.Contains(repoURL, "github.com") { @@ -34,10 +36,10 @@ func getAuthMethod(repoURL string, metadata map[string]any) (transport.AuthMetho }, nil } } - sshKeyPath := "" sshKeyStr, ok := metadata["sshKey"] if ok { + log.Debug().Str("op", "git-clone/auth").Msg("sshKey found") sshKeyPath = sshKeyStr.(string) } if sshKeyPath != "" { @@ -47,6 +49,6 @@ func getAuthMethod(repoURL string, metadata map[string]any) (transport.AuthMetho } return publicKeys, nil } - + log.Debug().Str("op", "git-clone/auth").Msg("no authentication method found") return nil, errors.New("no authentication method found") } diff --git a/internal/downloaders/gitclone/download.go b/internal/downloaders/git-clone/download.go similarity index 80% rename from internal/downloaders/gitclone/download.go rename to internal/downloaders/git-clone/download.go index 2640ae6..d8473ba 100644 --- a/internal/downloaders/gitclone/download.go +++ b/internal/downloaders/git-clone/download.go @@ -10,14 +10,15 @@ import ( "strings" "github.com/go-git/go-git/v5" + "github.com/rs/zerolog/log" "github.com/tanq16/danzo/internal/utils" ) -type gitCloneProgress struct { +type gitCloneProgressWriter struct { streamFunc func(string) } -func (p *gitCloneProgress) Write(data []byte) (int, error) { +func (p *gitCloneProgressWriter) Write(data []byte) (int, error) { message := strings.TrimSpace(string(data)) if message != "" && p.streamFunc != nil { p.streamFunc(message) @@ -28,45 +29,36 @@ func (p *gitCloneProgress) Write(data []byte) (int, error) { func (d *GitCloneDownloader) Download(job *utils.DanzoJob) error { cloneURL := job.Metadata["cloneURL"].(string) depth, _ := job.Metadata["depth"].(int) - - // Get authentication if available auth, err := getAuthMethod(cloneURL, job.Metadata) if err != nil && job.StreamFunc != nil { job.StreamFunc(fmt.Sprintf("Warning: %v", err)) } - - // Create progress writer - progress := &gitCloneProgress{ + progress := &gitCloneProgressWriter{ streamFunc: job.StreamFunc, } - - // Build clone options cloneOptions := &git.CloneOptions{ URL: cloneURL, Progress: progress, Auth: auth, } - + log.Debug().Str("op", "git-clone/download").Msg("clone options created") if depth > 0 { cloneOptions.Depth = depth } - - // Perform the clone if job.StreamFunc != nil { job.StreamFunc(fmt.Sprintf("Cloning %s", cloneURL)) } - + log.Info().Str("op", "git-clone/download").Msg("initiating clone") _, err = git.PlainClone(job.OutputPath, false, cloneOptions) if err != nil { + log.Error().Str("op", "git-clone/download").Msgf("git clone failed: %v", err) return fmt.Errorf("git clone failed: %v", err) } - - // Get directory size for final report + log.Info().Str("op", "git-clone/download").Msg("clone completed") size, err := getDirSize(job.OutputPath) if err == nil && job.StreamFunc != nil { job.StreamFunc(fmt.Sprintf("Clone complete - Total size: %s", utils.FormatBytes(uint64(size)))) } - return nil } @@ -85,7 +77,6 @@ func getDirSize(path string) (int64, error) { } } } - var size int64 err := filepath.Walk(path, func(_ string, info os.FileInfo, err error) error { if err != nil { diff --git a/internal/downloaders/gitclone/initial.go b/internal/downloaders/git-clone/initial.go similarity index 85% rename from internal/downloaders/gitclone/initial.go rename to internal/downloaders/git-clone/initial.go index 54db559..197233f 100644 --- a/internal/downloaders/gitclone/initial.go +++ b/internal/downloaders/git-clone/initial.go @@ -6,23 +6,21 @@ import ( "path/filepath" "strings" + "github.com/rs/zerolog/log" "github.com/tanq16/danzo/internal/utils" ) type GitCloneDownloader struct{} func (d *GitCloneDownloader) ValidateJob(job *utils.DanzoJob) error { - // Parse repository URL provider, owner, repo, err := parseGitURL(job.URL) if err != nil { return err } - - // Store parsed values job.Metadata["provider"] = provider job.Metadata["owner"] = owner job.Metadata["repo"] = repo - + log.Info().Str("op", "git-clone/initial").Msgf("job validated for %s/%s/%s", provider, owner, repo) return nil } @@ -30,27 +28,19 @@ func (d *GitCloneDownloader) BuildJob(job *utils.DanzoJob) error { provider := job.Metadata["provider"].(string) owner := job.Metadata["owner"].(string) repo := job.Metadata["repo"].(string) - - // Build actual clone URL cloneURL := fmt.Sprintf("https://%s/%s/%s", provider, owner, repo) job.Metadata["cloneURL"] = cloneURL - - // Set output path if not specified if job.OutputPath == "" { job.OutputPath = repo } - - // Check if directory already exists if info, err := os.Stat(job.OutputPath); err == nil && info.IsDir() { job.OutputPath = utils.RenewOutputPath(job.OutputPath) } - - // Create output directory outputDir := filepath.Dir(job.OutputPath) if err := os.MkdirAll(outputDir, 0755); err != nil { return fmt.Errorf("error creating output directory: %v", err) } - + log.Info().Str("op", "git-clone/initial").Msgf("job built for %s/%s/%s", provider, owner, repo) return nil } @@ -58,28 +48,19 @@ func parseGitURL(url string) (string, string, string, error) { url = strings.TrimSpace(url) url = strings.TrimSuffix(url, ".git") url = strings.TrimSuffix(url, "/") - - // Remove https:// prefix if present url = strings.TrimPrefix(url, "https://") url = strings.TrimPrefix(url, "http://") - - // Split by / parts := strings.Split(url, "/") if len(parts) < 3 { return "", "", "", fmt.Errorf("invalid git URL format, expected provider/owner/repo") } - provider := parts[0] owner := parts[1] repo := parts[2] - - // Validate provider switch provider { case "github.com", "gitlab.com", "bitbucket.org": - // Valid providers default: return "", "", "", fmt.Errorf("unsupported git provider: %s", provider) } - return provider, owner, repo, nil } diff --git a/internal/downloaders/ghrelease/download.go b/internal/downloaders/github-release/download.go similarity index 79% rename from internal/downloaders/ghrelease/download.go rename to internal/downloaders/github-release/download.go index f45d5b7..aef6749 100644 --- a/internal/downloaders/ghrelease/download.go +++ b/internal/downloaders/github-release/download.go @@ -3,6 +3,7 @@ package ghrelease import ( "time" + "github.com/rs/zerolog/log" danzohttp "github.com/tanq16/danzo/internal/downloaders/http" "github.com/tanq16/danzo/internal/utils" ) @@ -10,21 +11,19 @@ import ( func (d *GitReleaseDownloader) Download(job *utils.DanzoJob) error { downloadURL := job.Metadata["downloadURL"].(string) fileSize := job.Metadata["fileSize"].(int64) - client := utils.NewDanzoHTTPClient(job.HTTPClientConfig) - progressCh := make(chan int64) progressDone := make(chan struct{}) + log.Info().Str("op", "github-release/download").Msgf("downloading %s", downloadURL) // Progress tracking goroutine go func() { defer close(progressDone) var totalDownloaded int64 startTime := time.Now() - ticker := time.NewTicker(100 * time.Millisecond) defer ticker.Stop() - + log.Debug().Str("op", "github-release/download").Msg("progress tracking goroutine started") for { select { case bytes, ok := <-progressCh: @@ -35,7 +34,6 @@ func (d *GitReleaseDownloader) Download(job *utils.DanzoJob) error { return } totalDownloaded += bytes - case <-ticker.C: if job.ProgressFunc != nil { job.ProgressFunc(totalDownloaded, fileSize) @@ -45,10 +43,8 @@ func (d *GitReleaseDownloader) Download(job *utils.DanzoJob) error { } }() - // Perform download using simple HTTP download + log.Debug().Str("op", "github-release/download").Msg("calling simple download") err := danzohttp.PerformSimpleDownload(downloadURL, job.OutputPath, client, progressCh) - <-progressDone - return err } diff --git a/internal/downloaders/ghrelease/helpers.go b/internal/downloaders/github-release/helpers.go similarity index 70% rename from internal/downloaders/ghrelease/helpers.go rename to internal/downloaders/github-release/helpers.go index dd8ee60..6de798d 100644 --- a/internal/downloaders/ghrelease/helpers.go +++ b/internal/downloaders/github-release/helpers.go @@ -11,6 +11,7 @@ import ( "strconv" "strings" + "github.com/rs/zerolog/log" "github.com/tanq16/danzo/internal/utils" ) @@ -23,6 +24,15 @@ var assetSelectMap = map[string][]string{ "darwinarm64": {"darwin-arm64", "darwin_arm64", "darwin-aarch64", "darwin_aarch64", "arm64-darwin", "aarch64-darwin", "arm64_darwin", "aarch64_darwin"}, } +var assetSelectMapFallback = map[string][]string{ + "linuxamd64": {"linux", "gnu", "x86-64", "x86_64", "amd64", "amd"}, + "linuxarm64": {"linux", "gnu", "arm", "arm64"}, + "windowsamd64": {"exe", "x86-64", "x86_64", "amd64", "amd"}, + "windowsarm64": {"exe", "arm", "arm64"}, + "darwinamd64": {"darwin", "apple", "x86-64", "x86_64", "amd64", "amd"}, + "darwinarm64": {"darwin", "apple", "arm", "arm64"}, +} + var repoPatterns = []*regexp.Regexp{ regexp.MustCompile(`^https?://github\.com/([^/]+)/([^/]+)/?.*$`), regexp.MustCompile(`^github\.com/([^/]+)/([^/]+)/?.*$`), @@ -35,14 +45,12 @@ var ignoredAssets = []string{ func parseGitHubURL(url string) (string, string, error) { url = strings.TrimSuffix(strings.TrimSpace(url), "/") - for _, pattern := range repoPatterns { matches := pattern.FindStringSubmatch(url) if len(matches) >= 3 { return matches[1], matches[2], nil } } - return "", "", fmt.Errorf("invalid GitHub repository format: %s", url) } @@ -50,16 +58,20 @@ func getGitHubReleaseAssets(owner, repo string, client *utils.DanzoHTTPClient) ( apiURL := fmt.Sprintf("https://api.github.com/repos/%s/%s/releases/latest", owner, repo) req, err := http.NewRequest("GET", apiURL, nil) if err != nil { + log.Error().Str("op", "github-release/helpers").Msgf("error creating API request: %v", err) return nil, "", fmt.Errorf("error creating API request: %v", err) } resp, err := client.Do(req) if err != nil { + log.Error().Str("op", "github-release/helpers").Msgf("error making API request: %v", err) return nil, "", fmt.Errorf("error making API request: %v", err) } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { + log.Error().Str("op", "github-release/helpers").Msgf("API request failed with status code: %d", resp.StatusCode) return nil, "", fmt.Errorf("API request failed with status code: %d", resp.StatusCode) } + log.Debug().Str("op", "github-release/helpers").Msg("API request successful") var release map[string]any if err := json.NewDecoder(resp.Body).Decode(&release); err != nil { @@ -68,72 +80,10 @@ func getGitHubReleaseAssets(owner, repo string, client *utils.DanzoHTTPClient) ( tagName, _ := release["tag_name"].(string) assets, ok := release["assets"].([]any) if !ok { + log.Warn().Str("op", "github-release/helpers").Msg("no assets found in the release") return nil, "", fmt.Errorf("no assets found in the release") } - var assetList []map[string]any - for _, asset := range assets { - assetMap, ok := asset.(map[string]any) - if ok { - assetList = append(assetList, assetMap) - } - } - if len(assetList) == 0 { - return nil, "", fmt.Errorf("no assets found in the release") - } - return assetList, tagName, nil -} - -func askGitHubReleaseAssets(owner, repo string, client *utils.DanzoHTTPClient) ([]map[string]any, string, error) { - apiURL := fmt.Sprintf("https://api.github.com/repos/%s/%s/releases", owner, repo) - req, err := http.NewRequest("GET", apiURL, nil) - if err != nil { - return nil, "", fmt.Errorf("error creating API request: %v", err) - } - resp, err := client.Do(req) - if err != nil { - return nil, "", fmt.Errorf("error making API request: %v", err) - } - defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { - return nil, "", fmt.Errorf("API request failed with status code: %d", resp.StatusCode) - } - - var releases []map[string]any - if err := json.NewDecoder(resp.Body).Decode(&releases); err != nil { - return nil, "", fmt.Errorf("error decoding API response: %v", err) - } - if len(releases) == 0 { - return nil, "", fmt.Errorf("no releases found for the repository") - } - fmt.Printf("Available releases for %s/%s:\n", owner, repo) - for i, release := range releases { - tagName, _ := release["tag_name"].(string) - fmt.Printf("%d. %s\n", i+1, tagName) - } - fmt.Print("\nEnter the number of the release to download: ") - reader := bufio.NewReader(os.Stdin) - input, err := reader.ReadString('\n') - if err != nil { - return nil, "", fmt.Errorf("error reading input: %v", err) - } - - input = strings.TrimSpace(input) - selection, err := strconv.Atoi(input) - if err != nil { - return nil, "", fmt.Errorf("invalid selection: %v", err) - } - if selection < 1 || selection > len(releases) { - return nil, "", fmt.Errorf("selection out of range") - } - linesUsed := len(releases) + 3 // Releases list + Prompt line + Input line - fmt.Printf("\033[%dA\033[J", linesUsed) - - selectedRelease := releases[selection-1] - tagName, _ := selectedRelease["tag_name"].(string) - assets, ok := selectedRelease["assets"].([]any) - if !ok { - return nil, "", fmt.Errorf("no assets found in the release") - } + log.Info().Str("op", "github-release/helpers").Msgf("found %d assets in the release", len(assets)) var assetList []map[string]any for _, asset := range assets { assetMap, ok := asset.(map[string]any) @@ -171,6 +121,7 @@ func promptGitHubAssetSelection(assets []map[string]any, tagName string) (string } linesUsed := len(assets) + 4 // Assets list + Release line + Prompt line + Input line + newline fmt.Printf("\033[%dA\033[J", linesUsed) + log.Info().Str("op", "github-release/helpers").Msgf("selected asset: %s", assets[selection-1]["name"]) selectedAsset := assets[selection-1] downloadURL, _ := selectedAsset["browser_download_url"].(string) diff --git a/internal/downloaders/ghrelease/initial.go b/internal/downloaders/github-release/initial.go similarity index 65% rename from internal/downloaders/ghrelease/initial.go rename to internal/downloaders/github-release/initial.go index f6c0d5f..221b2d4 100644 --- a/internal/downloaders/ghrelease/initial.go +++ b/internal/downloaders/github-release/initial.go @@ -5,6 +5,7 @@ import ( "runtime" "strings" + "github.com/rs/zerolog/log" "github.com/tanq16/danzo/internal/utils" ) @@ -15,11 +16,9 @@ func (d *GitReleaseDownloader) ValidateJob(job *utils.DanzoJob) error { if err != nil { return err } - - // Store parsed values job.Metadata["owner"] = owner job.Metadata["repo"] = repo - + log.Info().Str("op", "github-release/initial").Msgf("job validated for %s/%s", owner, repo) return nil } @@ -27,56 +26,44 @@ func (d *GitReleaseDownloader) BuildJob(job *utils.DanzoJob) error { owner := job.Metadata["owner"].(string) repo := job.Metadata["repo"].(string) manual := job.Metadata["manual"].(bool) - client := utils.NewDanzoHTTPClient(job.HTTPClientConfig) - var assets []map[string]any var tagName string var err error - // Always get latest release first + log.Debug().Str("op", "github-release/initial").Msgf("fetching release info for %s/%s", owner, repo) assets, tagName, err = getGitHubReleaseAssets(owner, repo, client) if err != nil { return fmt.Errorf("error fetching release info: %v", err) } - - // Try auto-select first + // Try auto-select first; not working, prompt for manual or fail + log.Debug().Str("op", "github-release/initial").Msgf("auto-selecting asset for %s/%s", runtime.GOOS, runtime.GOARCH) downloadURL, size, err := selectGitHubLatestAsset(assets) if err != nil { return err } - - // If auto-select failed and manual not specified, fail if downloadURL == "" && !manual { + log.Error().Str("op", "github-release/initial").Msgf("could not automatically select asset for %s/%s, no --manual flag", runtime.GOOS, runtime.GOARCH) return fmt.Errorf("could not automatically select asset for platform %s/%s, use --manual flag", runtime.GOOS, runtime.GOARCH) } - - // If manual mode or auto-select failed with manual flag if manual { + log.Debug().Str("op", "github-release/initial").Msgf("prompting for manual asset selection for %s/%s", runtime.GOOS, runtime.GOARCH) job.PauseFunc() - - // Get user selection downloadURL, size, err = promptGitHubAssetSelection(assets, tagName) job.ResumeFunc() - if err != nil { return err } } - // Extract filename from URL urlParts := strings.Split(downloadURL, "/") filename := urlParts[len(urlParts)-1] - - // Set output path if not specified if job.OutputPath == "" { job.OutputPath = filename } - - // Store download info job.Metadata["downloadURL"] = downloadURL job.Metadata["fileSize"] = size job.Metadata["tagName"] = tagName - + log.Info().Str("op", "github-release/initial").Msgf("job built for %s/%s", owner, repo) return nil } diff --git a/internal/downloaders/gdrive/auth.go b/internal/downloaders/google-drive/auth.go similarity index 77% rename from internal/downloaders/gdrive/auth.go rename to internal/downloaders/google-drive/auth.go index 141c231..e875f07 100644 --- a/internal/downloaders/gdrive/auth.go +++ b/internal/downloaders/google-drive/auth.go @@ -8,7 +8,9 @@ import ( "os" "path/filepath" + "github.com/rs/zerolog/log" "github.com/tanq16/danzo/internal/output" + "github.com/tanq16/danzo/internal/utils" "golang.org/x/oauth2" "golang.org/x/oauth2/google" "golang.org/x/term" @@ -19,8 +21,10 @@ func getAccessTokenFromCredentials(credentialsFile string) (string, error) { if err != nil { return "", fmt.Errorf("unable to read credentials file: %v", err) } + log.Debug().Str("op", "google-drive/auth").Msgf("using credentials from %s", credentialsFile) config, err := google.ConfigFromJSON(b, "https://www.googleapis.com/auth/drive.readonly") if err != nil { + log.Error().Str("op", "google-drive/auth").Msgf("unable to parse client secret file: %v", err) return "", fmt.Errorf("unable to parse client secret file: %v", err) } @@ -39,6 +43,7 @@ func getAccessTokenFromCredentials(credentialsFile string) (string, error) { token = newToken // Save refreshed token if err := saveToken(tokenFile, token); err != nil { + log.Warn().Str("op", "google-drive/auth").Msgf("unable to save refreshed token: %v", err) } } else { return "", errors.New("OAuth token is expired and cannot be refreshed") @@ -50,8 +55,10 @@ func getAccessTokenFromCredentials(credentialsFile string) (string, error) { func getOAuthToken(config *oauth2.Config, tokenFile string) (*oauth2.Token, error) { token, err := tokenFromFile(tokenFile) if err == nil { + log.Debug().Str("op", "google-drive/auth").Msgf("existing token retrieved") return token, nil } + log.Debug().Str("op", "google-drive/auth").Msgf("no existing token retrieved, get new one with OAuth flow") authURL := config.AuthCodeURL("state-token", oauth2.AccessTypeOffline) output.PrintDetail("\nVisit this URL to get the authorization code:\n") fmt.Printf("%s\n", authURL) @@ -60,15 +67,19 @@ func getOAuthToken(config *oauth2.Config, tokenFile string) (*oauth2.Token, erro if _, err := fmt.Scan(&authCode); err != nil { return nil, fmt.Errorf("unable to read authorization code: %v", err) } + log.Debug().Str("op", "google-drive/auth").Msgf("exchanging scanned auth code for token") token, err = config.Exchange(context.Background(), authCode) if err != nil { return nil, fmt.Errorf("unable to exchange auth code for token: %v", err) } if err := saveToken(tokenFile, token); err != nil { + log.Warn().Str("op", "google-drive/auth").Msgf("unable to save new token: %v", err) } clearLength := 6 clearLength += len(authURL)/getTerminalWidth() + 1 - fmt.Printf("\033[%dA\033[J", clearLength) + if !utils.GlobalDebugFlag { + fmt.Printf("\033[%dA\033[J", clearLength) + } return token, nil } @@ -80,6 +91,7 @@ func tokenFromFile(file string) (*oauth2.Token, error) { defer f.Close() token := &oauth2.Token{} err = json.NewDecoder(f).Decode(token) + log.Debug().Str("op", "google-drive/auth").Msgf("token retrieved from file") return token, err } diff --git a/internal/downloaders/gdrive/download.go b/internal/downloaders/google-drive/download.go similarity index 86% rename from internal/downloaders/gdrive/download.go rename to internal/downloaders/google-drive/download.go index d17ee9c..3b93588 100644 --- a/internal/downloaders/gdrive/download.go +++ b/internal/downloaders/google-drive/download.go @@ -6,6 +6,7 @@ import ( "path/filepath" "strings" + "github.com/rs/zerolog/log" danzohttp "github.com/tanq16/danzo/internal/downloaders/http" "github.com/tanq16/danzo/internal/utils" ) @@ -15,6 +16,7 @@ func (d *GDriveDownloader) Download(job *utils.DanzoJob) error { isFolder := job.Metadata["isFolder"].(bool) totalSize := job.Metadata["totalSize"].(int64) client := utils.NewDanzoHTTPClient(job.HTTPClientConfig) + log.Info().Str("op", "google-drive/download").Msgf("downloading gdrive file; isFolder:%v", isFolder) if isFolder { return d.downloadFolder(job, token, client, totalSize) } else { @@ -34,8 +36,7 @@ func (d *GDriveDownloader) downloadFile(job *utils.DanzoJob, token string, clien } } }() - - config := utils.DownloadConfig{ + config := utils.HTTPDownloadConfig{ URL: job.URL, OutputPath: job.OutputPath, HTTPClientConfig: job.HTTPClientConfig, @@ -48,7 +49,6 @@ func (d *GDriveDownloader) downloadFolder(job *utils.DanzoJob, token string, cli if err := os.MkdirAll(job.OutputPath, 0755); err != nil { return fmt.Errorf("error creating folder: %v", err) } - var totalDownloaded int64 for _, file := range files { fileID := file["id"].(string) @@ -68,8 +68,7 @@ func (d *GDriveDownloader) downloadFolder(job *utils.DanzoJob, token string, cli } } }(progressCh) - - config := utils.DownloadConfig{ + config := utils.HTTPDownloadConfig{ URL: fmt.Sprintf("https://drive.google.com/file/d/%s/view", fileID), OutputPath: outputPath, HTTPClientConfig: job.HTTPClientConfig, @@ -82,7 +81,7 @@ func (d *GDriveDownloader) downloadFolder(job *utils.DanzoJob, token string, cli return nil } -func performGDriveDownload(config utils.DownloadConfig, token string, fileID string, client *utils.DanzoHTTPClient, progressCh chan<- int64) error { +func performGDriveDownload(config utils.HTTPDownloadConfig, token string, fileID string, client *utils.DanzoHTTPClient, progressCh chan<- int64) error { outputDir := filepath.Dir(config.OutputPath) if err := os.MkdirAll(outputDir, 0755); err != nil { return fmt.Errorf("error creating output directory: %v", err) @@ -95,6 +94,7 @@ func performGDriveDownload(config utils.DownloadConfig, token string, fileID str } else { downloadURL = fmt.Sprintf("%s/%s?alt=media&key=%s", driveAPIURL, fileID, token) } + log.Debug().Str("op", "google-drive/download").Msgf("performing simple http download for %s", downloadURL) err := danzohttp.PerformSimpleDownload(downloadURL, config.OutputPath, client, progressCh) if err != nil { return fmt.Errorf("error downloading Google Drive file: %v", err) diff --git a/internal/downloaders/gdrive/helpers.go b/internal/downloaders/google-drive/helpers.go similarity index 85% rename from internal/downloaders/gdrive/helpers.go rename to internal/downloaders/google-drive/helpers.go index f9907d6..304add5 100644 --- a/internal/downloaders/gdrive/helpers.go +++ b/internal/downloaders/google-drive/helpers.go @@ -8,6 +8,7 @@ import ( "regexp" "strings" + "github.com/rs/zerolog/log" "github.com/tanq16/danzo/internal/utils" ) @@ -44,8 +45,10 @@ func extractFileID(rawURL string) (string, error) { func getFileMetadata(rawURL string, client *utils.DanzoHTTPClient, token string) (map[string]any, string, error) { fileID, err := extractFileID(rawURL) if err != nil { + log.Error().Str("op", "google-drive/helpers").Msgf("error extracting file ID: %v", err) return nil, "", fmt.Errorf("error extracting file ID: %v", err) } + log.Debug().Str("op", "google-drive/helpers").Msgf("extracted file ID: %s", fileID) isOAuth := !strings.HasPrefix(token, "AIza") var metadataURL string if isOAuth { @@ -68,6 +71,7 @@ func getFileMetadata(rawURL string, client *utils.DanzoHTTPClient, token string) } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { + log.Error().Str("op", "google-drive/helpers").Msgf("failed to get file metadata, status: %d", resp.StatusCode) return nil, "", fmt.Errorf("failed to get file metadata, status: %d", resp.StatusCode) } var metadata map[string]any @@ -75,6 +79,7 @@ func getFileMetadata(rawURL string, client *utils.DanzoHTTPClient, token string) if err != nil { return nil, "", fmt.Errorf("error parsing metadata response: %v", err) } + log.Debug().Str("op", "google-drive/helpers").Msgf("file metadata retrieved") return metadata, fileID, nil } @@ -82,7 +87,7 @@ func listFolderContents(folderID, token string, client *utils.DanzoHTTPClient) ( var files []map[string]any pageToken := "" isOAuth := !strings.HasPrefix(token, "AIza") - + log.Debug().Str("op", "google-drive/helpers").Msgf("listing folder contents for %s", folderID) for { var url string if isOAuth { @@ -121,6 +126,7 @@ func listFolderContents(folderID, token string, client *utils.DanzoHTTPClient) ( return nil, err } if items, ok := result["files"].([]any); ok { + log.Debug().Str("op", "google-drive/helpers").Msgf("listing %d items", len(items)) for _, item := range items { if fileMap, ok := item.(map[string]any); ok { files = append(files, fileMap) @@ -130,6 +136,7 @@ func listFolderContents(folderID, token string, client *utils.DanzoHTTPClient) ( if nextToken, ok := result["nextPageToken"].(string); ok && nextToken != "" { pageToken = nextToken + log.Debug().Str("op", "google-drive/helpers").Msgf("listing next page") } else { break } diff --git a/internal/downloaders/gdrive/initial.go b/internal/downloaders/google-drive/initial.go similarity index 78% rename from internal/downloaders/gdrive/initial.go rename to internal/downloaders/google-drive/initial.go index 36ac13d..aab32fc 100644 --- a/internal/downloaders/gdrive/initial.go +++ b/internal/downloaders/google-drive/initial.go @@ -5,6 +5,7 @@ import ( "os" "strconv" + "github.com/rs/zerolog/log" "github.com/tanq16/danzo/internal/utils" ) @@ -30,6 +31,7 @@ func (d *GDriveDownloader) ValidateJob(job *utils.DanzoJob) error { return fmt.Errorf("credentials file not found: %v", err) } } + log.Info().Str("op", "google-drive/initial").Msgf("job validated for %s", job.URL) return nil } @@ -37,17 +39,19 @@ func (d *GDriveDownloader) BuildJob(job *utils.DanzoJob) error { fileID := job.Metadata["fileID"].(string) var token string var err error - if apiKey, ok := job.Metadata["apiKey"].(string); ok { + log.Debug().Str("op", "google-drive/initial").Msgf("using API key") token = apiKey } else if credFile, ok := job.Metadata["credentialsFile"].(string); ok { job.PauseFunc() + log.Debug().Str("op", "google-drive/initial").Msgf("using credentials file") token, err = getAccessTokenFromCredentials(credFile) job.ResumeFunc() if err != nil { return fmt.Errorf("error getting OAuth token: %v", err) } } + log.Debug().Str("op", "google-drive/initial").Msgf("token retrieved") job.Metadata["token"] = token client := utils.NewDanzoHTTPClient(job.HTTPClientConfig) @@ -55,18 +59,18 @@ func (d *GDriveDownloader) BuildJob(job *utils.DanzoJob) error { if err != nil { return fmt.Errorf("error getting metadata: %v", err) } + log.Debug().Str("op", "google-drive/initial").Msgf("retrieved item metadata") // Check if it's a folder mimeType, _ := metadata["mimeType"].(string) if mimeType == "application/vnd.google-apps.folder" { job.Metadata["isFolder"] = true + log.Debug().Str("op", "google-drive/initial").Msgf("detected folder, listing contents") files, err := listFolderContents(fileID, token, client) if err != nil { return fmt.Errorf("error listing folder contents: %v", err) } job.Metadata["folderFiles"] = files - - // Calculate total size var totalSize int64 for _, file := range files { if size, ok := file["size"].(string); ok { @@ -76,13 +80,13 @@ func (d *GDriveDownloader) BuildJob(job *utils.DanzoJob) error { } } job.Metadata["totalSize"] = totalSize - // Set output path as folder name + log.Debug().Str("op", "google-drive/initial").Msgf("recorded total size as %v", totalSize) if job.OutputPath == "" { job.OutputPath = metadata["name"].(string) } } else { + log.Debug().Str("op", "google-drive/initial").Msgf("detected file") job.Metadata["isFolder"] = false - // Single file if job.OutputPath == "" { job.OutputPath = metadata["name"].(string) } @@ -91,8 +95,6 @@ func (d *GDriveDownloader) BuildJob(job *utils.DanzoJob) error { job.Metadata["totalSize"] = size } } - - // Check if output exists if info, err := os.Stat(job.OutputPath); err == nil { if job.Metadata["isFolder"].(bool) && info.IsDir() { job.OutputPath = utils.RenewOutputPath(job.OutputPath) @@ -100,5 +102,6 @@ func (d *GDriveDownloader) BuildJob(job *utils.DanzoJob) error { job.OutputPath = utils.RenewOutputPath(job.OutputPath) } } + log.Info().Str("op", "google-drive/initial").Msgf("job built for gdrive %s", fileID) return nil } diff --git a/internal/downloaders/http/initial.go b/internal/downloaders/http/initial.go index 5462b66..3960ae5 100644 --- a/internal/downloaders/http/initial.go +++ b/internal/downloaders/http/initial.go @@ -12,6 +12,7 @@ import ( "strings" "time" + "github.com/rs/zerolog/log" "github.com/tanq16/danzo/internal/utils" ) @@ -25,22 +26,20 @@ func (d *HTTPDownloader) ValidateJob(job *utils.DanzoJob) error { if parsedURL.Scheme != "http" && parsedURL.Scheme != "https" { return fmt.Errorf("unsupported scheme: %s", parsedURL.Scheme) } - client := utils.NewDanzoHTTPClient(job.HTTPClientConfig) - req, err := http.NewRequest("HEAD", job.URL, nil) if err != nil { return fmt.Errorf("error creating request: %v", err) } - + log.Debug().Str("op", "http/initial").Msgf("Sending HEAD request to %s", job.URL) resp, err := client.Do(req) if err != nil { return fmt.Errorf("error checking URL: %v", err) } defer resp.Body.Close() - if resp.StatusCode == http.StatusMovedPermanently || resp.StatusCode == http.StatusFound { if location := resp.Header.Get("Location"); location != "" { + log.Debug().Str("op", "http/initial").Msgf("URL redirected to %s", location) job.URL = location } } else if resp.StatusCode == http.StatusNotFound { @@ -48,19 +47,18 @@ func (d *HTTPDownloader) ValidateJob(job *utils.DanzoJob) error { } else if resp.StatusCode >= 400 { return fmt.Errorf("server returned error: %d", resp.StatusCode) } - + log.Info().Str("op", "http/initial").Msgf("job validated for %s", job.URL) return nil } func (d *HTTPDownloader) BuildJob(job *utils.DanzoJob) error { job.HTTPClientConfig.HighThreadMode = job.Connections > 5 - client := utils.NewDanzoHTTPClient(job.HTTPClientConfig) - fileSize, fileName, err := getFileInfo(job.URL, client) if err != nil && err != utils.ErrRangeRequestsNotSupported { return fmt.Errorf("error getting file info: %v", err) } + log.Debug().Str("op", "http/initial").Msgf("File info retrieved: size=%d, name=%s, rangeSupported=%v", fileSize, fileName, err != utils.ErrRangeRequestsNotSupported) if job.OutputPath == "" && fileName != "" { job.OutputPath = fileName @@ -73,28 +71,24 @@ func (d *HTTPDownloader) BuildJob(job *utils.DanzoJob) error { } } - // Check existing file if existingFile, err := os.Stat(job.OutputPath); err == nil { if fileSize > 0 && existingFile.Size() == fileSize { return fmt.Errorf("file already exists with same size") } job.OutputPath = utils.RenewOutputPath(job.OutputPath) + log.Debug().Str("op", "http/initial").Msgf("Output path renewed to %s", job.OutputPath) } - job.Metadata["fileSize"] = fileSize job.Metadata["rangeSupported"] = err != utils.ErrRangeRequestsNotSupported - + log.Info().Str("op", "http/initial").Msgf("job built for %s", job.URL) return nil } func (d *HTTPDownloader) Download(job *utils.DanzoJob) error { client := utils.NewDanzoHTTPClient(job.HTTPClientConfig) - fileSize, _ := job.Metadata["fileSize"].(int64) rangeSupported, _ := job.Metadata["rangeSupported"].(bool) - progressCh := make(chan int64, 100) - progressDone := make(chan struct{}) startTime := time.Now() @@ -103,15 +97,12 @@ func (d *HTTPDownloader) Download(job *utils.DanzoJob) error { var totalDownloaded int64 var lastUpdate time.Time var lastBytes int64 - ticker := time.NewTicker(100 * time.Millisecond) defer ticker.Stop() - for { select { case bytes, ok := <-progressCh: if !ok { - // Final update when channel closes if job.ProgressFunc != nil { job.ProgressFunc(totalDownloaded, fileSize) } @@ -120,20 +111,16 @@ func (d *HTTPDownloader) Download(job *utils.DanzoJob) error { totalDownloaded += bytes case <-ticker.C: - // Periodic update for smooth progress display if totalDownloaded > lastBytes { if job.ProgressFunc != nil { job.ProgressFunc(totalDownloaded, fileSize) } - - // Calculate and store speed elapsed := time.Since(lastUpdate).Seconds() if elapsed > 0 { speed := float64(totalDownloaded-lastBytes) / elapsed job.Metadata["downloadSpeed"] = speed job.Metadata["elapsedTime"] = time.Since(startTime).Seconds() } - lastUpdate = time.Now() lastBytes = totalDownloaded } @@ -141,18 +128,16 @@ func (d *HTTPDownloader) Download(job *utils.DanzoJob) error { } }() - // Perform download var err error - - // Decide download strategy if !rangeSupported || job.Connections == 1 { + log.Debug().Str("op", "http/initial").Msg("Using simple downloader (range not supported or 1 connection)") err = PerformSimpleDownload(job.URL, job.OutputPath, client, progressCh) } else if fileSize/int64(job.Connections) < 2*utils.DefaultBufferSize { - // Chunk size would be too small, use simple download + log.Debug().Str("op", "http/initial").Msg("Using simple downloader (chunk size too small)") err = PerformSimpleDownload(job.URL, job.OutputPath, client, progressCh) } else { - // Use multi-connection download - config := utils.DownloadConfig{ + log.Debug().Str("op", "http/initial").Msg("Using multi-chunk downloader") + config := utils.HTTPDownloadConfig{ URL: job.URL, OutputPath: job.OutputPath, Connections: job.Connections, @@ -165,10 +150,8 @@ func (d *HTTPDownloader) Download(job *utils.DanzoJob) error { // close(progressCh) <-progressDone - // Store final statistics job.Metadata["totalDownloaded"] = fileSize job.Metadata["totalTime"] = time.Since(startTime).Seconds() - return err } diff --git a/internal/downloaders/http/multi-chunk-handlers.go b/internal/downloaders/http/multi-chunk-handlers.go index 7939c26..24ff7f9 100644 --- a/internal/downloaders/http/multi-chunk-handlers.go +++ b/internal/downloaders/http/multi-chunk-handlers.go @@ -10,10 +10,11 @@ import ( "sync" "time" + "github.com/rs/zerolog/log" "github.com/tanq16/danzo/internal/utils" ) -func chunkedDownload(job *utils.DownloadJob, chunk *utils.DownloadChunk, client *utils.DanzoHTTPClient, wg *sync.WaitGroup, progressCh chan<- int64, mutex *sync.Mutex) { +func chunkedDownload(job *utils.HTTPDownloadJob, chunk *utils.HTTPDownloadChunk, client *utils.DanzoHTTPClient, wg *sync.WaitGroup, progressCh chan<- int64, mutex *sync.Mutex) { defer wg.Done() tempDir := filepath.Join(filepath.Dir(job.Config.OutputPath), ".danzo-temp") tempFileName := filepath.Join(tempDir, fmt.Sprintf("%s.part%d", filepath.Base(job.Config.OutputPath), chunk.ID)) @@ -22,6 +23,7 @@ func chunkedDownload(job *utils.DownloadJob, chunk *utils.DownloadChunk, client if fileInfo, err := os.Stat(tempFileName); err == nil { resumeOffset = fileInfo.Size() if resumeOffset == expectedSize { + log.Debug().Str("op", "http/multi-chunk-handlers").Msgf("Chunk %d already completed", chunk.ID) mutex.Lock() job.TempFiles = append(job.TempFiles, tempFileName) mutex.Unlock() @@ -30,6 +32,7 @@ func chunkedDownload(job *utils.DownloadJob, chunk *utils.DownloadChunk, client progressCh <- resumeOffset return } else if resumeOffset > 0 && resumeOffset < expectedSize { + log.Debug().Str("op", "http/multi-chunk-handlers").Msgf("Resuming chunk %d from %d bytes", chunk.ID, resumeOffset) } else if chunk.Downloaded > 0 { os.Remove(tempFileName) resumeOffset = 0 @@ -38,6 +41,7 @@ func chunkedDownload(job *utils.DownloadJob, chunk *utils.DownloadChunk, client maxRetries := 5 for retry := range maxRetries { if retry > 0 { + log.Warn().Str("op", "http/multi-chunk-handlers").Msgf("Retrying chunk %d (attempt %d/%d)", chunk.ID, retry+1, maxRetries) time.Sleep(time.Duration(retry+1) * 500 * time.Millisecond) // Backoff if fileInfo, err := os.Stat(tempFileName); err == nil { currentSize := fileInfo.Size() @@ -50,18 +54,21 @@ func chunkedDownload(job *utils.DownloadJob, chunk *utils.DownloadChunk, client } } if err := downloadSingleChunk(job, chunk, client, tempFileName, progressCh, resumeOffset); err != nil { + log.Error().Str("op", "http/multi-chunk-handlers").Err(err).Msgf("Failed to download chunk %d", chunk.ID) continue } // On success + log.Debug().Str("op", "http/multi-chunk-handlers").Msgf("Chunk %d download successful", chunk.ID) mutex.Lock() job.TempFiles = append(job.TempFiles, tempFileName) mutex.Unlock() chunk.Completed = true return } + log.Error().Str("op", "http/multi-chunk-handlers").Msgf("Chunk %d failed after %d retries", chunk.ID, maxRetries) } -func downloadSingleChunk(job *utils.DownloadJob, chunk *utils.DownloadChunk, client *utils.DanzoHTTPClient, tempFileName string, progressCh chan<- int64, resumeOffset int64) error { +func downloadSingleChunk(job *utils.HTTPDownloadJob, chunk *utils.HTTPDownloadChunk, client *utils.DanzoHTTPClient, tempFileName string, progressCh chan<- int64, resumeOffset int64) error { flag := os.O_WRONLY | os.O_CREATE if resumeOffset > 0 { flag |= os.O_APPEND diff --git a/internal/downloaders/http/multi-downloader.go b/internal/downloaders/http/multi-downloader.go index bf7df6a..9f84e62 100644 --- a/internal/downloaders/http/multi-downloader.go +++ b/internal/downloaders/http/multi-downloader.go @@ -11,11 +11,12 @@ import ( "sync" "time" + "github.com/rs/zerolog/log" "github.com/tanq16/danzo/internal/utils" ) -func PerformMultiDownload(config utils.DownloadConfig, client *utils.DanzoHTTPClient, fileSize int64, progressCh chan<- int64) error { - job := utils.DownloadJob{ +func PerformMultiDownload(config utils.HTTPDownloadConfig, client *utils.DanzoHTTPClient, fileSize int64, progressCh chan<- int64) error { + job := utils.HTTPDownloadJob{ Config: config, FileSize: fileSize, StartTime: time.Now(), @@ -24,6 +25,7 @@ func PerformMultiDownload(config utils.DownloadConfig, client *utils.DanzoHTTPCl if err := os.MkdirAll(tempDir, 0755); err != nil { return fmt.Errorf("error creating temp directory: %v", err) } + log.Debug().Str("op", "http/multi-downloader").Msgf("Temporary directory created at %s", tempDir) // Setup chunks mutex := &sync.Mutex{} @@ -39,7 +41,7 @@ func PerformMultiDownload(config utils.DownloadConfig, client *utils.DanzoHTTPCl endByte = fileSize - 1 } if endByte >= startByte { - job.Chunks = append(job.Chunks, utils.DownloadChunk{ + job.Chunks = append(job.Chunks, utils.HTTPDownloadChunk{ ID: i, StartByte: startByte, EndByte: endByte, @@ -47,6 +49,7 @@ func PerformMultiDownload(config utils.DownloadConfig, client *utils.DanzoHTTPCl } currentPosition = endByte + 1 } + log.Debug().Str("op", "http/multi-downloader").Msgf("Created %d chunks for download", len(job.Chunks)) // Start connection goroutines var wg sync.WaitGroup @@ -56,6 +59,7 @@ func PerformMultiDownload(config utils.DownloadConfig, client *utils.DanzoHTTPCl } // Wait for all downloads to complete + log.Debug().Str("op", "http/multi-downloader").Msg("Waiting for all chunks to download") wg.Wait() close(progressCh) allCompleted := true @@ -71,14 +75,16 @@ func PerformMultiDownload(config utils.DownloadConfig, client *utils.DanzoHTTPCl } // Assemble the file + log.Info().Str("op", "http/multi-downloader").Msg("All chunks downloaded, assembling file") err := assembleFile(job) if err != nil { return fmt.Errorf("error assembling file: %v", err) } + log.Info().Str("op", "http/multi-downloader").Msg("File assembled successfully") return nil } -func assembleFile(job utils.DownloadJob) error { +func assembleFile(job utils.HTTPDownloadJob) error { allChunksCompleted := true for _, chunk := range job.Chunks { if !chunk.Completed { @@ -98,9 +104,6 @@ func assembleFile(job utils.DownloadJob) error { } return idI < idJ }) - // for _, file := range tempFiles { - // chunkID, _ := extractChunkID(file) - // } destFile, err := os.Create(job.Config.OutputPath) if err != nil { return err @@ -132,8 +135,10 @@ func assembleFile(job utils.DownloadJob) error { if totalWritten != job.FileSize { return fmt.Errorf("error: total written bytes (%d) doesn't match expected file size (%d)", totalWritten, job.FileSize) } + log.Debug().Str("op", "http/multi-downloader").Msgf("Successfully wrote %d bytes to %s", totalWritten, job.Config.OutputPath) // Cleanup temporary files + log.Debug().Str("op", "http/multi-downloader").Msg("Cleaning up temporary chunk files") for _, tempFilePath := range tempFiles { os.Remove(tempFilePath) } diff --git a/internal/downloaders/http/simple-downloader.go b/internal/downloaders/http/simple-downloader.go index eaf4522..75c2d93 100644 --- a/internal/downloaders/http/simple-downloader.go +++ b/internal/downloaders/http/simple-downloader.go @@ -6,25 +6,52 @@ import ( "net/http" "os" "path/filepath" + "time" + "github.com/rs/zerolog/log" "github.com/tanq16/danzo/internal/utils" ) func PerformSimpleDownload(url, outputPath string, client *utils.DanzoHTTPClient, progressCh chan<- int64) error { + defer close(progressCh) tempDir := filepath.Join(filepath.Dir(outputPath), ".danzo-temp") if err := os.MkdirAll(tempDir, 0755); err != nil { return fmt.Errorf("error creating temp directory: %v", err) } tempOutputPath := fmt.Sprintf("%s.part", filepath.Join(tempDir, filepath.Base(outputPath))) + maxRetries := 5 + var lastErr error + for retry := range maxRetries { + if retry > 0 { + log.Warn().Str("op", "http/simple-downloader").Msgf("Retrying download for %s (attempt %d/%d)", outputPath, retry+1, maxRetries) + time.Sleep(time.Duration(retry+1) * 500 * time.Millisecond) // Exponential backoff + } + err := downloadAttempt(url, tempOutputPath, client, progressCh) + if err != nil { + lastErr = err + log.Error().Str("op", "http/simple-downloader").Err(err).Msgf("Download attempt %d failed", retry+1) + continue + } + if err := os.Rename(tempOutputPath, outputPath); err != nil { + return fmt.Errorf("error renaming (finalizing) output file: %v", err) + } + log.Info().Str("op", "http/simple-downloader").Msgf("Simple download successful for %s", outputPath) + return nil + } + return fmt.Errorf("download failed after %d retries: %w", maxRetries, lastErr) +} + +func downloadAttempt(url, tempOutputPath string, client *utils.DanzoHTTPClient, progressCh chan<- int64) error { var resumeOffset int64 = 0 - var fileMode int = os.O_CREATE | os.O_WRONLY + fileMode := os.O_CREATE | os.O_WRONLY if fileInfo, err := os.Stat(tempOutputPath); err == nil { resumeOffset = fileInfo.Size() fileMode |= os.O_APPEND } else { fileMode |= os.O_TRUNC } + outFile, err := os.OpenFile(tempOutputPath, fileMode, 0644) if err != nil { return fmt.Errorf("error creating output file: %v", err) @@ -38,6 +65,7 @@ func PerformSimpleDownload(url, outputPath string, client *utils.DanzoHTTPClient if resumeOffset > 0 { req.Header.Set("Range", fmt.Sprintf("bytes=%d-", resumeOffset)) + log.Debug().Str("op", "http/simple-downloader").Msgf("Resuming download from offset %d", resumeOffset) } req.Header.Set("Connection", "keep-alive") resp, err := client.Do(req) @@ -45,16 +73,16 @@ func PerformSimpleDownload(url, outputPath string, client *utils.DanzoHTTPClient return fmt.Errorf("error executing GET request: %v", err) } defer resp.Body.Close() - defer close(progressCh) if resumeOffset > 0 { if resp.StatusCode != http.StatusPartialContent { + log.Warn().Str("op", "http/simple-downloader").Msgf("Server does not support resume (status %d). Restarting download.", resp.StatusCode) + // Reset and restart download from scratch outFile.Close() outFile, err = os.OpenFile(tempOutputPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644) if err != nil { return fmt.Errorf("error creating output file: %v", err) } - defer outFile.Close() resumeOffset = 0 } else { progressCh <- resumeOffset @@ -63,33 +91,22 @@ func PerformSimpleDownload(url, outputPath string, client *utils.DanzoHTTPClient return fmt.Errorf("unexpected status code: %d", resp.StatusCode) } buffer := make([]byte, utils.DefaultBufferSize) - var newBytes int64 = 0 - var totalDownloaded int64 = resumeOffset for { - bytesRead, err := resp.Body.Read(buffer) + bytesRead, readErr := resp.Body.Read(buffer) if bytesRead > 0 { _, writeErr := outFile.Write(buffer[:bytesRead]) if writeErr != nil { return fmt.Errorf("error writing to output file: %v", writeErr) } - newBytes += int64(bytesRead) - totalDownloaded += int64(bytesRead) progressCh <- int64(bytesRead) } - if err != nil { - if err == io.EOF { + if readErr != nil { + if readErr == io.EOF { break } - return fmt.Errorf("error reading response body: %v", err) + return fmt.Errorf("error reading response body: %v", readErr) } } - // Ensure file is synced and closed (while auto-handled in Unix, Windows needs this) outFile.Sync() - if err := outFile.Close(); err != nil { - return fmt.Errorf("error closing output file: %v", err) - } - if err := os.Rename(tempOutputPath, outputPath); err != nil { - return fmt.Errorf("error renaming (finalizing) output file: %v", err) - } return nil } diff --git a/internal/downloaders/m3u8/download.go b/internal/downloaders/live-stream/download.go similarity index 80% rename from internal/downloaders/m3u8/download.go rename to internal/downloaders/live-stream/download.go index 3903114..95c3d21 100644 --- a/internal/downloaders/m3u8/download.go +++ b/internal/downloaders/live-stream/download.go @@ -8,6 +8,7 @@ import ( "sync" "sync/atomic" + "github.com/rs/zerolog/log" "github.com/tanq16/danzo/internal/utils" ) @@ -17,29 +18,25 @@ func (d *M3U8Downloader) Download(job *utils.DanzoJob) error { return fmt.Errorf("error creating temp directory: %v", err) } defer os.RemoveAll(tempDir) - client := utils.NewDanzoHTTPClient(job.HTTPClientConfig) - - // Get manifest content + log.Debug().Str("op", "live-stream/download").Msgf("Fetching manifest from %s", job.URL) manifestContent, err := getM3U8Contents(job.URL, client) if err != nil { return fmt.Errorf("error fetching manifest: %v", err) } - - // Process manifest and get segment URLs segmentURLs, err := processM3U8Content(manifestContent, job.URL, client) if err != nil { return fmt.Errorf("error processing manifest: %v", err) } - if len(segmentURLs) == 0 { return fmt.Errorf("no segments found in manifest") } + log.Info().Str("op", "live-stream/download").Msgf("Found %d segments to download", len(segmentURLs)) - // Calculate total size by getting actual sizes of all segments totalSize, segmentSizes, err := calculateTotalSize(segmentURLs, job.Connections, client) if err != nil { // Fallback to estimate + log.Warn().Str("op", "live-stream/download").Msgf("Could not calculate total size accurately: %v. Using estimate.", err) totalSize = int64(len(segmentURLs)) * 1024 * 1024 // 1MB per segment estimate segmentSizes = make([]int64, len(segmentURLs)) for i := range segmentSizes { @@ -48,18 +45,18 @@ func (d *M3U8Downloader) Download(job *utils.DanzoJob) error { } job.Metadata["totalSize"] = totalSize job.Metadata["segmentSizes"] = segmentSizes + log.Debug().Str("op", "live-stream/download").Msgf("Total estimated size: %s", utils.FormatBytes(uint64(totalSize))) - // Download segments in parallel + log.Info().Str("op", "live-stream/download").Msg("Starting parallel download of segments") segmentFiles, err := downloadSegmentsParallel(segmentURLs, tempDir, job.Connections, client, job.ProgressFunc, totalSize) if err != nil { return fmt.Errorf("error downloading segments: %v", err) } - - // Merge segments + log.Info().Str("op", "live-stream/download").Msg("All segments downloaded, merging with ffmpeg") if err := mergeSegments(segmentFiles, job.OutputPath); err != nil { return fmt.Errorf("error merging segments: %v", err) } - + log.Info().Str("op", "live-stream/download").Msg("Segments merged successfully") return nil } @@ -68,31 +65,24 @@ func downloadSegmentsParallel(segmentURLs []string, outputDir string, numWorkers var mu sync.Mutex var totalDownloaded int64 var downloadErr error - - // Create segment jobs type segmentJob struct { index int url string } - jobCh := make(chan segmentJob, len(segmentURLs)) for i, url := range segmentURLs { jobCh <- segmentJob{index: i, url: url} } close(jobCh) - - // Pre-create ordered list downloadedFiles = make([]string, len(segmentURLs)) - // Start workers var wg sync.WaitGroup - for i := 0; i < numWorkers; i++ { + for range numWorkers { wg.Add(1) go func() { defer wg.Done() for job := range jobCh { outputPath := filepath.Join(outputDir, fmt.Sprintf("segment_%04d.ts", job.index)) - size, err := downloadSegment(job.url, outputPath, client) if err != nil { mu.Lock() @@ -102,12 +92,9 @@ func downloadSegmentsParallel(segmentURLs []string, outputDir string, numWorkers mu.Unlock() return } - mu.Lock() downloadedFiles[job.index] = outputPath mu.Unlock() - - // Update progress downloaded := atomic.AddInt64(&totalDownloaded, size) if progressFunc != nil { progressFunc(downloaded, totalSize) @@ -117,11 +104,9 @@ func downloadSegmentsParallel(segmentURLs []string, outputDir string, numWorkers } wg.Wait() - if downloadErr != nil { return nil, downloadErr } - return downloadedFiles, nil } @@ -132,12 +117,10 @@ func mergeSegments(segmentFiles []string, outputPath string) error { return fmt.Errorf("error creating segment list file: %v", err) } defer os.Remove(tempListFile) - for _, file := range segmentFiles { fmt.Fprintf(f, "file '%s'\n", file) } f.Close() - cmd := exec.Command( "ffmpeg", "-f", "concat", @@ -147,11 +130,10 @@ func mergeSegments(segmentFiles []string, outputPath string) error { "-y", outputPath, ) - + log.Debug().Str("op", "live-stream/download").Msgf("Executing ffmpeg command: %s", cmd.String()) output, err := cmd.CombinedOutput() if err != nil { return fmt.Errorf("ffmpeg error: %v\nOutput: %s", err, string(output)) } - return nil } diff --git a/internal/downloaders/m3u8/helpers.go b/internal/downloaders/live-stream/helpers.go similarity index 92% rename from internal/downloaders/m3u8/helpers.go rename to internal/downloaders/live-stream/helpers.go index 4cfdffd..93a507d 100644 --- a/internal/downloaders/m3u8/helpers.go +++ b/internal/downloaders/live-stream/helpers.go @@ -10,6 +10,7 @@ import ( "strings" "sync" + "github.com/rs/zerolog/log" "github.com/tanq16/danzo/internal/utils" ) @@ -18,22 +19,19 @@ func getM3U8Contents(manifestURL string, client *utils.DanzoHTTPClient) (string, if err != nil { return "", fmt.Errorf("error creating request: %v", err) } - resp, err := client.Do(req) if err != nil { return "", fmt.Errorf("error fetching m3u8 manifest: %v", err) } defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { return "", fmt.Errorf("server returned status code %d", resp.StatusCode) } - content, err := io.ReadAll(resp.Body) if err != nil { return "", fmt.Errorf("error reading manifest content: %v", err) } - + log.Debug().Str("op", "live-stream/helpers").Msgf("Successfully read manifest from %s", manifestURL) return string(content), nil } @@ -42,7 +40,6 @@ func processM3U8Content(content, manifestURL string, client *utils.DanzoHTTPClie if err != nil { return nil, fmt.Errorf("error parsing manifest URL: %v", err) } - scanner := bufio.NewScanner(strings.NewReader(content)) var segmentURLs []string var masterPlaylistURLs []string @@ -50,22 +47,18 @@ func processM3U8Content(content, manifestURL string, client *utils.DanzoHTTPClie for scanner.Scan() { line := strings.TrimSpace(scanner.Text()) - if line == "" || (strings.HasPrefix(line, "#") && !strings.Contains(line, "#EXT-X-STREAM-INF")) { continue } - if strings.Contains(line, "#EXT-X-STREAM-INF") { isMasterPlaylist = true continue } - if !strings.HasPrefix(line, "#") { segmentURL, err := resolveURL(baseURL, line) if err != nil { return nil, fmt.Errorf("error resolving URL: %v", err) } - if isMasterPlaylist { masterPlaylistURLs = append(masterPlaylistURLs, segmentURL) } else { @@ -73,20 +66,19 @@ func processM3U8Content(content, manifestURL string, client *utils.DanzoHTTPClie } } } - if err := scanner.Err(); err != nil { return nil, fmt.Errorf("error scanning m3u8 content: %v", err) } // For master playlist, fetch first playlist (highest quality) if isMasterPlaylist && len(masterPlaylistURLs) > 0 { + log.Debug().Str("op", "live-stream/helpers").Msgf("Detected master playlist, fetching sub-playlist: %s", masterPlaylistURLs[0]) subContent, err := getM3U8Contents(masterPlaylistURLs[0], client) if err != nil { return nil, fmt.Errorf("error fetching sub-playlist: %v", err) } return processM3U8Content(subContent, masterPlaylistURLs[0], client) } - return segmentURLs, nil } @@ -94,12 +86,10 @@ func resolveURL(baseURL *url.URL, urlStr string) (string, error) { if strings.HasPrefix(urlStr, "http://") || strings.HasPrefix(urlStr, "https://") { return urlStr, nil } - relURL, err := url.Parse(urlStr) if err != nil { return "", err } - absURL := baseURL.ResolveReference(relURL) return absURL.String(), nil } @@ -109,20 +99,18 @@ func calculateTotalSize(segmentURLs []string, numWorkers int, client *utils.Danz var totalSize int64 var mu sync.Mutex var sizeErr error - type sizeJob struct { index int url string } - jobCh := make(chan sizeJob, len(segmentURLs)) for i, url := range segmentURLs { jobCh <- sizeJob{index: i, url: url} } close(jobCh) - + log.Debug().Str("op", "live-stream/helpers").Msg("Calculating total size of all segments") var wg sync.WaitGroup - for i := 0; i < numWorkers; i++ { + for range numWorkers { wg.Add(1) go func() { defer wg.Done() @@ -136,7 +124,6 @@ func calculateTotalSize(segmentURLs []string, numWorkers int, client *utils.Danz mu.Unlock() continue } - mu.Lock() segmentSizes[job.index] = size totalSize += size @@ -144,13 +131,10 @@ func calculateTotalSize(segmentURLs []string, numWorkers int, client *utils.Danz } }() } - wg.Wait() - if sizeErr != nil { return 0, nil, sizeErr } - return totalSize, segmentSizes, nil } @@ -159,22 +143,18 @@ func getSize(url string, client *utils.DanzoHTTPClient) (int64, error) { if err != nil { return 0, err } - resp, err := client.Do(req) if err != nil { return 0, err } defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { return 0, fmt.Errorf("server returned status code %d", resp.StatusCode) } - contentLength := resp.Header.Get("Content-Length") if contentLength == "" { return 0, fmt.Errorf("no content length") } - var size int64 fmt.Sscanf(contentLength, "%d", &size) return size, nil @@ -185,27 +165,22 @@ func downloadSegment(segmentURL, outputPath string, client *utils.DanzoHTTPClien if err != nil { return 0, fmt.Errorf("error creating request: %v", err) } - resp, err := client.Do(req) if err != nil { return 0, fmt.Errorf("error downloading segment: %v", err) } defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { return 0, fmt.Errorf("server returned status code %d", resp.StatusCode) } - outFile, err := os.Create(outputPath) if err != nil { return 0, fmt.Errorf("error creating output file: %v", err) } defer outFile.Close() - written, err := io.Copy(outFile, resp.Body) if err != nil { return 0, fmt.Errorf("error writing segment: %v", err) } - return written, nil } diff --git a/internal/downloaders/m3u8/initial.go b/internal/downloaders/live-stream/initial.go similarity index 83% rename from internal/downloaders/m3u8/initial.go rename to internal/downloaders/live-stream/initial.go index fa3c695..ad14609 100644 --- a/internal/downloaders/m3u8/initial.go +++ b/internal/downloaders/live-stream/initial.go @@ -7,6 +7,7 @@ import ( "path/filepath" "time" + "github.com/rs/zerolog/log" "github.com/tanq16/danzo/internal/utils" ) @@ -20,6 +21,7 @@ func (d *M3U8Downloader) ValidateJob(job *utils.DanzoJob) error { if parsedURL.Scheme != "http" && parsedURL.Scheme != "https" { return fmt.Errorf("unsupported scheme: %s", parsedURL.Scheme) } + log.Info().Str("op", "live-stream/initial").Msgf("job validated for %s", job.URL) return nil } @@ -27,15 +29,11 @@ func (d *M3U8Downloader) BuildJob(job *utils.DanzoJob) error { if job.OutputPath == "" { job.OutputPath = fmt.Sprintf("stream_%s.mp4", time.Now().Format("2006-01-02_15-04")) } - - // Check if output file exists if existingFile, err := os.Stat(job.OutputPath); err == nil && existingFile != nil { job.OutputPath = utils.RenewOutputPath(job.OutputPath) } - - // Create temp directory path tempDir := filepath.Join(filepath.Dir(job.OutputPath), ".danzo-temp", "m3u8_"+time.Now().Format("20060102150405")) job.Metadata["tempDir"] = tempDir - + log.Info().Str("op", "live-stream/initial").Msgf("job built for %s", job.URL) return nil } diff --git a/internal/downloaders/s3/download.go b/internal/downloaders/s3/download.go index 8424a3b..e3cb2f4 100644 --- a/internal/downloaders/s3/download.go +++ b/internal/downloaders/s3/download.go @@ -7,6 +7,7 @@ import ( "sync" "sync/atomic" + "github.com/rs/zerolog/log" "github.com/tanq16/danzo/internal/utils" ) @@ -15,27 +16,23 @@ func (d *S3Downloader) Download(job *utils.DanzoJob) error { key := job.Metadata["key"].(string) fileType := job.Metadata["fileType"].(string) profile := job.Metadata["profile"].(string) - - // Get S3 client s3Client, err := getS3Client(profile) if err != nil { return fmt.Errorf("error creating S3 client: %v", err) } - if fileType == "folder" { + log.Info().Str("op", "s3/download").Msgf("Starting folder download for s3://%s/%s", bucket, key) return d.downloadFolder(job, bucket, key, s3Client) } else { + log.Info().Str("op", "s3/download").Msgf("Starting file download for s3://%s/%s", bucket, key) return d.downloadFile(job, bucket, key, s3Client) } } func (d *S3Downloader) downloadFile(job *utils.DanzoJob, bucket, key string, s3Client *S3Client) error { size := job.Metadata["size"].(int64) - progressCh := make(chan int64, 100) defer close(progressCh) - - // Progress tracking go func() { var totalDownloaded int64 for bytes := range progressCh { @@ -45,47 +42,36 @@ func (d *S3Downloader) downloadFile(job *utils.DanzoJob, bucket, key string, s3C } } }() - return performS3Download(bucket, key, job.OutputPath, s3Client, progressCh) } func (d *S3Downloader) downloadFolder(job *utils.DanzoJob, bucket, prefix string, s3Client *S3Client) error { - // List all objects in folder objects, err := listS3Objects(bucket, prefix, s3Client) if err != nil { return fmt.Errorf("error listing objects: %v", err) } - if len(objects) == 0 { return fmt.Errorf("no objects found in s3://%s/%s", bucket, prefix) } - - // Calculate total size + log.Debug().Str("op", "s3/download").Msgf("Found %d objects to download in folder", len(objects)) var totalSize int64 for _, obj := range objects { totalSize += obj.Size } - // Download objects in parallel var totalDownloaded int64 var mu sync.Mutex var downloadErr error - - // Create download jobs jobCh := make(chan s3Object, len(objects)) for _, obj := range objects { jobCh <- obj } close(jobCh) - - // Start workers - numWorkers := job.Connections - if numWorkers > len(objects) { - numWorkers = len(objects) - } + numWorkers := min(job.Connections, len(objects)) + log.Debug().Str("op", "s3/download").Msgf("Using %d parallel workers for folder download", numWorkers) var wg sync.WaitGroup - for i := 0; i < numWorkers; i++ { + for range numWorkers { wg.Add(1) go func() { defer wg.Done() @@ -94,7 +80,6 @@ func (d *S3Downloader) downloadFolder(job *utils.DanzoJob, bucket, prefix string relPath := strings.TrimPrefix(obj.Key, prefix) relPath = strings.TrimPrefix(relPath, "/") outputPath := filepath.Join(job.OutputPath, relPath) - // Create directory if needed if err := createDirectory(filepath.Dir(outputPath)); err != nil { mu.Lock() @@ -104,11 +89,8 @@ func (d *S3Downloader) downloadFolder(job *utils.DanzoJob, bucket, prefix string mu.Unlock() return } - - // Download individual file - progressCh := make(chan int64, 100) - // Track progress + progressCh := make(chan int64, 100) go func(ch <-chan int64) { for bytes := range ch { downloaded := atomic.AddInt64(&totalDownloaded, bytes) @@ -120,7 +102,6 @@ func (d *S3Downloader) downloadFolder(job *utils.DanzoJob, bucket, prefix string err := performS3Download(bucket, obj.Key, outputPath, s3Client, progressCh) close(progressCh) - if err != nil { mu.Lock() if downloadErr == nil { @@ -132,8 +113,6 @@ func (d *S3Downloader) downloadFolder(job *utils.DanzoJob, bucket, prefix string } }() } - wg.Wait() - return downloadErr } diff --git a/internal/downloaders/s3/helpers.go b/internal/downloaders/s3/helpers.go index 77764af..2317024 100644 --- a/internal/downloaders/s3/helpers.go +++ b/internal/downloaders/s3/helpers.go @@ -10,6 +10,7 @@ import ( "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/config" "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/rs/zerolog/log" "github.com/tanq16/danzo/internal/utils" ) @@ -23,6 +24,7 @@ type s3Object struct { } func getS3Client(profile string) (*S3Client, error) { + log.Debug().Str("op", "s3/helpers").Msgf("Loading AWS config with profile: %s", profile) cfg, err := config.LoadDefaultConfig(context.Background(), config.WithSharedConfigProfile(profile), config.WithRetryMode("adaptive"), @@ -38,52 +40,48 @@ func getS3Client(profile string) (*S3Client, error) { func getS3ObjectInfo(bucket, key string, client *S3Client) (string, int64, error) { // Try HEAD request first + log.Debug().Str("op", "s3/helpers").Msgf("Checking if s3://%s/%s is a file", bucket, key) headObj, err := client.client.HeadObject(context.Background(), &s3.HeadObjectInput{ Bucket: aws.String(bucket), Key: aws.String(key), }) - - if err == nil { - // It's a file + if err == nil { // It's a file size := int64(0) if headObj.ContentLength != nil { size = *headObj.ContentLength } + log.Debug().Str("op", "s3/helpers").Msgf("Object is a file with size %d", size) return "file", size, nil } - // Check if it's a folder by listing with prefix + log.Debug().Str("op", "s3/helpers").Msgf("Checking if s3://%s/%s is a folder", bucket, key) result, err := client.client.ListObjectsV2(context.Background(), &s3.ListObjectsV2Input{ Bucket: aws.String(bucket), Prefix: aws.String(key), MaxKeys: aws.Int32(1), }) - if err != nil { return "", 0, fmt.Errorf("error accessing S3 object: %v", err) } - if len(result.Contents) > 0 || len(result.CommonPrefixes) > 0 { + log.Debug().Str("op", "s3/helpers").Msg("Object is a folder") return "folder", -1, nil } - return "", 0, fmt.Errorf("S3 object not found") } func listS3Objects(bucket, prefix string, client *S3Client) ([]s3Object, error) { + log.Debug().Str("op", "s3/helpers").Msgf("Listing objects in s3://%s with prefix %s", bucket, prefix) var objects []s3Object - paginator := s3.NewListObjectsV2Paginator(client.client, &s3.ListObjectsV2Input{ Bucket: aws.String(bucket), Prefix: aws.String(prefix), }) - for paginator.HasMorePages() { page, err := paginator.NextPage(context.Background()) if err != nil { return nil, fmt.Errorf("error listing objects: %v", err) } - for _, obj := range page.Contents { if obj.Key != nil && obj.Size != nil { // Skip directories (0-byte objects ending with /) @@ -97,12 +95,11 @@ func listS3Objects(bucket, prefix string, client *S3Client) ([]s3Object, error) } } } - return objects, nil } func performS3Download(bucket, key, outputPath string, client *S3Client, progressCh chan<- int64) error { - // Get object + log.Debug().Str("op", "s3/helpers").Msgf("Downloading s3://%s/%s to %s", bucket, key, outputPath) result, err := client.client.GetObject(context.Background(), &s3.GetObjectInput{ Bucket: aws.String(bucket), Key: aws.String(key), @@ -111,8 +108,6 @@ func performS3Download(bucket, key, outputPath string, client *S3Client, progres return fmt.Errorf("error getting object: %v", err) } defer result.Body.Close() - - // Create output file file, err := os.Create(outputPath) if err != nil { return fmt.Errorf("error creating file: %v", err) @@ -137,11 +132,9 @@ func performS3Download(bucket, key, outputPath string, client *S3Client, progres return fmt.Errorf("error reading object: %v", err) } } - return nil } -// Helper functions that might be missing from utils package func directoryExists(path string) (bool, error) { info, err := os.Stat(path) if os.IsNotExist(err) { diff --git a/internal/downloaders/s3/initial.go b/internal/downloaders/s3/initial.go index b28475e..d6979e7 100644 --- a/internal/downloaders/s3/initial.go +++ b/internal/downloaders/s3/initial.go @@ -4,22 +4,20 @@ import ( "fmt" "strings" + "github.com/rs/zerolog/log" "github.com/tanq16/danzo/internal/utils" ) type S3Downloader struct{} func (d *S3Downloader) ValidateJob(job *utils.DanzoJob) error { - // Parse S3 URL - supports both s3://bucket/key and bucket/key formats bucket, key, err := parseS3URL(job.URL) if err != nil { return err } - - // Store parsed values job.Metadata["bucket"] = bucket job.Metadata["key"] = key - + log.Info().Str("op", "s3/initial").Msgf("job validated for s3://%s/%s", bucket, key) return nil } @@ -27,8 +25,6 @@ func (d *S3Downloader) BuildJob(job *utils.DanzoJob) error { bucket := job.Metadata["bucket"].(string) key := job.Metadata["key"].(string) profile := job.Metadata["profile"].(string) - - // Get S3 client with profile s3Client, err := getS3Client(profile) if err != nil { return fmt.Errorf("error creating S3 client: %v", err) @@ -39,57 +35,46 @@ func (d *S3Downloader) BuildJob(job *utils.DanzoJob) error { if err != nil { return fmt.Errorf("error getting S3 object info: %v", err) } - job.Metadata["fileType"] = fileType job.Metadata["size"] = size + log.Debug().Str("op", "s3/initial").Msgf("Determined object type: %s, size: %d", fileType, size) - // Set output path if not specified if job.OutputPath == "" { if fileType == "folder" { - // For folders, use the key as directory name parts := strings.Split(strings.TrimSuffix(key, "/"), "/") job.OutputPath = parts[len(parts)-1] if job.OutputPath == "" { job.OutputPath = bucket } } else { - // For files, use the filename parts := strings.Split(key, "/") job.OutputPath = parts[len(parts)-1] } } - // Check if output already exists if fileType == "folder" { - // For folders, check directory if exists, err := directoryExists(job.OutputPath); err == nil && exists { job.OutputPath = utils.RenewOutputPath(job.OutputPath) } } else { - // For files, check file if exists, err := fileExists(job.OutputPath); err == nil && exists { job.OutputPath = utils.RenewOutputPath(job.OutputPath) } } - + log.Info().Str("op", "s3/initial").Msgf("job built for s3://%s/%s", bucket, key) return nil } func parseS3URL(url string) (string, string, error) { - // Remove s3:// prefix if present url = strings.TrimPrefix(url, "s3://") - - // Split bucket and key parts := strings.SplitN(url, "/", 2) if len(parts) < 1 || parts[0] == "" { return "", "", fmt.Errorf("invalid S3 URL format") } - bucket := parts[0] key := "" if len(parts) > 1 { key = parts[1] } - return bucket, key, nil } diff --git a/internal/downloaders/youtube-music/download.go b/internal/downloaders/youtube-music/download.go index 27f2e84..b6b98cd 100644 --- a/internal/downloaders/youtube-music/download.go +++ b/internal/downloaders/youtube-music/download.go @@ -7,16 +7,14 @@ import ( "os/exec" "strings" + "github.com/rs/zerolog/log" "github.com/tanq16/danzo/internal/utils" ) func (d *YTMusicDownloader) Download(job *utils.DanzoJob) error { ytdlpPath := job.Metadata["ytdlpPath"].(string) ffmpegPath := job.Metadata["ffmpegPath"].(string) - - // Download audio using yt-dlp tempOutput := strings.TrimSuffix(job.OutputPath, ".m4a") - args := []string{ "--progress", "--newline", @@ -29,53 +27,53 @@ func (d *YTMusicDownloader) Download(job *utils.DanzoJob) error { "--no-playlist", job.URL, } - cmd := exec.Command(ytdlpPath, args...) + log.Debug().Str("op", "youtube-music/download").Msgf("Executing yt-dlp command: %s", cmd.String()) stdout, err := cmd.StdoutPipe() if err != nil { + log.Error().Str("op", "youtube-music/download").Err(err).Msg("Error creating stdout pipe") return fmt.Errorf("error creating stdout pipe: %v", err) } - stderr, err := cmd.StderrPipe() if err != nil { + log.Error().Str("op", "youtube-music/download").Err(err).Msg("Error creating stderr pipe") return fmt.Errorf("error creating stderr pipe: %v", err) } - if err := cmd.Start(); err != nil { + log.Error().Str("op", "youtube-music/download").Err(err).Msg("Error starting yt-dlp") return fmt.Errorf("error starting yt-dlp: %v", err) } - // Process output streams go processStream(stdout, job.StreamFunc) go processStream(stderr, job.StreamFunc) - if err := cmd.Wait(); err != nil { + log.Error().Str("op", "youtube-music/download").Err(err).Msg("yt-dlp command failed") return fmt.Errorf("yt-dlp failed: %v", err) } + log.Debug().Str("op", "youtube-music/download").Msgf("yt-dlp audio extraction completed for %s", job.URL) // Apply metadata if music client is specified if musicClient, ok := job.Metadata["musicClient"].(string); ok { musicID := job.Metadata["musicID"].(string) - if job.StreamFunc != nil { job.StreamFunc(fmt.Sprintf("Fetching metadata from %s...", musicClient)) } - // Ensure output path ends with .m4a finalPath := job.OutputPath if !strings.HasSuffix(finalPath, ".m4a") { finalPath = tempOutput + ".m4a" } - + log.Debug().Str("op", "youtube-music/download").Msgf("Applying music metadata from %s", musicClient) err := addMusicMetadata(tempOutput+".m4a", finalPath, musicClient, musicID, job.StreamFunc) if err != nil { + log.Warn().Str("op", "youtube-music/download").Err(err).Msg("Failed to add metadata") if job.StreamFunc != nil { job.StreamFunc(fmt.Sprintf("Warning: Failed to add metadata: %v", err)) } } } - + log.Info().Str("op", "youtube-music/download").Msgf("YouTube music download completed for %s", job.URL) return nil } diff --git a/internal/downloaders/youtube-music/initial.go b/internal/downloaders/youtube-music/initial.go index cf10376..69ddce1 100644 --- a/internal/downloaders/youtube-music/initial.go +++ b/internal/downloaders/youtube-music/initial.go @@ -7,6 +7,7 @@ import ( "path/filepath" "strings" + "github.com/rs/zerolog/log" "github.com/tanq16/danzo/internal/downloaders/youtube" "github.com/tanq16/danzo/internal/utils" ) @@ -14,14 +15,11 @@ import ( type YTMusicDownloader struct{} func (d *YTMusicDownloader) ValidateJob(job *utils.DanzoJob) error { - // Validate YouTube URL if !strings.Contains(job.URL, "youtube.com/watch") && !strings.Contains(job.URL, "youtu.be/") && !strings.Contains(job.URL, "music.youtube.com") { return fmt.Errorf("invalid YouTube URL") } - - // Validate music client if provided if client, ok := job.Metadata["musicClient"].(string); ok { if client != "deezer" && client != "apple" { return fmt.Errorf("unsupported music client: %s", client) @@ -30,26 +28,25 @@ func (d *YTMusicDownloader) ValidateJob(job *utils.DanzoJob) error { return fmt.Errorf("music ID required for %s", client) } } - + log.Info().Str("op", "youtube-music/initial").Msgf("job validated for %s", job.URL) return nil } func (d *YTMusicDownloader) BuildJob(job *utils.DanzoJob) error { - // Check for yt-dlp ytdlpPath, err := youtube.EnsureYtdlp() if err != nil { return fmt.Errorf("error ensuring yt-dlp: %v", err) } job.Metadata["ytdlpPath"] = ytdlpPath + log.Debug().Str("op", "youtube-music/initial").Msgf("Using yt-dlp at: %s", ytdlpPath) - // Check for ffmpeg (required for audio extraction and metadata) ffmpegPath, err := ensureFFmpeg() if err != nil { return fmt.Errorf("error ensuring ffmpeg: %v", err) } job.Metadata["ffmpegPath"] = ffmpegPath + log.Debug().Str("op", "youtube-music/initial").Msgf("Using ffmpeg at: %s", ffmpegPath) - // Set output path if not specified if job.OutputPath == "" { job.OutputPath = "%(title)s.m4a" } else if !strings.HasSuffix(job.OutputPath, ".m4a") { @@ -57,11 +54,11 @@ func (d *YTMusicDownloader) BuildJob(job *utils.DanzoJob) error { job.OutputPath = strings.TrimSuffix(job.OutputPath, filepath.Ext(job.OutputPath)) + ".m4a" } - // Check if output exists if info, err := os.Stat(job.OutputPath); err == nil && !info.IsDir() { job.OutputPath = utils.RenewOutputPath(job.OutputPath) + log.Debug().Str("op", "youtube-music/initial").Msgf("Output path renewed to %s", job.OutputPath) } - + log.Info().Str("op", "youtube-music/initial").Msgf("job built for %s", job.URL) return nil } @@ -70,5 +67,6 @@ func ensureFFmpeg() (string, error) { if err == nil { return path, nil } + log.Error().Str("op", "youtube-music/initial").Msg("ffmpeg not found in PATH. Please install it.") return "", fmt.Errorf("ffmpeg not found in PATH, please install manually") } diff --git a/internal/downloaders/youtube-music/metadata.go b/internal/downloaders/youtube-music/metadata.go index f9f0357..8da6c0d 100644 --- a/internal/downloaders/youtube-music/metadata.go +++ b/internal/downloaders/youtube-music/metadata.go @@ -13,6 +13,7 @@ import ( "time" "github.com/google/uuid" + "github.com/rs/zerolog/log" "github.com/tanq16/danzo/internal/utils" ) @@ -70,26 +71,29 @@ func addMusicMetadata(inputPath, outputPath, musicClient, musicId string, stream func addAppleMetadata(inputPath, outputPath, musicId string, streamFunc func(string)) error { client := utils.NewDanzoHTTPClient(httpConfig) apiURL := fmt.Sprintf("https://itunes.apple.com/lookup?id=%s&entity=song", musicId) - + log.Debug().Str("op", "youtube-music/metadata").Msgf("Fetching iTunes metadata from: %s", apiURL) req, _ := http.NewRequest("GET", apiURL, nil) resp, err := client.Do(req) if err != nil { + log.Error().Str("op", "youtube-music/metadata").Err(err).Msg("Error fetching iTunes metadata") return fmt.Errorf("error fetching metadata: %v", err) } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { + log.Error().Str("op", "youtube-music/metadata").Msgf("iTunes API request failed with status code %d", resp.StatusCode) return fmt.Errorf("API request failed with status code %d", resp.StatusCode) } - var itunesResp ITunesResponse if err := json.NewDecoder(resp.Body).Decode(&itunesResp); err != nil { + log.Error().Str("op", "youtube-music/metadata").Err(err).Msg("Error parsing iTunes response") return fmt.Errorf("error parsing response: %v", err) } - if itunesResp.ResultCount == 0 || len(itunesResp.Results) == 0 { + log.Warn().Str("op", "youtube-music/metadata").Msgf("No results found for iTunes ID: %s", musicId) return fmt.Errorf("no results found for iTunes ID: %s", musicId) } + log.Debug().Str("op", "youtube-music/metadata").Msg("Successfully fetched and parsed iTunes metadata") trackInfo := itunesResp.Results[0] tempDir := filepath.Join(filepath.Dir(outputPath), ".danzo-temp") @@ -97,44 +101,43 @@ func addAppleMetadata(inputPath, outputPath, musicId string, streamFunc func(str return fmt.Errorf("error creating temp directory: %v", err) } defer os.RemoveAll(tempDir) - fileMarker := uuid.New().String() var artworkPath string - // Download artwork if trackInfo.ArtworkUrl100 != "" { highResArtwork := strings.Replace(trackInfo.ArtworkUrl100, "100x100", "1000x1000", 1) artworkPath = filepath.Join(tempDir, fileMarker+".jpg") err := downloadFile(highResArtwork, artworkPath, client) if err != nil { + log.Warn().Str("op", "youtube-music/metadata").Err(err).Msg("Failed to download artwork") artworkPath = "" } } - - // Apply metadata return applyMetadataWithFFmpeg(inputPath, outputPath, trackInfo, artworkPath, streamFunc) } func addDeezerMetadata(inputPath, outputPath, musicId string, streamFunc func(string)) error { client := utils.NewDanzoHTTPClient(httpConfig) apiURL := fmt.Sprintf("https://api.deezer.com/track/%s", musicId) - + log.Debug().Str("op", "youtube-music/metadata").Msgf("Fetching Deezer metadata from: %s", apiURL) req, _ := http.NewRequest("GET", apiURL, nil) resp, err := client.Do(req) if err != nil { + log.Error().Str("op", "youtube-music/metadata").Err(err).Msg("Error fetching Deezer metadata") return fmt.Errorf("error fetching metadata: %v", err) } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { + log.Error().Str("op", "youtube-music/metadata").Msgf("Deezer API request failed with status code %d", resp.StatusCode) return fmt.Errorf("API request failed with status code %d", resp.StatusCode) } - var deezerResp DeezerResponse if err := json.NewDecoder(resp.Body).Decode(&deezerResp); err != nil { + log.Error().Str("op", "youtube-music/metadata").Err(err).Msg("Error parsing Deezer response") return fmt.Errorf("error parsing response: %v", err) } - + log.Debug().Str("op", "youtube-music/metadata").Msg("Successfully fetched and parsed Deezer metadata") tempDir := filepath.Join(filepath.Dir(outputPath), ".danzo-temp") if err := os.MkdirAll(tempDir, 0755); err != nil { return fmt.Errorf("error creating temp directory: %v", err) @@ -143,15 +146,14 @@ func addDeezerMetadata(inputPath, outputPath, musicId string, streamFunc func(st fileMarker := uuid.New().String() var artworkPath string - if deezerResp.Album.Cover != "" { artworkPath = filepath.Join(tempDir, fileMarker+".jpg") err := downloadFile(deezerResp.Album.Cover, artworkPath, client) if err != nil { + log.Warn().Str("op", "youtube-music/metadata").Err(err).Msg("Failed to download artwork") artworkPath = "" } } - return applyDeezerMetadataWithFFmpeg(inputPath, outputPath, deezerResp, artworkPath, streamFunc) } @@ -167,18 +169,15 @@ func applyMetadataWithFFmpeg(inputPath, outputPath string, trackInfo struct { DiscCount int `json:"discCount"` ArtworkUrl100 string `json:"artworkUrl100"` }, artworkPath string, streamFunc func(string)) error { - tempDir := filepath.Dir(artworkPath) if tempDir == "" { tempDir = filepath.Dir(outputPath) } - metadataPath := filepath.Join(tempDir, uuid.New().String()+".txt") escapeRegex := regexp.MustCompile(`[^a-zA-Z0-9\s\-_]`) escapeRE := func(s string) string { return escapeRegex.ReplaceAllString(s, "") } - metadataContent := fmt.Sprintf(";FFMETADATA1\ntitle=%s\nartist=%s\nalbum=%s\n", escapeRE(trackInfo.TrackName), escapeRE(trackInfo.ArtistName), escapeRE(trackInfo.CollectionName)) @@ -190,11 +189,9 @@ func applyMetadataWithFFmpeg(inputPath, outputPath string, trackInfo struct { metadataContent += fmt.Sprintf("date=%s\n", escapeRE(trackInfo.ReleaseDate)) } } - if trackInfo.PrimaryGenreName != "" { metadataContent += fmt.Sprintf("genre=%s\n", escapeRE(trackInfo.PrimaryGenreName)) } - if trackInfo.TrackNumber > 0 { if trackInfo.TrackCount > 0 { metadataContent += fmt.Sprintf("track=%d/%d\n", trackInfo.TrackNumber, trackInfo.TrackCount) @@ -202,7 +199,6 @@ func applyMetadataWithFFmpeg(inputPath, outputPath string, trackInfo struct { metadataContent += fmt.Sprintf("track=%d\n", trackInfo.TrackNumber) } } - if trackInfo.DiscNumber > 0 { if trackInfo.DiscCount > 0 { metadataContent += fmt.Sprintf("disc=%d/%d\n", trackInfo.DiscNumber, trackInfo.DiscCount) @@ -214,30 +210,25 @@ func applyMetadataWithFFmpeg(inputPath, outputPath string, trackInfo struct { if err := os.WriteFile(metadataPath, []byte(metadataContent), 0644); err != nil { return fmt.Errorf("error writing metadata file: %v", err) } - - args := []string{ - "-i", inputPath, - "-i", metadataPath, - } - + args := []string{"-i", inputPath, "-i", metadataPath} if artworkPath != "" { args = append(args, "-i", artworkPath, "-map", "0", "-map", "2") args = append(args, "-disposition:v:0", "attached_pic") } - args = append(args, "-map_metadata", "1", "-codec", "copy") args = append(args, "-id3v2_version", "3", "-y", outputPath) cmd := exec.Command("ffmpeg", args...) + log.Debug().Str("op", "youtube-music/metadata").Msgf("Applying metadata with ffmpeg: %s", cmd.String()) output, err := cmd.CombinedOutput() if err != nil { + log.Error().Str("op", "youtube-music/metadata").Err(err).Msgf("FFmpeg error: %s", string(output)) return fmt.Errorf("FFmpeg error: %v\nOutput: %s", err, string(output)) } - if streamFunc != nil { streamFunc("Metadata applied successfully") } - + log.Info().Str("op", "youtube-music/metadata").Msgf("Metadata successfully applied to %s", outputPath) return nil } @@ -246,13 +237,11 @@ func applyDeezerMetadataWithFFmpeg(inputPath, outputPath string, deezerResp Deez if tempDir == "" { tempDir = filepath.Dir(outputPath) } - metadataPath := filepath.Join(tempDir, uuid.New().String()+".txt") escapeRegex := regexp.MustCompile(`[^a-zA-Z0-9\s\-_]`) escapeRE := func(s string) string { return escapeRegex.ReplaceAllString(s, "") } - metadataContent := fmt.Sprintf(";FFMETADATA1\ntitle=%s\nartist=%s\nalbum=%s\n", escapeRE(deezerResp.Title), escapeRE(deezerResp.Artist.Name), escapeRE(deezerResp.Album.Title)) @@ -260,49 +249,41 @@ func applyDeezerMetadataWithFFmpeg(inputPath, outputPath string, deezerResp Deez metadataContent += fmt.Sprintf("date=%s\n", escapeRE(deezerResp.ReleaseDate)) } - // Find composer for _, contributor := range deezerResp.Contributors { if strings.Contains(strings.ToLower(contributor.Role), "compos") { metadataContent += fmt.Sprintf("composer=%s\n", escapeRE(contributor.Name)) break } } - if deezerResp.TrackNumber > 0 { metadataContent += fmt.Sprintf("track=%d\n", deezerResp.TrackNumber) } - if deezerResp.DiskNumber > 0 { metadataContent += fmt.Sprintf("disc=%d\n", deezerResp.DiskNumber) } - if err := os.WriteFile(metadataPath, []byte(metadataContent), 0644); err != nil { return fmt.Errorf("error writing metadata file: %v", err) } - args := []string{ - "-i", inputPath, - "-i", metadataPath, - } - + args := []string{"-i", inputPath, "-i", metadataPath} if artworkPath != "" { args = append(args, "-i", artworkPath, "-map", "0", "-map", "2") args = append(args, "-disposition:v:0", "attached_pic") } - args = append(args, "-map_metadata", "1", "-codec", "copy") args = append(args, "-id3v2_version", "3", "-y", outputPath) cmd := exec.Command("ffmpeg", args...) + log.Debug().Str("op", "youtube-music/metadata").Msgf("Applying metadata with ffmpeg: %s", cmd.String()) output, err := cmd.CombinedOutput() if err != nil { + log.Error().Str("op", "youtube-music/metadata").Err(err).Msgf("FFmpeg error: %s", string(output)) return fmt.Errorf("FFmpeg error: %v\nOutput: %s", err, string(output)) } - if streamFunc != nil { streamFunc("Metadata applied successfully") } - + log.Info().Str("op", "youtube-music/metadata").Msgf("Metadata successfully applied to %s", outputPath) return nil } @@ -313,17 +294,14 @@ func downloadFile(url, filepath string, client *utils.DanzoHTTPClient) error { return err } defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { return fmt.Errorf("bad status: %s", resp.Status) } - out, err := os.Create(filepath) if err != nil { return err } defer out.Close() - _, err = io.Copy(out, resp.Body) return err } diff --git a/internal/downloaders/youtube/download.go b/internal/downloaders/youtube/download.go index dde85c7..e05a61a 100644 --- a/internal/downloaders/youtube/download.go +++ b/internal/downloaders/youtube/download.go @@ -7,6 +7,7 @@ import ( "os/exec" "strings" + "github.com/rs/zerolog/log" "github.com/tanq16/danzo/internal/utils" ) @@ -14,8 +15,6 @@ func (d *YouTubeDownloader) Download(job *utils.DanzoJob) error { ytdlpPath := job.Metadata["ytdlpPath"].(string) ytdlpFormat := job.Metadata["ytdlpFormat"].(string) ffmpegPath := job.Metadata["ffmpegPath"].(string) - - // Build yt-dlp command args := []string{ "--progress", "--newline", @@ -26,34 +25,31 @@ func (d *YouTubeDownloader) Download(job *utils.DanzoJob) error { "--no-playlist", job.URL, } - cmd := exec.Command(ytdlpPath, args...) + log.Debug().Str("op", "youtube/download").Msgf("Executing yt-dlp command: %s", cmd.String()) - // Set up pipes for stdout and stderr stdout, err := cmd.StdoutPipe() if err != nil { + log.Error().Str("op", "youtube/download").Err(err).Msg("Error creating stdout pipe") return fmt.Errorf("error creating stdout pipe: %v", err) } - stderr, err := cmd.StderrPipe() if err != nil { + log.Error().Str("op", "youtube/download").Err(err).Msg("Error creating stderr pipe") return fmt.Errorf("error creating stderr pipe: %v", err) } - - // Start the command if err := cmd.Start(); err != nil { + log.Error().Str("op", "youtube/download").Err(err).Msg("Error starting yt-dlp") return fmt.Errorf("error starting yt-dlp: %v", err) } - // Process output streams go processStream(stdout, job.StreamFunc) go processStream(stderr, job.StreamFunc) - - // Wait for completion if err := cmd.Wait(); err != nil { + log.Error().Str("op", "youtube/download").Err(err).Msg("yt-dlp command failed") return fmt.Errorf("yt-dlp failed: %v", err) } - + log.Info().Str("op", "youtube/download").Msgf("yt-dlp download completed for %s", job.URL) return nil } diff --git a/internal/downloaders/youtube/helpers.go b/internal/downloaders/youtube/helpers.go index 52b4451..82b79cf 100644 --- a/internal/downloaders/youtube/helpers.go +++ b/internal/downloaders/youtube/helpers.go @@ -8,17 +8,19 @@ import ( "path/filepath" "runtime" + "github.com/rs/zerolog/log" "github.com/tanq16/danzo/internal/utils" ) func downloadYtdlp() (string, error) { goos := runtime.GOOS goarch := runtime.GOARCH - var filename string switch { case goos == "windows" && goarch == "amd64": filename = "yt-dlp.exe" + case goos == "windows" && goarch == "arm64": + filename = "yt-dlp_arm64.exe" case goos == "linux" && goarch == "amd64": filename = "yt-dlp_linux" case goos == "linux" && goarch == "arm64": @@ -31,53 +33,50 @@ func downloadYtdlp() (string, error) { tempDir := ".danzo-temp" if err := os.MkdirAll(tempDir, 0755); err != nil { + log.Error().Str("op", "youtube/helpers").Err(err).Msg("Error creating temp directory") return "", fmt.Errorf("error creating temp directory: %v", err) } - downloadURL := fmt.Sprintf("https://github.com/yt-dlp/yt-dlp/releases/latest/download/%s", filename) filePath := filepath.Join(tempDir, "yt-dlp") if goos == "windows" { filePath += ".exe" } - // Download file + log.Info().Str("op", "youtube/helpers").Msgf("Downloading yt-dlp from %s to %s", downloadURL, filePath) if err := downloadFile(downloadURL, filePath); err != nil { + log.Error().Str("op", "youtube/helpers").Err(err).Msg("Failed to download yt-dlp") return "", err } - - // Make executable on Unix systems if goos != "windows" { if err := os.Chmod(filePath, 0755); err != nil { + log.Error().Str("op", "youtube/helpers").Err(err).Msg("Failed to set permissions for yt-dlp") return "", fmt.Errorf("error setting permissions: %v", err) } } - + log.Info().Str("op", "youtube/helpers").Msg("yt-dlp downloaded successfully") return filePath, nil } func downloadFile(url, filepath string) error { + log.Debug().Str("op", "youtube/helpers").Msgf("Downloading file from %s to %s", url, filepath) client := utils.NewDanzoHTTPClient(utils.HTTPClientConfig{}) req, err := http.NewRequest("GET", url, nil) if err != nil { return err } - resp, err := client.Do(req) if err != nil { return err } defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { return fmt.Errorf("bad status: %s", resp.Status) } - out, err := os.Create(filepath) if err != nil { return err } defer out.Close() - _, err = io.Copy(out, resp.Body) return err } diff --git a/internal/downloaders/youtube/initial.go b/internal/downloaders/youtube/initial.go index 6ab1074..82e7fd7 100644 --- a/internal/downloaders/youtube/initial.go +++ b/internal/downloaders/youtube/initial.go @@ -8,6 +8,7 @@ import ( "runtime" "strings" + "github.com/rs/zerolog/log" "github.com/tanq16/danzo/internal/utils" ) @@ -28,69 +29,63 @@ var ytdlpFormats = map[string]string{ } func (d *YouTubeDownloader) ValidateJob(job *utils.DanzoJob) error { - // Validate URL if !strings.Contains(job.URL, "youtube.com/watch") && !strings.Contains(job.URL, "youtu.be/") && !strings.Contains(job.URL, "music.youtube.com") { return fmt.Errorf("invalid YouTube URL") } - - // Validate format if specified if format, ok := job.Metadata["format"].(string); ok { if _, exists := ytdlpFormats[format]; !exists { return fmt.Errorf("unsupported format: %s", format) } } - + log.Info().Str("op", "youtube/initial").Msgf("job validated for %s", job.URL) return nil } func (d *YouTubeDownloader) BuildJob(job *utils.DanzoJob) error { - // Set default format if not specified format, ok := job.Metadata["format"].(string) if !ok || format == "" { - format = "best" + format = "decent" job.Metadata["format"] = format } - - // Set ytdlp format string job.Metadata["ytdlpFormat"] = ytdlpFormats[format] + log.Debug().Str("op", "youtube/initial").Msgf("Using format key '%s' for yt-dlp format '%s'", format, ytdlpFormats[format]) - // Check for required tools ytdlpPath, err := EnsureYtdlp() if err != nil { return fmt.Errorf("error ensuring yt-dlp: %v", err) } job.Metadata["ytdlpPath"] = ytdlpPath + log.Debug().Str("op", "youtube/initial").Msgf("Using yt-dlp at: %s", ytdlpPath) ffmpegPath, err := EnsureFFmpeg() if err != nil { return fmt.Errorf("error ensuring ffmpeg: %v", err) } job.Metadata["ffmpegPath"] = ffmpegPath + log.Debug().Str("op", "youtube/initial").Msgf("Using ffmpeg at: %s", ffmpegPath) ffprobePath, err := ensureFFprobe() if err != nil { return fmt.Errorf("error ensuring ffprobe: %v", err) } job.Metadata["ffprobePath"] = ffprobePath + log.Debug().Str("op", "youtube/initial").Msgf("Using ffprobe at: %s", ffprobePath) - // Set output path if not specified if job.OutputPath == "" { job.OutputPath = "%(title)s.%(ext)s" } - + log.Info().Str("op", "youtube/initial").Msgf("job built for %s", job.URL) return nil } func EnsureYtdlp() (string, error) { - // Check if yt-dlp is in PATH path, err := exec.LookPath("yt-dlp") if err == nil { + log.Debug().Str("op", "youtube/initial").Msgf("yt-dlp found in PATH: %s", path) return path, nil } - - // Check in current directory execDir, err := os.Executable() if err == nil { ytdlpPath := filepath.Join(filepath.Dir(execDir), "yt-dlp") @@ -98,20 +93,20 @@ func EnsureYtdlp() (string, error) { ytdlpPath += ".exe" } if _, err := os.Stat(ytdlpPath); err == nil { + log.Debug().Str("op", "youtube/initial").Msgf("yt-dlp found in executable directory: %s", ytdlpPath) return ytdlpPath, nil } } - - // Download yt-dlp + log.Warn().Str("op", "youtube/initial").Msg("yt-dlp not found, attempting download") return downloadYtdlp() } func EnsureFFmpeg() (string, error) { path, err := exec.LookPath("ffmpeg") if err == nil { + log.Debug().Str("op", "youtube/initial").Msgf("ffmpeg found in PATH: %s", path) return path, nil } - execDir, err := os.Executable() if err == nil { ffmpegPath := filepath.Join(filepath.Dir(execDir), "ffmpeg") @@ -119,19 +114,20 @@ func EnsureFFmpeg() (string, error) { ffmpegPath += ".exe" } if _, err := os.Stat(ffmpegPath); err == nil { + log.Debug().Str("op", "youtube/initial").Msgf("ffmpeg found in executable directory: %s", ffmpegPath) return ffmpegPath, nil } } - + log.Error().Str("op", "youtube/initial").Msg("ffmpeg not found in PATH or executable directory. Please install it.") return "", fmt.Errorf("ffmpeg not found in PATH, please install manually") } func ensureFFprobe() (string, error) { path, err := exec.LookPath("ffprobe") if err == nil { + log.Debug().Str("op", "youtube/initial").Msgf("ffprobe found in PATH: %s", path) return path, nil } - execDir, err := os.Executable() if err == nil { ffprobePath := filepath.Join(filepath.Dir(execDir), "ffprobe") @@ -139,9 +135,10 @@ func ensureFFprobe() (string, error) { ffprobePath += ".exe" } if _, err := os.Stat(ffprobePath); err == nil { + log.Debug().Str("op", "youtube/initial").Msgf("ffprobe found in executable directory: %s", ffprobePath) return ffprobePath, nil } } - + log.Error().Str("op", "youtube/initial").Msg("ffprobe not found in PATH or executable directory. Please install it.") return "", fmt.Errorf("ffprobe not found in PATH, please install manually") } diff --git a/internal/output/manager.go b/internal/output/manager.go index 1ca314b..5007cd5 100644 --- a/internal/output/manager.go +++ b/internal/output/manager.go @@ -20,12 +20,13 @@ const ( ) type JobOutput struct { - ID int - Name string - Status JobStatus - Message string - StreamLines []string - StartTime time.Time + ID int + Name string + Status JobStatus + Message string + StreamLines []string + StartTime time.Time + CompletedTime time.Duration } type Manager struct { @@ -98,6 +99,7 @@ func (m *Manager) Complete(id int, message string) { defer m.mu.Unlock() if job, exists := m.jobs[id]; exists { job.Status = StatusSuccess + job.CompletedTime = time.Since(job.StartTime).Round(time.Second) job.StreamLines = []string{} // Clear streams on completion if message != "" { job.Message = message @@ -152,9 +154,12 @@ func (m *Manager) StartDisplay() { m.wg.Add(1) go func() { defer m.wg.Done() - ticker := time.NewTicker(300 * time.Millisecond) + timePerUpdate := 300 * time.Millisecond + if utils.GlobalDebugFlag { + timePerUpdate = 3 * time.Second // slow refresh for debug mode + } + ticker := time.NewTicker(timePerUpdate) defer ticker.Stop() - for { select { case <-ticker.C: @@ -179,7 +184,7 @@ func (m *Manager) StopDisplay() { func (m *Manager) updateDisplay() { m.mu.RLock() defer m.mu.RUnlock() - if m.lastLineCount > 0 { + if m.lastLineCount > 0 && !utils.GlobalDebugFlag { fmt.Printf("\033[%dA\033[J", m.lastLineCount) } @@ -225,7 +230,11 @@ func (m *Manager) updateDisplay() { completed = completed[len(completed)-8:] } for _, job := range completed { - totalTime := time.Since(job.StartTime).Round(time.Second) + totalTime := job.CompletedTime + if job.CompletedTime == 0 { + totalTime = time.Since(job.StartTime).Round(time.Second) + job.CompletedTime = totalTime + } style := successStyle if job.Status == StatusError { style = errorStyle @@ -253,7 +262,7 @@ func (m *Manager) showSummary() { } } fmt.Println() - fmt.Println(" " + success2Style.Render(fmt.Sprintf("Completed %d of %d", success, len(m.jobs)))) + fmt.Println(" " + successStyle.Render(fmt.Sprintf("Completed %d of %d", success, len(m.jobs)))) if errors > 0 { fmt.Println(" " + errorStyle.Render(fmt.Sprintf("Failed %d of %d", errors, len(m.jobs)))) } @@ -296,21 +305,3 @@ func printProgressBar(current, total int64, width int) string { bar += StyleSymbols["bullet"] return debugStyle.Render(fmt.Sprintf("%s %.1f%% %s ", bar, percent*100, StyleSymbols["bullet"])) } - -// TODO: Implement this at some point - -// func getTerminalWidth() int { -// width, _, err := term.GetSize(int(os.Stdout.Fd())) -// if err != nil || width <= 0 { -// return 80 // Default fallback width -// } -// return width -// } - -// func getTerminalHeight() int { -// height, _, err := term.GetSize(int(os.Stdout.Fd())) -// if err != nil || height <= 0 { -// return 24 // Default fallback height -// } -// return height -// } diff --git a/internal/output/vars.go b/internal/output/vars.go index 9f63f34..1dafe07 100644 --- a/internal/output/vars.go +++ b/internal/output/vars.go @@ -7,16 +7,14 @@ import ( ) var ( - successStyle = lipgloss.NewStyle().Foreground(lipgloss.Color("37")) // dark green - success2Style = lipgloss.NewStyle().Foreground(lipgloss.Color("2")) // green - errorStyle = lipgloss.NewStyle().Foreground(lipgloss.Color("9")) // red - warningStyle = lipgloss.NewStyle().Foreground(lipgloss.Color("11")) // yellow - pendingStyle = lipgloss.NewStyle().Foreground(lipgloss.Color("12")) // blue - infoStyle = lipgloss.NewStyle().Foreground(lipgloss.Color("14")) // cyan - debugStyle = lipgloss.NewStyle().Foreground(lipgloss.Color("250")) // light grey - detailStyle = lipgloss.NewStyle().Foreground(lipgloss.Color("13")) // purple - streamStyle = lipgloss.NewStyle().Foreground(lipgloss.Color("240")) // grey - headerStyle = lipgloss.NewStyle().Bold(true).Foreground(lipgloss.Color("69")) // purple + successStyle = lipgloss.NewStyle().Foreground(lipgloss.Color("37")) // dark green + errorStyle = lipgloss.NewStyle().Foreground(lipgloss.Color("9")) // red + warningStyle = lipgloss.NewStyle().Foreground(lipgloss.Color("11")) // yellow + pendingStyle = lipgloss.NewStyle().Foreground(lipgloss.Color("12")) // blue + infoStyle = lipgloss.NewStyle().Foreground(lipgloss.Color("14")) // cyan + debugStyle = lipgloss.NewStyle().Foreground(lipgloss.Color("250")) // light grey + detailStyle = lipgloss.NewStyle().Foreground(lipgloss.Color("13")) // purple + streamStyle = lipgloss.NewStyle().Foreground(lipgloss.Color("240")) // grey ) var StyleSymbols = map[string]string{ @@ -34,9 +32,6 @@ var StyleSymbols = map[string]string{ func PrintSuccess(text string) { fmt.Println(successStyle.Render(text)) } -func PrintSuccess2(text string) { - fmt.Println(success2Style.Render(text)) -} func PrintError(text string) { fmt.Println(errorStyle.Render(text)) } @@ -58,15 +53,9 @@ func PrintDetail(text string) { func PrintStream(text string) { fmt.Println(streamStyle.Render(text)) } -func PrintHeader(text string) { - fmt.Println(headerStyle.Render(text)) -} func FSuccess(text string) string { return successStyle.Render(text) } -func FSuccess2(text string) string { - return success2Style.Render(text) -} func FError(text string) string { return errorStyle.Render(text) } @@ -88,6 +77,3 @@ func FDetail(text string) string { func FStream(text string) string { return streamStyle.Render(text) } -func FHeader(text string) string { - return headerStyle.Render(text) -} diff --git a/internal/scheduler/scheduler.go b/internal/scheduler/scheduler.go index e3c1852..a3bc2bc 100644 --- a/internal/scheduler/scheduler.go +++ b/internal/scheduler/scheduler.go @@ -4,11 +4,12 @@ import ( "fmt" "sync" - "github.com/tanq16/danzo/internal/downloaders/gdrive" - "github.com/tanq16/danzo/internal/downloaders/ghrelease" - "github.com/tanq16/danzo/internal/downloaders/gitclone" + "github.com/rs/zerolog/log" + gitclone "github.com/tanq16/danzo/internal/downloaders/git-clone" + ghrelease "github.com/tanq16/danzo/internal/downloaders/github-release" + gdrive "github.com/tanq16/danzo/internal/downloaders/google-drive" httpDownloader "github.com/tanq16/danzo/internal/downloaders/http" - "github.com/tanq16/danzo/internal/downloaders/m3u8" + m3u8 "github.com/tanq16/danzo/internal/downloaders/live-stream" "github.com/tanq16/danzo/internal/downloaders/s3" "github.com/tanq16/danzo/internal/downloaders/youtube" youtubemusic "github.com/tanq16/danzo/internal/downloaders/youtube-music" @@ -24,23 +25,24 @@ type Scheduler struct { } var downloaderRegistry = map[string]utils.Downloader{ - "http": &httpDownloader.HTTPDownloader{}, - "s3": &s3.S3Downloader{}, - "gdrive": &gdrive.GDriveDownloader{}, - "gitclone": &gitclone.GitCloneDownloader{}, - "ghrelease": &ghrelease.GitReleaseDownloader{}, - "m3u8": &m3u8.M3U8Downloader{}, - "youtube": &youtube.YouTubeDownloader{}, - "ytmusic": &youtubemusic.YTMusicDownloader{}, + "http": &httpDownloader.HTTPDownloader{}, + "s3": &s3.S3Downloader{}, + "google-drive": &gdrive.GDriveDownloader{}, + "git-clone": &gitclone.GitCloneDownloader{}, + "github-release": &ghrelease.GitReleaseDownloader{}, + "live-stream": &m3u8.M3U8Downloader{}, + "youtube": &youtube.YouTubeDownloader{}, + "youtube-music": &youtubemusic.YTMusicDownloader{}, } -func Run(jobs []utils.DanzoJob, numWorkers int, fileLog bool) { +func Run(jobs []utils.DanzoJob, numWorkers int) { s := &Scheduler{ outputMgr: output.NewManager(), pauseRequestCh: make(chan struct{}), resumeRequestCh: make(chan struct{}), singleJobMode: len(jobs) == 1, } + log.Debug().Str("op", "scheduler").Msgf("Starting output manager") s.outputMgr.StartDisplay() defer s.outputMgr.StopDisplay() @@ -51,12 +53,14 @@ func Run(jobs []utils.DanzoJob, numWorkers int, fileLog bool) { go s.handlePauseResume() } + log.Debug().Str("op", "scheduler").Msgf("Send jobs to pipeline") jobCh := make(chan utils.DanzoJob, len(jobs)) for _, job := range jobs { jobCh <- job } close(jobCh) + log.Debug().Str("op", "scheduler").Msgf("Start %d workers", numWorkers) var wg sync.WaitGroup for range numWorkers { wg.Add(1) @@ -67,10 +71,14 @@ func Run(jobs []utils.DanzoJob, numWorkers int, fileLog bool) { } wg.Wait() + log.Debug().Str("op", "scheduler").Msgf("All workers done") if allSuccessful { + log.Debug().Str("op", "scheduler").Msgf("Clean up output dirs after successful jobs") for dir := range outputDirs { utils.CleanFunction(dir) } + } else { + log.Error().Str("op", "scheduler").Msgf("Not all jobs were successful") } } @@ -87,6 +95,7 @@ func (s *Scheduler) handlePauseResume() { func (s *Scheduler) processJobs(jobCh <-chan utils.DanzoJob, outputDirs *map[string]bool, allSuccessful *bool, mu *sync.Mutex) { for job := range jobCh { + log.Debug().Str("op", "scheduler/processJobs").Msgf("Processing job %s", job.OutputPath) funcID := s.outputMgr.RegisterFunction(job.OutputPath) downloader, exists := downloaderRegistry[job.JobType] if !exists { @@ -95,15 +104,19 @@ func (s *Scheduler) processJobs(jobCh <-chan utils.DanzoJob, outputDirs *map[str continue } + log.Debug().Str("op", "scheduler/processJobs").Msgf("Downloader found for %s", job.JobType) s.outputMgr.SetStatus(funcID, "pending") s.outputMgr.SetMessage(funcID, fmt.Sprintf("Validating %s job", job.JobType)) + log.Debug().Str("op", "scheduler/processJobs").Msgf("Validating job %s", job.OutputPath) err := downloader.ValidateJob(&job) if err != nil { + log.Error().Str("op", "scheduler/processJobs").Msgf("Validation failed for %s", job.OutputPath) s.outputMgr.ReportError(funcID, fmt.Errorf("validation failed: %v", err)) s.outputMgr.SetMessage(funcID, fmt.Sprintf("Validation failed for %s", job.OutputPath)) continue } + log.Info().Str("op", "scheduler/processJobs").Msgf("Preparing job %s", job.OutputPath) s.outputMgr.SetMessage(funcID, fmt.Sprintf("Preparing %s job", job.JobType)) if s.singleJobMode { job.PauseFunc = func() { s.pauseRequestCh <- struct{}{} } @@ -111,6 +124,7 @@ func (s *Scheduler) processJobs(jobCh <-chan utils.DanzoJob, outputDirs *map[str } err = downloader.BuildJob(&job) if err != nil { + log.Error().Str("op", "scheduler/processJobs").Msgf("Build failed for %s", job.OutputPath) if err.Error() == "file already exists with same size" { s.outputMgr.SetStatus(funcID, "success") s.outputMgr.SetMessage(funcID, fmt.Sprintf("File already exists: %s", job.OutputPath)) @@ -122,21 +136,25 @@ func (s *Scheduler) processJobs(jobCh <-chan utils.DanzoJob, outputDirs *map[str continue } - if job.ProgressType == "progress" { + log.Debug().Str("op", "scheduler/processJobs").Msgf("Setting progress type for %s", job.OutputPath) + switch job.ProgressType { + case "progress": job.ProgressFunc = func(downloaded, total int64) { if total > 0 { s.outputMgr.AddProgressBarToStream(funcID, downloaded, total) } } - } else if job.ProgressType == "stream" { + case "stream": job.StreamFunc = func(line string) { s.outputMgr.AddStreamLine(funcID, line) } } + log.Info().Str("op", "scheduler/processJobs").Msgf("Performing download for %s", job.OutputPath) s.outputMgr.SetMessage(funcID, fmt.Sprintf("Downloading %s", job.OutputPath)) err = downloader.Download(&job) if err != nil { + log.Error().Str("op", "scheduler/processJobs").Msgf("Download failed for %s", job.OutputPath) mu.Lock() *allSuccessful = false mu.Unlock() @@ -144,6 +162,7 @@ func (s *Scheduler) processJobs(jobCh <-chan utils.DanzoJob, outputDirs *map[str s.outputMgr.SetMessage(funcID, fmt.Sprintf("Download failed for %s", job.OutputPath)) continue } + log.Info().Str("op", "scheduler/processJobs").Msgf("Download completed for %s", job.OutputPath) mu.Lock() (*outputDirs)[job.OutputPath] = true mu.Unlock() diff --git a/internal/utils/functions.go b/internal/utils/functions.go index 5ea03e7..97570d7 100644 --- a/internal/utils/functions.go +++ b/internal/utils/functions.go @@ -12,27 +12,6 @@ func GetRandomUserAgent() string { return userAgents[time.Now().UnixNano()%int64(len(userAgents))] } -func DetermineDownloadType(url string) string { - if strings.HasPrefix(url, "https://drive.google.com") { - return "gdrive" - } else if strings.HasPrefix(url, "s3://") { - return "s3" - } else if strings.HasPrefix(url, "https://youtu.be") || strings.HasPrefix(url, "https://www.youtube.com") || strings.HasPrefix(url, "https://music.youtube.com") { - return "youtube" - } else if strings.HasPrefix(url, "ftp://") || strings.HasPrefix(url, "ftps://") { - return "ftp" - } else if strings.HasPrefix(url, "sftp://") { - return "sftp" - } else if strings.HasPrefix(url, "github://") { - return "gitrelease" - } else if strings.HasPrefix(url, "github.com") || strings.HasPrefix(url, "gitlab.com") || strings.HasPrefix(url, "bitbucket.org") || strings.HasPrefix(url, "git.com") { - return "gitclone" - } else if strings.HasPrefix(url, "m3u8://") { - return "m3u8" - } - return "http" -} - func RenewOutputPath(outputPath string) string { dir := filepath.Dir(outputPath) base := filepath.Base(outputPath) diff --git a/internal/utils/http-client.go b/internal/utils/http-client.go index 858d4af..950b676 100644 --- a/internal/utils/http-client.go +++ b/internal/utils/http-client.go @@ -6,6 +6,8 @@ import ( "net/url" "syscall" "time" + + "github.com/rs/zerolog/log" ) type HTTPClientConfig struct { @@ -54,6 +56,7 @@ func NewDanzoHTTPClient(cfg HTTPClientConfig) *DanzoHTTPClient { }) }, }).DialContext + log.Debug().Str("op", "utils/http-client").Msg("Using high thread mode") } if cfg.ProxyURL != "" { proxyURL, err := url.Parse(cfg.ProxyURL) @@ -65,6 +68,7 @@ func NewDanzoHTTPClient(cfg HTTPClientConfig) *DanzoHTTPClient { proxyURL.User = url.User(cfg.ProxyUsername) } } + log.Debug().Str("op", "utils/http-client").Msgf("Using proxy: %s", proxyURL.String()) transport.Proxy = http.ProxyURL(proxyURL) } } diff --git a/internal/utils/vars.go b/internal/utils/vars.go index f999e1d..e864e13 100644 --- a/internal/utils/vars.go +++ b/internal/utils/vars.go @@ -6,12 +6,15 @@ import ( "time" ) +var GlobalDebugFlag bool + type Downloader interface { Download(job *DanzoJob) error BuildJob(job *DanzoJob) error ValidateJob(job *DanzoJob) error } +// A single download job for Danzo type DanzoJob struct { ID string JobType string @@ -27,14 +30,14 @@ type DanzoJob struct { ResumeFunc func() // Request resume for output } -type DownloadConfig struct { +type HTTPDownloadConfig struct { URL string OutputPath string Connections int HTTPClientConfig HTTPClientConfig } -type DownloadChunk struct { +type HTTPDownloadChunk struct { ID int StartByte int64 EndByte int64 @@ -46,10 +49,10 @@ type DownloadChunk struct { FinishTime time.Time } -type DownloadJob struct { - Config DownloadConfig +type HTTPDownloadJob struct { + Config HTTPDownloadConfig FileSize int64 - Chunks []DownloadChunk + Chunks []HTTPDownloadChunk StartTime time.Time TempFiles []string } @@ -59,7 +62,6 @@ const LogFile = ".danzo.log" var ErrRangeRequestsNotSupported = errors.New("range requests are not supported") var ChunkIDRegex = regexp.MustCompile(`\.part(\d+)$`) -var PMDebug = false // Local-only User-Agent list var userAgents = []string{