-
Notifications
You must be signed in to change notification settings - Fork 0
feat(find_longtime_files): add new tool for analyzing log file proces… #10
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Merged
Merged
Changes from 1 commit
Commits
Show all changes
2 commits
Select commit
Hold shift + click to select a range
File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,9 @@ | ||
[package] | ||
name = "find_longtime_files" | ||
version = "0.1.0" | ||
edition = "2021" | ||
|
||
[dependencies] | ||
chrono = "0.4" | ||
regex = "1" | ||
indicatif = "0.17.11" |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,129 @@ | ||
use chrono::NaiveDateTime; | ||
use indicatif::{ProgressBar, ProgressStyle}; | ||
use regex::Regex; | ||
use std::env; | ||
use std::fs::File; | ||
use std::io::{BufRead, BufReader}; | ||
|
||
/// Remove ANSI escape codes from a string. | ||
fn remove_ansi_codes(s: &str) -> String { | ||
// This regex matches ANSI escape sequences. | ||
let ansi_re = Regex::new(r"\x1B\[[0-9;]*[a-zA-Z]").unwrap(); | ||
ansi_re.replace_all(s, "").to_string() | ||
} | ||
|
||
/// Extract the timestamp from a line (first two whitespace-separated tokens). | ||
fn extract_timestamp(line: &str) -> Option<String> { | ||
let parts: Vec<&str> = line.split_whitespace().collect(); | ||
if parts.len() < 2 { | ||
None | ||
} else { | ||
Some(format!("{} {}", parts[0], parts[1])) | ||
} | ||
} | ||
|
||
/// Extract the filename from the line using a regex. | ||
/// The regex captures the filename following "The format of" and before "is <format>". | ||
fn extract_filename(line: &str) -> Option<String> { | ||
let re = Regex::new(r"The format of\s+(\S+)\s+is\s+\S+").unwrap(); | ||
re.captures(line) | ||
.and_then(|caps| caps.get(1).map(|m| m.as_str().to_string())) | ||
} | ||
|
||
fn main() { | ||
sangshuduo marked this conversation as resolved.
Show resolved
Hide resolved
|
||
// Check command-line arguments. | ||
let args: Vec<String> = env::args().collect(); | ||
if args.len() != 3 { | ||
eprintln!("Usage: {} <number-of-files> <log-file>", args[0]); | ||
std::process::exit(1); | ||
} | ||
let num_files: usize = args[1].parse().expect("Invalid number-of-files"); | ||
let log_file = &args[2]; | ||
|
||
// Open the log file. | ||
let file = File::open(log_file).unwrap_or_else(|err| { | ||
eprintln!("Error opening {}: {}", log_file, err); | ||
std::process::exit(1); | ||
}); | ||
// Get file metadata to determine total file size. | ||
let metadata = file.metadata().expect("Failed to get metadata"); | ||
let total_size = metadata.len(); | ||
|
||
let reader = BufReader::new(file); | ||
|
||
// Create a progress bar based on total file size. | ||
let pb = ProgressBar::new(total_size); | ||
pb.set_style( | ||
ProgressStyle::default_bar() | ||
.template("[{elapsed_precise}] [{bar:40.cyan/blue}] {bytes}/{total_bytes} ({eta})") | ||
.unwrap() | ||
.progress_chars("##-"), | ||
); | ||
|
||
// We'll store (duration_in_seconds, filename) pairs. | ||
let mut diffs: Vec<(f64, String)> = Vec::new(); | ||
|
||
// Variables to hold the previous log entry's timestamp and file name. | ||
let mut prev_dt: Option<NaiveDateTime> = None; | ||
let mut prev_file: Option<String> = None; | ||
|
||
// Timestamp format for milliseconds (3 digits). | ||
let timestamp_format = "%Y-%m-%d %H:%M:%S.%3f"; | ||
sangshuduo marked this conversation as resolved.
Show resolved
Hide resolved
|
||
|
||
for line in reader.lines() { | ||
// Update progress bar with the length of the line plus newline. | ||
let line = match line { | ||
Ok(l) => l, | ||
Err(e) => { | ||
eprintln!("Error reading line: {}", e); | ||
continue; | ||
} | ||
}; | ||
pb.inc((line.len() + 1) as u64); | ||
|
||
// Remove ANSI escape sequences. | ||
let clean_line = remove_ansi_codes(&line); | ||
|
||
// Extract the timestamp string. | ||
let ts_str = match extract_timestamp(&clean_line) { | ||
Some(ts) => ts, | ||
None => continue, | ||
}; | ||
|
||
// Parse the timestamp as a NaiveDateTime (without timezone). | ||
let naive_dt = match NaiveDateTime::parse_from_str(&ts_str, timestamp_format) { | ||
Ok(dt) => dt, | ||
Err(e) => { | ||
eprintln!("Error parsing date '{}': {}", ts_str, e); | ||
continue; | ||
} | ||
}; | ||
|
||
// Extract the filename. | ||
let filename = match extract_filename(&clean_line) { | ||
Some(f) => f, | ||
None => continue, | ||
}; | ||
|
||
// If we have a previous timestamp, compute the duration. | ||
if let (Some(prev), Some(prev_filename)) = (prev_dt, &prev_file) { | ||
let duration = naive_dt.signed_duration_since(prev); | ||
let diff_seconds = duration.num_microseconds().unwrap_or(0) as f64 / 1_000_000.0; | ||
diffs.push((diff_seconds, prev_filename.clone())); | ||
} | ||
|
||
// Update the previous values. | ||
prev_dt = Some(naive_dt); | ||
prev_file = Some(filename); | ||
} | ||
|
||
pb.finish_with_message("Processing complete"); | ||
|
||
// Sort by processing time in descending order. | ||
diffs.sort_by(|a, b| b.0.partial_cmp(&a.0).unwrap()); | ||
|
||
println!("Top {} files with longest processing times:", num_files); | ||
for (i, (duration, file)) in diffs.iter().take(num_files).enumerate() { | ||
println!("{}. {} took {:.6} seconds", i + 1, file, duration); | ||
} | ||
} |
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Uh oh!
There was an error while loading. Please reload this page.