Skip to content

Commit b09f9d3

Browse files
authored
feat: exclude digest when calculating task id (#803)
Signed-off-by: Gaius <[email protected]>
1 parent 6762de4 commit b09f9d3

File tree

5 files changed

+177
-26
lines changed

5 files changed

+177
-26
lines changed

Cargo.lock

Lines changed: 8 additions & 8 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

Cargo.toml

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ members = [
1212
]
1313

1414
[workspace.package]
15-
version = "0.1.113"
15+
version = "0.1.114"
1616
authors = ["The Dragonfly Developers"]
1717
homepage = "https://d7y.io/"
1818
repository = "https://github.com/dragonflyoss/client.git"
@@ -22,13 +22,13 @@ readme = "README.md"
2222
edition = "2021"
2323

2424
[workspace.dependencies]
25-
dragonfly-client = { path = "dragonfly-client", version = "0.1.113" }
26-
dragonfly-client-core = { path = "dragonfly-client-core", version = "0.1.113" }
27-
dragonfly-client-config = { path = "dragonfly-client-config", version = "0.1.113" }
28-
dragonfly-client-storage = { path = "dragonfly-client-storage", version = "0.1.113" }
29-
dragonfly-client-backend = { path = "dragonfly-client-backend", version = "0.1.113" }
30-
dragonfly-client-util = { path = "dragonfly-client-util", version = "0.1.113" }
31-
dragonfly-client-init = { path = "dragonfly-client-init", version = "0.1.113" }
25+
dragonfly-client = { path = "dragonfly-client", version = "0.1.114" }
26+
dragonfly-client-core = { path = "dragonfly-client-core", version = "0.1.114" }
27+
dragonfly-client-config = { path = "dragonfly-client-config", version = "0.1.114" }
28+
dragonfly-client-storage = { path = "dragonfly-client-storage", version = "0.1.114" }
29+
dragonfly-client-backend = { path = "dragonfly-client-backend", version = "0.1.114" }
30+
dragonfly-client-util = { path = "dragonfly-client-util", version = "0.1.114" }
31+
dragonfly-client-init = { path = "dragonfly-client-init", version = "0.1.114" }
3232
thiserror = "1.0"
3333
dragonfly-api = "=2.0.167"
3434
reqwest = { version = "0.12.4", features = ["stream", "native-tls", "default-tls", "rustls-tls"] }

dragonfly-client-util/src/id_generator/mod.rs

Lines changed: 161 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -71,7 +71,6 @@ impl IDGenerator {
7171
pub fn task_id(
7272
&self,
7373
url: &str,
74-
digest: Option<&str>,
7574
tag: Option<&str>,
7675
application: Option<&str>,
7776
filtered_query_params: Vec<String>,
@@ -83,18 +82,24 @@ impl IDGenerator {
8382
.filter(|(k, _)| !filtered_query_params.contains(&k.to_string()));
8483

8584
let mut artifact_url = url.clone();
86-
artifact_url.query_pairs_mut().clear().extend_pairs(query);
85+
if query.clone().count() == 0 {
86+
artifact_url.set_query(None);
87+
} else {
88+
artifact_url.query_pairs_mut().clear().extend_pairs(query);
89+
}
90+
91+
let artifact_url_str = artifact_url.to_string();
92+
let final_url = if artifact_url_str.ends_with('/') && artifact_url.path() == "/" {
93+
artifact_url_str.trim_end_matches('/').to_string()
94+
} else {
95+
artifact_url_str
96+
};
8797

8898
// Initialize the hasher.
8999
let mut hasher = Sha256::new();
90100

91101
// Add the url to generate the task id.
92-
hasher.update(artifact_url.to_string());
93-
94-
// Add the digest to generate the task id.
95-
if let Some(digest) = digest {
96-
hasher.update(digest);
97-
}
102+
hasher.update(final_url);
98103

99104
// Add the tag to generate the task id.
100105
if let Some(tag) = tag {
@@ -165,3 +170,151 @@ impl IDGenerator {
165170
TaskType::Standard
166171
}
167172
}
173+
174+
#[cfg(test)]
175+
mod tests {
176+
use super::*;
177+
use std::fs::File;
178+
use std::io::Write;
179+
use tempfile::tempdir;
180+
181+
#[test]
182+
fn should_generate_host_id() {
183+
let test_cases = vec![
184+
(
185+
IDGenerator::new("127.0.0.1".to_string(), "localhost".to_string(), false),
186+
"127.0.0.1-localhost",
187+
),
188+
(
189+
IDGenerator::new("127.0.0.1".to_string(), "localhost".to_string(), true),
190+
"127.0.0.1-localhost-seed",
191+
),
192+
];
193+
194+
for (generator, expected) in test_cases {
195+
assert_eq!(generator.host_id(), expected);
196+
}
197+
}
198+
199+
#[test]
200+
fn should_generate_task_id() {
201+
let test_cases = vec![
202+
(
203+
IDGenerator::new("127.0.0.1".to_string(), "localhost".to_string(), false),
204+
"https://example.com",
205+
Some("foo"),
206+
Some("bar"),
207+
vec![],
208+
"160fa7f001d9d2e893130894fbb60a5fb006e1d61bff82955f2946582bc9de1d",
209+
),
210+
(
211+
IDGenerator::new("127.0.0.1".to_string(), "localhost".to_string(), false),
212+
"https://example.com",
213+
Some("foo"),
214+
None,
215+
vec![],
216+
"2773851c628744fb7933003195db436ce397c1722920696c4274ff804d86920b",
217+
),
218+
(
219+
IDGenerator::new("127.0.0.1".to_string(), "localhost".to_string(), false),
220+
"https://example.com",
221+
None,
222+
Some("bar"),
223+
vec![],
224+
"63dee2822037636b0109876b58e95692233840753a882afa69b9b5ee82a6c57d",
225+
),
226+
(
227+
IDGenerator::new("127.0.0.1".to_string(), "localhost".to_string(), false),
228+
"https://example.com?foo=foo&bar=bar",
229+
None,
230+
None,
231+
vec!["foo".to_string(), "bar".to_string()],
232+
"100680ad546ce6a577f42f52df33b4cfdca756859e664b8d7de329b150d09ce9",
233+
),
234+
];
235+
236+
for (generator, url, tag, application, filtered_query_params, expected_id) in test_cases {
237+
let task_id = generator
238+
.task_id(url, tag, application, filtered_query_params)
239+
.unwrap();
240+
assert_eq!(task_id, expected_id);
241+
}
242+
}
243+
244+
#[test]
245+
fn should_generate_persistent_cache_task_id() {
246+
let test_cases = vec![
247+
(
248+
IDGenerator::new("127.0.0.1".to_string(), "localhost".to_string(), false),
249+
"This is a test file",
250+
Some("tag1"),
251+
Some("app1"),
252+
"84ed9fca6c51c725c21ab005682509bc9f5a9e08779aa14039a1df41bd95bb9f",
253+
),
254+
(
255+
IDGenerator::new("127.0.0.1".to_string(), "localhost".to_string(), false),
256+
"This is a test file",
257+
None,
258+
Some("app1"),
259+
"c39ee7baea1df8276d16224b6bbe93d0abaedaa056e819bb1a6318e28cdde508",
260+
),
261+
(
262+
IDGenerator::new("127.0.0.1".to_string(), "localhost".to_string(), false),
263+
"This is a test file",
264+
Some("tag1"),
265+
None,
266+
"de692dcd9b6eace344140ef2718033527ee0a2e436c03044a771902bd536ae7d",
267+
),
268+
];
269+
270+
for (generator, file_content, tag, application, expected_id) in test_cases {
271+
let dir = tempdir().unwrap();
272+
let file_path = dir.path().join("testfile");
273+
let mut f = File::create(&file_path).unwrap();
274+
f.write_all(file_content.as_bytes()).unwrap();
275+
276+
let task_id = generator
277+
.persistent_cache_task_id(&file_path, tag, application)
278+
.unwrap();
279+
assert_eq!(task_id, expected_id);
280+
}
281+
}
282+
283+
#[test]
284+
fn should_generate_peer_id() {
285+
let test_cases = vec![
286+
(
287+
IDGenerator::new("127.0.0.1".to_string(), "localhost".to_string(), false),
288+
false,
289+
),
290+
(
291+
IDGenerator::new("127.0.0.1".to_string(), "localhost".to_string(), true),
292+
true,
293+
),
294+
];
295+
296+
for (generator, is_seed_peer) in test_cases {
297+
let peer_id = generator.peer_id();
298+
assert!(peer_id.starts_with("127.0.0.1-localhost-"));
299+
if is_seed_peer {
300+
assert!(peer_id.ends_with("-seed"));
301+
}
302+
}
303+
}
304+
305+
#[test]
306+
fn should_generate_task_type() {
307+
let test_cases = vec![
308+
("some-task-id", TaskType::Standard),
309+
(
310+
"some-task-id-persistent-cache-task",
311+
TaskType::PersistentCache,
312+
),
313+
];
314+
315+
let generator = IDGenerator::new("127.0.0.1".to_string(), "localhost".to_string(), false);
316+
for (id, expected_type) in test_cases {
317+
assert_eq!(generator.task_type(id), expected_type);
318+
}
319+
}
320+
}

dragonfly-client/src/grpc/dfdaemon_download.rs

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -199,7 +199,6 @@ impl DfdaemonDownload for DfdaemonDownloadServerHandler {
199199
.id_generator
200200
.task_id(
201201
download.url.as_str(),
202-
download.digest.as_deref(),
203202
download.tag.as_deref(),
204203
download.application.as_deref(),
205204
download.filtered_query_params.clone(),

dragonfly-client/src/grpc/dfdaemon_upload.rs

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -194,7 +194,6 @@ impl DfdaemonUpload for DfdaemonUploadServerHandler {
194194
.id_generator
195195
.task_id(
196196
download.url.as_str(),
197-
download.digest.as_deref(),
198197
download.tag.as_deref(),
199198
download.application.as_deref(),
200199
download.filtered_query_params.clone(),

0 commit comments

Comments
 (0)