-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathtar.rs
More file actions
2223 lines (2134 loc) · 89.2 KB
/
Copy pathtar.rs
File metadata and controls
2223 lines (2134 loc) · 89.2 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
//! Streaming tar extractor sink.
//!
//! Hand-rolled rather than wrapping the upstream `tar` crate so the
//! parser state can be (de)serialized into a checkpoint and resumed
//! from any byte boundary — the v6 [`crate::checkpoint::SinkState::Tar`]
//! `in_flight` trailer carries the full parser state and
//! [`TarSink::resume`] reconstructs it. Earlier rounds of the project
//! relied on the [`Sink::is_quiescent`] contract being "true only
//! between members"; with mid-member resume support the sink is
//! resumable from anywhere and `is_quiescent` is `true` whenever the
//! sink is healthy. The format is small and the parser is the kind
//! of code we want to be able to audit byte-for-byte.
//!
//! # Format support
//!
//! - **USTAR (POSIX.1-1988)** headers (`ustar\0` + `00` magic) and
//! **old-GNU** headers (`ustar \0` magic). The two layouts are
//! byte-for-byte compatible apart from those eight magic/version
//! bytes; the typeflag dispatch picks up the format-specific
//! extensions ('L' / 'K').
//! - **PAX (POSIX.1-2001) extended headers** (typeflag `x`) for the
//! `path` and `size` keys. `path` lifts the 100/255-byte length
//! limit; `size` lifts the 8 GiB octal-encoded limit and is the
//! mechanism §7.4 names for "ustar size limits" handling.
//! - **GNU base-256 (binary) numeric encoding** in the size field.
//! GNU tar (the default in most distros) switches the size field
//! from octal ASCII to a big-endian unsigned integer with the high
//! bit of the first byte set whenever the value exceeds the
//! 8 GiB octal limit, instead of (or in addition to) emitting a
//! PAX `size=` record. Chain-state snapshots that contain
//! individual files ≥ 8 GiB rely on this.
//! - **GNU long-name extensions** (typeflag `L`) for entries whose
//! path exceeds the 100/255-byte ustar limits. The bytes following
//! the `L` header are read as a NUL-terminated path and applied
//! to the next entry, matching what GNU `tar` does on extract.
//! `K` (long link target) is read the same way and applied to the
//! next entry's link target, so symlinks / hard links whose target
//! exceeds the 100-byte `linkname` field extract correctly.
//! - **Regular files** (`0`, `\0`, `7`), **directories** (`5`),
//! **symbolic links** (`2`) and **hard links** (`1`).
//!
//! Everything else — device nodes (`3`/`4`), FIFOs (`6`), PAX global
//! headers (`g`) — is rejected with [`SinkError::UnsupportedEntry`].
//!
//! # Path safety
//!
//! Entry names are resolved purely lexically against
//! [`TarSink::new`]'s root. The resolver rejects:
//!
//! - Absolute paths (`/etc/passwd`).
//! - Any component equal to `..`.
//! - Empty entry names.
//! - Entry names containing NUL bytes.
//!
//! Once the sink can create symbolic links, that lexical guarantee is
//! no longer *complete* on its own: a hostile archive can create a
//! symlink (`evil -> /`) and then write a later entry named
//! `evil/passwd`, whose name passes every lexical check yet lands
//! outside the root once the OS follows `evil`. peel defends against
//! this by **never following a symlink it (or anything else) created
//! when placing an entry**:
//!
//! - Parent directories are materialized one component at a time with
//! [`fs::create_dir`]; any existing component that `lstat`s as a
//! symlink aborts the entry with [`SinkError::SymlinkTraversal`]
//! rather than being traversed.
//! - A regular file / symlink / hard-link whose *final* path
//! component already exists as a symlink removes that symlink first,
//! so an `O_TRUNC` open can never write through it.
//! - Symbolic-link targets themselves are recreated **verbatim**,
//! exactly as archived — including absolute (`/usr/lib`) and
//! upward (`../sibling`) targets — because peel's own writes never
//! traverse them. This matches GNU `tar`'s faithful behavior; the
//! safety boundary is "peel writes nothing outside the root," not
//! "the root contains no pointers outside it."
//! - Hard-link targets, by contrast, *are* constrained: a hard link
//! must resolve to an already-extracted regular file inside the
//! root, reached without traversing any symlink, else
//! [`SinkError::UnsafeLink`].
//!
//! The lexical resolver still rejects archives whose entries cancel
//! out a `..` later — a stricter posture than `bsdtar`. The symlink
//! checks are `lstat`-based rather than `openat2`-based, so a *local*
//! attacker racing the extraction could in principle win a TOCTOU
//! window; the threat model peel defends is a hostile *archive*, not
//! a hostile concurrent process mutating the extraction root.
//!
//! # Streaming guarantees
//!
//! [`Sink::write`] accepts arbitrary chunk boundaries: feeding the same
//! archive byte-by-byte produces the same on-disk output as feeding it
//! all at once. The parser maintains a single 512-byte header buffer
//! and a per-entry data cursor, both of which advance independently of
//! the call boundaries.
use std::collections::HashSet;
use std::fs::{self, File, OpenOptions};
use std::io::Write;
use std::path::{Component, Path, PathBuf};
use crate::sink::{Sink, SinkError};
/// Tar block size. Headers are exactly one block; file data is padded
/// up to the next block boundary.
const BLOCK: usize = 512;
/// Stream into a directory tree on disk, member by member.
///
/// Construct with [`TarSink::new`]; feed the archive bytes via
/// [`Sink::write`]; finalize with [`Sink::close`]. The sink reports
/// [`Sink::is_quiescent`] as `true` only between members so the
/// coordinator can take checkpoints at restart-safe boundaries.
pub struct TarSink {
/// Extraction root. Every successfully written file's path lies
/// inside this directory.
root: PathBuf,
/// Driving state machine — see [`State`].
state: State,
/// Total bytes consumed from the archive so far. Used in error
/// messages to point the user at the failing record.
archive_offset: u64,
/// Number of *consecutive* zero blocks observed. Two of them mark
/// the end of the archive.
zero_blocks_seen: u8,
/// PAX `path=` override applying to the next non-PAX entry.
pending_path: Option<String>,
/// PAX `size=` override applying to the next non-PAX entry.
pending_size: Option<u64>,
/// Link-target override (PAX `linkpath=` or a GNU `K` long-link
/// extension) applying to the next symlink / hard-link entry.
pending_linkpath: Option<String>,
/// Directory paths already materialized as *real* directories
/// under [`Self::root`] and verified not to be symlinks. Lets the
/// per-entry parent-directory walk skip the `lstat` + `mkdir`
/// syscalls for ancestors it has already vetted (tar entries are
/// overwhelmingly grouped by directory, so the hit rate is high).
/// Not part of the checkpoint: a resumed sink rebuilds it lazily
/// from the on-disk tree, which is the authoritative state.
ensured_dirs: HashSet<PathBuf>,
/// Sticky failure flag. Once a write errors, every subsequent
/// write returns an error too — partial extraction is never silently
/// continued.
poisoned: bool,
}
/// Parser state machine.
///
/// Transitions:
/// ```text
/// Header(filled<512) ──bytes──▶ Header(filled+=)
/// Header(filled==512) ──parse──▶ {Header,File,PaxData,Finished}
/// File(remaining>0) ──bytes──▶ File(remaining-=)
/// File(remaining==0, padding>0) ──bytes──▶ File(padding-=)
/// File(remaining==0, padding==0) ──▶ Header(0)
/// PaxData ── analogous, then applies overrides and ──▶ Header(0)
/// Finished ── trailing bytes are an error
/// ```
enum State {
/// Filling a 512-byte header buffer.
Header {
/// Number of bytes received toward the next header. `0..=BLOCK`.
filled: usize,
/// The header buffer itself. Boxed so the `State` enum stays
/// small (the variant is only ~24 bytes instead of ~520).
buf: Box<[u8; BLOCK]>,
},
/// Writing a regular file's body to disk, then skipping its
/// 512-byte block padding.
File {
/// Bytes of file data still to receive.
remaining: u64,
/// Bytes of trailing zero padding still to consume.
padding: u16,
/// The file we are writing into.
file: File,
/// Resolved on-disk path; carried for error context only.
path: PathBuf,
/// Total payload size declared by the tar header. Used by
/// the checkpoint snapshotter to derive how many bytes have
/// already been written (`total_size - remaining`); not
/// otherwise consulted by the live parser.
total_size: u64,
},
/// Collecting a PAX 'x' extended header's body into a buffer.
PaxData {
/// Bytes of PAX body still to receive.
remaining: u64,
/// Bytes of trailing zero padding still to consume.
padding: u16,
/// Accumulator for the entry data; drained on completion.
buf: Vec<u8>,
},
/// Collecting a GNU long-name ('L') or long-link ('K') extension.
/// The body is a NUL-terminated path that overrides the *next*
/// entry's name ('L') or link-target ('K') field.
LongName {
/// Bytes of body still to receive.
remaining: u64,
/// Bytes of trailing zero padding still to consume.
padding: u16,
/// Accumulator for the long path / link target.
buf: Vec<u8>,
/// `true` for 'K' (long link target, applied to the next
/// entry's `pending_linkpath`), `false` for 'L' (long path,
/// applied to the next entry's `pending_path`).
is_link: bool,
},
/// End-of-archive marker observed; further bytes other than zeros
/// are an error.
Finished,
}
impl TarSink {
/// Construct a sink that extracts into `root`.
///
/// The directory must already exist; we never create the root
/// itself, only entries within it. Most test paths use
/// `fs::create_dir_all(&root)` first.
///
/// # Errors
///
/// Returns [`SinkError::Io`] if `root` cannot be canonicalized
/// (does not exist, permission denied, …). Canonicalizing once
/// up-front means later path-escape checks compare absolute paths
/// rather than relative-vs-relative segments.
pub fn new<P: AsRef<Path>>(root: P) -> Result<Self, SinkError> {
let root = root.as_ref();
let canonical = root.canonicalize().map_err(|source| SinkError::Io {
path: root.to_path_buf(),
source,
})?;
Ok(Self {
root: canonical,
state: State::Header {
filled: 0,
buf: Box::new([0u8; BLOCK]),
},
archive_offset: 0,
zero_blocks_seen: 0,
pending_path: None,
pending_size: None,
pending_linkpath: None,
ensured_dirs: HashSet::new(),
poisoned: false,
})
}
/// Construct a [`TarSink`] pre-seeded from a previously captured
/// [`crate::checkpoint::TarSinkState`].
///
/// `state` was produced by [`Sink::sink_state`] on the prior
/// (killed) run's sink, and the on-disk extraction is whatever
/// the killed run had durably written before the checkpoint
/// fired. The resume path:
///
/// 1. Canonicalizes `root` (the same way [`Self::new`] does).
/// 2. For [`crate::checkpoint::TarMemberState::File`]: opens
/// the recorded path under `root`, truncates it to
/// `total_size - remaining` (so any torn write past the
/// checkpoint is discarded), and seeks to that offset.
/// Restores the live `State::File` ready to receive the
/// payload's continuation.
/// 3. For other variants: rebuilds the `State` enum directly
/// from the saved fields.
///
/// # Errors
///
/// Returns [`SinkError::Io`] if `root` can't be canonicalized
/// or the in-flight file can't be reopened at the recorded
/// offset; [`SinkError::PathEscape`] if the recorded file path
/// lies outside `root`.
pub fn resume<P: AsRef<Path>>(
root: P,
state: &crate::checkpoint::TarSinkState,
) -> Result<Self, SinkError> {
use crate::checkpoint::TarMemberState;
let root_ref = root.as_ref();
let canonical_root = root_ref.canonicalize().map_err(|source| SinkError::Io {
path: root_ref.to_path_buf(),
source,
})?;
let live_state = match &state.state {
TarMemberState::Header { filled, buf: saved } => {
if *filled > BLOCK as u32 || saved.len() != *filled as usize {
return Err(SinkError::Io {
path: canonical_root.clone(),
source: std::io::Error::other(format!(
"resume: header filled={filled} buf.len()={}",
saved.len(),
)),
});
}
let mut buf = Box::new([0u8; BLOCK]);
buf[..saved.len()].copy_from_slice(saved);
State::Header {
filled: *filled as usize,
buf,
}
}
TarMemberState::File {
remaining,
padding,
path,
total_size,
} => {
if *remaining > *total_size {
return Err(SinkError::Io {
path: canonical_root.clone(),
source: std::io::Error::other(format!(
"resume: file remaining {remaining} > total_size {total_size}",
)),
});
}
let path_buf = PathBuf::from(path);
if !path_buf.starts_with(&canonical_root) {
return Err(SinkError::PathEscape {
entry: path.clone(),
root: canonical_root.clone(),
});
}
let already_written = total_size.saturating_sub(*remaining);
let mut file = OpenOptions::new()
.read(true)
.write(true)
.truncate(false)
.open(&path_buf)
.map_err(|source| SinkError::Io {
path: path_buf.clone(),
source,
})?;
// Truncate any tail past the checkpoint, then seek
// to the resume position. The truncate-then-seek
// shape mirrors `RawSink::resume`.
file.set_len(already_written)
.map_err(|source| SinkError::Io {
path: path_buf.clone(),
source,
})?;
use std::io::{Seek, SeekFrom};
file.seek(SeekFrom::Start(already_written))
.map_err(|source| SinkError::Io {
path: path_buf.clone(),
source,
})?;
State::File {
remaining: *remaining,
padding: u16::try_from(*padding).map_err(|_| SinkError::Io {
path: path_buf.clone(),
source: std::io::Error::other(format!(
"resume: file padding {padding} ≥ 65536",
)),
})?,
file,
path: path_buf,
total_size: *total_size,
}
}
TarMemberState::PaxData {
remaining,
padding,
buf,
} => State::PaxData {
remaining: *remaining,
padding: u16::try_from(*padding).map_err(|_| SinkError::Io {
path: canonical_root.clone(),
source: std::io::Error::other(
format!("resume: pax padding {padding} ≥ 65536",),
),
})?,
buf: buf.clone(),
},
TarMemberState::LongName {
remaining,
padding,
buf,
is_link,
} => State::LongName {
remaining: *remaining,
padding: u16::try_from(*padding).map_err(|_| SinkError::Io {
path: canonical_root.clone(),
source: std::io::Error::other(format!(
"resume: longname padding {padding} ≥ 65536",
)),
})?,
buf: buf.clone(),
is_link: *is_link,
},
TarMemberState::Finished => State::Finished,
};
Ok(Self {
root: canonical_root,
state: live_state,
archive_offset: state.archive_offset,
zero_blocks_seen: state.zero_blocks_seen,
pending_path: state.pending_path.clone(),
pending_size: state.pending_size,
pending_linkpath: state.pending_linkpath.clone(),
ensured_dirs: HashSet::new(),
poisoned: false,
})
}
/// Borrow the configured extraction root.
#[must_use]
pub fn root(&self) -> &Path {
&self.root
}
/// Feed bytes through the parser without the poisoning bookkeeping.
/// All public entry points wrap this in the sticky-failure check.
fn write_inner(&mut self, mut input: &[u8]) -> Result<(), SinkError> {
while !input.is_empty() {
// We only step as much as one state arm can consume; the
// outer loop re-enters with the remainder so a long input
// can cross arbitrarily many state transitions in one call.
let consumed = match &mut self.state {
State::Header { filled, buf } => {
let want = BLOCK - *filled;
let take = input.len().min(want);
buf[*filled..*filled + take].copy_from_slice(&input[..take]);
*filled += take;
if *filled == BLOCK {
// The buffer is complete — process it. We
// can't keep `filled`/`buf` borrowed across the
// call, so move the buffer out, drop the
// borrow, then transition. The state will be
// overwritten before the function returns.
let header_buf = std::mem::replace(buf, Box::new([0u8; BLOCK]));
*filled = 0;
self.process_header(*header_buf)?;
}
take
}
State::File {
remaining,
padding,
file,
path,
total_size: _,
} => {
if *remaining > 0 {
let want = usize::try_from(*remaining)
.unwrap_or(usize::MAX)
.min(input.len());
file.write_all(&input[..want])
.map_err(|source| SinkError::Io {
path: path.clone(),
source,
})?;
*remaining -= want as u64;
// Transition immediately when both data and
// padding are exhausted. Without this guard,
// a file whose size is an exact multiple of
// 512 (so `padding == 0`) would loop into the
// "both zero" arm below, return 0 consumed,
// and trip the outer no-progress check.
if *remaining == 0 && *padding == 0 {
self.finish_file_state();
}
want
} else if *padding > 0 {
let want = usize::from(*padding).min(input.len());
// The padding is supposed to be zero bytes; we
// do not enforce that — real-world archives
// produced by `gnu tar` zero them out, but the
// spec only says "padding to a 512-byte
// boundary" and we are forgiving.
*padding -= want as u16;
if *padding == 0 {
self.finish_file_state();
}
want
} else {
// Both zero — should have transitioned out
// already.
self.finish_file_state();
0
}
}
State::PaxData {
remaining,
padding,
buf,
} => {
if *remaining > 0 {
let want = usize::try_from(*remaining)
.unwrap_or(usize::MAX)
.min(input.len());
buf.extend_from_slice(&input[..want]);
*remaining -= want as u64;
// Same alignment-fix rationale as the File
// arm: a PAX header whose size is a multiple
// of 512 would otherwise stall the parser.
if *remaining == 0 && *padding == 0 {
self.finish_pax_state()?;
}
want
} else if *padding > 0 {
let want = usize::from(*padding).min(input.len());
*padding -= want as u16;
if *padding == 0 {
self.finish_pax_state()?;
}
want
} else {
self.finish_pax_state()?;
0
}
}
State::LongName {
remaining,
padding,
buf,
is_link: _,
} => {
if *remaining > 0 {
let want = usize::try_from(*remaining)
.unwrap_or(usize::MAX)
.min(input.len());
// Both 'L' (long path) and 'K' (long link
// target) buffer the payload; the allocation
// is capped at `process_header` time so an
// oversized extension cannot grow the buffer
// unbounded.
buf.extend_from_slice(&input[..want]);
*remaining -= want as u64;
if *remaining == 0 && *padding == 0 {
self.finish_long_name_state()?;
}
want
} else if *padding > 0 {
let want = usize::from(*padding).min(input.len());
*padding -= want as u16;
if *padding == 0 {
self.finish_long_name_state()?;
}
want
} else {
self.finish_long_name_state()?;
0
}
}
State::Finished => {
// After the end-of-archive marker, we tolerate
// additional zero bytes (real-world archives often
// pad to a 10 KiB block) but reject anything else.
let nz = input.iter().position(|&b| b != 0).unwrap_or(input.len());
if nz < input.len() {
return Err(SinkError::TrailingData {
archive_offset: self.archive_offset + nz as u64,
});
}
nz
}
};
self.archive_offset += consumed as u64;
input = &input[consumed..];
// Defensive: a state transition that consumes zero bytes
// and does not change state would loop forever. The arms
// above either consume or transition; the only place that
// can return 0 is the Finished arm with empty `input`,
// which is exited by the outer while.
if consumed == 0 && !input.is_empty() {
return Err(SinkError::MalformedHeader {
archive_offset: self.archive_offset,
reason: "parser made no progress (internal invariant)".into(),
});
}
}
Ok(())
}
/// Process a fully-buffered 512-byte header.
fn process_header(&mut self, header: [u8; BLOCK]) -> Result<(), SinkError> {
if header.iter().all(|&b| b == 0) {
self.zero_blocks_seen = self.zero_blocks_seen.saturating_add(1);
if self.zero_blocks_seen >= 2 {
self.state = State::Finished;
}
// Single zero block: we stay in Header, waiting to see if
// the second one follows.
return Ok(());
}
self.zero_blocks_seen = 0;
let header_offset = self
.archive_offset
.checked_sub(BLOCK as u64 - 1)
.map_or(self.archive_offset, |o| o.saturating_sub(1));
// The header occupies [header_offset, header_offset+512); the
// saturating arithmetic keeps the diagnostic value sane even on
// pathological offsets.
validate_magic(&header, header_offset)?;
validate_checksum(&header, header_offset)?;
let parsed = ParsedHeader::from_bytes(&header, header_offset)?;
let type_flag = parsed.type_flag;
let raw_size = parsed.size;
// PAX overrides are applied on top of the parsed header so the
// actual on-disk size and name reflect the override-not-the-
// raw-header.
let entry_size = self.pending_size.take().unwrap_or(raw_size);
let entry_name = match self.pending_path.take() {
Some(p) => p,
None => parsed.combined_name()?,
};
// The link-target override (PAX `linkpath=` / GNU `K`) is
// consumed for *every* entry, not just link entries, so a
// stray override can't leak forward onto an unrelated later
// entry. Only the symlink / hard-link arms below read it.
let entry_link = self.pending_linkpath.take();
match type_flag {
// Regular file. '\0' is the historical encoding, '0' the
// POSIX one; both mean the same thing. '7' (contiguous
// file) is treated identically — the distinction is
// semantic-free on every modern filesystem.
0 | b'0' | b'7' => {
let path = self.resolve_entry_path(&entry_name)?;
self.ensure_parent_dirs(&entry_name, &path)?;
// If the final component already exists as a symlink
// (left by an earlier entry, or pre-seeded), remove it
// first so the `O_TRUNC` open below writes a fresh
// regular file instead of following the link and
// truncating its target.
self.unlink_if_symlink(&path)?;
let file = OpenOptions::new()
.write(true)
.create(true)
.truncate(true)
.open(&path)
.map_err(|source| SinkError::Io {
path: path.clone(),
source,
})?;
let padding = padding_for(entry_size);
self.state = if entry_size == 0 && padding == 0 {
// Zero-length file with no padding to skip;
// transition straight back to Header.
drop(file);
State::Header {
filled: 0,
buf: Box::new([0u8; BLOCK]),
}
} else {
State::File {
remaining: entry_size,
padding,
file,
path,
total_size: entry_size,
}
};
Ok(())
}
// Directory.
b'5' => {
let path = self.resolve_entry_path(&entry_name)?;
self.ensure_dir(&entry_name, &path)?;
// Directory entries should declare size=0; some
// archives in the wild ignore that. We honor whatever
// the header (or PAX) said and skip that many bytes
// before resuming.
let padding = padding_for(entry_size);
self.state = if entry_size == 0 && padding == 0 {
State::Header {
filled: 0,
buf: Box::new([0u8; BLOCK]),
}
} else {
State::PaxData {
// Reuse the PaxData state as a "skip these
// bytes" buffer for non-zero-size directory
// entries; we discard the bytes by checking
// the buf len in finish_pax_state but it is
// simpler to just throw away the bytes inline.
// Use a dedicated skip path:
remaining: entry_size,
padding,
// An empty Vec means "discard incoming bytes";
// see finish_pax_state for the cap.
buf: Vec::new(),
}
};
Ok(())
}
// Symbolic link. The target lives in the `linkname` field
// (or a PAX `linkpath=` / GNU `K` override) and is
// recreated verbatim — absolute and upward targets
// included — because peel never *follows* a symlink it
// created when placing a later entry (see the module-level
// path-safety notes). Symlinks declare size 0; we tolerate
// (and skip) a stray body the way the directory arm does.
b'2' => {
let target = self.link_target(&entry_name, entry_link, &parsed)?;
let path = self.resolve_entry_path(&entry_name)?;
self.ensure_parent_dirs(&entry_name, &path)?;
// Clear any file or symlink already sitting at the
// final path so `symlink(2)` does not fail with
// EEXIST (last-entry-wins, matching the regular-file
// arm's `O_TRUNC`).
self.remove_existing_nondir(&path)?;
create_symlink(Path::new(&target), &path).map_err(|source| SinkError::Io {
path: path.clone(),
source,
})?;
self.state = self.skip_body_state(entry_size);
Ok(())
}
// Hard link. Unlike a symlink, the target must resolve to
// an already-extracted regular file *inside* the root,
// reached without crossing a symlink — otherwise `link(2)`
// could alias a file the archive never legitimately
// produced. Hard links declare size 0.
b'1' => {
let target = self.link_target(&entry_name, entry_link, &parsed)?;
let link_path = self.resolve_entry_path(&entry_name)?;
let target_path = self.resolve_hardlink_target(&entry_name, &target)?;
self.ensure_parent_dirs(&entry_name, &link_path)?;
self.remove_existing_nondir(&link_path)?;
fs::hard_link(&target_path, &link_path).map_err(|source| SinkError::Io {
path: link_path.clone(),
source,
})?;
self.state = self.skip_body_state(entry_size);
Ok(())
}
// PAX extended header for the next entry. Body is a
// sequence of `<len> <key>=<value>\n` records we'll parse
// in finish_pax_state.
b'x' => {
let padding = padding_for(entry_size);
self.state = if entry_size == 0 && padding == 0 {
State::Header {
filled: 0,
buf: Box::new([0u8; BLOCK]),
}
} else {
State::PaxData {
remaining: entry_size,
padding,
// We pre-allocate the exact size — bounded
// small in practice (a few KiB at most).
buf: Vec::with_capacity(usize::try_from(entry_size).unwrap_or(0)),
}
};
Ok(())
}
// GNU long-name ('L') / long-link ('K') extensions. The
// header's "./@LongLink" name is ignored; the body holds
// the real path. 'L' overrides the next entry's name; 'K'
// overrides the next entry's link target. Pre-cap the
// allocation so a hostile archive can't ask us to reserve
// gigabytes of memory.
b'L' | b'K' => {
let is_link = type_flag == b'K';
let padding = padding_for(entry_size);
self.state = if entry_size == 0 && padding == 0 {
State::Header {
filled: 0,
buf: Box::new([0u8; BLOCK]),
}
} else {
let cap_hint = usize::try_from(entry_size)
.unwrap_or(usize::MAX)
.min(64 * 1024);
State::LongName {
remaining: entry_size,
padding,
buf: Vec::with_capacity(cap_hint),
is_link,
}
};
Ok(())
}
other => Err(SinkError::UnsupportedEntry {
type_flag: other,
entry: entry_name,
}),
}
}
fn finish_file_state(&mut self) {
// Drop the file, transition home.
self.state = State::Header {
filled: 0,
buf: Box::new([0u8; BLOCK]),
};
}
fn finish_long_name_state(&mut self) -> Result<(), SinkError> {
let State::LongName {
remaining: _,
padding: _,
buf,
is_link,
} = std::mem::replace(
&mut self.state,
State::Header {
filled: 0,
buf: Box::new([0u8; BLOCK]),
},
)
else {
// INVARIANT: only called from within the LongName arm.
return Ok(());
};
// GNU stores the path NUL-terminated and pads to a 512-byte
// boundary with zeros. Trim at the first NUL so the override
// we apply matches what `tar` itself would.
let trimmed = buf.iter().position(|&b| b == 0).unwrap_or(buf.len());
let bytes = &buf[..trimmed];
let kind = if is_link { "long-link" } else { "long-name" };
let value = std::str::from_utf8(bytes).map_err(|_| SinkError::MalformedHeader {
archive_offset: self.archive_offset,
reason: format!("GNU {kind} payload is not valid UTF-8"),
})?;
// PAX 'path=' / 'linkpath=' take precedence if both the PAX
// and GNU extensions are present for the same entry — PAX is
// the modern spec and any archive emitting both is signaling
// the PAX value as authoritative.
if is_link {
if self.pending_linkpath.is_none() {
self.pending_linkpath = Some(value.to_string());
}
} else if self.pending_path.is_none() {
self.pending_path = Some(value.to_string());
}
Ok(())
}
fn finish_pax_state(&mut self) -> Result<(), SinkError> {
let State::PaxData {
remaining: _,
padding: _,
buf,
} = std::mem::replace(
&mut self.state,
State::Header {
filled: 0,
buf: Box::new([0u8; BLOCK]),
},
)
else {
// INVARIANT: only called from within the PaxData arm.
return Ok(());
};
if !buf.is_empty() {
let records = parse_pax_records(&buf, self.archive_offset)?;
for (key, value) in records {
match key.as_str() {
"path" => self.pending_path = Some(value),
"linkpath" => self.pending_linkpath = Some(value),
"size" => {
self.pending_size =
Some(value.parse::<u64>().map_err(|_| SinkError::MalformedPax {
archive_offset: self.archive_offset,
reason: format!("size value {value:?} is not a u64"),
})?);
}
_ => {
// Unknown keys are silently ignored — PAX
// requires that an extractor not break on
// unknown extensions.
}
}
}
}
Ok(())
}
// `unsafe_component_reason` is a free fn (not a method) so it
// can be unit-tested without constructing a `TarSink`.
/// Resolve an entry name to an absolute path under `self.root`,
/// rejecting anything that escapes.
///
/// Tar archives in the wild commonly include a self-referential
/// directory entry like `./` (or just `.`) representing the
/// archive root itself — `bsdtar` and GNU `tar` both emit it on
/// `tar cf out.tar ./`. Those entries resolve to `self.root` and
/// are extracted as a no-op `mkdir -p` of the (already-existing)
/// output directory. Treating that as a path-escape would refuse
/// every Arbitrum snapshot and many real-world tarballs; accept
/// it instead.
fn resolve_entry_path(&self, entry: &str) -> Result<PathBuf, SinkError> {
if entry.is_empty() || entry.contains('\0') {
return Err(SinkError::PathEscape {
entry: entry.to_string(),
root: self.root.clone(),
});
}
if entry.starts_with('/') {
return Err(SinkError::PathEscape {
entry: entry.to_string(),
root: self.root.clone(),
});
}
let mut out = self.root.clone();
for component in entry.split('/') {
if component.is_empty() || component == "." {
continue;
}
if component == ".." {
return Err(SinkError::PathEscape {
entry: entry.to_string(),
root: self.root.clone(),
});
}
// Reject any component containing a path separator — on
// Unix this is just a NUL guard (already checked) and
// the std `PathBuf::push` already documents it as an
// append. We do an extra check for robustness against any
// future cross-platform expansion.
if Path::new(component)
.components()
.any(|c| !matches!(c, Component::Normal(_)))
{
return Err(SinkError::PathEscape {
entry: entry.to_string(),
root: self.root.clone(),
});
}
// NTFS-portable safety check (`PLAN_v3_windows.md` §8).
// Applied on every platform so an archive's behavior is
// uniform across hosts: a path that would silently
// collide on Windows is rejected on Linux too.
if let Some(reason) = unsafe_component_reason(component) {
return Err(SinkError::UnsafePath {
entry: entry.to_string(),
component: component.to_string(),
reason,
});
}
out.push(component);
}
// Empty resolved path means the entry was `.`, `./`, or
// similar — that's the archive root itself. Return
// `self.root`; the caller's directory-ensure is a no-op
// because the root already exists. A file entry whose name
// resolves to the root (vanishingly rare; would have to be
// `./` with type-flag `0`) would fail at `OpenOptions::write`
// with a clean "Is a directory" IO error — matched behaviour
// is fine.
Ok(out)
}
/// Resolve the link target for a symlink / hard-link entry.
///
/// Precedence: a `pending_linkpath` override (PAX `linkpath=` or a
/// GNU `K` long-link) wins over the 100-byte header `linkname`
/// field, matching what GNU `tar` applies on extract. The string
/// is returned *verbatim*; the caller decides how to interpret it
/// (a symlink writes it as-is, a hard link resolves it under the
/// root). Empty or NUL-bearing targets are malformed.
fn link_target(
&self,
entry: &str,
pending: Option<String>,
parsed: &ParsedHeader<'_>,
) -> Result<String, SinkError> {
let target = match pending {
Some(t) => t,
None => parsed.link_name_str()?,
};
if target.is_empty() {
return Err(SinkError::MalformedHeader {
archive_offset: self.archive_offset,
reason: format!("link entry {entry:?} has an empty target"),
});
}
if target.contains('\0') {
return Err(SinkError::MalformedHeader {
archive_offset: self.archive_offset,
reason: format!("link entry {entry:?} target contains a NUL byte"),
});
}
Ok(target)
}
/// The parser state to enter after a header whose body the sink
/// wants to *skip*. Symlinks / hard links should declare size 0,
/// but we tolerate (and discard) a stray body the way the
/// directory arm does. Returns `Header` when there is nothing to
/// skip, else a discard buffer sized to the body.
fn skip_body_state(&self, entry_size: u64) -> State {
let padding = padding_for(entry_size);
if entry_size == 0 && padding == 0 {
State::Header {
filled: 0,
buf: Box::new([0u8; BLOCK]),
}
} else {
// An empty `buf` marks PaxData as a pure discard sink (see
// finish_pax_state); the bytes are dropped on the floor.
State::PaxData {