Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 5 additions & 1 deletion foyer-storage/src/engine/block/engine.rs
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ use crate::{
},
error::{Error, Result},
filter::conditions::IoThrottle,
io::{bytes::IoSliceMut, PAGE},
io::{bytes::IoSliceMut, device::DeviceCaps, PAGE},
keeper::PieceRef,
runtime::Runtime,
serde::EntryDeserializer,
Expand Down Expand Up @@ -332,6 +332,10 @@ where
let device = self.device;
let block_size = self.block_size;

match device.caps() {
DeviceCaps::Block(_) => {}
}

let mut tombstones = vec![];

let tombstone_log = if self.enable_tombstone_log {
Expand Down
18 changes: 17 additions & 1 deletion foyer-storage/src/io/device/combined.rs
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ use std::sync::{Arc, RwLock};

use crate::{
io::{
device::{Device, DeviceBuilder, Partition, PartitionId},
device::{BlockCaps, Device, DeviceBuilder, DeviceCaps, Partition, PartitionId},
error::{IoError, IoResult},
},
RawFile, Statistics, Throttle,
Expand Down Expand Up @@ -65,13 +65,24 @@ impl CombinedDeviceBuilder {

impl DeviceBuilder for CombinedDeviceBuilder {
fn build(self) -> IoResult<Arc<dyn Device>> {
let mut alignment = 0;
for device in &self.devices {
match device.caps() {
DeviceCaps::Block(block) => alignment = alignment.max(block.alignment),
}
}
if self.devices.is_empty() {
return Err(IoError::other("combined device requires at least one inner device"));
}

let device = CombinedDevice {
devices: self.devices,
statistics: Arc::new(Statistics::new(self.throttle)),
inner: RwLock::new(Inner {
partitions: vec![],
next: 0,
}),
caps: DeviceCaps::Block(BlockCaps { alignment }),
};
let device = Arc::new(device);
Ok(device)
Expand All @@ -90,9 +101,14 @@ pub struct CombinedDevice {
devices: Vec<Arc<dyn Device>>,
inner: RwLock<Inner>,
statistics: Arc<Statistics>,
caps: DeviceCaps,
}

impl Device for CombinedDevice {
fn caps(&self) -> &DeviceCaps {
&self.caps
}

fn capacity(&self) -> usize {
self.devices.iter().map(|d| d.capacity()).sum()
}
Expand Down
11 changes: 10 additions & 1 deletion foyer-storage/src/io/device/file.rs
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,10 @@ use fs4::free_space;

use crate::{
io::{
device::{statistics::Statistics, throttle::Throttle, Device, DeviceBuilder, Partition, PartitionId},
device::{
statistics::Statistics, throttle::Throttle, BlockCaps, Device, DeviceBuilder, DeviceCaps, Partition,
PartitionId,
},
error::IoResult,
PAGE,
},
Expand Down Expand Up @@ -119,6 +122,7 @@ impl DeviceBuilder for FileDeviceBuilder {
capacity,
statistics,
partitions: RwLock::new(vec![]),
caps: DeviceCaps::Block(BlockCaps::default()),
};
let device: Arc<dyn Device> = Arc::new(device);
Ok(device)
Expand All @@ -132,9 +136,14 @@ pub struct FileDevice {
capacity: usize,
partitions: RwLock<Vec<Arc<FilePartition>>>,
statistics: Arc<Statistics>,
caps: DeviceCaps,
}

impl Device for FileDevice {
fn caps(&self) -> &DeviceCaps {
&self.caps
}

fn capacity(&self) -> usize {
self.capacity
}
Expand Down
11 changes: 10 additions & 1 deletion foyer-storage/src/io/device/fs.rs
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,10 @@ use fs4::free_space;

use crate::{
io::{
device::{statistics::Statistics, throttle::Throttle, Device, DeviceBuilder, Partition, PartitionId},
device::{
statistics::Statistics, throttle::Throttle, BlockCaps, Device, DeviceBuilder, DeviceCaps, Partition,
PartitionId,
},
error::IoResult,
PAGE,
},
Expand Down Expand Up @@ -103,6 +106,7 @@ impl DeviceBuilder for FsDeviceBuilder {
#[cfg(target_os = "linux")]
direct: self.direct,
partitions: RwLock::new(vec![]),
caps: DeviceCaps::Block(BlockCaps::default()),
};
let device: Arc<dyn Device> = Arc::new(device);
Ok(device)
Expand All @@ -118,6 +122,7 @@ pub struct FsDevice {
#[cfg(target_os = "linux")]
direct: bool,
partitions: RwLock<Vec<Arc<FsPartition>>>,
caps: DeviceCaps,
}

impl FsDevice {
Expand All @@ -128,6 +133,10 @@ impl FsDevice {
}

impl Device for FsDevice {
fn caps(&self) -> &DeviceCaps {
&self.caps
}

fn capacity(&self) -> usize {
self.capacity
}
Expand Down
25 changes: 24 additions & 1 deletion foyer-storage/src/io/device/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ pub mod throttle;

use std::{any::Any, fmt::Debug, sync::Arc};

use crate::io::{device::statistics::Statistics, error::IoResult};
use crate::io::{device::statistics::Statistics, error::IoResult, PAGE};

pub type PartitionId = u32;

Expand All @@ -42,6 +42,26 @@ pub trait DeviceBuilder: Send + Sync + 'static + Debug {
fn build(self) -> IoResult<Arc<dyn Device>>;
}

/// The capabilities of a device.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum DeviceCaps {
/// A block-addressable device.
Block(BlockCaps),
}

/// Capabilities for block devices.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct BlockCaps {
/// The required alignment for block I/O.
pub alignment: usize,
}

impl Default for BlockCaps {
fn default() -> Self {
Self { alignment: PAGE }
}
}

/// Partition is a logical segment of a device.
pub trait Partition: Send + Sync + 'static + Debug + Any {
/// Get the id of the partition.
Expand All @@ -61,6 +81,9 @@ pub trait Partition: Send + Sync + 'static + Debug + Any {

/// Device trait.
pub trait Device: Send + Sync + 'static + Debug + Any {
/// Get the capabilities of the device.
fn caps(&self) -> &DeviceCaps;

/// Get the capacity of the device.
///
/// NOTE: `capacity` must be 4K aligned.
Expand Down
11 changes: 10 additions & 1 deletion foyer-storage/src/io/device/noop.rs
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,10 @@ use std::sync::{Arc, RwLock};

use crate::{
io::{
device::{statistics::Statistics, throttle::Throttle, Device, DeviceBuilder, Partition, PartitionId},
device::{
statistics::Statistics, throttle::Throttle, BlockCaps, Device, DeviceBuilder, DeviceCaps, Partition,
PartitionId,
},
error::IoResult,
},
RawFile,
Expand Down Expand Up @@ -48,6 +51,7 @@ impl DeviceBuilder for NoopDeviceBuilder {
partitions: RwLock::new(vec![]),
capacity: self.capacity,
statistics,
caps: DeviceCaps::Block(BlockCaps::default()),
}))
}
}
Expand All @@ -59,9 +63,14 @@ pub struct NoopDevice {
capacity: usize,

statistics: Arc<Statistics>,
caps: DeviceCaps,
}

impl Device for NoopDevice {
fn caps(&self) -> &DeviceCaps {
&self.caps
}

fn capacity(&self) -> usize {
self.capacity
}
Expand Down
11 changes: 10 additions & 1 deletion foyer-storage/src/io/device/partial.rs
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ use std::sync::{Arc, RwLock};

use crate::{
io::{
device::{statistics::Statistics, Device, DeviceBuilder, Partition, PartitionId},
device::{statistics::Statistics, Device, DeviceBuilder, DeviceCaps, Partition, PartitionId},
error::IoResult,
},
IoError,
Expand Down Expand Up @@ -51,10 +51,14 @@ impl PartialDeviceBuilder {

impl DeviceBuilder for PartialDeviceBuilder {
fn build(self) -> IoResult<Arc<dyn Device>> {
let caps = match self.device.caps() {
DeviceCaps::Block(block) => *block,
};
Ok(Arc::new(PartialDevice {
inner: self.device,
capacity: self.capacity,
partitions: RwLock::new(vec![]),
caps: DeviceCaps::Block(caps),
}))
}
}
Expand All @@ -65,9 +69,14 @@ pub struct PartialDevice {
inner: Arc<dyn Device>,
capacity: usize,
partitions: RwLock<Vec<Arc<PartialPartition>>>,
caps: DeviceCaps,
}

impl Device for PartialDevice {
fn caps(&self) -> &DeviceCaps {
&self.caps
}

fn capacity(&self) -> usize {
self.capacity
}
Expand Down
2 changes: 1 addition & 1 deletion foyer-storage/src/prelude.rs
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ pub use crate::{
partial::PartialDeviceBuilder,
statistics::Statistics,
throttle::{IopsCounter, Throttle},
Device, DeviceBuilder, RawFile,
BlockCaps, Device, DeviceBuilder, DeviceCaps, RawFile,
},
engine::{
noop::{NoopIoEngine, NoopIoEngineBuilder},
Expand Down
Loading