Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 7 additions & 1 deletion backend/config/test.exs
Original file line number Diff line number Diff line change
Expand Up @@ -124,6 +124,10 @@ config :edgehog,
:astarte_file_transfer_capabilities_module,
Edgehog.Astarte.Device.FileTransferCapabilitiesMock

config :edgehog,
:astarte_file_download_request_module,
Edgehog.Astarte.Device.FileDownloadRequestMock

config :edgehog, :astarte_forwarder_session_module, Edgehog.Astarte.Device.ForwarderSessionMock
config :edgehog, :astarte_geolocation_module, Edgehog.Astarte.Device.GeolocationMock
config :edgehog, :astarte_hardware_info_module, Edgehog.Astarte.Device.HardwareInfoMock
Expand All @@ -140,8 +144,10 @@ config :edgehog, :astarte_trigger_data_layer, Edgehog.Astarte.Trigger.MockDataLa
config :edgehog, :astarte_wifi_scan_result_module, Edgehog.Astarte.Device.WiFiScanResultMock
config :edgehog, :base_images_storage_module, Edgehog.BaseImages.StorageMock
config :edgehog, :container_reconciler, Edgehog.Containers.ReconcilerMock
config :edgehog, :files_storage_module, Edgehog.StorageMock
config :edgehog, :files_storage_module, Edgehog.Files.StorageMock
config :edgehog, :files_ephemeral_file_module, Edgehog.Files.EphemeralFileMock
config :edgehog, :os_management_ephemeral_image_module, Edgehog.OSManagement.EphemeralImageMock
config :edgehog, :presigned_urls_storage_module, Edgehog.StorageMock
config :edgehog, :reconciler_module, Edgehog.Tenants.ReconcilerMock

# Enable s3 storage since we're using mocks for it
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@ defimpl Edgehog.Campaigns.CampaignMechanism.Core,
def mark_operation_as_timed_out!(_mechanism, operation_id, tenant_id) do
file_download_request = Files.fetch_file_download_request!(operation_id, tenant: tenant_id)

case Files.set_response(
case Files.set_file_download_response(
file_download_request,
%{status: :failed, response_code: -1, response_message: "Request timed out"},
tenant: tenant_id
Expand Down
73 changes: 73 additions & 0 deletions backend/lib/edgehog/files/compressor.ex
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
#
# This file is part of Edgehog.
#
# Copyright 2026 SECO Mind Srl
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# SPDX-License-Identifier: Apache-2.0
#

defmodule Edgehog.Files.Compressor do
@moduledoc false

def compress(%Plug.Upload{} = upload, :gz) do
compressed_path =
Path.join(System.tmp_dir!(), "#{Ash.UUIDv7.generate()}.gz")

input = File.stream!(upload.path, 64_000, [])
output = File.open!(compressed_path, [:write, :binary])

z = :zlib.open()
:ok = :zlib.deflateInit(z, :default, :deflated, 31, 8, :default)

Enum.each(input, fn chunk ->
compressed = :zlib.deflate(z, chunk)
IO.binwrite(output, compressed)
end)

final = :zlib.deflate(z, <<>>, :finish)
IO.binwrite(output, final)

:zlib.close(z)
File.close(output)

{:ok,
%Plug.Upload{
path: compressed_path,
filename: upload.filename <> ".gz",
content_type: "application/gzip"
}}
rescue
error -> {:error, error}
end

def compress(%Plug.Upload{} = upload, :lz4) do
compressed_path =
Path.join(System.tmp_dir!(), "#{Ash.UUIDv7.generate()}.lz4")

upload.path
|> File.read!()
|> NimbleLZ4.compress_frame()
|> then(&File.write!(compressed_path, &1))

{:ok,
%Plug.Upload{
path: compressed_path,
filename: upload.filename <> ".lz4",
content_type: "application/x-lz4"
}}
rescue
error -> {:error, error}
end
end
36 changes: 36 additions & 0 deletions backend/lib/edgehog/files/digest.ex
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
#
# This file is part of Edgehog.
#
# Copyright 2026 SECO Mind Srl
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# SPDX-License-Identifier: Apache-2.0
#

defmodule Edgehog.Files.Digest do
@moduledoc false

@sha256_prefix "sha256:"

def file_sha256!(file_path) do
hash =
file_path
|> File.stream!(2048)
|> Enum.reduce(:crypto.hash_init(:sha256), &:crypto.hash_update(&2, &1))
|> :crypto.hash_final()
|> Base.encode16(case: :lower)

@sha256_prefix <> hash
end
end
76 changes: 76 additions & 0 deletions backend/lib/edgehog/files/file/bucket_storage.ex
Original file line number Diff line number Diff line change
@@ -0,0 +1,76 @@
#
# This file is part of Edgehog.
#
# Copyright 2026 SECO Mind Srl
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# SPDX-License-Identifier: Apache-2.0
#

defmodule Edgehog.Files.File.BucketStorage do
@moduledoc """
Implementation of `Edgehog.Files.File.Storage` that uses a bucket storage (e.g., S3, MinIO) to store files.
"""

@behaviour Edgehog.Files.File.Storage

alias Edgehog.Files.File
alias Edgehog.Files.File.Storage
alias Edgehog.Files.Uploaders

@impl Storage
def store(tenant_id, file_name, repository_id, encoding, %Plug.Upload{} = upload) do
scope = %{
tenant_id: tenant_id,
file_name: file_name,
repository_id: repository_id,
encoding: encoding
}

with {:ok, stored_name} <- Uploaders.File.store({upload, scope}) do
file_url = Uploaders.File.url({stored_name, scope})
{:ok, file_url}
end
end

@impl Storage
def delete(%File{} = file, encoding) when encoding in [nil, :gz, :lz4] do
{file_field, encoding_value} =
case encoding do
nil -> {:base_file, nil}
:gz -> {:gz_file, :gz}
:lz4 -> {:lz4_file, :lz4}
end

%File{
tenant_id: tenant_id,
name: file_name,
repository_id: repository_id
} = file

url =
file
|> Map.fetch!(file_field)
|> Map.fetch!(:url)

scope = %{
tenant_id: tenant_id,
file_name: file_name,
repository_id: repository_id,
encoding: encoding_value
}

Uploaders.File.delete({url, scope})
end
end
60 changes: 0 additions & 60 deletions backend/lib/edgehog/files/file/calculations/get_presigned_url.ex

This file was deleted.

60 changes: 0 additions & 60 deletions backend/lib/edgehog/files/file/calculations/put_presigned_url.ex

This file was deleted.

Original file line number Diff line number Diff line change
Expand Up @@ -18,12 +18,23 @@
# SPDX-License-Identifier: Apache-2.0
#

defmodule Edgehog.Files.File.Changes.DeleteFile do
defmodule Edgehog.Files.File.Changes.HandleFileDeletion do
@moduledoc """
Deletes a file from the storage backend that was uploaded via a presigned URL.
Ash change to handle file deletion after a file record is deleted.
"""

use Ash.Resource.Change

alias Edgehog.Files.File.BucketStorage

require Logger

@storage_module Application.compile_env(
:edgehog,
:files_storage_module,
BucketStorage
)

@impl Ash.Resource.Change
def change(%Ash.Changeset{valid?: false} = changeset, _opts, _context), do: changeset

Expand All @@ -34,13 +45,9 @@ defmodule Edgehog.Files.File.Changes.DeleteFile do
end

defp delete_old_file({:ok, file} = result) do
tenant_id = file.tenant_id
repository_id = file.repository_id
filename = file.name

file_path = "uploads/tenants/#{tenant_id}/repositories/#{repository_id}/files/#{filename}"

_ = Edgehog.Storage.delete(file_path)
_ = @storage_module.delete(file, nil)
_ = @storage_module.delete(file, :gz)
_ = @storage_module.delete(file, :lz4)

result
end
Expand Down
Loading
Loading