Skip to content

Commit 084aff9

Browse files
committed
AzureBlobTarget: use file basename for the temp download_file_location
When the target blob is inside a path (for instance "path/to/target") and we use `download_when_reading=True`, the directory layout doesn't exist and it fails with: ``` File "src/luigi/luigi/contrib/azureblob.py", line 194, in __enter__ self.client.download_as_file(self.container, self.blob, self.download_file_location) File "src/luigi/luigi/contrib/azureblob.py", line 101, in download_as_file return self.connection.get_blob_to_path(container, blob, location) File "src/luigi/venv/lib/python3.6/site-packages/azure/storage/blob/baseblobservice.py", line 1765, in get_blob_to_path with open(file_path, open_mode) as stream: FileNotFoundError: [Errno 2] No such file or directory: '/var/folders/7q/l5knvjqx3pg569hwsdrzjw480000gn/T/2020-10-12 14:20:01.950869689rh5q2/path/to/movie-cheesy.txt' ``` Use `os.path.basename(blob)` to keep only the blob name as file name instead of the full path (it's written in a temporary directory anyway)
1 parent cf2abbd commit 084aff9

File tree

2 files changed

+12
-2
lines changed

2 files changed

+12
-2
lines changed

luigi/contrib/azureblob.py

+2-1
Original file line numberDiff line numberDiff line change
@@ -179,7 +179,8 @@ def __init__(self, container, blob, client, download_when_reading, **kwargs):
179179
self.closed = False
180180
self.download_when_reading = download_when_reading
181181
self.azure_blob_options = kwargs
182-
self.download_file_location = os.path.join(tempfile.mkdtemp(prefix=str(datetime.datetime.utcnow())), blob)
182+
self.download_file_location = os.path.join(tempfile.mkdtemp(prefix=str(datetime.datetime.utcnow())),
183+
os.path.basename(blob))
183184
self.fid = None
184185

185186
def read(self, n=None):

test/contrib/azureblob_test.py

+10-1
Original file line numberDiff line numberDiff line change
@@ -117,7 +117,7 @@ def test_upload_copy_move_remove_blob(self):
117117

118118
class MovieScriptTask(luigi.Task):
119119
def output(self):
120-
return AzureBlobTarget("luigi-test", "movie-cheesy.txt", client, download_when_reading=False)
120+
return AzureBlobTarget("luigi-test", "path/to/movie-cheesy.txt", client, download_when_reading=False)
121121

122122
def run(self):
123123
client.connection.create_container("luigi-test")
@@ -170,3 +170,12 @@ def tearDown(self):
170170

171171
def test_AzureBlobTarget(self):
172172
luigi.build([FinalTask()], local_scheduler=True, log_level='NOTSET')
173+
174+
def test_AzureBlobTarget_download_when_reading(self):
175+
task = MovieScriptTask()
176+
task.run()
177+
target = task.output()
178+
# Test reading a target blob with a context manager and download_when_reading=True
179+
target.download_when_reading = True
180+
with target.open("r") as f:
181+
assert "James Bond" in f.read()

0 commit comments

Comments
 (0)