[importer_direct_upload] file storage location is changed to tempfile.NamedTemporaryFile

Author: agl29

desktop/core/src/desktop/settings.py

@@ -115,8 +115,7 @@
 # URL that handles the media served from MEDIA_ROOT. Make sure to use a
 # trailing slash.
 # Examples: "http://media.lawrence.com/media/", "http://example.com/media/"
-MEDIA_ROOT = os.path.join(BASE_DIR, 'media')
-MEDIA_URL = '/media/'
+MEDIA_URL = ''
 
 
 ############################################################

desktop/libs/indexer/src/indexer/api3.py

@@ -25,11 +25,11 @@
 import logging
 import urllib.error
 import sys
+import tempfile
 import uuid
 
 from django.urls import reverse
 from django.views.decorators.http import require_POST
-from django.core.files.storage import FileSystemStorage
 
 LOG = logging.getLogger(__name__)
 
@@ -44,7 +44,6 @@
 from desktop.lib.i18n import smart_unicode
 from desktop.lib.python_util import check_encoding
 from desktop.models import Document2
-from desktop.settings import BASE_DIR
 from kafka.kafka_api import get_topics, get_topic_data
 from notebook.connectors.base import get_api, Notebook
 from notebook.decorators import api_error_handler
@@ -231,7 +230,7 @@ def guess_field_types(request):
   if file_format['inputFormat'] == 'localfile':
     path = urllib_unquote(file_format['path'])
 
-    with open(BASE_DIR + path, 'r') as local_file:
+    with open(path, 'r') as local_file:
 
       reader = csv.reader(local_file)
       csv_data = list(reader)
@@ -742,11 +741,11 @@ def save_pipeline(request):
 def upload_local_file(request):
 
   upload_file = request.FILES['inputfile']
-  fs = FileSystemStorage()
   username = request.user.username
-  filename = "%s_%s.%s" % (username, uuid.uuid4(), 'csv')
-  name = fs.save(filename, upload_file)
-
-  local_file_url = fs.url(name)
+  filename = "%s_%s" % (username, uuid.uuid4())
+  temp_file = tempfile.NamedTemporaryFile(prefix=filename, suffix='.csv', delete=False)
+  temp_file.write(upload_file.read())
+  local_file_url = temp_file.name
+  temp_file.close()
 
   return JsonResponse({'local_file_url': local_file_url})

desktop/libs/indexer/src/indexer/indexers/sql.py

@@ -352,7 +352,7 @@ def create_table_from_local_file(self, source, destination, start_time=-1):
     path = urllib_unquote(source['path'])
 
     if path:                                                  # data insertion
-      with open(BASE_DIR + path, 'r') as local_file:
+      with open(path, 'r') as local_file:
         reader = csv.reader(local_file)
         _csv_rows = []
 

desktop/libs/indexer/src/indexer/indexers/sql_tests.py

@@ -23,6 +23,7 @@
 from nose.tools import assert_equal, assert_true
 
 from desktop.lib.django_test_util import make_logged_in_client
+from desktop.settings import BASE_DIR
 from useradmin.models import User
 
 from azure.conf import ABFS_CLUSTERS
@@ -841,7 +842,7 @@ def test_create_table_from_local_mysql():
   with patch('indexer.indexers.sql.get_interpreter') as get_interpreter:
     get_interpreter.return_value = {'Name': 'MySQL', 'dialect': 'mysql'}
     source = {
-      'path': '/apps/beeswax/data/tables/us_population.csv',
+      'path': BASE_DIR + '/apps/beeswax/data/tables/us_population.csv',
       'sourceType': 'mysql',
       'format': {'hasHeader': False}
     }
@@ -875,7 +876,7 @@ def test_create_table_from_local_phoenix():
   with patch('indexer.indexers.sql.get_interpreter') as get_interpreter:
     get_interpreter.return_value = {'Name': 'Phoenix', 'dialect': 'phoenix'}
     source = {
-      'path': '/apps/beeswax/data/tables/us_population.csv',
+      'path': BASE_DIR + '/apps/beeswax/data/tables/us_population.csv',
       'sourceType': 'phoenix',
       'format': {'hasHeader': False}
     }
@@ -926,7 +927,7 @@ def test_create_table_from_local_impala():
   with patch('indexer.indexers.sql.get_interpreter') as get_interpreter:
     get_interpreter.return_value = {'Name': 'Impala', 'dialect': 'impala'}
     source = {
-      'path': '/apps/beeswax/data/tables/flights.csv',
+      'path': BASE_DIR + '/apps/beeswax/data/tables/flights.csv',
       'sourceType': 'impala',
       'format': {'hasHeader': True}
     }