Skip to content

Commit 76c0865

Browse files
KatharineKatharine
committed
Merge pull request #303 from pebble/bugfix/js_import
PBL-33672: Github import bug
2 parents 8795e49 + af78a7f commit 76c0865

File tree

1 file changed

+128
-121
lines changed

1 file changed

+128
-121
lines changed

‎ide/tasks/archive.py‎

Lines changed: 128 additions & 121 deletions
Original file line numberDiff line numberDiff line change
@@ -105,7 +105,7 @@ def get_filename_variant(file_name, resource_suffix_map):
105105
split = file_name_parts[0].split("~")
106106
tags = split[1:]
107107
try:
108-
ids = [resource_suffix_map['~'+tag] for tag in tags]
108+
ids = [resource_suffix_map['~' + tag] for tag in tags]
109109
except KeyError as key:
110110
raise ValueError('Unrecognised tag %s' % key)
111111
root_file_name = split[0] + file_name_parts[1]
@@ -146,7 +146,6 @@ def do_import_archive(project_id, archive, delete_project=False):
146146
raise Exception("Too many files in zip file.")
147147
file_list = [x.filename for x in contents]
148148

149-
150149
base_dir = find_project_root(file_list)
151150
dir_end = len(base_dir)
152151

@@ -163,130 +162,114 @@ def make_valid_filename(zip_entry):
163162
raise Exception("Excessively large compressed file.")
164163
return entry_filename
165164

166-
# Now iterate over the things we found
165+
# Now iterate over the things we found, filter out invalid files and look for the manifest.
166+
filtered_contents = []
167+
manifest_entry = None
168+
for entry in contents:
169+
filename = make_valid_filename(entry)
170+
if not filename:
171+
continue
172+
if filename == MANIFEST:
173+
manifest_entry = entry
174+
else:
175+
filtered_contents.append((filename, entry))
176+
if not manifest_entry:
177+
# If this ever happens, find_project_root() is broken.
178+
raise Exception(_("No manifest file found"))
179+
167180
with transaction.atomic():
168-
for entry in contents:
169-
filename = make_valid_filename(entry)
170-
if not filename:
171-
continue
172-
173-
if filename == MANIFEST:
174-
# We have a resource map! We can now try importing things from it.
175-
with z.open(entry) as f:
176-
m = json.loads(f.read())
177-
178-
project.app_uuid = m['uuid']
179-
project.app_short_name = m['shortName']
180-
project.app_long_name = m['longName']
181-
project.app_company_name = m['companyName']
182-
project.app_version_label = m['versionLabel']
183-
project.sdk_version = m.get('sdkVersion', '2')
184-
project.app_is_watchface = m.get('watchapp', {}).get('watchface', False)
185-
project.app_is_hidden = m.get('watchapp', {}).get('hiddenApp', False)
186-
project.app_is_shown_on_communication = m.get('watchapp', {}).get('onlyShownOnCommunication', False)
187-
project.app_capabilities = ','.join(m.get('capabilities', []))
188-
project.app_modern_multi_js = m.get('enableMultiJS', False)
189-
if 'targetPlatforms' in m:
190-
project.app_platforms = ','.join(m['targetPlatforms'])
191-
project.app_keys = dict_to_pretty_json(m.get('appKeys', {}))
192-
project.project_type = m.get('projectType', 'native')
193-
if project.project_type not in [x[0] for x in Project.PROJECT_TYPES]:
194-
raise Exception("Illegal project type %s" % project.project_type)
195-
media_map = m['resources']['media']
196-
197-
tag_map = {v: k for k, v in ResourceVariant.VARIANT_STRINGS.iteritems() if v}
198-
199-
desired_resources = {}
200-
resources_files = {}
201-
resource_variants = {}
202-
file_exists_for_root = {}
203-
204-
# Go through the media map and look for resources
205-
for resource in media_map:
206-
file_name = resource['file']
207-
identifier = resource['name']
208-
# Pebble.js and simply.js both have some internal resources that we don't import.
209-
if project.project_type in {'pebblejs', 'simplyjs'}:
210-
if identifier in {'MONO_FONT_14', 'IMAGE_MENU_ICON', 'IMAGE_LOGO_SPLASH', 'IMAGE_TILE_SPLASH'}:
211-
continue
212-
tags, root_file_name = get_filename_variant(file_name, tag_map)
213-
if (len(tags) != 0):
214-
raise ValueError("Generic resource filenames cannot contain a tilde (~)")
215-
if file_name not in desired_resources:
216-
desired_resources[root_file_name] = []
217-
print "Desired resource: %s" % root_file_name
218-
desired_resources[root_file_name].append(resource)
219-
file_exists_for_root[root_file_name] = False
220-
221-
for zipitem in contents:
222-
# Let's just try opening the file
223-
filename = make_valid_filename(zipitem)
224-
if filename is False or not filename.startswith(RES_PATH):
225-
continue
226-
filename = filename[len(RES_PATH)+1:]
227-
try:
228-
extracted = z.open("%s%s/%s"%(base_dir, RES_PATH, filename))
229-
except KeyError:
230-
print "Failed to open %s" % filename
231-
continue
232-
233-
# Now we know the file exists and is in the resource directory - is it one we want?
234-
tags, root_file_name = get_filename_variant(filename, tag_map)
235-
tags_string = ",".join(str(int(t)) for t in tags)
236-
237-
print "Importing file %s with root %s " % (zipitem.filename, root_file_name)
238-
239-
if root_file_name in desired_resources:
240-
medias = desired_resources[root_file_name]
241-
print "Looking for variants of %s" % root_file_name
242-
243-
# Because 'kind' and 'is_menu_icons' are properties of ResourceFile in the database,
244-
# we just use the first one.
245-
resource = medias[0]
246-
# Make only one resource file per base resource.
247-
if root_file_name not in resources_files:
248-
kind = resource['type']
249-
is_menu_icon = resource.get('menuIcon', False)
250-
resources_files[root_file_name] = ResourceFile.objects.create(
251-
project=project,
252-
file_name=os.path.basename(root_file_name),
253-
kind=kind,
254-
is_menu_icon=is_menu_icon)
255-
256-
# But add a resource variant for every file
257-
print "Adding variant %s with tags [%s]" % (root_file_name, tags_string)
258-
actual_file_name = resource['file']
259-
resource_variants[actual_file_name] = ResourceVariant.objects.create(resource_file=resources_files[root_file_name], tags=tags_string)
260-
resource_variants[actual_file_name].save_file(extracted)
261-
file_exists_for_root[root_file_name] = True
262-
263-
# Now add all the resource identifiers
264-
for root_file_name in desired_resources:
265-
for resource in desired_resources[root_file_name]:
266-
target_platforms = json.dumps(resource['targetPlatforms']) if 'targetPlatforms' in resource else None
267-
ResourceIdentifier.objects.create(
268-
resource_file=resources_files[root_file_name],
269-
resource_id=resource['name'],
270-
target_platforms=target_platforms,
271-
# Font options
272-
character_regex=resource.get('characterRegex', None),
273-
tracking=resource.get('trackingAdjust', None),
274-
compatibility=resource.get('compatibility', None),
275-
# Bitmap options
276-
memory_format=resource.get('memoryFormat', None),
277-
storage_format=resource.get('storageFormat', None),
278-
space_optimisation=resource.get('spaceOptimization', None)
279-
)
280-
281-
# Check that at least one variant of each specified resource exists.
282-
for root_file_name, loaded in file_exists_for_root.iteritems():
283-
if not loaded:
284-
raise KeyError("No file was found to satisfy the manifest filename: {}".format(root_file_name))
181+
# We have a resource map! We can now try importing things from it.
182+
with z.open(manifest_entry) as f:
183+
m = json.loads(f.read())
184+
185+
project.app_uuid = m['uuid']
186+
project.app_short_name = m['shortName']
187+
project.app_long_name = m['longName']
188+
project.app_company_name = m['companyName']
189+
project.app_version_label = m['versionLabel']
190+
project.sdk_version = m.get('sdkVersion', '2')
191+
project.app_is_watchface = m.get('watchapp', {}).get('watchface', False)
192+
project.app_is_hidden = m.get('watchapp', {}).get('hiddenApp', False)
193+
project.app_is_shown_on_communication = m.get('watchapp', {}).get('onlyShownOnCommunication', False)
194+
project.app_capabilities = ','.join(m.get('capabilities', []))
195+
project.app_modern_multi_js = m.get('enableMultiJS', False)
196+
if 'targetPlatforms' in m:
197+
project.app_platforms = ','.join(m['targetPlatforms'])
198+
project.app_keys = dict_to_pretty_json(m.get('appKeys', {}))
199+
project.project_type = m.get('projectType', 'native')
200+
if project.project_type not in [x[0] for x in Project.PROJECT_TYPES]:
201+
raise Exception("Illegal project type %s" % project.project_type)
202+
media_map = m['resources']['media']
203+
204+
tag_map = {v: k for k, v in ResourceVariant.VARIANT_STRINGS.iteritems() if v}
205+
206+
desired_resources = {}
207+
resources_files = {}
208+
resource_variants = {}
209+
file_exists_for_root = {}
210+
211+
# Go through the media map and look for resources
212+
for resource in media_map:
213+
file_name = resource['file']
214+
identifier = resource['name']
215+
# Pebble.js and simply.js both have some internal resources that we don't import.
216+
if project.project_type in {'pebblejs', 'simplyjs'}:
217+
if identifier in {'MONO_FONT_14', 'IMAGE_MENU_ICON', 'IMAGE_LOGO_SPLASH', 'IMAGE_TILE_SPLASH'}:
218+
continue
219+
tags, root_file_name = get_filename_variant(file_name, tag_map)
220+
if (len(tags) != 0):
221+
raise ValueError("Generic resource filenames cannot contain a tilde (~)")
222+
if file_name not in desired_resources:
223+
desired_resources[root_file_name] = []
224+
print "Desired resource: %s" % root_file_name
225+
desired_resources[root_file_name].append(resource)
226+
file_exists_for_root[root_file_name] = False
227+
228+
# Go through the zip file process all resource and source files.
229+
for filename, entry in filtered_contents:
230+
if filename.startswith(RES_PATH):
231+
base_filename = filename[len(RES_PATH) + 1:]
232+
# Let's just try opening the file
233+
try:
234+
extracted = z.open("%s%s/%s" % (base_dir, RES_PATH, base_filename))
235+
except KeyError:
236+
print "Failed to open %s" % base_filename
237+
continue
238+
239+
# Now we know the file exists and is in the resource directory - is it the one we want?
240+
tags, root_file_name = get_filename_variant(base_filename, tag_map)
241+
tags_string = ",".join(str(int(t)) for t in tags)
242+
243+
print "Importing file %s with root %s " % (entry.filename, root_file_name)
244+
245+
if root_file_name in desired_resources:
246+
medias = desired_resources[root_file_name]
247+
print "Looking for variants of %s" % root_file_name
248+
249+
# Because 'kind' and 'is_menu_icons' are properties of ResourceFile in the database,
250+
# we just use the first one.
251+
resource = medias[0]
252+
# Make only one resource file per base resource.
253+
if root_file_name not in resources_files:
254+
kind = resource['type']
255+
is_menu_icon = resource.get('menuIcon', False)
256+
resources_files[root_file_name] = ResourceFile.objects.create(
257+
project=project,
258+
file_name=os.path.basename(root_file_name),
259+
kind=kind,
260+
is_menu_icon=is_menu_icon)
261+
262+
# But add a resource variant for every file
263+
print "Adding variant %s with tags [%s]" % (root_file_name, tags_string)
264+
actual_file_name = resource['file']
265+
resource_variants[actual_file_name] = ResourceVariant.objects.create(resource_file=resources_files[root_file_name], tags=tags_string)
266+
resource_variants[actual_file_name].save_file(extracted)
267+
file_exists_for_root[root_file_name] = True
285268

286269
elif filename.startswith(SRC_DIR):
287270
if (not filename.startswith('.')) and (filename.endswith('.c') or filename.endswith('.h') or filename.endswith('.js')):
288271
base_filename = filename[len(SRC_DIR):]
289-
if project.app_modern_multi_js and filename.endswith('.js') and filename.startswith('js/'):
272+
if project.app_modern_multi_js and base_filename.endswith('.js') and base_filename.startswith('js/'):
290273
base_filename = base_filename[len('js/'):]
291274
source = SourceFile.objects.create(project=project, file_name=base_filename)
292275
with z.open(entry.filename) as f:
@@ -297,6 +280,30 @@ def make_valid_filename(zip_entry):
297280
source = SourceFile.objects.create(project=project, file_name=base_filename, target='worker')
298281
with z.open(entry.filename) as f:
299282
source.save_file(f.read().decode('utf-8'))
283+
284+
# Now add all the resource identifiers
285+
for root_file_name in desired_resources:
286+
for resource in desired_resources[root_file_name]:
287+
target_platforms = json.dumps(resource['targetPlatforms']) if 'targetPlatforms' in resource else None
288+
ResourceIdentifier.objects.create(
289+
resource_file=resources_files[root_file_name],
290+
resource_id=resource['name'],
291+
target_platforms=target_platforms,
292+
# Font options
293+
character_regex=resource.get('characterRegex', None),
294+
tracking=resource.get('trackingAdjust', None),
295+
compatibility=resource.get('compatibility', None),
296+
# Bitmap options
297+
memory_format=resource.get('memoryFormat', None),
298+
storage_format=resource.get('storageFormat', None),
299+
space_optimisation=resource.get('spaceOptimization', None)
300+
)
301+
302+
# Check that at least one variant of each specified resource exists.
303+
for root_file_name, loaded in file_exists_for_root.iteritems():
304+
if not loaded:
305+
raise KeyError("No file was found to satisfy the manifest filename: {}".format(root_file_name))
306+
300307
project.save()
301308
send_td_event('cloudpebble_zip_import_succeeded', project=project)
302309

0 commit comments

Comments
 (0)