Skip to content

Commit ce35017

Browse files
KapiainenKapiainen
authored
Merge pull request #30 from Kapiainen/encoding-fix
Encoding fix
2 parents a08e2c0 + 639ad85 commit ce35017

File tree

3 files changed

+144
-125
lines changed

3 files changed

+144
-125
lines changed

‎README.md

+6
Original file line numberDiff line numberDiff line change
@@ -225,6 +225,12 @@ Single file build system and a batch build variant.
225225

226226
## **Changelog**
227227

228+
Version 2.6.2 - YYYY/MM/DD:
229+
230+
**Skyrim**
231+
- Added exception handling to the plugin and linter to catch UnicodeDecodeError errors raised when attempting to read scripts while using the default encoding of the operating system's locale.
232+
- Added a confirmation dialog to the 'Generate completions' command when attempting to process more than 100 scripts.
233+
228234
Version 2.6.1 - 2016/10/21:
229235

230236
**Skyrim**

‎Source/Modules/Skyrim/Linter.py

+83-78
Original file line numberDiff line numberDiff line change
@@ -1547,86 +1547,91 @@ def CacheScript(self, name, path = None, line = None):
15471547
fullPath = self.GetPath(name)
15481548
if not fullPath:
15491549
self.Abort("Could not find the '%s' script." % name, line)
1550-
with open(fullPath) as f:
1551-
scriptContents = f.read()
1552-
lines = []
1553-
tokens = []
1554-
try:
1555-
for token in self.lex.Process(scriptContents):
1556-
if token.type == self.lex.NEWLINE:
1557-
if tokens:
1558-
lines.append(tokens)
1559-
tokens = []
1560-
elif token.type != self.COMMENT_LINE and token.type != self.COMMENT_BLOCK:
1561-
tokens.append(token)
1562-
except LexicalError as e:
1563-
self.Abort("Found a lexical error in the '%s' script." % name)
1564-
extends = []
1565-
functions = {}
1566-
properties = {}
1567-
states = {}
1568-
statements = []
1569-
try:
1570-
for line in lines:
1571-
stat = self.syn.Process(line)
1572-
if stat:
1573-
statements.append(stat)
1574-
except SyntacticError as e:
1575-
self.Abort("Found a syntactic error in the '%s' script." % name)
1576-
header = False
1577-
if statements[0].type == self.STAT_SCRIPTHEADER:
1578-
if statements[0].data.parent:
1579-
header = True
1580-
extends = self.GetLineage(statements[0].data.parent)
1581-
parent = self.GetCachedScript(statements[0].data.parent)
1582-
functions.update(parent.functions)
1583-
properties.update(parent.properties)
1584-
states.update(parent.states)
1585-
functions["GOTOSTATE"] = Statement(self.STAT_FUNCTIONDEF, 0, FunctionDef(None, None, False, "GOTOSTATE", "GoToState", [ParameterDef(self.KW_STRING, "String", False, "ASNEWSTATE", "asNewState", None)], []))
1586-
functions["GETSTATE"] = Statement(self.STAT_FUNCTIONDEF, 0, FunctionDef(self.KW_STRING, "String", False, "GETSTATE", "GetState", [], []))
1587-
functions["ONINIT"] = Statement(self.STAT_EVENTDEF, 0, EventDef(None, "ONINIT", "OnInit", [], []))
1588-
functions["ONBEGINSTATE"] = Statement(self.STAT_EVENTDEF, 0, EventDef(None, "ONBEGINSTATE", "OnBeginState", [], []))
1589-
functions["ONENDSTATE"] = Statement(self.STAT_EVENTDEF, 0, EventDef(None, "ONENDSTATE", "OnEndState", [], []))
1590-
i = 0
1591-
while i < len(statements):
1592-
if statements[i].type == self.STAT_FUNCTIONDEF or statements[i].type == self.STAT_EVENTDEF:
1593-
start = statements[i]
1594-
functions[statements[i].data.name] = start
1595-
if not self.KW_NATIVE in statements[i].data.flags:
1596-
while i < len(statements) and not (statements[i].type == self.STAT_KEYWORD and (statements[i].data.type == self.KW_ENDFUNCTION or statements[i].data.type == self.KW_ENDEVENT)):
1597-
if start.type == self.STAT_FUNCTIONDEF and not start.data.docstring:
1598-
if statements[i].type == self.STAT_DOCUMENTATION:
1599-
start.data.docstring = statements[i]
1600-
i += 1
1601-
else:
1602-
if start.type == self.STAT_FUNCTIONDEF and i + 1 < len(statements) and statements[i + 1].type == self.STAT_DOCUMENTATION:
1603-
start.data.docstring = statements[i + 1]
1604-
i += 1
1605-
elif statements[i].type == self.STAT_PROPERTYDEF:
1606-
start = statements[i]
1607-
properties[statements[i].data.name] = start
1608-
if not self.KW_AUTO in statements[i].data.flags and not self.KW_AUTOREADONLY in statements[i].data.flags:
1609-
while i < len(statements) and not (statements[i].type == self.STAT_KEYWORD and statements[i].data.type == self.KW_ENDPROPERTY):
1610-
if not start.data.docstring:
1611-
if statements[i].type == self.STAT_DOCUMENTATION:
1612-
start.data.docstring = statements[i]
1613-
i += 1
1614-
else:
1615-
if i + 1 < len(statements) and statements[i + 1].type == self.STAT_DOCUMENTATION:
1616-
start.data.docstring = statements[i + 1]
1617-
i += 1
1618-
elif statements[i].type == self.STAT_STATEDEF:
1619-
start = statements[i]
1620-
states[statements[i].data.name] = start
1621-
while i < len(statements) and not (statements[i].type == self.STAT_KEYWORD and statements[i].data.type == self.KW_ENDSTATE):
1550+
scriptContents = ""
1551+
try:
1552+
with open(fullPath) as f:
1553+
scriptContents = f.read()
1554+
except UnicodeDecodeError:
1555+
with open(fullPath, encoding="utf8") as f:
1556+
scriptContents = f.read()
1557+
lines = []
1558+
tokens = []
1559+
try:
1560+
for token in self.lex.Process(scriptContents):
1561+
if token.type == self.lex.NEWLINE:
1562+
if tokens:
1563+
lines.append(tokens)
1564+
tokens = []
1565+
elif token.type != self.COMMENT_LINE and token.type != self.COMMENT_BLOCK:
1566+
tokens.append(token)
1567+
except LexicalError as e:
1568+
self.Abort("Found a lexical error in the '%s' script." % name)
1569+
extends = []
1570+
functions = {}
1571+
properties = {}
1572+
states = {}
1573+
statements = []
1574+
try:
1575+
for line in lines:
1576+
stat = self.syn.Process(line)
1577+
if stat:
1578+
statements.append(stat)
1579+
except SyntacticError as e:
1580+
self.Abort("Found a syntactic error in the '%s' script." % name)
1581+
header = False
1582+
if statements[0].type == self.STAT_SCRIPTHEADER:
1583+
if statements[0].data.parent:
1584+
header = True
1585+
extends = self.GetLineage(statements[0].data.parent)
1586+
parent = self.GetCachedScript(statements[0].data.parent)
1587+
functions.update(parent.functions)
1588+
properties.update(parent.properties)
1589+
states.update(parent.states)
1590+
functions["GOTOSTATE"] = Statement(self.STAT_FUNCTIONDEF, 0, FunctionDef(None, None, False, "GOTOSTATE", "GoToState", [ParameterDef(self.KW_STRING, "String", False, "ASNEWSTATE", "asNewState", None)], []))
1591+
functions["GETSTATE"] = Statement(self.STAT_FUNCTIONDEF, 0, FunctionDef(self.KW_STRING, "String", False, "GETSTATE", "GetState", [], []))
1592+
functions["ONINIT"] = Statement(self.STAT_EVENTDEF, 0, EventDef(None, "ONINIT", "OnInit", [], []))
1593+
functions["ONBEGINSTATE"] = Statement(self.STAT_EVENTDEF, 0, EventDef(None, "ONBEGINSTATE", "OnBeginState", [], []))
1594+
functions["ONENDSTATE"] = Statement(self.STAT_EVENTDEF, 0, EventDef(None, "ONENDSTATE", "OnEndState", [], []))
1595+
i = 0
1596+
while i < len(statements):
1597+
if statements[i].type == self.STAT_FUNCTIONDEF or statements[i].type == self.STAT_EVENTDEF:
1598+
start = statements[i]
1599+
functions[statements[i].data.name] = start
1600+
if not self.KW_NATIVE in statements[i].data.flags:
1601+
while i < len(statements) and not (statements[i].type == self.STAT_KEYWORD and (statements[i].data.type == self.KW_ENDFUNCTION or statements[i].data.type == self.KW_ENDEVENT)):
1602+
if start.type == self.STAT_FUNCTIONDEF and not start.data.docstring:
1603+
if statements[i].type == self.STAT_DOCUMENTATION:
1604+
start.data.docstring = statements[i]
16221605
i += 1
1623-
elif statements[i].type == self.STAT_SCRIPTHEADER:
1624-
if not header and statements[i].data.parent:
1625-
header = True
1626-
extends = self.GetLineage(statements[i].data.parent)
1627-
i += 1
1606+
else:
1607+
if start.type == self.STAT_FUNCTIONDEF and i + 1 < len(statements) and statements[i + 1].type == self.STAT_DOCUMENTATION:
1608+
start.data.docstring = statements[i + 1]
1609+
i += 1
1610+
elif statements[i].type == self.STAT_PROPERTYDEF:
1611+
start = statements[i]
1612+
properties[statements[i].data.name] = start
1613+
if not self.KW_AUTO in statements[i].data.flags and not self.KW_AUTOREADONLY in statements[i].data.flags:
1614+
while i < len(statements) and not (statements[i].type == self.STAT_KEYWORD and statements[i].data.type == self.KW_ENDPROPERTY):
1615+
if not start.data.docstring:
1616+
if statements[i].type == self.STAT_DOCUMENTATION:
1617+
start.data.docstring = statements[i]
1618+
i += 1
1619+
else:
1620+
if i + 1 < len(statements) and statements[i + 1].type == self.STAT_DOCUMENTATION:
1621+
start.data.docstring = statements[i + 1]
1622+
i += 1
1623+
elif statements[i].type == self.STAT_STATEDEF:
1624+
start = statements[i]
1625+
states[statements[i].data.name] = start
1626+
while i < len(statements) and not (statements[i].type == self.STAT_KEYWORD and statements[i].data.type == self.KW_ENDSTATE):
1627+
i += 1
1628+
elif statements[i].type == self.STAT_SCRIPTHEADER:
1629+
if not header and statements[i].data.parent:
1630+
header = True
1631+
extends = self.GetLineage(statements[i].data.parent)
1632+
i += 1
16281633

1629-
self.cache[name] = CachedScript(extends, properties, functions, states)
1634+
self.cache[name] = CachedScript(extends, properties, functions, states)
16301635
return True
16311636

16321637
def GetCachedScript(self, name, line = None):

‎Source/Modules/Skyrim/Plugin.py

+55-47
Original file line numberDiff line numberDiff line change
@@ -55,56 +55,64 @@ def generate_completions(self):
5555
syn = Linter.Syntactic()
5656
sem = Linter.Semantic()
5757
files = [f for f in os.listdir(self.path) if ".psc" in f]
58+
if len(files) > 100:
59+
if not sublime.ok_cancel_dialog("You are about to generate static completions for %d scripts.\n\nAre you sure you want to continue?" % len(files)):
60+
return
5861
for file in files:
5962
path = os.path.join(self.path, file)
6063
scriptName = file[:-4]
61-
with open(path) as fi:
62-
scriptContents = fi.read()
63-
if scriptContents:
64-
lines = []
65-
tokens = []
66-
try:
67-
for token in lex.Process(scriptContents):
68-
if token.type == lex.NEWLINE:
69-
if tokens:
70-
lines.append(tokens)
71-
tokens = []
72-
elif token.type != lex.COMMENT_LINE and token.type != lex.COMMENT_BLOCK:
73-
tokens.append(token)
74-
except Linter.LexicalError as e:
75-
if PYTHON_VERSION[0] == 2:
76-
print("SublimePapyrus - Lexical error on line %d, column %d in '%s': %s" % (e.line, e.column, path, e.message))
77-
elif PYTHON_VERSION[0] >= 3:
78-
SublimePapyrus.ShowMessage("Error on line %d, column %d in '%s': %s" % (e.line, e.column, path, e.message))
79-
return
80-
if lines:
81-
statements = []
82-
for line in lines:
83-
try:
84-
stat = syn.Process(line)
85-
if stat and (stat.type == sem.STAT_FUNCTIONDEF or stat.type == sem.STAT_EVENTDEF):
86-
statements.append(stat)
87-
except Linter.SyntacticError as e:
88-
if PYTHON_VERSION[0] == 2:
89-
print("SublimePapyrus - Syntactic error on line %d in '%s': %s" % (e.line, path, e.message))
90-
elif PYTHON_VERSION[0] >= 3:
91-
SublimePapyrus.ShowMessage("Error on line %d in '%s': %s" % (e.line, path, e.message))
92-
return
93-
scriptNameLower = scriptName.lower()
94-
completions = [{"trigger": "%s\t%s" % (scriptNameLower, "script"), "contents": scriptName}]
95-
for stat in statements:
96-
if stat.type == sem.STAT_FUNCTIONDEF:
97-
temp = SublimePapyrus.MakeFunctionCompletion(stat, sem, script=scriptNameLower)
98-
completions.append({"trigger": temp[0], "contents": temp[1]})
99-
elif stat.type == sem.STAT_EVENTDEF:
100-
temp = SublimePapyrus.MakeEventCompletion(stat, sem, calling=False, script=scriptNameLower)
101-
completions.append({"trigger": temp[0], "contents": temp[1]})
102-
output = {
103-
"scope": VALID_SCOPE,
104-
"completions": completions
105-
}
106-
with open(os.path.join(outputFolder, "SublimePapyrus - Skyrim - %s.sublime-completions" % scriptName), "w") as fo:
107-
json.dump(output, fo, indent=2)
64+
scriptContents = ""
65+
try:
66+
with open(path) as fi:
67+
scriptContents = fi.read()
68+
except UnicodeDecodeError:
69+
with open(path, encoding="utf8") as fi:
70+
scriptContents = fi.read()
71+
if scriptContents:
72+
lines = []
73+
tokens = []
74+
try:
75+
for token in lex.Process(scriptContents):
76+
if token.type == lex.NEWLINE:
77+
if tokens:
78+
lines.append(tokens)
79+
tokens = []
80+
elif token.type != lex.COMMENT_LINE and token.type != lex.COMMENT_BLOCK:
81+
tokens.append(token)
82+
except Linter.LexicalError as e:
83+
if PYTHON_VERSION[0] == 2:
84+
print("SublimePapyrus - Lexical error on line %d, column %d in '%s': %s" % (e.line, e.column, path, e.message))
85+
elif PYTHON_VERSION[0] >= 3:
86+
SublimePapyrus.ShowMessage("Error on line %d, column %d in '%s': %s" % (e.line, e.column, path, e.message))
87+
return
88+
if lines:
89+
statements = []
90+
for line in lines:
91+
try:
92+
stat = syn.Process(line)
93+
if stat and (stat.type == sem.STAT_FUNCTIONDEF or stat.type == sem.STAT_EVENTDEF):
94+
statements.append(stat)
95+
except Linter.SyntacticError as e:
96+
if PYTHON_VERSION[0] == 2:
97+
print("SublimePapyrus - Syntactic error on line %d in '%s': %s" % (e.line, path, e.message))
98+
elif PYTHON_VERSION[0] >= 3:
99+
SublimePapyrus.ShowMessage("Error on line %d in '%s': %s" % (e.line, path, e.message))
100+
return
101+
scriptNameLower = scriptName.lower()
102+
completions = [{"trigger": "%s\t%s" % (scriptNameLower, "script"), "contents": scriptName}]
103+
for stat in statements:
104+
if stat.type == sem.STAT_FUNCTIONDEF:
105+
temp = SublimePapyrus.MakeFunctionCompletion(stat, sem, script=scriptNameLower)
106+
completions.append({"trigger": temp[0], "contents": temp[1]})
107+
elif stat.type == sem.STAT_EVENTDEF:
108+
temp = SublimePapyrus.MakeEventCompletion(stat, sem, calling=False, script=scriptNameLower)
109+
completions.append({"trigger": temp[0], "contents": temp[1]})
110+
output = {
111+
"scope": VALID_SCOPE,
112+
"completions": completions
113+
}
114+
with open(os.path.join(outputFolder, "SublimePapyrus - Skyrim - %s.sublime-completions" % scriptName), "w") as fo:
115+
json.dump(output, fo, indent=2)
108116
print("SublimePapyrus - Finished generating completions for scripts in '%s'" % self.path)
109117

110118
linterCache = {}

0 commit comments

Comments
 (0)