diff --git a/Makefile b/Makefile index 9398f9e7..11233580 100644 --- a/Makefile +++ b/Makefile @@ -130,7 +130,7 @@ ACCEPT_PY_PASS_FILES := $(patsubst %,build/%_py.pass,$(filter-out %/native_test, BENCHMARKS := $(patsubst %.py,%,$(wildcard benchmarks/*.py)) BENCHMARK_BINS := $(patsubst %,build/%_benchmark,$(BENCHMARKS)) -TOOL_BINS = $(patsubst %,build/bin/%,benchcmp coverparse diffrange genmake pydeps) +TOOL_BINS = $(patsubst %,build/bin/%,benchcmp coverparse diffrange genmake gentest pydeps) GOLINT_BIN = build/bin/golint PYLINT_BIN = build/bin/pylint @@ -294,6 +294,7 @@ build/stdlib.mk: build/bin/genmake | $(STDLIB_SRCS) -include build/stdlib.mk $(patsubst %,build/src/__python__/%/module.go,$(STDLIB_PACKAGES)): $(COMPILER) +$(patsubst %,build/src/__python__/%/module_test.go,$(STDLIB_PACKAGES)): build/bin/gentest $(COMPILER) $(patsubst %,build/src/__python__/%/module.d,$(STDLIB_PACKAGES)): build/bin/pydeps $(PYTHONPARSER_SRCS) $(COMPILER) $(patsubst %,$(PKG_DIR)/__python__/%.a,$(STDLIB_PACKAGES)): $(RUNTIME) diff --git a/tools/genmake b/tools/genmake index c13465a5..06c396e9 100755 --- a/tools/genmake +++ b/tools/genmake @@ -28,11 +28,13 @@ parser.add_argument('-all_target', default='all', help='make target that will build all modules') -def _PrintRule(target, prereqs, rules): +def _PrintRule(target, prereqs, rules, echo_debug=False): print '{}: {}'.format(target, ' '.join(prereqs)) if rules: print '\t@mkdir -p $(@D)' for rule in rules: + if echo_debug: + print '\t@echo @{}'.format(rule) print '\t@{}'.format(rule) print @@ -67,9 +69,19 @@ def main(args): modname = basename.replace(os.sep, '.') ar_name = os.path.join(pkg_dir, '__python__', basename + '.a') go_file = os.path.join(pydir, basename, 'module.go') + gotest_file = os.path.join(pydir, basename, 'module_test.go') + ar_deps = [go_file] _PrintRule(go_file, [os.path.join(dirpath, filename)], - ['grumpc -modname={} $< > $@'.format(modname)]) + ['grumpc -modname={} $< > $@'.format(modname)], + echo_debug=True) + if modname.endswith(('_test', '_tests')): + ar_deps.append(gotest_file) + _PrintRule(gotest_file, + [os.path.join(dirpath, filename)], + ['gentest -modname={} > $@'.format(modname)], + echo_debug=True) + recipe = (r"""pydeps -modname=%s $< | awk '{gsub(/\./, "/", $$0); """ r"""print "%s: %s/__python__/" $$0 ".a"}' > $@""") dep_file = os.path.join(pydir, basename, 'module.d') @@ -77,7 +89,7 @@ def main(args): [recipe % (modname, ar_name, pkg_dir)]) go_package = '__python__/' + basename.replace(os.sep, '/') recipe = 'go tool compile -o $@ -p {} -complete -I {} -pack $<' - _PrintRule(ar_name, [go_file], [recipe.format(go_package, pkg_dir)]) + _PrintRule(ar_name, ar_deps, [recipe.format(go_package, pkg_dir)]) if args.all_target: _PrintRule(args.all_target, [ar_name], []) print '-include {}\n'.format(dep_file) diff --git a/tools/gentest b/tools/gentest new file mode 100755 index 00000000..ae957270 --- /dev/null +++ b/tools/gentest @@ -0,0 +1,100 @@ +#!/usr/bin/env python + +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""gentest creates a Golang module_test.go from a Python file, for coverage. + +Usage: $ gentest -m # Create test from the named module. +""" + +import argparse +import os +import sys +import textwrap + +from grumpy.compiler import imputil + +parser = argparse.ArgumentParser() +parser.add_argument('-modname', required=True, + help='Module to generate a _test.go file to') + + +template = textwrap.dedent(""" + package %s + import ( + \t"testing" + \t"grumpy" + \t%s + ) + func TestRunCode(t *testing.T) { + \tgrumpy.ImportModule(grumpy.NewRootFrame(), "traceback") + \tif grumpy.RunMain(Code) != 0 { + \t\tt.Fail() + \t} + } +""") + + +def _package_name(modname): + if modname.startswith('__go__/'): + return '__python__/' + modname + return '__python__/' + modname.replace('.', '/') + + +def _get_gopath(): + gopath = os.getenv('GOPATH', None) + if not gopath: + print >> sys.stderr, 'GOPATH not set' + raise RuntimeError('GOPATH not set') + return gopath + + +def main(args): + modname = args.modname + gopath = _get_gopath() + workdir = 'build' # It is ok _right now_ + py_dir = os.path.join(workdir, 'src', '__python__') + mod_dir = os.path.join(py_dir, modname.replace('.', '/')) + + script = os.path.join(py_dir, '%s.py' % modname.replace('.', '/')) + gopath = gopath + os.pathsep + workdir + testfile_modname = modname[:-5] if modname.endswith('_test') else modname + testfile_modname = testfile_modname.split('.')[-1] + + if not os.path.isfile(script): + return # The script does not exist. And is OK! + + names = imputil.calculate_transitive_deps(modname, script, gopath) + + # Find the script associated with the given module. + for d in gopath.split(os.pathsep): + script = imputil.find_script(os.path.join(d, 'src', '__python__'), modname) + if script: + break + else: + raise RuntimeError("can't find module %s", modname) + + names = imputil.calculate_transitive_deps(modname, script, gopath) + # Make sure traceback is available in all Python binaries. + names.add('traceback') + + imports = '\n\t'.join('_ "%s"' % _package_name(name) for name in names) + + testfile_contents = template % (testfile_modname, imports) + with open(os.path.join(mod_dir, 'module_test.go'), 'w') as go_testfile: + go_testfile.write(testfile_contents) + + +if __name__ == '__main__': + sys.exit(main(parser.parse_args())) diff --git a/tools/grumpc b/tools/grumpc index 53837738..d3ddb9fa 100755 --- a/tools/grumpc +++ b/tools/grumpc @@ -23,6 +23,7 @@ import argparse import os import sys import textwrap +from StringIO import StringIO from grumpy.compiler import block from grumpy.compiler import imputil @@ -34,6 +35,10 @@ from grumpy import pythonparser parser = argparse.ArgumentParser() parser.add_argument('script', help='Python source filename') parser.add_argument('-modname', default='__main__', help='Python module name') +parser.add_argument('-output', default=None, + help='Golang output file path (defaults to stdout)') +parser.add_argument('-astest', default=False, action='store_true', + help='Output as a test suite.') def main(args): @@ -49,8 +54,9 @@ def main(args): with open(args.script) as py_file: py_contents = py_file.read() + try: - mod = pythonparser.parse(py_contents) + parsed_module = pythonparser.parse(py_contents) except SyntaxError as e: print >> sys.stderr, '{}: line {}: invalid syntax: {}'.format( e.filename, e.lineno, e.text) @@ -58,7 +64,7 @@ def main(args): # Do a pass for compiler directives from `from __future__ import *` statements try: - future_node, future_features = imputil.parse_future_features(mod) + future_node, future_features = imputil.parse_future_features(parsed_module) except util.CompileError as e: print >> sys.stderr, str(e) return 2 @@ -66,39 +72,80 @@ def main(args): importer = imputil.Importer(gopath, args.modname, args.script, future_features.absolute_import) full_package_name = args.modname.replace('.', '/') - mod_block = block.ModuleBlock(importer, full_package_name, args.script, - py_contents, future_features) + module_block = block.ModuleBlock(importer, full_package_name, args.script, + py_contents, future_features) - visitor = stmt.StatementVisitor(mod_block, future_node) + visitor = stmt.StatementVisitor(module_block, future_node) # Indent so that the module body is aligned with the goto labels. with visitor.writer.indent_block(): try: - visitor.visit(mod) + visitor.visit(parsed_module) except util.ParseError as e: print >> sys.stderr, str(e) return 2 - writer = util.Writer(sys.stdout) + go_result = StringIO() + writer = util.Writer(go_result) tmpl = textwrap.dedent("""\ package $package import πg "grumpy" + $import_testing var Code *πg.Code func init() { \tCode = πg.NewCode("", $script, nil, 0, func(πF *πg.Frame, _ []*πg.Object) (*πg.Object, *πg.BaseException) { \t\tvar πR *πg.Object; _ = πR \t\tvar πE *πg.BaseException; _ = πE""") - writer.write_tmpl(tmpl, package=args.modname.split('.')[-1], + + go_package_name = args.modname.split('.')[-1] + + # HACK: Should suppress _test from package name + # See: https://github.com/google/grumpy/issues/383#issuecomment-353394740 + if go_package_name.endswith('_test'): + final_package_name = go_package_name[:-5] + else: + final_package_name = go_package_name + + modname = util.go_str(args.modname) + writer.write_tmpl(tmpl, package=final_package_name, + import_testing='import πt "testing"' if args.astest else '', script=util.go_str(args.script)) with writer.indent_block(2): - for s in sorted(mod_block.strings): + for s in sorted(module_block.strings): writer.write('ß{} := πg.InternStr({})'.format(s, util.go_str(s))) - writer.write_temp_decls(mod_block) - writer.write_block(mod_block, visitor.writer.getvalue()) + writer.write_temp_decls(module_block) + writer.write_block(module_block, visitor.writer.getvalue()) writer.write_tmpl(textwrap.dedent("""\ \t\treturn nil, πE \t}) \tπg.RegisterModule($modname, Code) - }"""), modname=util.go_str(args.modname)) + }"""), modname=modname) + + if args.astest: + + tmpl = textwrap.dedent("""\ + + func TestRunCode(t *testing.T) { + \tinit() + \tπg.ImportModule(grumpy.NewRootFrame(), "traceback") + \t//πg.ImportModule(grumpy.NewRootFrame(), $modname) + \tπg.RunMain(Code) + } + """) + + writer.write_tmpl(tmpl, modname=modname) + + try: + if args.output: + go_output = open(args.output, 'w') + else: + go_output = sys.stdout + except IOError: + print >> sys.stderr, str(e) + return 2 + + go_result.seek(0) + go_output.write(go_result.read()) + go_output.close() return 0