From f919d4902b584dc0d9e36158c4716e34a60d96f5 Mon Sep 17 00:00:00 2001 From: vinod shelke Date: Thu, 14 May 2020 03:10:09 -0700 Subject: [PATCH 1/7] change to address transitive dependency for vulcan --- src/engine/SCons/Spirent/__init__.py | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/src/engine/SCons/Spirent/__init__.py b/src/engine/SCons/Spirent/__init__.py index ae4dc63cb5..bb240ca898 100644 --- a/src/engine/SCons/Spirent/__init__.py +++ b/src/engine/SCons/Spirent/__init__.py @@ -136,15 +136,26 @@ def _write_edge(self, f, from_id, to_id): if from_id: f.write(' n%x -> n%x;\n' % (from_id, to_id)) - def _walk_tree_write_nodes(self, f, parent_id, node, visited, written): + def _is_shared_lib(self, node): + sh_lib_extn = ['.so', '.lib'] + filename = str(node.rfile()) + extension = os.path.splitext(str(node.rfile()))[1] + return extension in sh_lib_extn or ".so." in filename + + def _walk_tree_write_nodes(self, f, parent_id, node, visited, written, transitive=False): node = node.disambiguate() node_id = id(node) + path = str(node) + if transitive : + for child in node.all_children(): + if self._is_shared_lib(child): + self._walk_tree_write_nodes(f, parent_id, child, visited, written, True) + self._walk_tree_write_nodes(f, parent_id, node, visited, written) if node_id in visited: if node_id in written: self._write_edge(f, parent_id, node_id) return visited.add(node_id) - path = str(node) if node.has_builder(): self._write_edge(f, parent_id, node_id) if not isinstance(node, SCons.Node.FS.Dir): @@ -164,13 +175,15 @@ def _walk_tree_write_nodes(self, f, parent_id, node, visited, written): self._write_edge(f, node_id, builder_id) for child in node.all_children(): self._walk_tree_write_nodes(f, builder_id, child, visited, written) + if self._is_shared_lib(child): + self._walk_tree_write_nodes(f, builder_id, child, visited, written, True) else: self._write_dir_node(f, node_id, path, parent_id is None) written.add(node_id) self._write_edge(f, parent_id, node_id) for child in node.all_children(): self._walk_tree_write_nodes(f, node_id, child, visited, written) - elif isinstance(node, SCons.Node.FS.File) and node.exists() and not os.path.isabs(path): + elif isinstance(node, SCons.Node.FS.File) and node.rfile().exists() and not os.path.isabs(path): self._write_edge(f, parent_id, node_id) self._write_source_node(f, node_id, path, self._get_node_hash(path), self._get_node_mode(path)) written.add(node_id) From 1da6209cd1cd88c751fa8018db5b9f214526c7c0 Mon Sep 17 00:00:00 2001 From: vinod shelke Date: Fri, 15 May 2020 05:17:14 -0700 Subject: [PATCH 2/7] resolved review comments --- src/engine/SCons/Spirent/__init__.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/engine/SCons/Spirent/__init__.py b/src/engine/SCons/Spirent/__init__.py index bb240ca898..3fcd991efd 100644 --- a/src/engine/SCons/Spirent/__init__.py +++ b/src/engine/SCons/Spirent/__init__.py @@ -136,25 +136,25 @@ def _write_edge(self, f, from_id, to_id): if from_id: f.write(' n%x -> n%x;\n' % (from_id, to_id)) - def _is_shared_lib(self, node): + def _has_transitive_dependencies(self, node): sh_lib_extn = ['.so', '.lib'] filename = str(node.rfile()) extension = os.path.splitext(str(node.rfile()))[1] return extension in sh_lib_extn or ".so." in filename - def _walk_tree_write_nodes(self, f, parent_id, node, visited, written, transitive=False): + def _walk_tree_write_nodes(self, f, parent_id, node, visited, written, transitive): node = node.disambiguate() node_id = id(node) - path = str(node) if transitive : for child in node.all_children(): - if self._is_shared_lib(child): + if self._has_transitive_dependencies(child): self._walk_tree_write_nodes(f, parent_id, child, visited, written, True) - self._walk_tree_write_nodes(f, parent_id, node, visited, written) + self._walk_tree_write_nodes(f, parent_id, node, visited, written, False) if node_id in visited: if node_id in written: self._write_edge(f, parent_id, node_id) return + path = str(node) visited.add(node_id) if node.has_builder(): self._write_edge(f, parent_id, node_id) @@ -174,21 +174,21 @@ def _walk_tree_write_nodes(self, f, parent_id, node, visited, written, transitiv written.add(builder_id) self._write_edge(f, node_id, builder_id) for child in node.all_children(): - self._walk_tree_write_nodes(f, builder_id, child, visited, written) - if self._is_shared_lib(child): + self._walk_tree_write_nodes(f, builder_id, child, visited, written, False) + if self._has_transitive_dependencies(child): self._walk_tree_write_nodes(f, builder_id, child, visited, written, True) else: self._write_dir_node(f, node_id, path, parent_id is None) written.add(node_id) self._write_edge(f, parent_id, node_id) for child in node.all_children(): - self._walk_tree_write_nodes(f, node_id, child, visited, written) + self._walk_tree_write_nodes(f, node_id, child, visited, written, False) elif isinstance(node, SCons.Node.FS.File) and node.rfile().exists() and not os.path.isabs(path): self._write_edge(f, parent_id, node_id) self._write_source_node(f, node_id, path, self._get_node_hash(path), self._get_node_mode(path)) written.add(node_id) for child in node.all_children(): - self._walk_tree_write_nodes(f, node_id, child, visited, written) + self._walk_tree_write_nodes(f, node_id, child, visited, written, False) def write(self, path, t): progress_display = SCons.SConf.progress_display @@ -208,7 +208,7 @@ def write(self, path, t): f.write('strict digraph {\n') if self._env: f.write(' graph [env=%s]\n' % json.dumps(json.dumps(dict(self._env)))) - self._walk_tree_write_nodes(f, None, t, set(), set()) + self._walk_tree_write_nodes(f, None, t, set(), set(), False) f.write('}\n') # pr.disable() From a9f8040088799c9575fc7d0375440c6ee04513af Mon Sep 17 00:00:00 2001 From: vinod shelke Date: Tue, 19 May 2020 03:02:24 -0700 Subject: [PATCH 3/7] regex and cliff's fix for bll vulcan build --- src/engine/SCons/Spirent/__init__.py | 46 +++++++++++++++++++--------- 1 file changed, 31 insertions(+), 15 deletions(-) diff --git a/src/engine/SCons/Spirent/__init__.py b/src/engine/SCons/Spirent/__init__.py index 3fcd991efd..2c4bd89875 100644 --- a/src/engine/SCons/Spirent/__init__.py +++ b/src/engine/SCons/Spirent/__init__.py @@ -10,6 +10,8 @@ import SCons.Node import SCons.SConf import SCons.Subst +import re +import ntpath def vulcan_builder(fs, options, graph): # Get the vulcan command line options @@ -137,25 +139,35 @@ def _write_edge(self, f, from_id, to_id): f.write(' n%x -> n%x;\n' % (from_id, to_id)) def _has_transitive_dependencies(self, node): - sh_lib_extn = ['.so', '.lib'] - filename = str(node.rfile()) - extension = os.path.splitext(str(node.rfile()))[1] - return extension in sh_lib_extn or ".so." in filename - - def _walk_tree_write_nodes(self, f, parent_id, node, visited, written, transitive): + filename = ntpath.basename(str(node.rfile())) + return re.search("([a-zA-Z0-9\s_\\.\-\+])+[\.](so|lib)",filename) + + def _write_transitive_dependencies(self, f, parent_id, node, visited, edge_written, node_written): node = node.disambiguate() node_id = id(node) - if transitive : + + if node_id in visited: + return + visited.add(node_id) + + if node_id not in edge_written and node_id in node_written: + self._write_edge(f, parent_id, node_id) + edge_written.add(node_id) + + if node.has_builder() and not isinstance(node, SCons.Node.FS.Dir): for child in node.all_children(): if self._has_transitive_dependencies(child): - self._walk_tree_write_nodes(f, parent_id, child, visited, written, True) - self._walk_tree_write_nodes(f, parent_id, node, visited, written, False) + self._write_transitive_dependencies(f, parent_id, child, visited, edge_written, node_written) + + def _walk_tree_write_nodes(self, f, parent_id, node, visited, written): + node = node.disambiguate() + node_id = id(node) if node_id in visited: if node_id in written: self._write_edge(f, parent_id, node_id) return - path = str(node) visited.add(node_id) + path = str(node) if node.has_builder(): self._write_edge(f, parent_id, node_id) if not isinstance(node, SCons.Node.FS.Dir): @@ -174,21 +186,25 @@ def _walk_tree_write_nodes(self, f, parent_id, node, visited, written, transitiv written.add(builder_id) self._write_edge(f, node_id, builder_id) for child in node.all_children(): - self._walk_tree_write_nodes(f, builder_id, child, visited, written, False) + self._walk_tree_write_nodes(f, builder_id, child, visited, written) + trans_visited = set() + trans_written = set() + for child in node.all_children(): if self._has_transitive_dependencies(child): - self._walk_tree_write_nodes(f, builder_id, child, visited, written, True) + trans_written.add(id(child.disambiguate())) + self._write_transitive_dependencies(f, builder_id, child, trans_visited, trans_written, written) else: self._write_dir_node(f, node_id, path, parent_id is None) written.add(node_id) self._write_edge(f, parent_id, node_id) for child in node.all_children(): - self._walk_tree_write_nodes(f, node_id, child, visited, written, False) + self._walk_tree_write_nodes(f, node_id, child, visited, written) elif isinstance(node, SCons.Node.FS.File) and node.rfile().exists() and not os.path.isabs(path): self._write_edge(f, parent_id, node_id) self._write_source_node(f, node_id, path, self._get_node_hash(path), self._get_node_mode(path)) written.add(node_id) for child in node.all_children(): - self._walk_tree_write_nodes(f, node_id, child, visited, written, False) + self._walk_tree_write_nodes(f, node_id, child, visited, written) def write(self, path, t): progress_display = SCons.SConf.progress_display @@ -208,7 +224,7 @@ def write(self, path, t): f.write('strict digraph {\n') if self._env: f.write(' graph [env=%s]\n' % json.dumps(json.dumps(dict(self._env)))) - self._walk_tree_write_nodes(f, None, t, set(), set(), False) + self._walk_tree_write_nodes(f, None, t, set(), set()) f.write('}\n') # pr.disable() From 2f1ab177b41df110d5c4cb5a397630b523ec3976 Mon Sep 17 00:00:00 2001 From: vinod shelke Date: Tue, 19 May 2020 10:04:57 -0700 Subject: [PATCH 4/7] regex and cliff's fix for bll vulcan build --- src/engine/SCons/Spirent/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/engine/SCons/Spirent/__init__.py b/src/engine/SCons/Spirent/__init__.py index 2c4bd89875..4479e09207 100644 --- a/src/engine/SCons/Spirent/__init__.py +++ b/src/engine/SCons/Spirent/__init__.py @@ -11,8 +11,8 @@ import SCons.SConf import SCons.Subst import re -import ntpath +REX_SH_LIBRARY_OBJ = re.compile(r"([a-zA-Z0-9\s_\\.\-\+])+[\.](so|lib|dll)") def vulcan_builder(fs, options, graph): # Get the vulcan command line options vulcan_options = options.vulcan_opts or os.environ.get("SCONS_VULCAN_OPTS") or "" @@ -139,8 +139,8 @@ def _write_edge(self, f, from_id, to_id): f.write(' n%x -> n%x;\n' % (from_id, to_id)) def _has_transitive_dependencies(self, node): - filename = ntpath.basename(str(node.rfile())) - return re.search("([a-zA-Z0-9\s_\\.\-\+])+[\.](so|lib)",filename) + path, filename = os.path.split(str(node.rfile())) + return REX_SH_LIBRARY_OBJ.search(filename) def _write_transitive_dependencies(self, f, parent_id, node, visited, edge_written, node_written): node = node.disambiguate() From ef6cbb88c0aa5614131b6248391125a09b86c884 Mon Sep 17 00:00:00 2001 From: vinod shelke Date: Tue, 19 May 2020 10:26:13 -0700 Subject: [PATCH 5/7] regex and cliff's fix for bll vulcan build --- src/engine/SCons/Spirent/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/engine/SCons/Spirent/__init__.py b/src/engine/SCons/Spirent/__init__.py index 4479e09207..7177d44d12 100644 --- a/src/engine/SCons/Spirent/__init__.py +++ b/src/engine/SCons/Spirent/__init__.py @@ -13,6 +13,7 @@ import re REX_SH_LIBRARY_OBJ = re.compile(r"([a-zA-Z0-9\s_\\.\-\+])+[\.](so|lib|dll)") + def vulcan_builder(fs, options, graph): # Get the vulcan command line options vulcan_options = options.vulcan_opts or os.environ.get("SCONS_VULCAN_OPTS") or "" @@ -139,7 +140,7 @@ def _write_edge(self, f, from_id, to_id): f.write(' n%x -> n%x;\n' % (from_id, to_id)) def _has_transitive_dependencies(self, node): - path, filename = os.path.split(str(node.rfile())) + path, filename = os.path.split(str(node.rfile())) return REX_SH_LIBRARY_OBJ.search(filename) def _write_transitive_dependencies(self, f, parent_id, node, visited, edge_written, node_written): From 58ed81e1eec5764c54f7a89da3e31ade75f18aa6 Mon Sep 17 00:00:00 2001 From: vinod shelke Date: Tue, 19 May 2020 13:50:09 -0700 Subject: [PATCH 6/7] regex and cliff's fix for bll vulcan build --- src/engine/SCons/Spirent/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/engine/SCons/Spirent/__init__.py b/src/engine/SCons/Spirent/__init__.py index 7177d44d12..68c5b7dda2 100644 --- a/src/engine/SCons/Spirent/__init__.py +++ b/src/engine/SCons/Spirent/__init__.py @@ -12,7 +12,7 @@ import SCons.Subst import re -REX_SH_LIBRARY_OBJ = re.compile(r"([a-zA-Z0-9\s_\\.\-\+])+[\.](so|lib|dll)") +REX_SH_LIBRARY_OBJ = re.compile(r"([a-zA-Z0-9\_\\.\-\+])+[\.](so|lib|dll)") def vulcan_builder(fs, options, graph): # Get the vulcan command line options From 213b747e058ac9f97b67f641d3d6f5f9420d4237 Mon Sep 17 00:00:00 2001 From: vinod shelke Date: Tue, 19 May 2020 22:51:11 -0700 Subject: [PATCH 7/7] regex corrected --- src/engine/SCons/Spirent/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/engine/SCons/Spirent/__init__.py b/src/engine/SCons/Spirent/__init__.py index 68c5b7dda2..68167900b8 100644 --- a/src/engine/SCons/Spirent/__init__.py +++ b/src/engine/SCons/Spirent/__init__.py @@ -5,14 +5,14 @@ import os.path import time import subprocess +import re import SCons.Action import SCons.Node import SCons.SConf import SCons.Subst -import re -REX_SH_LIBRARY_OBJ = re.compile(r"([a-zA-Z0-9\_\\.\-\+])+[\.](so|lib|dll)") +REX_SH_LIBRARY_OBJ = re.compile(r"([a-zA-Z0-9\_\.\-\+])+[\.](so|lib|dll)") def vulcan_builder(fs, options, graph): # Get the vulcan command line options