Rust: extract sources of crates

rust/extractor/src/translate/base.rs

@@ -16,7 +16,7 @@ use ra_ap_ide_db::RootDatabase;
 use ra_ap_ide_db::line_index::{LineCol, LineIndex};
 use ra_ap_parser::SyntaxKind;
 use ra_ap_span::TextSize;
-use ra_ap_syntax::ast::HasName;
+use ra_ap_syntax::ast::{Const, Fn, HasName, Static};
 use ra_ap_syntax::{
     AstNode, NodeOrToken, SyntaxElementChildren, SyntaxError, SyntaxNode, SyntaxToken, TextRange,
     ast,
@@ -598,7 +598,34 @@ impl<'a> Translator<'a> {
         })();
     }
 
-    pub(crate) fn should_be_excluded(&self, item: &impl ast::HasAttrs) -> bool {
+    pub(crate) fn should_be_excluded(&self, item: &(impl ast::HasAttrs + ast::AstNode)) -> bool {
+        if "true"
+            == std::env::var("CODEQL_EXTRACTOR_RUST_OPTION_EXCLUDE_BODIES").unwrap_or_default()
+        {
+            let syntax = item.syntax();
+            if let Some(body) = syntax.parent().and_then(Fn::cast).and_then(|x| x.body()) {
+                if body.syntax() == syntax {
+                    tracing::debug!("Skipping Fn body");
+                    return true;
+                }
+            }
+            if let Some(body) = syntax.parent().and_then(Const::cast).and_then(|x| x.body()) {
+                if body.syntax() == syntax {
+                    tracing::debug!("Skipping Const body");
+                    return true;
+                }
+            }
+            if let Some(body) = syntax
+                .parent()
+                .and_then(Static::cast)
+                .and_then(|x| x.body())
+            {
+                if body.syntax() == syntax {
+                    tracing::debug!("Skipping Static body");
+                    return true;
+                }
+            }
+        }
         self.semantics.is_some_and(|sema| {
             item.attrs().any(|attr| {
                 attr.as_simple_call().is_some_and(|(name, tokens)| {

rust/tools/BUILD.bazel

@@ -2,7 +2,7 @@ load("//misc/bazel:pkg.bzl", "codeql_pkg_files")
 
 codeql_pkg_files(
     name = "tools",
-    srcs = glob(["*.cmd"]),
+    srcs = glob(["*.cmd", "autobuild.py"]),
     exes = glob(["*.sh"]),
     visibility = ["//rust:__pkg__"],
 )

rust/tools/autobuild.py

@@ -0,0 +1,46 @@
+
+import os
+import json
+import subprocess
+
+""" 
+run "cargo metadata --format-version=1" 
+"""
+
+
+def get_cargo_metadata():
+    metadata = json.loads(subprocess.check_output(
+        ["cargo", "metadata", "--format-version=1"]))
+    return metadata
+
+
+CODEQL_EXTRACTOR_RUST_ROOT = os.environ.get("CODEQL_EXTRACTOR_RUST_ROOT")
+CODEQL_PLATFORM = os.environ.get("CODEQL_PLATFORM")
+database = os.environ.get("CODEQL_EXTRACTOR_RUST_WIP_DATABASE")
+scratch_dir = os.environ.get("CODEQL_EXTRACTOR_RUST_SCRATCH_DIR")
+metadata = get_cargo_metadata()
+metadata_file = os.path.join(scratch_dir, "metadata_file.yaml")
+os.makedirs(scratch_dir, exist_ok=True)
+with open(metadata_file, "w") as f:
+    f.write("---\n")
+    f.write(json.dumps(metadata, indent=4))
+
+subprocess.run(["codeql", "database", "index-files", database,
+               "-lyaml", "--working-dir", scratch_dir, "--include", "metadata_file.yaml"])
+paths = set()
+for package in metadata['packages']:
+    for target in package['targets']:
+        if 'lib' in target['kind'] or 'bin' in target['kind']:
+            src_path = target['src_path']
+            paths.add(os.path.dirname(src_path))
+
+autobuild = "{root}/tools/{platform}/autobuild".format(
+    root=CODEQL_EXTRACTOR_RUST_ROOT, platform=CODEQL_PLATFORM)
+
+for path in paths:
+    if not path.startswith(os.getcwd()):
+        env = os.environ.copy()
+        env['CODEQL_EXTRACTOR_RUST_OPTION_EXCLUDE_BODIES'] = "true"
+    else:
+        env = None
+    subprocess.run([autobuild], cwd=path, env=env)

rust/tools/autobuild.sh

@@ -3,4 +3,6 @@
 set -eu
 
 export RUST_BACKTRACE=1
-exec "$CODEQL_EXTRACTOR_RUST_ROOT/tools/$CODEQL_PLATFORM/autobuild"
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+
+exec /usr/bin/env python3 "${SCRIPT_DIR}/autobuild.py"

rust/tools/qltest.sh

@@ -7,15 +7,28 @@ export RUST_BACKTRACE=full
 QLTEST_LOG="$CODEQL_EXTRACTOR_RUST_LOG_DIR/qltest.log"
 QLTEST_COLORED_LOG="$CODEQL_EXTRACTOR_RUST_SCRATCH_DIR/qltest.log"
 dirname "$QLTEST_LOG" "$QLTEST_COLORED_LOG" | xargs mkdir -p
-# put full-color output on the side, but remove the color codes from the log file
-# also, print (colored) output only in case of failure
-if ! "$CODEQL_EXTRACTOR_RUST_ROOT/tools/$CODEQL_PLATFORM/extractor" \
-     --qltest \
-     --logging-verbosity=progress+ \
-     2>&1 \
-     | tee "$QLTEST_COLORED_LOG" \
-     | sed 's/\x1B\[[0-9;]\{1,\}[A-Za-z]//g' \
-     > "$QLTEST_LOG"; then
-  cat "$QLTEST_COLORED_LOG"
-  exit 1
+
+if [ -f "Cargo.lock" ]; then
+  SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+  if ! /usr/bin/env python3 "${SCRIPT_DIR}/autobuild.py" \
+      2>&1 \
+      | tee "$QLTEST_COLORED_LOG" \
+      | sed 's/\x1B\[[0-9;]\{1,\}[A-Za-z]//g' \
+      > "$QLTEST_LOG"; then
+    cat "$QLTEST_COLORED_LOG"
+    exit 1
+  fi
+else
+  # put full-color output on the side, but remove the color codes from the log file
+  # also, print (colored) output only in case of failure
+  if ! "$CODEQL_EXTRACTOR_RUST_ROOT/tools/$CODEQL_PLATFORM/extractor" \
+      --qltest \
+      --logging-verbosity=progress+ \
+      2>&1 \
+      | tee "$QLTEST_COLORED_LOG" \
+      | sed 's/\x1B\[[0-9;]\{1,\}[A-Za-z]//g' \
+      > "$QLTEST_LOG"; then
+    cat "$QLTEST_COLORED_LOG"
+    exit 1
+  fi
 fi