Merge branch 'main' into redsun82/kotlin

Author: redsun82

.bazelrc

@@ -11,7 +11,8 @@ common --override_module=semmle_code=%workspace%/misc/bazel/semmle_code_stub
 build --repo_env=CC=clang --repo_env=CXX=clang++
 
 build:linux --cxxopt=-std=c++20
-build:macos --cxxopt=-std=c++20 --cpu=darwin_x86_64
+# we currently cannot built the swift extractor for ARM
+build:macos --cxxopt=-std=c++20 --copt=-arch --copt=x86_64 --linkopt=-arch --linkopt=x86_64
 build:windows --cxxopt=/std:c++20 --cxxopt=/Zc:preprocessor
 
 # this requires developer mode, but is required to have pack installer functioning
@@ -21,4 +22,6 @@ common --enable_runfiles
 common --registry=file:///%workspace%/misc/bazel/registry
 common --registry=https://bcr.bazel.build
 
+common --@rules_dotnet//dotnet/settings:strict_deps=false
+
 try-import %workspace%/local.bazelrc

.bazelrc.internal

@@ -2,3 +2,9 @@
 
 common --registry=file:///%workspace%/ql/misc/bazel/registry
 common --registry=https://bcr.bazel.build
+
+# See bazelbuild/rules_dotnet#413: strict_deps in C# also appliy to 3rd-party deps, and when we pull
+# in (for example) the xunit package, there's no code in this at all, it just depends transitively on
+# its implementation packages without providing any code itself.
+# We either can depend on internal implementation details, or turn of strict deps.
+common --@rules_dotnet//dotnet/settings:strict_deps=false

.gitattributes

@@ -79,3 +79,5 @@
 /csharp/paket.lock linguist-generated=true
 # needs eol=crlf, as `paket` touches this file and saves it as crlf
 /csharp/.paket/Paket.Restore.targets linguist-generated=true eol=crlf
+/csharp/paket.main.bzl linguist-generated=true
+/csharp/paket.main_extension.bzl linguist-generated=true

.github/workflows/build-ripunzip.yml

@@ -0,0 +1,74 @@
+name: Build runzip
+
+on:
+  workflow_dispatch:
+    inputs:
+      ripunzip-version:
+        description: "what reference to checktout from google/runzip"
+        required: false
+        default: v1.2.1
+      openssl-version:
+        description: "what reference to checkout from openssl/openssl for Linux"
+        required: false
+        default: openssl-3.3.0
+
+jobs:
+  build:
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-20.04, macos-12, windows-2019]
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          repository: google/ripunzip
+          ref: ${{ inputs.ripunzip-version }}
+      # we need to avoid ripunzip dynamically linking into libssl
+      # see https://github.com/sfackler/rust-openssl/issues/183
+      - if: runner.os == 'Linux'
+        name: checkout openssl
+        uses: actions/checkout@v4
+        with:
+          repository: openssl/openssl
+          path: openssl
+          ref: ${{ inputs.openssl-version }}
+      - if: runner.os == 'Linux'
+        name: build and install openssl with fPIC
+        shell: bash
+        working-directory: openssl
+        run: |
+          ./config -fPIC --prefix=$HOME/.local --openssldir=$HOME/.local/ssl
+          make -j $(nproc)
+          make install_sw -j $(nproc)
+      - if: runner.os == 'Linux'
+        name: build (linux)
+        shell: bash
+        run: |
+          env OPENSSL_LIB_DIR=$HOME/.local/lib64 OPENSSL_INCLUDE_DIR=$HOME/.local/include OPENSSL_STATIC=yes cargo build --release
+          mv target/release/ripunzip ripunzip-linux
+      - if: runner.os == 'Windows'
+        name: build (windows)
+        shell: bash
+        run: |
+          cargo build --release
+          mv target/release/ripunzip ripunzip-windows
+      - name: build (macOS)
+        if: runner.os == 'macOS'
+        shell: bash
+        run: |
+          rustup target install x86_64-apple-darwin
+          rustup target install aarch64-apple-darwin
+          cargo build --target x86_64-apple-darwin --release
+          cargo build --target aarch64-apple-darwin --release
+          lipo -create -output ripunzip-macos \
+            -arch x86_64 target/x86_64-apple-darwin/release/ripunzip \
+            -arch arm64 target/aarch64-apple-darwin/release/ripunzip
+      - uses: actions/upload-artifact@v4
+        with:
+          name: ripunzip-${{ runner.os }}
+          path: ripunzip-*
+      - name: Check built binary
+        shell: bash
+        run: |
+          ./ripunzip-* --version

BUILD.bazel

@@ -0,0 +1 @@
+exports_files(["LICENSE"])

MODULE.bazel

@@ -24,9 +24,19 @@ bazel_dep(name = "nlohmann_json", version = "3.11.3", repo_name = "json")
 bazel_dep(name = "fmt", version = "10.0.0")
 bazel_dep(name = "rules_kotlin", version = "1.9.4-codeql.1")
 bazel_dep(name = "gazelle", version = "0.36.0")
+bazel_dep(name = "rules_dotnet", version = "0.15.1")
 
 bazel_dep(name = "buildifier_prebuilt", version = "6.4.0", dev_dependency = True)
 
+dotnet = use_extension("@rules_dotnet//dotnet:extensions.bzl", "dotnet")
+dotnet.toolchain(dotnet_version = "8.0.101")
+use_repo(dotnet, "dotnet_toolchains")
+
+register_toolchains("@dotnet_toolchains//:all")
+
+csharp_main_extension = use_extension("//csharp:paket.main_extension.bzl", "main_extension")
+use_repo(csharp_main_extension, "paket.main")
+
 pip = use_extension("@rules_python//python/extensions:pip.bzl", "pip")
 pip.parse(
     hub_name = "codegen_deps",

config/identical-files.json

@@ -364,5 +364,9 @@
   "Python model summaries test extension": [
     "python/ql/test/library-tests/dataflow/model-summaries/InlineTaintTest.ext.yml",
     "python/ql/test/library-tests/dataflow/model-summaries/NormalDataflowTest.ext.yml"
+  ],
+  "shared tree-sitter extractor cargo.toml": [
+    "shared/tree-sitter-extractor/Cargo.toml",
+    "ruby/extractor/codeql-extractor-fake-crate/Cargo.toml"
   ]
 }

cpp/ql/lib/change-notes/2024-05-23-Version1.md

@@ -0,0 +1,4 @@
+---
+category: breaking
+---
+* CodeQL package management is now generally available, and all GitHub-produced CodeQL packages have had their version numbers increased to 1.0.0.

cpp/ql/lib/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/cpp-all
-version: 0.13.2-dev
+version: 1.0.0-dev
 groups: cpp
 dbscheme: semmlecode.cpp.dbscheme
 extractor: cpp

cpp/ql/lib/semmle/code/cpp/controlflow/IRGuards.qll

@@ -565,7 +565,7 @@ class IRGuardCondition extends Instruction {
   /** Holds if (determined by this guard) `op == k` evaluates to `areEqual` if this expression evaluates to `value`. */
   cached
   predicate comparesEq(Operand op, int k, boolean areEqual, AbstractValue value) {
-    compares_eq(this, op, k, areEqual, value)
+    unary_compares_eq(this, op, k, areEqual, false, value)
   }
 
   /**
@@ -586,7 +586,7 @@ class IRGuardCondition extends Instruction {
   cached
   predicate ensuresEq(Operand op, int k, IRBlock block, boolean areEqual) {
     exists(AbstractValue value |
-      compares_eq(this, op, k, areEqual, value) and this.valueControls(block, value)
+      unary_compares_eq(this, op, k, areEqual, false, value) and this.valueControls(block, value)
     )
   }
 
@@ -611,7 +611,7 @@ class IRGuardCondition extends Instruction {
   cached
   predicate ensuresEqEdge(Operand op, int k, IRBlock pred, IRBlock succ, boolean areEqual) {
     exists(AbstractValue value |
-      compares_eq(this, op, k, areEqual, value) and
+      unary_compares_eq(this, op, k, areEqual, false, value) and
       this.valueControlsEdge(pred, succ, value)
     )
   }
@@ -737,26 +737,66 @@ private predicate compares_eq(
   )
 }
 
-/** Holds if `op == k` is `areEqual` given that `test` is equal to `value`. */
-private predicate compares_eq(
-  Instruction test, Operand op, int k, boolean areEqual, AbstractValue value
+/**
+ * Holds if `op == k` is `areEqual` given that `test` is equal to `value`.
+ *
+ * Many internal predicates in this file have a `inNonZeroCase` column.
+ * Ideally, the `k` column would be a type such as `Option<int>::Option`, to
+ * represent whether we have a concrete value `k` such that `op == k`, or whether
+ * we only know that `op != 0`.
+ * However, cannot instantiate `Option` with an infinite type. Thus the boolean
+ * `inNonZeroCase` is used to distinquish the `Some` (where we have a concrete
+ * value `k`) and `None` cases (where we only know that `op != 0`).
+ *
+ * Thus, if `inNonZeroCase = true` then `op != 0` and the value of `k` is
+ * meaningless.
+ *
+ * To see why `inNonZeroCase` is needed consider the following C program:
+ * ```c
+ * char* p = ...;
+ * if(p) {
+ *   use(p);
+ * }
+ * ```
+ * in C++ there would be an int-to-bool conversion on `p`. However, since C
+ * does not have booleans there is no conversion. We want to be able to
+ * conclude that `p` is non-zero in the true branch, so we need to give `k`
+ * some value. However, simply setting `k = 1` would make the rest of the
+ * analysis think that `k == 1` holds inside the branch. So we distinquish
+ * between the above case and
+ * ```c
+ * if(p == 1) {
+ *   use(p)
+ * }
+ * ```
+ * by setting `inNonZeroCase` to `true` in the former case, but not in the
+ * latter.
+ */
+private predicate unary_compares_eq(
+  Instruction test, Operand op, int k, boolean areEqual, boolean inNonZeroCase, AbstractValue value
 ) {
   /* The simple case where the test *is* the comparison so areEqual = testIsTrue xor eq. */
-  exists(AbstractValue v | simple_comparison_eq(test, op, k, v) |
+  exists(AbstractValue v | unary_simple_comparison_eq(test, op, k, inNonZeroCase, v) |
     areEqual = true and value = v
     or
     areEqual = false and value = v.getDualValue()
   )
   or
-  complex_eq(test, op, k, areEqual, value)
+  unary_complex_eq(test, op, k, areEqual, inNonZeroCase, value)
   or
   /* (x is true => (op == k)) => (!x is false => (op == k)) */
-  exists(AbstractValue dual | value = dual.getDualValue() |
-    compares_eq(test.(LogicalNotInstruction).getUnary(), op, k, areEqual, dual)
+  exists(AbstractValue dual, boolean inNonZeroCase0 |
+    value = dual.getDualValue() and
+    unary_compares_eq(test.(LogicalNotInstruction).getUnary(), op, k, inNonZeroCase0, areEqual, dual)
+  |
+    k = 0 and inNonZeroCase = inNonZeroCase0
+    or
+    k != 0 and inNonZeroCase = true
   )
   or
   // ((test is `areEqual` => op == const + k2) and const == `k1`) =>
   // test is `areEqual` => op == k1 + k2
+  inNonZeroCase = false and
   exists(int k1, int k2, ConstantInstruction const |
     compares_eq(test, op, const.getAUse(), k2, areEqual, value) and
     int_value(const) = k1 and
@@ -781,35 +821,53 @@ private predicate simple_comparison_eq(
   value.(BooleanValue).getValue() = false
 }
 
-/** Rearrange various simple comparisons into `op == k` form. */
-private predicate simple_comparison_eq(Instruction test, Operand op, int k, AbstractValue value) {
+/**
+ * Holds if `test` is an instruction that is part of test that eventually is
+ * used in a conditional branch.
+ */
+private predicate relevantUnaryComparison(Instruction test) {
+  not test instanceof CompareInstruction and
+  exists(IRType type, ConditionalBranchInstruction branch |
+    type instanceof IRAddressType or type instanceof IRIntegerType
+  |
+    type = test.getResultIRType() and
+    branch.getCondition() = test
+  )
+  or
+  exists(LogicalNotInstruction logicalNot |
+    relevantUnaryComparison(logicalNot) and
+    test = logicalNot.getUnary()
+  )
+}
+
+/**
+ * Rearrange various simple comparisons into `op == k` form.
+ */
+private predicate unary_simple_comparison_eq(
+  Instruction test, Operand op, int k, boolean inNonZeroCase, AbstractValue value
+) {
   exists(SwitchInstruction switch, CaseEdge case |
     test = switch.getExpression() and
     op.getDef() = test and
     case = value.(MatchValue).getCase() and
     exists(switch.getSuccessor(case)) and
-    case.getValue().toInt() = k
+    case.getValue().toInt() = k and
+    inNonZeroCase = false
   )
   or
   // There's no implicit CompareInstruction in files compiled as C since C
   // doesn't have implicit boolean conversions. So instead we check whether
   // there's a branch on a value of pointer or integer type.
-  exists(ConditionalBranchInstruction branch, IRType type |
-    not test instanceof CompareInstruction and
-    type = test.getResultIRType() and
-    (type instanceof IRAddressType or type instanceof IRIntegerType) and
-    test = branch.getCondition() and
-    op.getDef() = test
-  |
-    // We'd like to also include a case such as:
-    // ```
-    // k = 1 and
-    // value.(BooleanValue).getValue() = true
-    // ```
-    // but all we know is that the value is non-zero in the true branch.
-    // So we can only conclude something in the false branch.
+  relevantUnaryComparison(test) and
+  op.getDef() = test and
+  (
+    k = 1 and
+    value.(BooleanValue).getValue() = true and
+    inNonZeroCase = true
+    or
     k = 0 and
-    value.(BooleanValue).getValue() = false
+    value.(BooleanValue).getValue() = false and
+    inNonZeroCase = false
   )
 }
 
@@ -821,12 +879,12 @@ private predicate complex_eq(
   add_eq(cmp, left, right, k, areEqual, value)
 }
 
-private predicate complex_eq(
-  Instruction test, Operand op, int k, boolean areEqual, AbstractValue value
+private predicate unary_complex_eq(
+  Instruction test, Operand op, int k, boolean areEqual, boolean inNonZeroCase, AbstractValue value
 ) {
-  sub_eq(test, op, k, areEqual, value)
+  unary_sub_eq(test, op, k, areEqual, inNonZeroCase, value)
   or
-  add_eq(test, op, k, areEqual, value)
+  unary_add_eq(test, op, k, areEqual, inNonZeroCase, value)
 }
 
 /*
@@ -1090,16 +1148,20 @@ private predicate sub_eq(
 }
 
 // op - x == c => op == (c+x)
-private predicate sub_eq(Instruction test, Operand op, int k, boolean areEqual, AbstractValue value) {
+private predicate unary_sub_eq(
+  Instruction test, Operand op, int k, boolean areEqual, boolean inNonZeroCase, AbstractValue value
+) {
+  inNonZeroCase = false and
   exists(SubInstruction sub, int c, int x |
-    compares_eq(test, sub.getAUse(), c, areEqual, value) and
+    unary_compares_eq(test, sub.getAUse(), c, areEqual, inNonZeroCase, value) and
     op = sub.getLeftOperand() and
     x = int_value(sub.getRight()) and
     k = c + x
   )
   or
+  inNonZeroCase = false and
   exists(PointerSubInstruction sub, int c, int x |
-    compares_eq(test, sub.getAUse(), c, areEqual, value) and
+    unary_compares_eq(test, sub.getAUse(), c, areEqual, inNonZeroCase, value) and
     op = sub.getLeftOperand() and
     x = int_value(sub.getRight()) and
     k = c + x
@@ -1153,11 +1215,13 @@ private predicate add_eq(
 }
 
 // left + x == right + c => left == right + (c-x)
-private predicate add_eq(
-  Instruction test, Operand left, int k, boolean areEqual, AbstractValue value
+private predicate unary_add_eq(
+  Instruction test, Operand left, int k, boolean areEqual, boolean inNonZeroCase,
+  AbstractValue value
 ) {
+  inNonZeroCase = false and
   exists(AddInstruction lhs, int c, int x |
-    compares_eq(test, lhs.getAUse(), c, areEqual, value) and
+    unary_compares_eq(test, lhs.getAUse(), c, areEqual, inNonZeroCase, value) and
     (
       left = lhs.getLeftOperand() and x = int_value(lhs.getRight())
       or
@@ -1166,8 +1230,9 @@ private predicate add_eq(
     k = c - x
   )
   or
+  inNonZeroCase = false and
   exists(PointerAddInstruction lhs, int c, int x |
-    compares_eq(test, lhs.getAUse(), c, areEqual, value) and
+    unary_compares_eq(test, lhs.getAUse(), c, areEqual, inNonZeroCase, value) and
     (
       left = lhs.getLeftOperand() and x = int_value(lhs.getRight())
       or