Skip to content

Commit 79f4073

Browse files
tinder-maxwellelliotttinder-maxwellelliott
authored
WIP: Reenable modified filepaths entry (#193)
* WIP: Reenable modified filepaths entry This should greatly speed up hash calculation time more updates * more updates * build
1 parent eedaeba commit 79f4073

9 files changed

+121
-32
lines changed

‎cli/BUILD

+43
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,13 @@ genrule(
1919
stamp = 1,
2020
)
2121

22+
config_setting(
23+
name = "macos",
24+
constraint_values = [
25+
"@@platforms//os:macos",
26+
],
27+
)
28+
2229
java_binary(
2330
name = "bazel-diff",
2431
jvm_flags = select({
@@ -48,6 +55,10 @@ kt_jvm_library(
4855

4956
kt_jvm_test(
5057
name = "BuildGraphHasherTest",
58+
jvm_flags = select({
59+
":macos": ["-Djava.security.manager=allow"],
60+
"//conditions:default": [],
61+
}),
5162
test_class = "com.bazel_diff.hash.BuildGraphHasherTest",
5263
runtime_deps = [":cli-test-lib"],
5364
)
@@ -57,42 +68,70 @@ kt_jvm_test(
5768
data = [
5869
":src/test/kotlin/com/bazel_diff/hash/fixture/foo.ts",
5970
],
71+
jvm_flags = select({
72+
":macos": ["-Djava.security.manager=allow"],
73+
"//conditions:default": [],
74+
}),
6075
test_class = "com.bazel_diff.hash.SourceFileHasherTest",
6176
runtime_deps = [":cli-test-lib"],
6277
)
6378

6479
kt_jvm_test(
6580
name = "CalculateImpactedTargetsInteractorTest",
81+
jvm_flags = select({
82+
":macos": ["-Djava.security.manager=allow"],
83+
"//conditions:default": [],
84+
}),
6685
test_class = "com.bazel_diff.interactor.CalculateImpactedTargetsInteractorTest",
6786
runtime_deps = [":cli-test-lib"],
6887
)
6988

7089
kt_jvm_test(
7190
name = "NormalisingPathConverterTest",
91+
jvm_flags = select({
92+
":macos": ["-Djava.security.manager=allow"],
93+
"//conditions:default": [],
94+
}),
7295
test_class = "com.bazel_diff.cli.converter.NormalisingPathConverterTest",
7396
runtime_deps = [":cli-test-lib"],
7497
)
7598

7699
kt_jvm_test(
77100
name = "OptionsConverterTest",
101+
jvm_flags = select({
102+
":macos": ["-Djava.security.manager=allow"],
103+
"//conditions:default": [],
104+
}),
78105
test_class = "com.bazel_diff.cli.converter.OptionsConverterTest",
79106
runtime_deps = [":cli-test-lib"],
80107
)
81108

82109
kt_jvm_test(
83110
name = "DeserialiseHashesInteractorTest",
111+
jvm_flags = select({
112+
":macos": ["-Djava.security.manager=allow"],
113+
"//conditions:default": [],
114+
}),
84115
test_class = "com.bazel_diff.interactor.DeserialiseHashesInteractorTest",
85116
runtime_deps = [":cli-test-lib"],
86117
)
87118

88119
kt_jvm_test(
89120
name = "BazelRuleTest",
121+
jvm_flags = select({
122+
":macos": ["-Djava.security.manager=allow"],
123+
"//conditions:default": [],
124+
}),
90125
test_class = "com.bazel_diff.bazel.BazelRuleTest",
91126
runtime_deps = [":cli-test-lib"],
92127
)
93128

94129
kt_jvm_test(
95130
name = "E2ETest",
131+
jvm_flags = select({
132+
":macos": ["-Djava.security.manager=allow"],
133+
"//conditions:default": [],
134+
}),
96135
test_class = "com.bazel_diff.e2e.E2ETest",
97136
runtime_deps = [":cli-test-lib"],
98137
)
@@ -103,6 +142,10 @@ kt_jvm_test(
103142
":src/test/kotlin/com/bazel_diff/io/fixture/correct.json",
104143
":src/test/kotlin/com/bazel_diff/io/fixture/wrong.json",
105144
],
145+
jvm_flags = select({
146+
":macos": ["-Djava.security.manager=allow"],
147+
"//conditions:default": [],
148+
}),
106149
test_class = "com.bazel_diff.io.ContentHashProviderTest",
107150
runtime_deps = [
108151
":cli-test-lib",

‎cli/src/main/kotlin/com/bazel_diff/bazel/BazelClient.kt

+3-3
Original file line numberDiff line numberDiff line change
@@ -2,9 +2,10 @@ package com.bazel_diff.bazel
22

33
import com.bazel_diff.log.Logger
44
import com.google.devtools.build.lib.query2.proto.proto2api.Build
5-
import org.koin.core.component.KoinComponent
6-
import org.koin.core.component.inject
5+
import java.nio.file.Path
76
import java.util.*
7+
import org.koin.core.component.inject
8+
import org.koin.core.component.KoinComponent
89

910
class BazelClient(private val useCquery: Boolean, private val fineGrainedHashExternalRepos: Set<String>) : KoinComponent {
1011
private val logger: Logger by inject()
@@ -54,4 +55,3 @@ class BazelClient(private val useCquery: Boolean, private val fineGrainedHashExt
5455
}
5556
}
5657
}
57-

‎cli/src/main/kotlin/com/bazel_diff/bazel/BazelQueryService.kt

+9-2
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,10 @@ class BazelQueryService(
2424
) : KoinComponent {
2525
private val logger: Logger by inject()
2626

27-
suspend fun query(query: String, useCquery: Boolean = false): List<Build.Target> {
27+
suspend fun query(
28+
query: String,
29+
useCquery: Boolean = false)
30+
: List<Build.Target> {
2831
// Unfortunately, there is still no direct way to tell if a target is compatible or not with the proto output
2932
// by itself. So we do an extra cquery with the trick at
3033
// https://bazel.build/extending/platforms#cquery-incompatible-target-detection to first find all compatible
@@ -58,7 +61,11 @@ class BazelQueryService(
5861
}
5962

6063
@OptIn(ExperimentalCoroutinesApi::class)
61-
private suspend fun runQuery(query: String, useCquery: Boolean, outputCompatibleTargets: Boolean = false): File {
64+
private suspend fun runQuery(
65+
query: String,
66+
useCquery: Boolean,
67+
outputCompatibleTargets: Boolean = false
68+
): File {
6269
val queryFile = Files.createTempFile(null, ".txt").toFile()
6370
queryFile.deleteOnExit()
6471
val outputFile = Files.createTempFile(null, ".bin").toFile()

‎cli/src/main/kotlin/com/bazel_diff/cli/GenerateHashesCommand.kt

+7-1
Original file line numberDiff line numberDiff line change
@@ -136,6 +136,12 @@ class GenerateHashesCommand : Callable<Int> {
136136
)
137137
var ignoredRuleHashingAttributes: Set<String> = emptySet()
138138

139+
@CommandLine.Option(
140+
names = ["-m", "--modified-filepaths"],
141+
description = ["Experimental: A text file containing a newline separated list of filepaths (relative to the workspace) these filepaths should represent the modified files between the specified revisions and will be used to scope what files are hashed during hash generation."]
142+
)
143+
var modifiedFilepaths: File? = null
144+
139145
@CommandLine.Spec
140146
lateinit var spec: CommandLine.Model.CommandSpec
141147

@@ -160,7 +166,7 @@ class GenerateHashesCommand : Callable<Int> {
160166
)
161167
}
162168

163-
return when (GenerateHashesInteractor().execute(seedFilepaths, outputPath, ignoredRuleHashingAttributes, targetType, includeTargetType)) {
169+
return when (GenerateHashesInteractor().execute(seedFilepaths, outputPath, ignoredRuleHashingAttributes, targetType, includeTargetType, modifiedFilepaths)) {
164170
true -> CommandLine.ExitCode.OK
165171
false -> CommandLine.ExitCode.SOFTWARE
166172
}.also { stopKoin() }

‎cli/src/main/kotlin/com/bazel_diff/hash/BuildGraphHasher.kt

+22-14
Original file line numberDiff line numberDiff line change
@@ -8,18 +8,19 @@ import com.bazel_diff.extensions.toHexString
88
import com.bazel_diff.log.Logger
99
import com.google.common.collect.Sets
1010
import com.google.devtools.build.lib.query2.proto.proto2api.Build
11-
import kotlinx.coroutines.Dispatchers
12-
import kotlinx.coroutines.async
13-
import kotlinx.coroutines.runBlocking
14-
import org.koin.core.component.KoinComponent
15-
import org.koin.core.component.inject
11+
import java.io.File
1612
import java.nio.file.Path
13+
import java.util.Calendar
14+
import java.util.concurrent.atomic.AtomicReference
1715
import java.util.concurrent.ConcurrentHashMap
1816
import java.util.concurrent.ConcurrentMap
19-
import java.util.concurrent.atomic.AtomicReference
2017
import java.util.stream.Collectors
2118
import kotlin.io.path.readBytes
22-
import java.util.Calendar
19+
import kotlinx.coroutines.async
20+
import kotlinx.coroutines.Dispatchers
21+
import kotlinx.coroutines.runBlocking
22+
import org.koin.core.component.inject
23+
import org.koin.core.component.KoinComponent
2324

2425
class BuildGraphHasher(private val bazelClient: BazelClient) : KoinComponent {
2526
private val targetHasher: TargetHasher by inject()
@@ -28,7 +29,8 @@ class BuildGraphHasher(private val bazelClient: BazelClient) : KoinComponent {
2829

2930
fun hashAllBazelTargetsAndSourcefiles(
3031
seedFilepaths: Set<Path> = emptySet(),
31-
ignoredAttrs: Set<String> = emptySet()
32+
ignoredAttrs: Set<String> = emptySet(),
33+
modifiedFilepaths: Set<Path> = emptySet()
3234
): Map<String, TargetHash> {
3335
val (sourceDigests, allTargets) = runBlocking {
3436
val targetsTask = async(Dispatchers.IO) {
@@ -39,7 +41,7 @@ class BuildGraphHasher(private val bazelClient: BazelClient) : KoinComponent {
3941

4042
val sourceDigestsFuture = async(Dispatchers.IO) {
4143
val sourceHashDurationEpoch = Calendar.getInstance().getTimeInMillis()
42-
val sourceFileTargets = hashSourcefiles(sourceTargets)
44+
val sourceFileTargets = hashSourcefiles(sourceTargets, modifiedFilepaths)
4345
val sourceHashDuration = Calendar.getInstance().getTimeInMillis() - sourceHashDurationEpoch
4446
logger.i { "Source file hashes calculated in $sourceHashDuration" }
4547
sourceFileTargets
@@ -52,11 +54,15 @@ class BuildGraphHasher(private val bazelClient: BazelClient) : KoinComponent {
5254
seedForFilepaths,
5355
sourceDigests,
5456
allTargets,
55-
ignoredAttrs
57+
ignoredAttrs,
58+
modifiedFilepaths
5659
)
5760
}
5861

59-
private fun hashSourcefiles(targets: List<BazelTarget.SourceFile>): ConcurrentMap<String, ByteArray> {
62+
private fun hashSourcefiles(
63+
targets: List<BazelTarget.SourceFile>,
64+
modifiedFilepaths: Set<Path>
65+
): ConcurrentMap<String, ByteArray> {
6066
val exception = AtomicReference<Exception?>(null)
6167
val result: ConcurrentMap<String, ByteArray> = targets.parallelStream()
6268
.map { sourceFile: BazelTarget.SourceFile ->
@@ -68,7 +74,7 @@ class BuildGraphHasher(private val bazelClient: BazelClient) : KoinComponent {
6874
}
6975
try {
7076
val sourceFileTarget = BazelSourceFileTarget(sourceFile.name, seed)
71-
Pair(sourceFileTarget.name, sourceFileHasher.digest(sourceFileTarget))
77+
Pair(sourceFileTarget.name, sourceFileHasher.digest(sourceFileTarget, modifiedFilepaths))
7278
} catch (e: Exception) {
7379
exception.set(e)
7480
null
@@ -90,7 +96,8 @@ class BuildGraphHasher(private val bazelClient: BazelClient) : KoinComponent {
9096
seedHash: ByteArray,
9197
sourceDigests: ConcurrentMap<String, ByteArray>,
9298
allTargets: List<BazelTarget>,
93-
ignoredAttrs: Set<String>
99+
ignoredAttrs: Set<String>,
100+
modifiedFilepaths: Set<Path>
94101
): Map<String, TargetHash> {
95102
val ruleHashes: ConcurrentMap<String, ByteArray> = ConcurrentHashMap()
96103
val targetToRule: MutableMap<String, BazelRule> = HashMap()
@@ -104,7 +111,8 @@ class BuildGraphHasher(private val bazelClient: BazelClient) : KoinComponent {
104111
sourceDigests,
105112
ruleHashes,
106113
seedHash,
107-
ignoredAttrs
114+
ignoredAttrs,
115+
modifiedFilepaths
108116
)
109117
Pair(target.name, TargetHash(target.javaClass.name.substringAfterLast('$'), targetDigest.toHexString()))
110118
}

‎cli/src/main/kotlin/com/bazel_diff/hash/RuleHasher.kt

+6-3
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ import com.google.common.annotations.VisibleForTesting
77
import org.koin.core.component.KoinComponent
88
import org.koin.core.component.inject
99
import java.util.concurrent.ConcurrentMap
10+
import java.nio.file.Path
1011

1112
class RuleHasher(private val useCquery: Boolean, private val fineGrainedHashExternalRepos: Set<String>) : KoinComponent {
1213
private val logger: Logger by inject()
@@ -31,7 +32,8 @@ class RuleHasher(private val useCquery: Boolean, private val fineGrainedHashExte
3132
sourceDigests: ConcurrentMap<String, ByteArray>,
3233
seedHash: ByteArray?,
3334
depPath: LinkedHashSet<String>?,
34-
ignoredAttrs: Set<String>
35+
ignoredAttrs: Set<String>,
36+
modifiedFilepaths: Set<Path>
3537
): ByteArray {
3638
val depPathClone = if (depPath != null) LinkedHashSet(depPath) else LinkedHashSet()
3739
if (depPathClone.contains(rule.name)) {
@@ -61,13 +63,14 @@ class RuleHasher(private val useCquery: Boolean, private val fineGrainedHashExte
6163
sourceDigests,
6264
seedHash,
6365
depPathClone,
64-
ignoredAttrs
66+
ignoredAttrs,
67+
modifiedFilepaths
6568
)
6669
safePutBytes(ruleInputHash)
6770
}
6871

6972
else -> {
70-
val heuristicDigest = sourceFileHasher.softDigest(BazelSourceFileTarget(ruleInput, ByteArray(0)))
73+
val heuristicDigest = sourceFileHasher.softDigest(BazelSourceFileTarget(ruleInput, ByteArray(0)), modifiedFilepaths)
7174
when {
7275
heuristicDigest != null -> {
7376
logger.i { "Source file $ruleInput picked up as an input for rule ${rule.name}" }

‎cli/src/main/kotlin/com/bazel_diff/hash/SourceFileHasher.kt

+11-4
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,10 @@ class SourceFileHasher : KoinComponent {
3838
this.externalRepoResolver = externalRepoResolver
3939
}
4040

41-
fun digest(sourceFileTarget: BazelSourceFileTarget): ByteArray {
41+
fun digest(
42+
sourceFileTarget: BazelSourceFileTarget,
43+
modifiedFilepaths: Set<Path> = emptySet()
44+
): ByteArray {
4245
return sha256 {
4346
val name = sourceFileTarget.name
4447
val filenamePath = if (name.startsWith("//")) {
@@ -69,7 +72,11 @@ class SourceFileHasher : KoinComponent {
6972
val file = absoluteFilePath.toFile()
7073
if (file.exists()) {
7174
if (file.isFile) {
72-
putFile(file)
75+
if (modifiedFilepaths.isEmpty()) {
76+
putFile(file)
77+
} else if (modifiedFilepaths.stream().anyMatch { workingDirectory.resolve(it) == file }) {
78+
putFile(file)
79+
}
7380
}
7481
} else {
7582
logger.w { "File $absoluteFilePath not found" }
@@ -80,7 +87,7 @@ class SourceFileHasher : KoinComponent {
8087
}
8188
}
8289

83-
fun softDigest(sourceFileTarget: BazelSourceFileTarget): ByteArray? {
90+
fun softDigest(sourceFileTarget: BazelSourceFileTarget, modifiedFilepaths: Set<Path> = emptySet()): ByteArray? {
8491
val name = sourceFileTarget.name
8592
if (!name.startsWith("//")) return null
8693

@@ -90,6 +97,6 @@ class SourceFileHasher : KoinComponent {
9097
val file = absoluteFilePath.toFile()
9198
if (!file.exists() || !file.isFile) return null
9299

93-
return digest(sourceFileTarget)
100+
return digest(sourceFileTarget, modifiedFilepaths)
94101
}
95102
}

‎cli/src/main/kotlin/com/bazel_diff/hash/TargetHasher.kt

+7-3
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import com.bazel_diff.bazel.BazelTarget
55
import org.koin.core.component.KoinComponent
66
import org.koin.core.component.inject
77
import java.util.concurrent.ConcurrentMap
8+
import java.nio.file.Path
89

910
class TargetHasher : KoinComponent {
1011
private val ruleHasher: RuleHasher by inject()
@@ -15,7 +16,8 @@ class TargetHasher : KoinComponent {
1516
sourceDigests: ConcurrentMap<String, ByteArray>,
1617
ruleHashes: ConcurrentMap<String, ByteArray>,
1718
seedHash: ByteArray?,
18-
ignoredAttrs: Set<String>
19+
ignoredAttrs: Set<String>,
20+
modifiedFilepaths: Set<Path>
1921
): ByteArray {
2022
return when (target) {
2123
is BazelTarget.GeneratedFile -> {
@@ -32,7 +34,8 @@ class TargetHasher : KoinComponent {
3234
sourceDigests,
3335
seedHash,
3436
depPath = null,
35-
ignoredAttrs
37+
ignoredAttrs,
38+
modifiedFilepaths
3639
)
3740
}
3841
}
@@ -44,7 +47,8 @@ class TargetHasher : KoinComponent {
4447
sourceDigests,
4548
seedHash,
4649
depPath = null,
47-
ignoredAttrs
50+
ignoredAttrs,
51+
modifiedFilepaths
4852
)
4953
}
5054
is BazelTarget.SourceFile -> sha256 {

0 commit comments

Comments
 (0)