Skip to content

Commit 772cbed

Browse files
cdlearycopybara-github
cdleary
authored and
copybara-github
committed
[solvers] Add counterexample extraction helper.
Right now this only works on bits types -- starting simple to hook up to the DSLX quickcheck prover as a next step so it can dump out counterexamples in unit test form. PiperOrigin-RevId: 623952106
1 parent b4e5542 commit 772cbed

4 files changed

+367
-0
lines changed

‎xls/solvers/BUILD

+40
Original file line numberDiff line numberDiff line change
@@ -311,3 +311,43 @@ cc_test(
311311
"//xls/common:xls_gunit_main",
312312
],
313313
)
314+
315+
cc_library(
316+
name = "z3_extract_counterexample",
317+
srcs = ["z3_extract_counterexample.cc"],
318+
hdrs = ["z3_extract_counterexample.h"],
319+
deps = [
320+
"@com_google_absl//absl/base:nullability",
321+
"@com_google_absl//absl/container:flat_hash_map",
322+
"@com_google_absl//absl/status",
323+
"@com_google_absl//absl/status:statusor",
324+
"@com_google_absl//absl/strings",
325+
"@com_google_absl//absl/strings:str_format",
326+
"@com_google_absl//absl/types:span",
327+
"//xls/common/status:ret_check",
328+
"//xls/common/status:status_macros",
329+
"//xls/ir:bits",
330+
"//xls/ir:format_preference",
331+
"//xls/ir:number_parser",
332+
"//xls/ir:type",
333+
"//xls/ir:value",
334+
],
335+
)
336+
337+
cc_test(
338+
name = "z3_extract_counterexample_test",
339+
srcs = ["z3_extract_counterexample_test.cc"],
340+
deps = [
341+
":z3_extract_counterexample",
342+
"@com_google_absl//absl/container:flat_hash_map",
343+
"@com_google_absl//absl/status",
344+
"@com_google_absl//absl/status:statusor",
345+
"//xls/common:xls_gunit",
346+
"//xls/common:xls_gunit_main",
347+
"//xls/common/status:matchers",
348+
"//xls/ir",
349+
"//xls/ir:bits",
350+
"//xls/ir:value",
351+
"@com_google_fuzztest//fuzztest",
352+
],
353+
)

‎xls/solvers/z3_extract_counterexample.cc

+160
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,160 @@
1+
// Copyright 2024 The XLS Authors
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
#include "xls/solvers/z3_extract_counterexample.h"
16+
17+
#include <string>
18+
#include <string_view>
19+
#include <utility>
20+
#include <vector>
21+
22+
#include "absl/base/nullability.h"
23+
#include "absl/container/flat_hash_map.h"
24+
#include "absl/status/status.h"
25+
#include "absl/status/statusor.h"
26+
#include "absl/strings/ascii.h"
27+
#include "absl/strings/escaping.h"
28+
#include "absl/strings/match.h"
29+
#include "absl/strings/str_format.h"
30+
#include "absl/strings/str_split.h"
31+
#include "absl/types/span.h"
32+
#include "xls/common/status/ret_check.h"
33+
#include "xls/common/status/status_macros.h"
34+
#include "xls/ir/bits.h"
35+
#include "xls/ir/format_preference.h"
36+
#include "xls/ir/number_parser.h"
37+
#include "xls/ir/type.h"
38+
#include "xls/ir/value.h"
39+
40+
namespace xls::solvers::z3 {
41+
namespace {
42+
43+
absl::StatusOr<Value> Z3ValueToXlsValue(std::string_view z3_value_text,
44+
const BitsType& bits_type) {
45+
if (!absl::StartsWith(z3_value_text, "#x")) {
46+
return absl::InvalidArgumentError(
47+
absl::StrFormat("Expect Z3 value to start with '#x'; got: %s",
48+
absl::CEscape(z3_value_text)));
49+
}
50+
51+
std::string_view bits_text = z3_value_text.substr(2);
52+
XLS_ASSIGN_OR_RETURN(
53+
Bits bits, ParseUnsignedNumberWithoutPrefix(
54+
bits_text, FormatPreference::kHex, bits_type.bit_count()));
55+
return Value(bits);
56+
}
57+
58+
} // namespace
59+
60+
absl::StatusOr<absl::flat_hash_map<std::string, Value>> ExtractCounterexample(
61+
std::string_view message, absl::Span<const IrParamSpec> params) {
62+
// Split on the opening ``` fence.
63+
std::vector<std::string_view> pieces =
64+
absl::StrSplit(message, absl::MaxSplits("```", 1));
65+
if (pieces.size() != 2) {
66+
return absl::InvalidArgumentError(
67+
absl::StrFormat("Could not find model within solver message: `%s`",
68+
absl::CEscape(message)));
69+
}
70+
71+
// Split against the closing ``` fence, the model data resides in the middle.
72+
pieces = absl::StrSplit(pieces[1], absl::MaxSplits("```", 1));
73+
if (pieces.size() != 2) {
74+
return absl::InvalidArgumentError(absl::StrFormat(
75+
"Could not find model (closing fence) within solver message: `%s`",
76+
absl::CEscape(message)));
77+
}
78+
std::string_view model_text = pieces[0];
79+
80+
// Since the parameters can be presented out of order we build up a map from
81+
// parameter name to the spec (which includes the type) for easy lookup in
82+
// arbitrary order.
83+
absl::flat_hash_map<std::string, absl::Nonnull<const IrParamSpec*>>
84+
name_to_spec;
85+
for (const IrParamSpec& spec : params) {
86+
auto [it, inserted] = name_to_spec.insert({spec.name, &spec});
87+
XLS_RET_CHECK(inserted);
88+
}
89+
90+
// Accumulate values as we iterate through lines -- note that when there are
91+
// more than one parameter there will be more than one line specifying data.
92+
absl::flat_hash_map<std::string, Value> results;
93+
results.reserve(params.size());
94+
95+
// Go through each line, skipping empty ones, to ensure we understand all the
96+
// data the model presents.
97+
std::vector<std::string_view> model_lines = absl::StrSplit(model_text, '\n');
98+
for (std::string_view line : model_lines) {
99+
if (line.empty()) {
100+
continue;
101+
}
102+
103+
// On each line split the parameter name from the data so we can parse the
104+
// data out.
105+
std::vector<std::string_view> sides =
106+
absl::StrSplit(line, absl::MaxSplits(" -> ", 1));
107+
if (sides.size() != 2) {
108+
return absl::InvalidArgumentError(absl::StrFormat(
109+
"Could not parse line in solver model: `%s`", absl::CEscape(line)));
110+
}
111+
112+
// Ensure the parameter name matches our understanding of what parameter
113+
// we're currently processing (since we're placing it in a vector directly).
114+
std::string_view param_name = sides[0];
115+
param_name = absl::StripAsciiWhitespace(param_name);
116+
117+
auto it = name_to_spec.find(param_name);
118+
if (it == name_to_spec.end()) {
119+
return absl::InvalidArgumentError(
120+
absl::StrFormat("ExtractCounterexample; could not find parameter "
121+
"name from model in user-provided spec: `%s`",
122+
param_name));
123+
}
124+
125+
absl::Nonnull<const IrParamSpec*> spec = it->second;
126+
127+
// Use the type to guide how we parse out the solver-presented data item --
128+
// right now it must be a bits type, we give an unimplemented error for
129+
// unsupported types.
130+
absl::Nonnull<const Type*> type = spec->type;
131+
auto* bits_type = dynamic_cast<const BitsType*>(type);
132+
if (bits_type == nullptr) {
133+
return absl::UnimplementedError(
134+
absl::StrFormat("ExtractCounterexample; only bits-typed parameters "
135+
"are currently supported; got: %s",
136+
type->ToString()));
137+
}
138+
XLS_ASSIGN_OR_RETURN(Value value, Z3ValueToXlsValue(sides[1], *bits_type));
139+
140+
auto it2 = results.find(param_name);
141+
if (it2 != results.end()) {
142+
return absl::InvalidArgumentError(absl::StrFormat(
143+
"ExtractCounterexample; saw duplicate param value for `%s`: %s",
144+
param_name, absl::CEscape(model_text)));
145+
}
146+
results.emplace_hint(it2, std::string{param_name}, std::move(value));
147+
}
148+
149+
// Validate that we populated the right number of values for the parameter
150+
// specification.
151+
if (results.size() != params.size()) {
152+
return absl::InvalidArgumentError(absl::StrFormat(
153+
"Z3 solver model counterexample did not include all parameters: %s",
154+
absl::CEscape(model_text)));
155+
}
156+
157+
return results;
158+
}
159+
160+
} // namespace xls::solvers::z3

‎xls/solvers/z3_extract_counterexample.h

+46
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
// Copyright 2024 The XLS Authors
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
#ifndef XLS_SOLVERS_Z3_EXTRACT_COUNTEREXAMPLE_H_
16+
#define XLS_SOLVERS_Z3_EXTRACT_COUNTEREXAMPLE_H_
17+
18+
#include <string>
19+
#include <string_view>
20+
21+
#include "absl/base/nullability.h"
22+
#include "absl/container/flat_hash_map.h"
23+
#include "absl/status/statusor.h"
24+
#include "absl/types/span.h"
25+
#include "xls/ir/type.h"
26+
#include "xls/ir/value.h"
27+
28+
namespace xls::solvers::z3 {
29+
30+
struct IrParamSpec {
31+
std::string name;
32+
absl::Nonnull<const Type*> type;
33+
};
34+
35+
// Given a message that Z3 produces when finding a counterexample, attempts to
36+
// extract the model as an XLS value representation.
37+
//
38+
// Returns a map from parameter name to the value that was parsed for that
39+
// value. Note that if not all parameters are obsered in the counterexample
40+
// message text, an error is returned.
41+
absl::StatusOr<absl::flat_hash_map<std::string, Value>> ExtractCounterexample(
42+
std::string_view message, absl::Span<const IrParamSpec> params);
43+
44+
} // namespace xls::solvers::z3
45+
46+
#endif // XLS_SOLVERS_Z3_EXTRACT_COUNTEREXAMPLE_H_

‎xls/solvers/z3_extract_counterexample_test.cc

+121
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,121 @@
1+
// Copyright 2024 The XLS Authors
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
#include "xls/solvers/z3_extract_counterexample.h"
16+
17+
#include <string_view>
18+
#include <vector>
19+
20+
#include "gmock/gmock.h"
21+
#include "gtest/gtest.h"
22+
#include "fuzztest/fuzztest.h"
23+
#include "absl/container/flat_hash_map.h"
24+
#include "absl/status/status.h"
25+
#include "absl/status/statusor.h"
26+
#include "xls/common/status/matchers.h"
27+
#include "xls/ir/bits.h"
28+
#include "xls/ir/package.h"
29+
#include "xls/ir/value.h"
30+
31+
namespace xls::solvers::z3 {
32+
namespace {
33+
34+
// Avoids commas in macros.
35+
using NameToValue = absl::flat_hash_map<std::string, Value>;
36+
37+
TEST(ExtractCounterexampleTest, SingleBitsParam) {
38+
constexpr std::string_view kMessage = R"( Model:
39+
```f -> #x0
40+
```
41+
)";
42+
Package package("test_package");
43+
std::vector<IrParamSpec> signature = {
44+
IrParamSpec{.name = "f", .type = package.GetBitsType(5)}};
45+
46+
XLS_ASSERT_OK_AND_ASSIGN(NameToValue values,
47+
ExtractCounterexample(kMessage, signature));
48+
EXPECT_EQ(values.size(), 1);
49+
EXPECT_EQ(values.at("f"), Value(UBits(0, 5)));
50+
}
51+
52+
// The values given back are in hex.
53+
TEST(ExtractCounterexampleTest, SingleBitsParamHexValue) {
54+
constexpr std::string_view kMessage = R"( Model:
55+
```f -> #xa
56+
```
57+
)";
58+
Package package("test_package");
59+
std::vector<IrParamSpec> signature = {
60+
IrParamSpec{.name = "f", .type = package.GetBitsType(5)}};
61+
62+
XLS_ASSERT_OK_AND_ASSIGN(NameToValue values,
63+
ExtractCounterexample(kMessage, signature));
64+
EXPECT_EQ(values.size(), 1);
65+
EXPECT_EQ(values.at("f"), Value(UBits(0xa, 5)));
66+
}
67+
68+
TEST(ExtractCounterexampleTest, FakeMessageTooManyParams) {
69+
constexpr std::string_view kMessage = R"( Model:
70+
```f -> #xa
71+
g -> #xa
72+
```
73+
)";
74+
Package package("test_package");
75+
std::vector<IrParamSpec> signature = {
76+
IrParamSpec{.name = "f", .type = package.GetBitsType(5)}};
77+
78+
EXPECT_THAT(ExtractCounterexample(kMessage, signature),
79+
status_testing::StatusIs(
80+
absl::StatusCode::kInvalidArgument,
81+
testing::HasSubstr("could not find parameter name from model "
82+
"in user-provided spec: `g`")));
83+
}
84+
85+
// Note: Z3 can (and sometimes does) give us the parameters in an arbitrary
86+
// order in its message.
87+
TEST(ExtractCounterexampleTest, BackwardsParams) {
88+
constexpr std::string_view kMessage = R"( Model:
89+
```param_1 -> #x00
90+
param_0 -> #xa0
91+
```
92+
)";
93+
Package package("test_package");
94+
std::vector<IrParamSpec> signature = {
95+
IrParamSpec{.name = "param_0", .type = package.GetBitsType(8)},
96+
IrParamSpec{.name = "param_1", .type = package.GetBitsType(8)},
97+
};
98+
99+
XLS_ASSERT_OK_AND_ASSIGN(NameToValue values,
100+
ExtractCounterexample(kMessage, signature));
101+
EXPECT_EQ(values.size(), 2);
102+
EXPECT_EQ(values.at("param_0"), Value(UBits(0xa0, 8)));
103+
EXPECT_EQ(values.at("param_1"), Value(UBits(0x00, 8)));
104+
}
105+
106+
void DoesNotCrashWithBitsTypeParam(std::string_view input) {
107+
Package package("test_package");
108+
std::vector<IrParamSpec> signature = {
109+
IrParamSpec{.name = "x", .type = package.GetBitsType(2)}};
110+
absl::StatusOr<NameToValue> results = ExtractCounterexample(input, signature);
111+
if (results.ok()) {
112+
EXPECT_EQ(results->size(), 1);
113+
EXPECT_TRUE(results->contains("x"));
114+
} else {
115+
EXPECT_EQ(results.status().code(), absl::StatusCode::kInvalidArgument);
116+
}
117+
}
118+
FUZZ_TEST(ExtractCounterexampleFuzzTest, DoesNotCrashWithBitsTypeParam);
119+
120+
} // namespace
121+
} // namespace xls::solvers::z3

0 commit comments

Comments
 (0)