Skip to content

Commit 6d8896f

Browse files
authored
Add /warnVacuousProofs option (#1016)
This option enables `/trackVerificationCoverage` and automatically warns if it detects vacuous proofs. This is intended for the less-common case where a front end doesn't add IDs to program elements and then post-process the results.
1 parent adaa245 commit 6d8896f

File tree

7 files changed

+312
-8
lines changed

7 files changed

+312
-8
lines changed

Source/Core/CoverageAnnotator.cs

Lines changed: 115 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,115 @@
1+
using System.Collections.Generic;
2+
using System.Diagnostics;
3+
4+
namespace Microsoft.Boogie;
5+
6+
/// <summary>
7+
/// Add `{:id ...}` attributes to all assertions, assumptions, requires
8+
/// clauses, ensures clauses, and call statements so that verification
9+
/// coverage tracking is possible. This exists primarily to support the
10+
/// automatic detection of vacuous proofs in the case where no front
11+
/// end has added these already.
12+
/// </summary>
13+
public class CoverageAnnotator : StandardVisitor
14+
{
15+
private int idCount = 0;
16+
private string currentImplementation;
17+
private Dictionary<string, ISet<string>> implementationGoalIds = new();
18+
private Dictionary<string, Absy> idMap = new();
19+
20+
private void AddImplementationGoalId(string id)
21+
{
22+
implementationGoalIds[currentImplementation].Add(id);
23+
}
24+
25+
private void AddIdIfMissing(ICarriesAttributes node, bool isGoal)
26+
{
27+
Absy absy = node as Absy;
28+
if (absy == null) {
29+
return;
30+
}
31+
var idStr = node.FindStringAttribute("id");
32+
if (idStr == null) {
33+
idStr = $"id_l{absy.tok.line}_c{absy.tok.col}_{NodeType(node)}_{idCount}";
34+
idCount++;
35+
}
36+
idMap.Add(idStr, absy);
37+
if (isGoal) {
38+
AddImplementationGoalId(idStr);
39+
}
40+
node.AddStringAttribute(absy.tok, "id", idStr);
41+
}
42+
43+
private string NodeType(ICarriesAttributes node)
44+
{
45+
return node switch
46+
{
47+
Requires _ => "requires",
48+
Ensures _ => "ensures",
49+
AssertCmd _ => "assert",
50+
AssumeCmd _ => "assume",
51+
CallCmd _ => "call",
52+
_ => "other"
53+
};
54+
}
55+
56+
/// <summary>
57+
/// Get the set of IDs that correspond to goals within the named
58+
/// implementation.
59+
/// </summary>
60+
/// <param name="implName">The name of the implementation.</param>
61+
/// <returns>The IDs for all goal elements within the implementation.</returns>
62+
public ISet<string> GetImplementationGoalIds(string implName) => implementationGoalIds[implName];
63+
64+
/// <summary>
65+
/// Get the AST node corresponding to the given ID.
66+
/// </summary>
67+
/// <param name="idStr">The `id` attribute placed on an AST node.</param>
68+
/// <returns>The node where that `id` occurs.</returns>
69+
public Absy GetIdNode(string idStr) => idMap[idStr];
70+
71+
public override Implementation VisitImplementation(Implementation node)
72+
{
73+
currentImplementation = node.Name;
74+
implementationGoalIds.TryAdd(currentImplementation, new HashSet<string>());
75+
return base.VisitImplementation(node);
76+
}
77+
78+
public override Cmd VisitAssertCmd(AssertCmd node)
79+
{
80+
if (node.Expr is LiteralExpr {IsTrue: true}) {
81+
return node;
82+
}
83+
84+
AddIdIfMissing(node, true);
85+
return base.VisitAssertCmd(node);
86+
}
87+
88+
public override Cmd VisitAssumeCmd(AssumeCmd node)
89+
{
90+
AddIdIfMissing(node, false);
91+
return base.VisitAssumeCmd(node);
92+
}
93+
94+
public override Cmd VisitCallCmd(CallCmd node)
95+
{
96+
AddIdIfMissing(node, false);
97+
return base.VisitCallCmd(node);
98+
}
99+
100+
public override Requires VisitRequires(Requires requires)
101+
{
102+
AddIdIfMissing(requires, false);
103+
return base.VisitRequires(requires);
104+
}
105+
106+
public override Ensures VisitEnsures(Ensures ensures)
107+
{
108+
if (ensures.Free) {
109+
return ensures;
110+
}
111+
112+
AddIdIfMissing(ensures, true);
113+
return base.VisitEnsures(ensures);
114+
}
115+
}

Source/Directory.Build.props

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22

33
<!-- Target framework and package configuration -->
44
<PropertyGroup>
5-
<Version>3.5.1</Version>
5+
<Version>3.5.2</Version>
66
<TargetFramework>net8.0</TargetFramework>
77
<GeneratePackageOnBuild>false</GeneratePackageOnBuild>
88
<Authors>Boogie</Authors>

Source/ExecutionEngine/CommandLineOptions.cs

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -317,11 +317,14 @@ public bool UseUnsatCoreForContractInfer {
317317

318318
public bool PrintAssignment { get; set; }
319319

320-
public bool TrackVerificationCoverage {
321-
get => trackVerificationCoverage;
320+
public bool TrackVerificationCoverage
321+
{
322+
get => trackVerificationCoverage || WarnVacuousProofs;
322323
set => trackVerificationCoverage = value;
323324
}
324325

326+
public bool WarnVacuousProofs { get; set; }
327+
325328
public int InlineDepth { get; set; } = -1;
326329

327330
public bool UseProverEvaluate {
@@ -1373,6 +1376,7 @@ protected override bool ParseOption(string name, CommandLineParseState ps)
13731376
ps.CheckBooleanFlag("useUnsatCoreForContractInfer", x => useUnsatCoreForContractInfer = x) ||
13741377
ps.CheckBooleanFlag("printAssignment", x => PrintAssignment = x) ||
13751378
ps.CheckBooleanFlag("trackVerificationCoverage", x => trackVerificationCoverage = x) ||
1379+
ps.CheckBooleanFlag("warnVacuousProofs", x => WarnVacuousProofs = x) ||
13761380
ps.CheckBooleanFlag("useProverEvaluate", x => useProverEvaluate = x) ||
13771381
ps.CheckBooleanFlag("deterministicExtractLoops", x => DeterministicExtractLoops = x) ||
13781382
ps.CheckBooleanFlag("verifySeparately", x => VerifySeparately = x) ||
@@ -2005,7 +2009,11 @@ Track and report which program elements labeled with an
20052009
assignments, and calls can be labeled for inclusion in this
20062010
report. This generalizes and replaces the previous
20072011
(undocumented) `/printNecessaryAssertions` option.
2008-
2012+
/warnVacuousProofs
2013+
Automatically add missing `{:id ...}` attributes to assumptions,
2014+
assertions, requires clauses, ensures clauses, and calls; enable the
2015+
`/trackVerificationCoverage` option; and warn when proof goals are
2016+
not covered by a proof.
20092017
/keepQuantifier
20102018
If pool-based quantifier instantiation creates instances of a quantifier
20112019
then keep the quantifier along with the instances. By default, the quantifier

Source/ExecutionEngine/ExecutionEngine.cs

Lines changed: 52 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -611,6 +611,9 @@ private Task<ProcessedProgram> PreProcessProgramVerification(Program program, Ca
611611
}
612612

613613
var processedProgram = Options.ExtractLoops ? ExtractLoops(program) : new ProcessedProgram(program);
614+
if (Options.WarnVacuousProofs) {
615+
processedProgram = AddVacuityChecking(processedProgram);
616+
}
614617

615618
if (Options.PrintInstrumented)
616619
{
@@ -635,6 +638,55 @@ private ProcessedProgram ExtractLoops(Program program)
635638
});
636639
}
637640

641+
private ProcessedProgram AddVacuityChecking(ProcessedProgram processedProgram)
642+
{
643+
Program program = processedProgram.Program;
644+
CoverageAnnotator annotator = new CoverageAnnotator();
645+
foreach (var impl in program.Implementations) {
646+
annotator.VisitImplementation(impl);
647+
}
648+
return new ProcessedProgram(program, (vcgen, impl, result) =>
649+
{
650+
CheckVacuity(annotator, impl, result);
651+
processedProgram.PostProcessResult(vcgen, impl, result);
652+
}
653+
);
654+
}
655+
656+
private void CheckVacuity(CoverageAnnotator annotator, Implementation impl, ImplementationRunResult verificationResult)
657+
{
658+
var covered = verificationResult
659+
.RunResults
660+
.SelectMany(r => r.CoveredElements)
661+
.ToHashSet();
662+
foreach (var goalId in annotator.GetImplementationGoalIds(impl.Name)) {
663+
if (!CoveredId(goalId, covered)) {
664+
var node = annotator.GetIdNode(goalId);
665+
Options.Printer.AdvisoryWriteLine(Options.OutputWriter,
666+
$"{node.tok.filename}({node.tok.line},{node.tok.col - 1}): Warning: Proved vacuously");
667+
}
668+
}
669+
}
670+
671+
private static bool CoveredId(string goalId, HashSet<TrackedNodeComponent> covered)
672+
{
673+
foreach (var component in covered) {
674+
if (component is LabeledNodeComponent { id: var id } && id == goalId) {
675+
return true;
676+
}
677+
678+
if (component is TrackedInvariantEstablished { invariantId: var eid } && eid == goalId) {
679+
return true;
680+
}
681+
682+
if (component is TrackedInvariantMaintained { invariantId: var mid } && mid == goalId) {
683+
return true;
684+
}
685+
}
686+
687+
return false;
688+
}
689+
638690
private Implementation[] GetPrioritizedImplementations(Program program)
639691
{
640692
var impls = program.Implementations.Where(
@@ -964,7 +1016,6 @@ private Task<ImplementationRunResult> VerifyImplementationWithLargeThread(Proces
9641016
return resultTask;
9651017
}
9661018

967-
9681019
#region Houdini
9691020

9701021
private async Task<PipelineOutcome> RunHoudini(Program program, PipelineStatistics stats, ErrorReporterDelegate er,

Source/Provers/SMTLib/SMTLibOptions.cs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ public interface SMTLibOptions : CoreOptions
1414
bool ImmediatelyAcceptCommands { get; }
1515
bool RunningBoogieFromCommandLine { get; }
1616
bool TrackVerificationCoverage { get; }
17+
bool WarnVacuousProofs { get; }
1718
string ProverPreamble { get; }
1819
bool TraceDiagnosticsOnTimeout { get; }
1920
uint TimeLimitPerAssertionInPercent { get; }

Test/coverage/verificationCoverage.bpl

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -100,8 +100,8 @@
100100
// RUN: %diff "%s.expect" "%t.coverage-p"
101101
// RUN: %boogie -trackVerificationCoverage -normalizeDeclarationOrder:1 -prune:1 "%s" > "%t.coverage-d"
102102
// RUN: %diff "%s.expect" "%t.coverage-d"
103-
// RUN: %boogie -trackVerificationCoverage -normalizeNames:1 -prune:1 "%s" > "%t.coverage-n"
104-
// RUN: %diff "%s.expect" "%t.coverage-n"
103+
// RUN: %boogie -trackVerificationCoverage -warnVacuousProofs -trace -normalizeNames:1 -prune:1 "%s" | grep -v "solver resource count" | grep -v "batch mode" > "%t.coverage-n"
104+
// RUN: %diff "%s.expect-trace" "%t.coverage-n"
105105

106106
procedure testRequiresAssign(n: int)
107107
requires {:id "r0"} n > 0; // covered
@@ -216,4 +216,4 @@ procedure usesSomeInteger() returns (r: bool)
216216
ensures r;
217217
{
218218
r := someInteger(7) == 3;
219-
}
219+
}
Lines changed: 129 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,129 @@
1+
Parsing verificationCoverage.bpl
2+
Coalescing blocks...
3+
Inlining...
4+
5+
Verifying testRequiresAssign ...
6+
[TRACE] Using prover: z3
7+
Proof dependencies:
8+
a0
9+
assert_a0
10+
assert_r0
11+
r0
12+
13+
Verifying sum ...
14+
Proof dependencies:
15+
id_l127_c3_assume_0
16+
id_l127_c3_assume_1
17+
invariant sinv_not_1 established
18+
invariant sinv_not_1 maintained
19+
invariant sinv1 assumed in body
20+
invariant sinv1 established
21+
invariant sinv1 maintained
22+
invariant sinv2 assumed in body
23+
invariant sinv2 established
24+
invariant sinv2 maintained
25+
spost
26+
spre1
27+
28+
Verifying contradictoryAssume ...
29+
Proof dependencies:
30+
cont_assume_1
31+
cont_assume_2
32+
verificationCoverage.bpl(143,4): Warning: Proved vacuously
33+
34+
Verifying falseRequires ...
35+
Proof dependencies:
36+
false_req
37+
verificationCoverage.bpl(150,4): Warning: Proved vacuously
38+
39+
Verifying contradictoryRequires ...
40+
Proof dependencies:
41+
cont_req_1
42+
cont_req_2
43+
verificationCoverage.bpl(158,4): Warning: Proved vacuously
44+
45+
Verifying assumeFalse ...
46+
Proof dependencies:
47+
assumeFalse
48+
verificationCoverage.bpl(163,2): Warning: Proved vacuously
49+
50+
Verifying testEnsuresCallee ...
51+
Proof dependencies:
52+
tee0
53+
tee1
54+
ter0
55+
56+
Verifying testEnsuresCaller ...
57+
Proof dependencies:
58+
call2_tee1
59+
ensures clause tee0 from call call1
60+
ensures clause tee0 from call call2
61+
ensures clause tee1 from call call2
62+
requires clause ter0 proved for call call1
63+
requires clause ter0 proved for call call2
64+
tee_not_1
65+
ter1
66+
xy_sum
67+
68+
Verifying obviouslyUnconstrainedCode ...
69+
Proof dependencies:
70+
a_gt_10
71+
constrained
72+
x_gt_10
73+
74+
Verifying callContradictoryFunction ...
75+
Proof dependencies:
76+
ensures clause cont_ens_abs from call call_cont
77+
requires clause xpos_abs proved for call call_cont
78+
xpos_caller
79+
verificationCoverage.bpl(203,2): Warning: Proved vacuously
80+
81+
Verifying usesSomeInteger ...
82+
Proof dependencies:
83+
id_l216_c3_ensures_2
84+
someInteger_value_axiom
85+
Proof dependencies of whole program:
86+
a_gt_10
87+
a0
88+
assert_a0
89+
assert_r0
90+
assumeFalse
91+
call2_tee1
92+
constrained
93+
cont_assume_1
94+
cont_assume_2
95+
cont_req_1
96+
cont_req_2
97+
ensures clause cont_ens_abs from call call_cont
98+
ensures clause tee0 from call call1
99+
ensures clause tee0 from call call2
100+
ensures clause tee1 from call call2
101+
false_req
102+
id_l127_c3_assume_0
103+
id_l127_c3_assume_1
104+
id_l216_c3_ensures_2
105+
invariant sinv_not_1 established
106+
invariant sinv_not_1 maintained
107+
invariant sinv1 assumed in body
108+
invariant sinv1 established
109+
invariant sinv1 maintained
110+
invariant sinv2 assumed in body
111+
invariant sinv2 established
112+
invariant sinv2 maintained
113+
r0
114+
requires clause ter0 proved for call call1
115+
requires clause ter0 proved for call call2
116+
requires clause xpos_abs proved for call call_cont
117+
someInteger_value_axiom
118+
spost
119+
spre1
120+
tee_not_1
121+
tee0
122+
tee1
123+
ter0
124+
ter1
125+
x_gt_10
126+
xpos_caller
127+
xy_sum
128+
129+
Boogie program verifier finished with 11 verified, 0 errors

0 commit comments

Comments
 (0)