-
Notifications
You must be signed in to change notification settings - Fork 51
Expand file tree
/
Copy patherror_detection.go
More file actions
352 lines (305 loc) · 13.5 KB
/
Copy patherror_detection.go
File metadata and controls
352 lines (305 loc) · 13.5 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
package pdpv0
import (
"encoding/hex"
"strings"
"github.com/ethereum/go-ethereum/accounts/abi"
"github.com/filecoin-project/curio/pdp/contract"
"github.com/filecoin-project/curio/pdp/contract/FWSS"
)
// Known contract errors indicating permanent dataset termination.
var (
ErrFWSSDataSetPaymentBeyondEndEpoch abi.Error
ErrFWSSDataSetPaymentAlreadyTerminated abi.Error
ErrPDPVerifierDataSetNotFound abi.Error
)
// PDP proving/initPP/nextPP custom errors that should be classified before
// falling back to generic contract-revert alert/retry handling.
var (
ErrFWSSProofAlreadySubmitted abi.Error
ErrFWSSProvingNotStarted abi.Error
ErrFWSSChallengeWindowTooEarly abi.Error
ErrFWSSProvingPeriodPassed abi.Error
ErrFWSSInvalidChallengeEpoch abi.Error
ErrFWSSNextProvingPeriodAlreadyCalled abi.Error
ErrFWSSProvingPeriodNotInitialized abi.Error
ErrPDPVerifierDataSetNotLive abi.Error
ErrPDPVerifierInsufficientChallengeDelay abi.Error
ErrPDPVerifierPendingPieceDeletions abi.Error
ErrPDPVerifierNoPiecesToProve abi.Error
// Unexpected proving invariant errors. Curio should not produce these in
// normal PDPv0 initPP/nextPP/prove flow; classify them explicitly so they
// alert and require investigation instead of entering recovery/backoff paths.
ErrPDPVerifierExcessiveChallengeDelay abi.Error
ErrFWSSOnlyPDPVerifierAllowed abi.Error
ErrFWSSInvalidChallengeCount abi.Error
)
// PDPVerifier proving-flow revert reason strings. These are Solidity
// require(..., "reason") failures, so they do not have ABI custom-error selectors.
const (
provingRevertOnlyStorageProviderCanProve = "Only the storage provider can prove possession"
provingRevertPrematureProof = "premature proof"
provingRevertNoChallengeScheduled = "no challenge scheduled"
provingRevertLeafIndexOutOfBounds = "Leaf index out of bounds"
provingRevertProofDidNotVerify = "proof did not verify"
provingRevertNoLeavesForProvingPeriod = "can only start proving once leaves are added"
)
func init() {
parsedPDPVerifier, err := contract.PDPVerifierMetaData.GetAbi()
if err != nil {
panic("failed to parse PDPVerifier ABI: " + err.Error())
}
var ok bool
ErrPDPVerifierDataSetNotFound, ok = parsedPDPVerifier.Errors["DataSetNotFound"]
if !ok {
panic("PDPVerifier ABI missing DataSetNotFound error")
}
ErrPDPVerifierDataSetNotLive, ok = parsedPDPVerifier.Errors["DataSetNotLive"]
if !ok {
panic("PDPVerifier ABI missing DataSetNotLive error")
}
ErrPDPVerifierInsufficientChallengeDelay, ok = parsedPDPVerifier.Errors["InsufficientChallengeDelay"]
if !ok {
panic("PDPVerifier ABI missing InsufficientChallengeDelay error")
}
ErrPDPVerifierExcessiveChallengeDelay, ok = parsedPDPVerifier.Errors["ExcessiveChallengeDelay"]
if !ok {
panic("PDPVerifier ABI missing ExcessiveChallengeDelay error")
}
ErrPDPVerifierPendingPieceDeletions, ok = parsedPDPVerifier.Errors["PendingPieceDeletions"]
if !ok {
panic("PDPVerifier removal ABI missing PendingPieceDeletions error")
}
ErrPDPVerifierNoPiecesToProve, ok = parsedPDPVerifier.Errors["NoPiecesToProve"]
if !ok {
panic("PDPVerifier removal ABI missing NoPiecesToProve error")
}
parsedFWSS, err := FWSS.FilecoinWarmStorageServiceMetaData.GetAbi()
if err != nil {
panic("failed to parse FWSS ABI: " + err.Error())
}
ErrFWSSDataSetPaymentBeyondEndEpoch, ok = parsedFWSS.Errors["DataSetPaymentBeyondEndEpoch"]
if !ok {
panic("FWSS ABI missing DataSetPaymentBeyondEndEpoch error")
}
ErrFWSSDataSetPaymentAlreadyTerminated, ok = parsedFWSS.Errors["DataSetPaymentAlreadyTerminated"]
if !ok {
panic("FWSS ABI missing DataSetPaymentAlreadyTerminated error")
}
ErrFWSSOnlyPDPVerifierAllowed, ok = parsedFWSS.Errors["OnlyPDPVerifierAllowed"]
if !ok {
panic("FWSS ABI missing OnlyPDPVerifierAllowed error")
}
ErrFWSSProofAlreadySubmitted, ok = parsedFWSS.Errors["ProofAlreadySubmitted"]
if !ok {
panic("FWSS ABI missing ProofAlreadySubmitted error")
}
ErrFWSSInvalidChallengeCount, ok = parsedFWSS.Errors["InvalidChallengeCount"]
if !ok {
panic("FWSS ABI missing InvalidChallengeCount error")
}
ErrFWSSProvingNotStarted, ok = parsedFWSS.Errors["ProvingNotStarted"]
if !ok {
panic("FWSS ABI missing ProvingNotStarted error")
}
ErrFWSSChallengeWindowTooEarly, ok = parsedFWSS.Errors["ChallengeWindowTooEarly"]
if !ok {
panic("FWSS ABI missing ChallengeWindowTooEarly error")
}
ErrFWSSProvingPeriodPassed, ok = parsedFWSS.Errors["ProvingPeriodPassed"]
if !ok {
panic("FWSS ABI missing ProvingPeriodPassed error")
}
ErrFWSSInvalidChallengeEpoch, ok = parsedFWSS.Errors["InvalidChallengeEpoch"]
if !ok {
panic("FWSS ABI missing InvalidChallengeEpoch error")
}
ErrFWSSNextProvingPeriodAlreadyCalled, ok = parsedFWSS.Errors["NextProvingPeriodAlreadyCalled"]
if !ok {
panic("FWSS ABI missing NextProvingPeriodAlreadyCalled error")
}
parsedFWSSStateView, err := FWSS.FilecoinWarmStorageServiceStateViewMetaData.GetAbi()
if err != nil {
panic("failed to parse FWSS state view ABI: " + err.Error())
}
ErrFWSSProvingPeriodNotInitialized, ok = parsedFWSSStateView.Errors["ProvingPeriodNotInitialized"]
if !ok {
panic("FWSS state view ABI missing ProvingPeriodNotInitialized error")
}
}
func contractErrorSelector(errDef abi.Error) string {
return hex.EncodeToString(errDef.ID[:4])
}
// IsUnrecoverableError returns true if the error contains a known unrecoverable
// error selector. These errors indicate the dataset should be permanently terminated
// and proving should stop immediately.
func IsUnrecoverableError(err error) bool {
if err == nil {
return false
}
errStr := strings.ToLower(err.Error())
return strings.Contains(errStr, contractErrorSelector(ErrFWSSDataSetPaymentBeyondEndEpoch)) ||
strings.Contains(errStr, contractErrorSelector(ErrFWSSDataSetPaymentAlreadyTerminated)) ||
strings.Contains(errStr, contractErrorSelector(ErrPDPVerifierDataSetNotLive))
}
// IsRetrySameProvingPeriodError returns true for prove timing reverts where
// the current prove task should retry without changing dataset failure state.
func IsRetrySameProvingPeriodError(err error) bool {
if err == nil {
return false
}
errStr := strings.ToLower(err.Error())
return strings.Contains(errStr, contractErrorSelector(ErrFWSSChallengeWindowTooEarly)) ||
strings.Contains(errStr, strings.ToLower(provingRevertPrematureProof))
}
// IsInsufficientChallengeDelayError returns true when initPP/nextPP used a
// challenge epoch too close to the current block. Waiting and resending the same
// transaction cannot fix this because the delay only decreases; the task must
// recompute the challenge epoch from fresh chain/listener state.
func IsInsufficientChallengeDelayError(err error) bool {
if err == nil {
return false
}
return strings.Contains(strings.ToLower(err.Error()), contractErrorSelector(ErrPDPVerifierInsufficientChallengeDelay))
}
// IsSkipCurrentProvingPeriodError returns true when provePossession no longer
// needs to submit a proof for the current proving period.
func IsSkipCurrentProvingPeriodError(err error) bool {
if err == nil {
return false
}
errStr := strings.ToLower(err.Error())
return strings.Contains(errStr, contractErrorSelector(ErrFWSSProofAlreadySubmitted)) ||
strings.Contains(errStr, contractErrorSelector(ErrFWSSProvingPeriodPassed)) ||
strings.Contains(errStr, strings.ToLower(provingRevertNoChallengeScheduled))
}
// IsNextProvingPeriodAlreadyCalledError returns true when initPP/nextPP learns
// that FWSS has already advanced the proving period.
func IsNextProvingPeriodAlreadyCalledError(err error) bool {
if err == nil {
return false
}
return strings.Contains(strings.ToLower(err.Error()), contractErrorSelector(ErrFWSSNextProvingPeriodAlreadyCalled))
}
// IsProvingPeriodNotInitializedError returns true when nextPP discovers local
// state has a prove schedule but FWSS has no initialized proving period.
func IsProvingPeriodNotInitializedError(err error) bool {
if err == nil {
return false
}
return strings.Contains(strings.ToLower(err.Error()), contractErrorSelector(ErrFWSSProvingPeriodNotInitialized))
}
// IsNextProvingPeriodEmptyDatasetError returns true when PDPVerifier refuses to
// start the next proving period because the current proving set has no leaves.
func IsNextProvingPeriodEmptyDatasetError(err error) bool {
if err == nil {
return false
}
errStr := strings.ToLower(err.Error())
return strings.Contains(errStr, strings.ToLower(provingRevertNoLeavesForProvingPeriod)) ||
strings.Contains(errStr, contractErrorSelector(ErrPDPVerifierNoPiecesToProve))
}
// IsPendingPieceDeletionsError returns true when nextProvingPeriod (or initPP)
// refuses to roll over because the data set still has scheduled removals
// queued. This is recoverable: the process deletions task processes the queue and the
// proving-period task retries.
func IsPendingPieceDeletionsError(err error) bool {
if err == nil {
return false
}
return strings.Contains(strings.ToLower(err.Error()), contractErrorSelector(ErrPDPVerifierPendingPieceDeletions))
}
func IsPDPVerifierDataSetNotLive(err error) bool {
if err == nil {
return false
}
return strings.Contains(strings.ToLower(err.Error()), contractErrorSelector(ErrPDPVerifierDataSetNotLive))
}
// IsRefreshProvingStateError returns true when initPP/nextPP selected a
// challenge epoch that FWSS no longer accepts. The scheduler should retry so it
// recomputes the proving-period calldata from fresh chain/listener state.
func IsRefreshProvingStateError(err error) bool {
if err == nil {
return false
}
errStr := strings.ToLower(err.Error())
return strings.Contains(errStr, contractErrorSelector(ErrFWSSInvalidChallengeEpoch))
}
// IsUnexpectedProvingInvariantError returns true for contract reverts that are
// impossible in Curio's normal PDPv0 path if Curio and the deployed contracts
// agree on listener wiring, schedule math, and proof challenge count.
//
// These are intentionally kept out of recovery categories:
// - ExcessiveChallengeDelay: initPP/nextPP should use listener-derived
// challenge epochs inside PDPVerifier's allowed finality window.
// - OnlyPDPVerifierAllowed: Curio sends PDPVerifier calls; FWSS callbacks
// should be invoked by PDPVerifier, not Curio.
// - InvalidChallengeCount: Curio generates contract.NumChallenges proofs,
// which should match FWSS CHALLENGES_PER_PROOF.
// - Leaf index out of bounds: Curio chooses challenges from PDPVerifier's
// challenge range before prove send; seeing this during prove send indicates
// verifier state drift, not normal proving-state refresh.
func IsUnexpectedProvingInvariantError(err error) bool {
if err == nil {
return false
}
errStr := strings.ToLower(err.Error())
return strings.Contains(errStr, contractErrorSelector(ErrPDPVerifierExcessiveChallengeDelay)) ||
strings.Contains(errStr, contractErrorSelector(ErrFWSSOnlyPDPVerifierAllowed)) ||
strings.Contains(errStr, contractErrorSelector(ErrFWSSInvalidChallengeCount)) ||
strings.Contains(errStr, strings.ToLower(provingRevertLeafIndexOutOfBounds))
}
// IsFWSSProvingNotStartedError returns true when PDPVerifier had a non-zero
// challenge epoch and called FWSS possessionProven, but FWSS had no active
// proving deadline. This indicates local/PDPVerifier/FWSS proving state
// divergence, not a timing retry or dataset termination. The prove handler
// should complete the current proof attempt and reset the dataset to initPP
// scheduling so Curio re-establishes a proving deadline before proving again.
func IsFWSSProvingNotStartedError(err error) bool {
if err == nil {
return false
}
return strings.Contains(strings.ToLower(err.Error()), contractErrorSelector(ErrFWSSProvingNotStarted))
}
// IsOperatorAttentionProvingError returns true for prove call authorization
// errors that should be surfaced instead of retried as normal proving flow.
func IsOperatorAttentionProvingError(err error) bool {
if err == nil {
return false
}
errStr := strings.ToLower(err.Error())
return strings.Contains(errStr, strings.ToLower(provingRevertOnlyStorageProviderCanProve))
}
// IsProofGenerationFailureError returns true when the contract rejected the
// generated proof itself.
func IsProofGenerationFailureError(err error) bool {
if err == nil {
return false
}
return strings.Contains(strings.ToLower(err.Error()), strings.ToLower(provingRevertProofDidNotVerify))
}
// IsPDPVerifierDataSetNotFound returns true if PDPVerifier reports that a data
// set no longer exists. In prove preflight this is terminal for local proving;
// in the deletion pipeline it means on-chain cleanup has finalized.
func IsPDPVerifierDataSetNotFound(err error) bool {
if err == nil {
return false
}
return strings.Contains(strings.ToLower(err.Error()), contractErrorSelector(ErrPDPVerifierDataSetNotFound))
}
// IsContractRevert returns true if the error indicates a contract revert.
// Contract reverts mean the on-chain state is rejecting the call - retrying
// immediately is pointless. This includes gas estimation failures due to
// reverts, which is how most failures manifest.
func IsContractRevert(err error) bool {
if err == nil {
return false
}
errStr := strings.ToLower(err.Error())
// Common patterns indicating contract reverts
return strings.Contains(errStr, "execution reverted") ||
strings.Contains(errStr, "vm execution error") ||
strings.Contains(errStr, "revert reason") ||
strings.Contains(errStr, "retcode=33") || // EVM revert exit code
strings.Contains(errStr, "(exit=[33]") || // Filecoin EVM revert format
strings.Contains(errStr, "contract reverted")
}