txscript: reject OP_CODESEPARATOR in unexecuted branches for non-segwit

In this commit, we fix a policy-level divergence with Bitcoin Core when
handling OP_CODESEPARATOR inside unexecuted OP_IF branches in non-segwit
scripts with the ScriptVerifyConstScriptCode flag.

Bitcoin Core's EvalScript (interpreter.cpp:474-476) places the
SCRIPT_VERIFY_CONST_SCRIPTCODE check for OP_CODESEPARATOR before the
fExec branch-execution gate, causing it to fire unconditionally on every
OP_CODESEPARATOR encountered during script iteration -- even inside
OP_FALSE OP_IF ... OP_ENDIF envelopes.

Previously, btcd's equivalent check lived inside the opcodeCodeSeparator
handler, which was never reached for opcodes in unexecuted branches due
to the early return in executeOpcode that skips non-conditional opcodes
when isBranchExecuting() is false. This meant a script like:

  OP_FALSE OP_IF OP_CODESEPARATOR OP_ENDIF <validation>

would be rejected by Bitcoin Core's mempool but accepted by btcd's.

The fix moves the check before the branch-execution gate in
executeOpcode, matching Bitcoin Core's structure. This follows the
existing pattern in btcd where isOpcodeDisabled and isOpcodeAlwaysIllegal
checks already fire regardless of branch execution state.

Note: SCRIPT_VERIFY_CONST_SCRIPTCODE is purely a policy flag (included
in STANDARD_SCRIPT_VERIFY_FLAGS but not MANDATORY_SCRIPT_VERIFY_FLAGS),
so this was not a consensus divergence. Both implementations would accept
such transactions if mined in a block.

Found via differential fuzzing by Bruno from bitcoinfuzz.

Author: Roasbeef

txscript/engine.go

@@ -485,6 +485,18 @@ func (vm *Engine) executeOpcode(op *opcode, data []byte) error {
 		return scriptError(ErrElementTooBig, str)
 	}
 
+	// With ScriptVerifyConstScriptCode, OP_CODESEPARATOR in non-segwit
+	// script is rejected even in an unexecuted branch. This mirrors
+	// Bitcoin Core's behavior where the check fires unconditionally
+	// before the branch execution gate.
+	if op.value == OP_CODESEPARATOR && vm.taprootCtx == nil &&
+		vm.witnessProgram == nil &&
+		vm.hasFlag(ScriptVerifyConstScriptCode) {
+
+		str := "OP_CODESEPARATOR used in non-segwit script"
+		return scriptError(ErrCodeSeparator, str)
+	}
+
 	// Nothing left to do when this is not a conditional opcode and it is
 	// not in an executing branch.
 	if !vm.isBranchExecuting() && !isOpcodeConditional(op.value) {

txscript/engine_test.go

@@ -426,3 +426,113 @@ func TestCheckSignatureEncoding(t *testing.T) {
 		}
 	}
 }
+
+// TestCodeSepUnexecutedBranch ensures that OP_CODESEPARATOR is rejected in
+// non-segwit scripts even when inside an unexecuted OP_IF branch, when the
+// ScriptVerifyConstScriptCode flag is set. This matches Bitcoin Core's
+// behavior where the SCRIPT_VERIFY_CONST_SCRIPTCODE check fires
+// unconditionally before the branch execution gate.
+func TestCodeSepUnexecutedBranch(t *testing.T) {
+	t.Parallel()
+
+	// A minimal transaction for script execution.
+	tx := &wire.MsgTx{
+		Version: 1,
+		TxIn: []*wire.TxIn{{
+			PreviousOutPoint: wire.OutPoint{
+				Hash: chainhash.Hash([32]byte{
+					0xc9, 0x97, 0xa5, 0xe5,
+					0x6e, 0x10, 0x41, 0x02,
+					0xfa, 0x20, 0x9c, 0x6a,
+					0x85, 0x2d, 0xd9, 0x06,
+					0x60, 0xa2, 0x0b, 0x2d,
+					0x9c, 0x35, 0x24, 0x23,
+					0xed, 0xce, 0x25, 0x85,
+					0x7f, 0xcd, 0x37, 0x04,
+				}),
+				Index: 0,
+			},
+			SignatureScript: mustParseShortForm("TRUE"),
+			Sequence:        4294967295,
+		}},
+		TxOut: []*wire.TxOut{{
+			Value:    1000000000,
+			PkScript: nil,
+		}},
+		LockTime: 0,
+	}
+
+	tests := []struct {
+		name    string
+		script  string
+		flags   ScriptFlags
+		wantErr bool
+		errCode ErrorCode
+	}{
+		{
+			// OP_CODESEPARATOR inside an unexecuted branch with
+			// the const scriptcode flag set should be rejected.
+			name:    "codesep in unexecuted IF with const scriptcode",
+			script:  "0 IF CODESEPARATOR ENDIF TRUE",
+			flags:   ScriptVerifyConstScriptCode,
+			wantErr: true,
+			errCode: ErrCodeSeparator,
+		},
+		{
+			// Without the flag, OP_CODESEPARATOR in an unexecuted
+			// branch should be fine (consensus behavior).
+			name:    "codesep in unexecuted IF without const scriptcode",
+			script:  "0 IF CODESEPARATOR ENDIF TRUE",
+			flags:   0,
+			wantErr: false,
+		},
+		{
+			// OP_CODESEPARATOR in an executed branch with the
+			// const scriptcode flag should also be rejected.
+			name:    "codesep in executed branch with const scriptcode",
+			script:  "CODESEPARATOR TRUE",
+			flags:   ScriptVerifyConstScriptCode,
+			wantErr: true,
+			errCode: ErrCodeSeparator,
+		},
+		{
+			// Nested unexecuted branches should still be caught.
+			name:    "codesep in nested unexecuted IF with const scriptcode",
+			script:  "0 IF 1 IF CODESEPARATOR ENDIF ENDIF TRUE",
+			flags:   ScriptVerifyConstScriptCode,
+			wantErr: true,
+			errCode: ErrCodeSeparator,
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			pkScript := mustParseShortForm(test.script)
+			vm, err := NewEngine(
+				pkScript, tx, 0, test.flags, nil, nil,
+				-1, nil,
+			)
+			if err != nil {
+				t.Fatalf("failed to create engine: %v", err)
+			}
+
+			err = vm.Execute()
+
+			switch {
+			case test.wantErr && err == nil:
+				t.Fatal("expected error but execution " +
+					"succeeded")
+
+			case !test.wantErr && err != nil:
+				t.Fatalf("unexpected error: %v", err)
+
+			case test.wantErr && err != nil:
+				if !IsErrorCode(err, test.errCode) {
+					t.Fatalf("expected error code "+
+						"%v, got: %v",
+						test.errCode, err)
+				}
+			}
+		})
+	}
+}