diff --git a/pkg/sip/media_port.go b/pkg/sip/media_port.go
index 7d6c04074..6891b1c24 100644
--- a/pkg/sip/media_port.go
+++ b/pkg/sip/media_port.go
@@ -670,6 +670,11 @@ func (p *MediaPort) SetConfig(c *MediaConf) error {
 	)
 
 	p.port.SetDst(c.Remote)
+	if p.opts.IgnorePreanswerData {
+		// this needs to happen before the SRTP session is created, otherwise the read deadline will be
+		// overwritten and we may get stuck in the discard loop
+		p.port.stopDiscarding()
+	}
 	var (
 		sess rtp.Session
 		err  error
@@ -784,9 +789,6 @@ func (p *MediaPort) setupOutput(tid traceid.ID) error {
 	if p.closed.IsBroken() {
 		return errors.New("media is already closed")
 	}
-	if p.opts.IgnorePreanswerData {
-		p.port.stopDiscarding()
-	}
 	go p.rtpLoop(tid, p.sess)
 	w, err := p.sess.OpenWriteStream()
 	if err != nil {