feat: add sliding sync (MSC4186) implementation

.gitignore

@@ -81,3 +81,6 @@ go.work*
 
 # helm chart
 helm/dendrite/charts/
+
+# Built binary in root (from local development)
+/dendrite

appservice/appservice_test.go

@@ -446,7 +446,7 @@ func TestOutputAppserviceEvent(t *testing.T) {
 		}
 
 		usrAPI := userapi.NewInternalAPI(processCtx, cfg, cm, natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff)
-		clientapi.AddPublicRoutes(processCtx, routers, cfg, natsInstance, nil, rsAPI, nil, nil, nil, usrAPI, nil, nil, caching.DisableMetrics)
+		clientapi.AddPublicRoutes(processCtx, routers, cfg, natsInstance, nil, rsAPI, nil, nil, nil, usrAPI, nil, nil, nil, caching.DisableMetrics)
 		createAccessTokens(t, accessTokens, usrAPI, processCtx.Context(), routers)
 
 		room := test.NewRoom(t, alice)

clientapi/auth/user_interactive.go

@@ -248,7 +248,7 @@ func (u *UserInteractive) Verify(ctx context.Context, bodyBytes []byte, device *
 		if !u.IsSingleStageFlow(authType) {
 			return nil, &util.JSONResponse{
 				Code: http.StatusBadRequest,
-				JSON: spec.Unknown("The auth.session is missing or unknown."),
+				JSON: spec.MissingParam("The auth.session is missing or unknown."),
 			}
 		}
 	}

clientapi/clientapi.go

@@ -7,6 +7,7 @@
 package clientapi
 
 import (
+	"github.com/element-hq/dendrite/internal/caching"
 	"github.com/element-hq/dendrite/internal/httputil"
 	"github.com/element-hq/dendrite/setup/config"
 	"github.com/element-hq/dendrite/setup/process"
@@ -36,7 +37,9 @@ func AddPublicRoutes(
 	fsAPI federationAPI.ClientFederationAPI,
 	userAPI userapi.ClientUserAPI,
 	userDirectoryProvider userapi.QuerySearchProfilesAPI,
-	extRoomsProvider api.ExtraPublicRoomsProvider, enableMetrics bool,
+	extRoomsProvider api.ExtraPublicRoomsProvider,
+	caches *caching.Caches,
+	enableMetrics bool,
 ) {
 	js, natsClient := natsInstance.Prepare(processContext, &cfg.Global.JetStream)
 
@@ -55,6 +58,6 @@ func AddPublicRoutes(
 		cfg, rsAPI, asAPI,
 		userAPI, userDirectoryProvider, federation,
 		syncProducer, transactionsCache, fsAPI,
-		extRoomsProvider, natsClient, enableMetrics,
+		extRoomsProvider, caches, natsClient, enableMetrics,
 	)
 }

clientapi/httputil/httputil.go

@@ -8,6 +8,7 @@ package httputil
 
 import (
 	"encoding/json"
+	"errors"
 	"io"
 	"net/http"
 	"unicode/utf8"
@@ -52,3 +53,36 @@ func UnmarshalJSON(body []byte, iface interface{}) *util.JSONResponse {
 	}
 	return nil
 }
+
+// MatrixErrorResponse converts a spec.MatrixError to a util.JSONResponse with the
+// appropriate HTTP status code based on the error code. This helper prevents error
+// codes from being lost when errors are wrapped or passed through handler chains.
+//
+// If the error is not a spec.MatrixError, it returns nil (caller should handle as internal error).
+//
+// HTTP status code mapping follows the Matrix spec:
+//   - M_FORBIDDEN, M_UNABLE_TO_AUTHORISE_JOIN -> 403
+//   - M_NOT_FOUND, M_UNRECOGNIZED -> 404
+//   - All other Matrix errors -> 400 (bad request)
+func MatrixErrorResponse(err error) *util.JSONResponse {
+	var matrixErr spec.MatrixError
+	if !errors.As(err, &matrixErr) {
+		return nil
+	}
+
+	var httpCode int
+	switch matrixErr.ErrCode {
+	case spec.ErrorForbidden, spec.ErrorUnableToAuthoriseJoin:
+		httpCode = http.StatusForbidden
+	case spec.ErrorNotFound, spec.ErrorUnrecognized:
+		httpCode = http.StatusNotFound
+	default:
+		// Most Matrix errors are client errors (bad request)
+		httpCode = http.StatusBadRequest
+	}
+
+	return &util.JSONResponse{
+		Code: httpCode,
+		JSON: matrixErr,
+	}
+}

clientapi/routing/directory.go

@@ -185,7 +185,7 @@ func SetLocalAlias(
 	if err != nil {
 		return util.JSONResponse{
 			Code: http.StatusInternalServerError,
-			JSON: spec.Unknown("internal server error"),
+			JSON: spec.InternalServerError{},
 		}
 	}
 
@@ -194,13 +194,13 @@ func SetLocalAlias(
 		util.GetLogger(req.Context()).WithError(err).Error("QuerySenderIDForUser failed")
 		return util.JSONResponse{
 			Code: http.StatusInternalServerError,
-			JSON: spec.Unknown("internal server error"),
+			JSON: spec.InternalServerError{},
 		}
 	} else if senderID == nil {
 		util.GetLogger(req.Context()).WithField("roomID", *roomID).WithField("userID", *userID).Error("Sender ID not found")
 		return util.JSONResponse{
 			Code: http.StatusInternalServerError,
-			JSON: spec.Unknown("internal server error"),
+			JSON: spec.InternalServerError{},
 		}
 	}
 
@@ -216,7 +216,7 @@ func SetLocalAlias(
 	if aliasAlreadyExists {
 		return util.JSONResponse{
 			Code: http.StatusConflict,
-			JSON: spec.Unknown("The alias " + alias + " already exists."),
+			JSON: spec.RoomInUse("The alias " + alias + " already exists."),
 		}
 	}
 
@@ -273,7 +273,7 @@ func RemoveLocalAlias(
 		util.GetLogger(req.Context()).WithError(err).Error("roomserverAPI.QueryMembershipForUser failed")
 		return util.JSONResponse{
 			Code: http.StatusInternalServerError,
-			JSON: spec.Unknown("internal server error"),
+			JSON: spec.InternalServerError{},
 		}
 	}
 	if !queryResp.IsInRoom {
@@ -294,7 +294,7 @@ func RemoveLocalAlias(
 	if deviceSenderID == nil {
 		return util.JSONResponse{
 			Code: http.StatusInternalServerError,
-			JSON: spec.Unknown("internal server error"),
+			JSON: spec.InternalServerError{},
 		}
 	}
 
@@ -303,7 +303,7 @@ func RemoveLocalAlias(
 		util.GetLogger(req.Context()).WithError(err).Error("aliasAPI.RemoveRoomAlias failed")
 		return util.JSONResponse{
 			Code: http.StatusInternalServerError,
-			JSON: spec.Unknown("internal server error"),
+			JSON: spec.InternalServerError{},
 		}
 	}
 

clientapi/routing/joined_rooms.go

@@ -30,7 +30,7 @@ func GetJoinedRooms(
 		util.GetLogger(req.Context()).WithError(err).Error("Invalid device user ID")
 		return util.JSONResponse{
 			Code: http.StatusInternalServerError,
-			JSON: spec.Unknown("internal server error"),
+			JSON: spec.InternalServerError{},
 		}
 	}
 
@@ -39,7 +39,7 @@ func GetJoinedRooms(
 		util.GetLogger(req.Context()).WithError(err).Error("QueryRoomsForUser failed")
 		return util.JSONResponse{
 			Code: http.StatusInternalServerError,
-			JSON: spec.Unknown("internal server error"),
+			JSON: spec.InternalServerError{},
 		}
 	}
 

clientapi/routing/joinroom.go

@@ -28,6 +28,15 @@ func JoinRoomByIDOrAlias(
 	profileAPI api.ClientUserAPI,
 	roomIDOrAlias string,
 ) util.JSONResponse {
+	// MSC3706: Trace join timing for diagnostics
+	joinStartTime := time.Now()
+	logger := util.GetLogger(req.Context()).WithFields(map[string]interface{}{
+		"room_id_or_alias": roomIDOrAlias,
+		"user_id":          device.UserID,
+		"trace":            "join_timing",
+	})
+	logger.Debug("Join request received")
+
 	// Prepare to ask the roomserver to perform the room join.
 	joinReq := roomserverAPI.PerformJoinRequest{
 		RoomIDOrAlias: roomIDOrAlias,
@@ -96,7 +105,7 @@ func JoinRoomByIDOrAlias(
 		case roomserverAPI.ErrInvalidID:
 			response = util.JSONResponse{
 				Code: http.StatusBadRequest,
-				JSON: spec.Unknown(e.Error()),
+				JSON: spec.InvalidParam(e.Error()),
 			}
 		case roomserverAPI.ErrNotAllowed:
 			jsonErr := spec.Forbidden(e.Error())
@@ -118,9 +127,14 @@ func JoinRoomByIDOrAlias(
 				JSON: spec.NotFound(e.Error()),
 			}
 		default:
-			response = util.JSONResponse{
-				Code: http.StatusInternalServerError,
-				JSON: spec.InternalServerError{},
+			// Check if this is already a Matrix error and preserve its error code
+			if resp := httputil.MatrixErrorResponse(err); resp != nil {
+				response = *resp
+			} else {
+				response = util.JSONResponse{
+					Code: http.StatusInternalServerError,
+					JSON: spec.InternalServerError{},
+				}
 			}
 		}
 		done <- response
@@ -131,6 +145,10 @@ func JoinRoomByIDOrAlias(
 	timer := time.NewTimer(time.Second * 20)
 	select {
 	case <-timer.C:
+		logger.WithFields(map[string]interface{}{
+			"duration_ms": time.Since(joinStartTime).Milliseconds(),
+			"result":      "timeout_202",
+		}).Debug("Join request timeout - returning 202 (join continues in background)")
 		return util.JSONResponse{
 			Code: http.StatusAccepted,
 			JSON: spec.Unknown("The room join will continue in the background."),
@@ -140,6 +158,10 @@ func JoinRoomByIDOrAlias(
 		if !timer.Stop() {
 			<-timer.C
 		}
+		logger.WithFields(map[string]interface{}{
+			"duration_ms": time.Since(joinStartTime).Milliseconds(),
+			"result_code": result.Code,
+		}).Debug("Join request completed")
 		return result
 	}
 }

clientapi/routing/leaveroom.go

@@ -9,6 +9,7 @@ package routing
 import (
 	"net/http"
 
+	"github.com/element-hq/dendrite/clientapi/httputil"
 	roomserverAPI "github.com/element-hq/dendrite/roomserver/api"
 	"github.com/element-hq/dendrite/userapi/api"
 	"github.com/matrix-org/gomatrixserverlib/spec"
@@ -25,7 +26,7 @@ func LeaveRoomByID(
 	if err != nil {
 		return util.JSONResponse{
 			Code: http.StatusBadRequest,
-			JSON: spec.Unknown("device userID is invalid"),
+			JSON: spec.InvalidParam("device userID is invalid"),
 		}
 	}
 
@@ -44,9 +45,22 @@ func LeaveRoomByID(
 				JSON: spec.LeaveServerNoticeError(),
 			}
 		}
-		return util.JSONResponse{
-			Code: http.StatusBadRequest,
-			JSON: spec.Unknown(err.Error()),
+		// Check if this is already a Matrix error and preserve its error code
+		if resp := httputil.MatrixErrorResponse(err); resp != nil {
+			return *resp
+		}
+		// Check for specific error types from roomserver
+		switch e := err.(type) {
+		case roomserverAPI.ErrNotAllowed:
+			return util.JSONResponse{
+				Code: http.StatusForbidden,
+				JSON: spec.Forbidden(e.Error()),
+			}
+		default:
+			return util.JSONResponse{
+				Code: http.StatusBadRequest,
+				JSON: spec.Unknown(err.Error()),
+			}
 		}
 	}
 

clientapi/routing/login_test.go

@@ -52,7 +52,7 @@ func TestLogin(t *testing.T) {
 		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff)
 
 		// We mostly need the userAPI for this test, so nil for other APIs/caches etc.
-		Setup(routers, cfg, nil, nil, userAPI, nil, nil, nil, nil, nil, nil, nil, caching.DisableMetrics)
+		Setup(routers, cfg, nil, nil, userAPI, nil, nil, nil, nil, nil, nil, nil, nil, caching.DisableMetrics)
 
 		// Create password
 		password := util.RandomString(8)

clientapi/routing/membership.go

@@ -214,7 +214,7 @@ func SendKick(
 	if queryRes.Membership != spec.Join && queryRes.Membership != spec.Invite {
 		return util.JSONResponse{
 			Code: http.StatusForbidden,
-			JSON: spec.Unknown("cannot /kick banned or left users"),
+			JSON: spec.Forbidden("cannot /kick banned or left users"),
 		}
 	}
 	// TODO: should we be using SendLeave instead?
@@ -269,8 +269,8 @@ func SendUnban(
 	// unban is only valid if the user is currently banned
 	if queryRes.Membership != spec.Ban {
 		return util.JSONResponse{
-			Code: http.StatusBadRequest,
-			JSON: spec.Unknown("can only /unban users that are banned"),
+			Code: http.StatusForbidden,
+			JSON: spec.Forbidden("can only /unban users that are banned"),
 		}
 	}
 	// TODO: should we be using SendLeave instead?
@@ -386,7 +386,7 @@ func sendInvite(
 	case roomserverAPI.ErrInvalidID:
 		return util.JSONResponse{
 			Code: http.StatusBadRequest,
-			JSON: spec.Unknown(e.Error()),
+			JSON: spec.InvalidParam(e.Error()),
 		}, e
 	case roomserverAPI.ErrNotAllowed:
 		return util.JSONResponse{
@@ -647,7 +647,7 @@ func SendForget(
 	if membershipRes.IsInRoom {
 		return util.JSONResponse{
 			Code: http.StatusBadRequest,
-			JSON: spec.Unknown(fmt.Sprintf("User %s is in room %s", device.UserID, roomID)),
+			JSON: spec.Forbidden(fmt.Sprintf("User %s is still in room %s", device.UserID, roomID)),
 		}
 	}
 

clientapi/routing/presence.go

@@ -58,7 +58,7 @@ func SetPresence(
 	if !ok {
 		return util.JSONResponse{
 			Code: http.StatusBadRequest,
-			JSON: spec.Unknown(fmt.Sprintf("Unknown presence '%s'.", presence.Presence)),
+			JSON: spec.InvalidParam(fmt.Sprintf("Unknown presence '%s'.", presence.Presence)),
 		}
 	}
 	err := producer.SendPresence(req.Context(), userID, presenceStatus, presence.StatusMsg)

clientapi/routing/profile.go

@@ -247,7 +247,7 @@ func updateProfile(
 	if err != nil {
 		return util.JSONResponse{
 			Code: http.StatusInternalServerError,
-			JSON: spec.Unknown("internal server error"),
+			JSON: spec.InternalServerError{},
 		}, err
 	}
 

clientapi/routing/redaction.go

@@ -73,7 +73,7 @@ func SendRedaction(
 		util.GetLogger(req.Context()).WithField("userID", *deviceUserID).WithField("roomID", roomID).Error("missing sender ID for user, despite having membership")
 		return util.JSONResponse{
 			Code: http.StatusInternalServerError,
-			JSON: spec.Unknown("internal server error"),
+			JSON: spec.InternalServerError{},
 		}
 	}