Skip to content

mismatch between {emmeans}\s and model_parameters()'s adjusted p-values #1194

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
strengejacke merged 9 commits into from
Feb 24, 2026
Merged

mismatch between {emmeans}\s and model_parameters()'s adjusted p-values #1194

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
65b8f0a
mismatch between `{emmeans}`\s and `model_parameters()`'s adjusted p-…
strengejacke Feb 24, 2026
64e224e
fix, tests
strengejacke Feb 24, 2026
7ed984d
fix
strengejacke Feb 24, 2026
2ef2146
fix tests
strengejacke Feb 24, 2026
0752811
fix
strengejacke Feb 24, 2026
56f6929
fix test
strengejacke Feb 24, 2026
c58bb08
fix test
strengejacke Feb 24, 2026
a18eb97
suppress warnings from summary
strengejacke Feb 24, 2026
41ba6d3
typo
strengejacke Feb 24, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion DESCRIPTION
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
Type: Package
Package: parameters
Title: Processing of Model Parameters
Version: 0.28.3.3
Version: 0.28.3.4
Authors@R:
c(person(given = "Daniel",
family = "Lüdecke",
Expand Down
2 changes: 2 additions & 0 deletions NEWS.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,8 @@
* Fixed issue where wrong (non-robust) standard errors were calculated for
`coxph` and `svycoxph` objects.

* Fixed issues with Tukey-p-value adjustment for *emmeans* objects.

# parameters 0.28.3

* fixed bug in `standardize_info(<fixest>)` that was preventing
Expand Down
96 changes: 71 additions & 25 deletions R/format_p_adjust.R
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,8 @@
format_p_adjust <- function(method) {
method <- tolower(method)

switch(method,
switch(
method,
holm = "Holm (1979)",
hochberg = "Hochberg (1988)",
hommel = "Hommel (1988)",
Expand Down Expand Up @@ -44,7 +45,7 @@
# for interaction terms, e.g. for "by" argument in emmeans
# pairwise comparison, we have to adjust the rank resp. the
# number of estimates in a comparison family
rank_adjust <- .p_adjust_rank(model, params)
rank_adjust <- .p_adjust_rank(model, params, tolower(p_adjust))

# only proceed if valid argument-value
if (tolower(p_adjust) %in% tolower(all_methods)) {
Expand All @@ -68,13 +69,19 @@
} else if (tolower(p_adjust) == "sidak") {
# sidak adjustment
params$p <- 1 - (1 - params$p)^(nrow(params) / rank_adjust)
} else if (tolower(p_adjust) == "sup-t") {
} else if (tolower(p_adjust) == "sup-t") {
# sup-t adjustment
params <- .p_adjust_supt(model, params)
}

if (isTRUE(all(old_p_vals == params$p)) && !identical(p_adjust, "none") && verbose) {
insight::format_warning(paste0("Could not apply ", p_adjust, "-adjustment to p-values. Either something went wrong, or the non-adjusted p-values were already very large.")) # nolint
if (
isTRUE(all(old_p_vals == params$p)) && !identical(p_adjust, "none") && verbose
) {
insight::format_warning(paste0(
"Could not apply ",
p_adjust,
"-adjustment to p-values. Either something went wrong, or the non-adjusted p-values were already very large."
)) # nolint
}
} else if (verbose) {
insight::format_alert(paste0("`p_adjust` must be one of ", toString(all_methods)))
Expand All @@ -86,14 +93,31 @@

# calculate rank adjustment -----

.p_adjust_rank <- function(model, params) {
.p_adjust_rank <- function(model, params, adjust = "tukey") {
tryCatch(
{
correction <- 1
by_vars <- model@misc$by.vars
if (!is.null(by_vars) && by_vars %in% colnames(params)) {
correction <- insight::n_unique(params[[by_vars]])
if (
!is.null(by_vars) && length(by_vars) > 0 && all(by_vars %in% colnames(params))
) {
if (length(by_vars) == 1) {
correction <- insight::n_unique(params[[by_vars]])
} else {
# compute unique combinations of multiple by-variables
by_data <- params[by_vars]
by_groups <- interaction(by_data, drop = TRUE)
correction <- insight::n_unique(by_groups)
}
} else if (identical(adjust, "tukey")) {
# correction <- .safe(prod(vapply(model@model.info$xlev, length, numeric(1))))
correction <- .safe(insight::n_unique(unlist(strsplit(
model@levels$contrast,
" - ",
fixed = TRUE
))))
}

correction
},
error = function(e) {
Expand All @@ -106,22 +130,26 @@
# tukey adjustment -----

.p_adjust_tukey <- function(params, stat_column, rank_adjust = 1, verbose = TRUE) {
df_column <- colnames(params)[stats::na.omit(match(c("df", "df_error"), colnames(params)))][1]
df_column <- colnames(params)[stats::na.omit(match(
c("df", "df_error"),
colnames(params)
))][1]
if (!is.na(df_column) && length(stat_column)) {
params$p <- suppressWarnings(stats::ptukey(
sqrt(2) * abs(params[[stat_column]]),
nmeans = nrow(params) / rank_adjust,
nmeans = rank_adjust,
df = params[[df_column]],
lower.tail = FALSE
))
# for specific contrasts, ptukey might fail, and the tukey-adjustement
# could just be simple p-value calculation
if (all(is.na(params$p))) {
params$p <- 2 * stats::pt(
abs(params[[stat_column]]),
df = params[[df_column]],
lower.tail = FALSE
)
params$p <- 2 *
stats::pt(
abs(params[[stat_column]]),
df = params[[df_column]],
lower.tail = FALSE
)
verbose <- FALSE
}
}
Expand All @@ -132,7 +160,10 @@
# scheffe adjustment -----

.p_adjust_scheffe <- function(model, params, stat_column, rank_adjust = 1) {
df_column <- colnames(params)[stats::na.omit(match(c("df", "df_error"), colnames(params)))][1]
df_column <- colnames(params)[stats::na.omit(match(
c("df", "df_error"),
colnames(params)
))][1]
if (!is.na(df_column) && length(stat_column)) {
# 1st try
scheffe_ranks <- try(qr(model@linfct)$rank, silent = TRUE)
Expand All @@ -146,7 +177,8 @@
scheffe_ranks <- nrow(params)
}
scheffe_ranks <- scheffe_ranks / rank_adjust
params$p <- stats::pf(params[[stat_column]]^2 / scheffe_ranks,
params$p <- stats::pf(
params[[stat_column]]^2 / scheffe_ranks,
df1 = scheffe_ranks,
df2 = params[[df_column]],
lower.tail = FALSE
Expand Down Expand Up @@ -182,7 +214,9 @@
# get correlation matrix, based on the covariance matrix
vc <- .safe(stats::cov2cor(insight::get_varcov(model, component = component)))
if (is.null(vc)) {
insight::format_warning("Could not calculate covariance matrix for `sup-t` adjustment.")
insight::format_warning(
"Could not calculate covariance matrix for `sup-t` adjustment."
)
return(params)
}
# get confidence interval level, or set default
Expand All @@ -197,18 +231,30 @@
}
# calculate updated confidence interval level, based on simultaenous
# confidence intervals (https://onlinelibrary.wiley.com/doi/10.1002/jae.2656)
crit <- mvtnorm::qmvt(ci_level, df = params[[df_column]][1], tail = "both.tails", corr = vc)$quantile
crit <- mvtnorm::qmvt(
ci_level,
df = params[[df_column]][1],
tail = "both.tails",
corr = vc
)$quantile
# update confidence intervals
params$CI_low <- params$Coefficient - crit * params$SE
params$CI_high <- params$Coefficient + crit * params$SE
# udpate p-values
for (i in 1:nrow(params)) {

Check warning on line 244 in R/format_p_adjust.R

View workflow job for this annotation

GitHub Actions / lint-changed-files / lint-changed-files 

file=R/format_p_adjust.R,line=244,col=13,[seq_linter] Use seq_len(nrow(...)) instead of 1:nrow(...), which is likely to be wrong in the empty edge case.
params$p[i] <- 1 - mvtnorm::pmvt(
lower = rep(-abs(stats::qt(params$p[i] / 2, df = params[[df_column]][i])), nrow(vc)),
upper = rep(abs(stats::qt(params$p[i] / 2, df = params[[df_column]][i])), nrow(vc)),
corr = vc,
df = params[[df_column]][i]
)
params$p[i] <- 1 -
mvtnorm::pmvt(
lower = rep(
-abs(stats::qt(params$p[i] / 2, df = params[[df_column]][i])),
nrow(vc)
),
upper = rep(
abs(stats::qt(params$p[i] / 2, df = params[[df_column]][i])),
nrow(vc)
),
corr = vc,
df = params[[df_column]][i]
)
}
params
}
4 changes: 2 additions & 2 deletions R/methods_glmmTMB.R
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -240,7 +240,7 @@
}

# fix argument, if model has only conditional component
cs <- stats::coef(summary(model))
cs <- suppressWarnings(stats::coef(summary(model)))
has_zeroinf <- modelinfo$is_zero_inflated
has_disp <- is.list(cs) && !is.null(cs$disp)

Expand Down Expand Up @@ -595,7 +595,7 @@

if (effects == "random") {
.se_random_effects_glmmTMB(model)
} else if (!is.null(vcov)) {

Check warning on line 598 in R/methods_glmmTMB.R

View workflow job for this annotation

GitHub Actions / lint-changed-files / lint-changed-files 

file=R/methods_glmmTMB.R,line=598,col=14,[if_not_else_linter] Prefer `if (A) x else y` to the less-readable `if (!A) y else x` in a simple if/else statement.
.se_robust_glmmTMB(model, component, vcov, vcov_args, verbose, ...)
} else {
.se_fixed_effects_glmmTMB(model, component, method, verbose)
Expand All @@ -616,7 +616,7 @@
return(se_kenward(model, component = "conditional"))
}

cs <- insight::compact_list(stats::coef(summary(model)))
cs <- suppressWarnings(insight::compact_list(stats::coef(summary(model))))
x <- lapply(names(cs), function(i) {
.data_frame(
Parameter = insight::find_parameters(
Expand All @@ -643,7 +643,7 @@
.se_robust_glmmTMB <- function(
model,
component = "all",
vcov,

Check warning on line 646 in R/methods_glmmTMB.R

View workflow job for this annotation

GitHub Actions / lint-changed-files / lint-changed-files 

file=R/methods_glmmTMB.R,line=646,col=3,[function_argument_linter] Arguments without defaults should come before arguments with defaults.
vcov_args = NULL,
verbose = TRUE,
...
Expand Down
120 changes: 60 additions & 60 deletions tests/testthat/_snaps/glmmTMB.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,78 +7,78 @@
[2] ""
[3] "Parameter | Log-Mean | SE | 95% CI | z | p"
[4] "----------------------------------------------------------------"
[5] "child | -1.09 | 0.10 | [-1.28, -0.90] | -11.09 | < .001"
[6] "camper [1] | 0.27 | 0.10 | [ 0.07, 0.47] | 2.70 | 0.007 "
[5] "child | -1.04 | 0.09 | [-1.22, -0.87] | -11.60 | < .001"
[6] "camper [1] | 0.27 | 0.09 | [ 0.08, 0.45] | 2.83 | 0.005 "

# print-model_parameters glmmTMB digits

Code
out[-c(5, 14)]
Output
[1] "# Fixed Effects (Count Model)"
[2] ""
[3] "Parameter | Log-Mean | SE | 95% CI | z | p"
[4] "--------------------------------------------------------------------------"
[5] "child | -1.0875 | 0.0981 | [-1.27967, -0.89528] | -11.0901 | < .001"
[6] "camper [1] | 0.2723 | 0.1009 | [ 0.07461, 0.46999] | 2.6997 | 0.007 "
[7] ""
[8] "# Fixed Effects (Zero-Inflation Component)"
[9] ""
[10] "Parameter | Log-Odds | SE | 95% CI | z | p"
[11] "-----------------------------------------------------------------------"
[12] "(Intercept) | 1.8896 | 0.6642 | [ 0.58780, 3.19147] | 2.8449 | 0.004"
[13] "camper [1] | -0.1701 | 0.3869 | [-0.92836, 0.58822] | -0.4396 | 0.660"
[14] ""
[15] "# Random Effects Variances"
[16] ""
[17] "Parameter | Coefficient | 95% CI"
[18] "---------------------------------------------------------------"
[19] "SD (Intercept: persons) | 3.4056 | [ 1.64567, 7.04777]"
[20] "SD (xb: persons) | 1.2132 | [ 0.59190, 2.48650]"
[21] "Cor (Intercept~xb: persons) | -1.0000 | [-1.00000, 1.00000]"
[22] ""
[23] "# Random Effects (Zero-Inflation Component)"
[24] ""
[25] "Parameter | Coefficient | 95% CI"
[26] "---------------------------------------------------------------"
[27] "SD (Intercept: persons) | 2.7358 | [ 1.16329, 6.43414]"
[28] "SD (zg: persons) | 1.5683 | [ 0.64246, 3.82852]"
[29] "Cor (Intercept~zg: persons) | 1.0000 | [-1.00000, 1.00000]"
[1] "# Fixed Effects (Count Model)"
[2] ""
[3] "Parameter | Log-Mean | SE | 95% CI | z | p"
[4] "--------------------------------------------------------------------------"
[5] "child | -1.0426 | 0.0899 | [-1.21878, -0.86645] | -11.5996 | < .001"
[6] "camper [1] | 0.2684 | 0.0948 | [ 0.08262, 0.45421] | 2.8315 | 0.005 "
[7] ""
[8] "# Fixed Effects (Zero-Inflation Component)"
[9] ""
[10] "Parameter | Log-Odds | SE | 95% CI | z | p"
[11] "-----------------------------------------------------------------------------"
[12] "(Intercept) | -4.0420 | 0.2038 | [-4.44154, -3.64250] | -19.8292 | < .001"
[13] "camper [1] | -4.8290 | 0.0002 | [-4.82949, -4.82861] | -21474.3719 | < .001"
[14] ""
[15] "# Random Effects Variances"
[16] ""
[17] "Parameter | Coefficient | 95% CI"
[18] "--------------------------------------------------------------"
[19] "SD (Intercept: persons) | 2.1800 | [1.15990, 4.09711]"
[20] "SD (xb: persons) | 1.0592 | [0.59457, 1.88686]"
[21] "Cor (Intercept~xb: persons) | -0.9883 | "
[22] ""
[23] "# Random Effects (Zero-Inflation Component)"
[24] ""
[25] "Parameter | Coefficient | 95% CI"
[26] "----------------------------------------------------------------"
[27] "SD (Intercept: persons) | 5.8635 | [ 3.83067, 8.97520]"
[28] "SD (zg: persons) | 12.5180 | [10.94956, 14.31115]"
[29] "Cor (Intercept~zg: persons) | 0.9790 | "

---

Code
out[-c(5, 14)]
Output
[1] "# Fixed Effects (Count Model)"
[2] ""
[3] "Parameter | Log-Mean | SE | 95% CI | z | p"
[4] "--------------------------------------------------------------------------"
[5] "child | -1.0875 | 0.0981 | [-1.27967, -0.89528] | -11.0901 | < .001"
[6] "camper [1] | 0.2723 | 0.1009 | [ 0.07461, 0.46999] | 2.6997 | 0.007 "
[7] ""
[8] "# Fixed Effects (Zero-Inflation Component)"
[9] ""
[10] "Parameter | Log-Odds | SE | 95% CI | z | p"
[11] "-----------------------------------------------------------------------"
[12] "(Intercept) | 1.8896 | 0.6642 | [ 0.58780, 3.19147] | 2.8449 | 0.004"
[13] "camper [1] | -0.1701 | 0.3869 | [-0.92836, 0.58822] | -0.4396 | 0.660"
[14] ""
[15] "# Random Effects Variances"
[16] ""
[17] "Parameter | Coefficient | 95% CI"
[18] "---------------------------------------------------------------"
[19] "SD (Intercept: persons) | 3.4056 | [ 1.64567, 7.04777]"
[20] "SD (xb: persons) | 1.2132 | [ 0.59190, 2.48650]"
[21] "Cor (Intercept~xb: persons) | -1.0000 | [-1.00000, 1.00000]"
[22] ""
[23] "# Random Effects (Zero-Inflation Component)"
[24] ""
[25] "Parameter | Coefficient | 95% CI"
[26] "---------------------------------------------------------------"
[27] "SD (Intercept: persons) | 2.7358 | [ 1.16329, 6.43414]"
[28] "SD (zg: persons) | 1.5683 | [ 0.64246, 3.82852]"
[29] "Cor (Intercept~zg: persons) | 1.0000 | [-1.00000, 1.00000]"
[1] "# Fixed Effects (Count Model)"
[2] ""
[3] "Parameter | Log-Mean | SE | 95% CI | z | p"
[4] "--------------------------------------------------------------------------"
[5] "child | -1.0426 | 0.0899 | [-1.21878, -0.86645] | -11.5996 | < .001"
[6] "camper [1] | 0.2684 | 0.0948 | [ 0.08262, 0.45421] | 2.8315 | 0.005 "
[7] ""
[8] "# Fixed Effects (Zero-Inflation Component)"
[9] ""
[10] "Parameter | Log-Odds | SE | 95% CI | z | p"
[11] "-----------------------------------------------------------------------------"
[12] "(Intercept) | -4.0420 | 0.2038 | [-4.44154, -3.64250] | -19.8292 | < .001"
[13] "camper [1] | -4.8290 | 0.0002 | [-4.82949, -4.82861] | -21474.3719 | < .001"
[14] ""
[15] "# Random Effects Variances"
[16] ""
[17] "Parameter | Coefficient | 95% CI"
[18] "--------------------------------------------------------------"
[19] "SD (Intercept: persons) | 2.1800 | [1.15990, 4.09711]"
[20] "SD (xb: persons) | 1.0592 | [0.59457, 1.88686]"
[21] "Cor (Intercept~xb: persons) | -0.9883 | "
[22] ""
[23] "# Random Effects (Zero-Inflation Component)"
[24] ""
[25] "Parameter | Coefficient | 95% CI"
[26] "----------------------------------------------------------------"
[27] "SD (Intercept: persons) | 5.8635 | [ 3.83067, 8.97520]"
[28] "SD (zg: persons) | 12.5180 | [10.94956, 14.31115]"
[29] "Cor (Intercept~zg: persons) | 0.9790 | "

# print-model_parameters glmmTMB CI alignment

Expand Down
Loading
Loading