Can't use offsets with offset = argument, only with offset() term

DESCRIPTION

@@ -1,7 +1,7 @@
 Type: Package
 Package: modelbased
 Title: Estimation of Model-Based Predictions, Contrasts and Means
-Version: 0.17.0.6
+Version: 0.17.0.7
 Authors@R:
     c(person(given = "Dominique",
              family = "Makowski",

NEWS.md

@@ -25,6 +25,12 @@
   when some of the indexed coefficients contained the same digit (in this case,
   `1` in `b1` and `b12`).
 
+* Fixed issues in offset handling when the offset was specified via the `offset`
+  argument in the model call instead of inside the formula.
+
+* Fixed issues with using offsets and predicting grand mean (i.e. when model
+  contains offset and `by = NULL`).
+
 # modelbased 0.17.0
 
 ## Breaking Changes

R/checks.R

@@ -141,47 +141,64 @@
   estimate,
   offset = NULL,
   my_args = NULL,
+  dots = NULL,
   verbose = TRUE
 ) {
-  model_offset <- insight::find_offset(model)
   # check if model has an offset at all
-  if (!is.null(model_offset) && !any(startsWith(my_args$by, model_offset)) && verbose) {
+  model_offset <- insight::find_offset(model)
+
+  # if we have no offset at all, or offset is explicitly specified, we don't
+  # need a warning here...
+  if (is.null(model_offset) || !is.null(dots$offset)) {
+    return(NULL)
+  }
+
+  # check if a) we have "newdata" and b) offset was specified in "newdata".
+  # if not, "offset_miss_1" flags that offset is not specified or provided,
+  # we need a warning
+  offset_miss_1 <- !is.null(dots$newdata) && !model_offset %in% colnames(dots$newdata)
+
+  # if no newdata provided, offset_miss_1 is FALSE, indicating no problems yet.
+  # We only need to start the 2nd check if we have no newdata
+  offset_miss_2 <- is.null(dots$newdata) &&
+    !is.null(my_args$by) &&
+    !any(startsWith(my_args$by, model_offset))
+
+  if (verbose && is.null(offset) && (offset_miss_1 || offset_miss_2)) {
     msg <- NULL
-    if (is.null(offset)) {
-      # if no offset argument was specified, tell user what this means
-      msg <- switch(
-        estimate,
-        specific = ,
-        typical = paste(
-          "Model contains an offset-term, which is set to its mean value.",
-          "If you want to average predictions over the distribution of the offset",
-          "(if appropriate), use `estimate = \"average\"` or `estimate = \"population\"`.",
-          "If you want to fix the offset to a specific value, for instance `1`,",
-          "use `offset = 1`."
-        ),
-        average = ,
-        population = paste(
-          "Model contains an offset-term and you average predictions over the",
-          "distribution of that offset. If you want to fix the offset to a",
-          "specific value, for instance `1`, use `offset = 1`."
-        )
+    # if no offset argument was specified, tell user what this means
+    msg <- switch(
+      estimate,
+      specific = ,
+      typical = paste(
+        "Model contains an offset-term, which is set to its mean value.",
+        "If you want to average predictions over the distribution of the offset",
+        "(if appropriate), use `estimate = \"average\"` or `estimate = \"population\"`.",
+        "If you want to fix the offset to a specific value, for instance `1`,",
+        "use `offset = 1`."
+      ),
+      average = ,
+      population = paste(
+        "Model contains an offset-term and you average predictions over the",
+        "distribution of that offset. If you want to fix the offset to a",
+        "specific value, for instance `1`, use `offset = 1`."
       )
-      # if offset term is log-transformed, tell user. offset should be fixed then
-      log_offset <- insight::find_transformation(insight::find_offset(
-        model,
-        as_term = TRUE
-      ))
-      if (!is.null(log_offset) && startsWith(log_offset, "log")) {
-        msg <- c(
-          msg,
-          paste(
-            "We also found that the model has a log-transformed offset term.",
-            "If you use the `offset` argument, the log-transformation will",
-            "automatically be applied to the provided offset-value. I.e., consider",
-            "using, for instance, `offset = 10` and not `offset = log(10)`."
-          )
+    )
+    # if offset term is log-transformed, tell user. offset should be fixed then
+    log_offset <- insight::find_transformation(insight::find_offset(
+      model,
+      as_term = TRUE
+    ))
+    if (!is.null(log_offset) && startsWith(log_offset, "log")) {
+      msg <- c(
+        msg,
+        paste(
+          "\nWe also found that the model has a log-transformed offset term.",
+          "If you use the `offset` argument, the log-transformation will",
+          "automatically be applied to the provided offset-value. I.e., consider",
+          "using, for instance, `offset = 10` and not `offset = log(10)`."
         )
-      }
+      )
     }
     if (!is.null(msg)) {
       insight::format_alert(msg)

R/get_marginalmeans.R

@@ -55,10 +55,6 @@ get_marginalmeans <- function(
   # Guess arguments
   my_args <- .guess_marginaleffects_arguments(model, by, verbose = verbose, ...)
 
-  # inform user about appropriate use of offset-terms
-  model_offset <- dots$offset
-  .check_offset(model, estimate, offset = model_offset, my_args, verbose = verbose)
-
   # find default response-type, and get information about back transformation
   predict_args <- .get_marginaleffects_type_argument(
     model,
@@ -73,6 +69,10 @@ get_marginalmeans <- function(
   # was "data" argument used? if so, it replaces "newdata" in dots
   dots <- .check_dots_data(dots, verbose)
 
+  # inform user about appropriate use of offset-terms
+  model_offset <- dots$offset
+  .check_offset(model, estimate, offset = model_offset, my_args, dots, verbose = verbose)
+
   # Second step: create a data grid -------------------------------------------
   # ---------------------------------------------------------------------------
 
@@ -503,14 +503,28 @@ get_marginalmeans <- function(
   dots[c("by", "factors", "include_random", "verbose")] <- NULL
   dg_args <- insight::compact_list(c(dg_args, dots))
 
+  # we first need to determine whether we have an offset term at all
+  # ==========================================================================
+
   # for estimate = "population", we need the offset in `by`, thus, we have to
   # add it before we call data grid
   model_offset <- insight::find_offset(model)
-  needs_offset <- !is.null(dots$offset) && !is.null(model_offset)
 
-  if (
-    needs_offset && estimate == "population" && !any(startsWith(dg_args$by, model_offset))
-  ) {
+  # offset_arg is a flag for the "offset" argument, which is used by users
+  # to define a specific offset-value
+  offset_arg <- (!is.null(dots$offset) && !is.null(model_offset))
+
+  # check if offset was specified via `offset` argument in the model call, and
+  # not inside formula. We need to handle this differently.
+  call_offset <- !is.null(insight::get_call(model)$offset)
+
+  # by_miss_offset is a flag that checks whether the offset-term was
+  # specified in the "by" argument or not
+  by_miss_offset <- (!is.null(dg_args$by) &&
+    !is.null(model_offset) &&
+    !any(startsWith(dg_args$by, model_offset)))
+
+  if (offset_arg && estimate == "population" && by_miss_offset) {
     dg_args$by <- c(dg_args$by, paste(model_offset, "=", dots$offset))
   }
 
@@ -519,8 +533,35 @@ get_marginalmeans <- function(
   datagrid_info <- attributes(datagrid)
 
   # handle offsets for other estimate-options
-  if (needs_offset && estimate != "population") {
+  if (offset_arg && estimate != "population") {
+    # if user provided an offset-argument, we use this value in the datagrid
     datagrid[[model_offset]] <- dots$offset
+  } else if (call_offset && estimate != "population") {
+    # else, we take the mean-value of the offset from the data
+    model_data <- insight::get_data(model)
+    datagrid[[model_offset]] <- mean(model_data[[model_offset]], na.rmn = TRUE)
+  }
+
+  # check if we have a "newdata" argument, where offset is already specified
+  # if so, we use that data and don't need to do anything else. However, if
+  # a) model has an offset, b) newdata is provided, and c) model-offset is
+  # not yet included in the newdata, or explicitly specied in the offset-argument,
+  # we add/overwrite that value. "datagrid" is already handled above
+  if (
+    !is.null(model_offset) &&
+      !is.null(dots$newdata) &&
+      (!model_offset %in% colnames(dots$newdata) || offset_arg)
+  ) {
+    if (offset_arg) {
+      # use user-specified offset-value
+      offset_val <- dots$offset
+    } else {
+      # take mean from model data
+      model_data <- insight::get_data(model)
+      offset_val <- mean(model_data[[model_offset]], na.rmn = TRUE)
+    }
+    # add offset-term to newdata
+    dots$newdata[[model_offset]] <- offset_val
   }
 
   # restore data types -  if we have defined numbers in `by`, like

tests/testthat/test-offset.R

@@ -1,7 +1,7 @@
 skip_if_not_installed("marginaleffects", minimum_version = "0.29.0")
-skip_if_not_installed("MASS")
 
-test_that("verbose", {
+test_that("offset works", {
+  skip_if_not_installed("MASS")
   set.seed(1)
   newdata <- data.frame(
     y = c(602, 38, 616, 256, 21, 723, 245, 176, 89, 1614, 31, 27, 313, 251, 345),
@@ -80,7 +80,71 @@ test_that("verbose", {
 })
 
 
+test_that("offset works when not in formula", {
+  set.seed(1)
+  newdata <- data.frame(
+    y = c(602, 38, 616, 256, 21, 723, 245, 176, 89, 1614, 31, 27, 313, 251, 345),
+    x = as.factor(sample(letters[1:3], 15, replace = TRUE)),
+    offset_1 = c(72, 50, 31, 30, 16, 25, 75, 16, 78, 40, 68, 25, 71, 52, 17)
+  )
+  moff <- glm(y ~ x, offset = log(offset_1), data = newdata, family = poisson())
+  expect_message(
+    {
+      out <- estimate_means(moff, "x")
+    },
+    regex = "Model contains"
+  )
+  expect_identical(dim(out), c(3L, 4L))
+  expect_equal(out$Mean, c(304.31461, 272.69787, 456.2186), tolerance = 1e-3)
+
+  expect_message(
+    {
+      out <- estimate_means(moff, "x", estimate = "average")
+    },
+    regex = "and you average"
+  )
+  expect_equal(out$Mean, c(264.40874, 173.47822, 493.28517), tolerance = 1e-3)
+
+  expect_silent({
+    out <- estimate_means(moff, "x", offset = 5)
+  })
+  expect_equal(out$Mean, c(34.26966, 30.70922, 51.37597), tolerance = 1e-3)
+})
+
+
+test_that("offset checks work", {
+  set.seed(1)
+  newdata <- data.frame(
+    y = c(602, 38, 616, 256, 21, 723, 245, 176, 89, 1614, 31, 27, 313, 251, 345),
+    x = as.factor(sample(letters[1:3], 15, replace = TRUE)),
+    offset_1 = c(72, 50, 31, 30, 16, 25, 75, 16, 78, 40, 68, 25, 71, 52, 17)
+  )
+  moff <- glm(y ~ x, offset = log(offset_1), data = newdata, family = poisson())
+
+  # offset term is in newdata, no message
+  expect_silent(estimate_means(moff, "x", data = newdata))
+
+  # offset term is explicitly specified, no message
+  expect_silent(estimate_means(moff, "x", data = newdata, offset = 1))
+
+  # offset term is neither in newdata, nor as offset-argument, so tell user
+  # that average is taken
+  expect_message(
+    {
+      out <- estimate_means(moff, "x", data = newdata[1:2])
+    },
+    regex = "Model contains"
+  )
+  # grand mean works
+  expect_silent({
+    out <- estimate_means(moff, by = NULL)
+  })
+  expect_identical(dim(out), c(1L, 3L))
+})
+
+
 test_that("offset, estimate_relation", {
+  skip_if_not_installed("MASS")
   set.seed(1)
   newdata <- data.frame(
     y = c(602, 38, 616, 256, 21, 723, 245, 176, 89, 1614, 31, 27, 313, 251, 345),