diff --git a/.changesets/make-it-easy-to-know-if-wrong-api-key-is-being-used.md b/.changesets/make-it-easy-to-know-if-wrong-api-key-is-being-used.md
new file mode 100644
index 0000000..3a1c9b8
--- /dev/null
+++ b/.changesets/make-it-easy-to-know-if-wrong-api-key-is-being-used.md
@@ -0,0 +1,6 @@
+---
+bump: patch
+type: add
+---
+
+When a request to AppSignal returns a `401 Unauthorized` error status code, display a log message asking the customer to ensure they're using an app-level push API key.
\ No newline at end of file
diff --git a/src/main.rs b/src/main.rs
index b817c58..0160eef 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -754,11 +754,27 @@ async fn send_batch(
 
     let response = client.post(url.clone()).body(batch_bytes).send().await?;
 
-    info!(
-        "Batch of {} metrics sent: {:?}",
-        batch.get_metrics().len(),
-        response.status()
-    );
+    let status = response.status();
+
+    if status.is_success() {
+        info!(
+            "Batch of {} metrics sent successfully (HTTP response status: {})",
+            batch.get_metrics().len(),
+            status
+        );
+    } else if status == 401 {
+        warn!(
+            "Batch of {} metrics failed to send (HTTP response status: {}) - make sure you're using an *app-level* push API key",
+            batch.get_metrics().len(),
+            status
+        );
+    } else {
+        warn!(
+            "Batch of {} metrics failed to send (HTTP response status: {})",
+            batch.get_metrics().len(),
+            status
+        );
+    }
 
     Ok(response)
 }