Skip to content

Commit cd41d40

Browse files
authored
Update queue bindings (#335)
* Update queues * Update queue bindings * Fix tests and lint * Fix tests * Add support for message batches and retry / message options * Add queue example * Address comments
1 parent fbdaca4 commit cd41d40

File tree

13 files changed

+901
-95
lines changed

13 files changed

+901
-95
lines changed

Cargo.lock

Lines changed: 10 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

README.md

Lines changed: 25 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -308,20 +308,41 @@ pub async fn main(message_batch: MessageBatch<MyType>, env: Env, _ctx: Context)
308308
// Log the message and meta data
309309
console_log!(
310310
"Got message {:?}, with id {} and timestamp: {}",
311-
message.body,
312-
message.id,
313-
message.timestamp.to_string()
311+
message.body(),
312+
message.id(),
313+
message.timestamp().to_string()
314314
);
315315

316316
// Send the message body to the other queue
317-
my_queue.send(&message.body).await?;
317+
my_queue.send(message.body()).await?;
318+
319+
// Ack individual message
320+
message.ack();
321+
322+
// Retry individual message
323+
message.retry();
318324
}
319325

320326
// Retry all messages
321327
message_batch.retry_all();
328+
// Ack all messages
329+
message_batch.ack_all();
322330
Ok(())
323331
}
324332
```
333+
You'll need to ensure you have the correct bindings in your `wrangler.toml`:
334+
```toml
335+
# ...
336+
[[queues.consumers]]
337+
queue = "myqueueotherqueue"
338+
max_batch_size = 10
339+
max_batch_timeout = 30
340+
341+
342+
[[queues.producers]]
343+
queue = "myqueue"
344+
binding = "my_queue"
345+
```
325346

326347
## Testing with Miniflare
327348

examples/queue/Cargo.toml

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
[package]
2+
name = "queue-on-workers"
3+
version = "0.1.0"
4+
edition = "2021"
5+
6+
[package.metadata.release]
7+
release = false
8+
9+
# https://github.com/rustwasm/wasm-pack/issues/1247
10+
[package.metadata.wasm-pack.profile.release]
11+
wasm-opt = false
12+
13+
[lib]
14+
crate-type = ["cdylib"]
15+
16+
[dependencies]
17+
serde = "1"
18+
worker = { workspace = true, features = ["queue"] }
19+
wasm-bindgen = { workspace = true }
20+
js-sys = { workspace = true }

examples/queue/README.md

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
# Using Queues on Cloudflare Workers
2+
3+
Demonstration of using `worker::Queue`

examples/queue/package.json

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
{
2+
"name": "queue-on-workers",
3+
"version": "0.0.0",
4+
"private": true,
5+
"scripts": {
6+
"deploy": "wrangler deploy",
7+
"dev": "wrangler dev --local"
8+
},
9+
"devDependencies": {
10+
"wrangler": "^3"
11+
}
12+
}

examples/queue/src/lib.rs

Lines changed: 149 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,149 @@
1+
use serde::{Deserialize, Serialize};
2+
use wasm_bindgen::JsValue;
3+
use worker::*;
4+
5+
const MY_MESSAGES_BINDING_NAME: &str = "my_messages";
6+
const MY_MESSAGES_QUEUE_NAME: &str = "mymessages";
7+
8+
const RAW_MESSAGES_BINDING_NAME: &str = "raw_messages";
9+
const RAW_MESSAGES_QUEUE_NAME: &str = "rawmessages";
10+
11+
#[derive(Serialize, Debug, Clone, Deserialize)]
12+
pub struct MyType {
13+
foo: String,
14+
bar: u32,
15+
}
16+
17+
#[event(fetch)]
18+
async fn main(_req: Request, env: Env, _: worker::Context) -> Result<Response> {
19+
let my_messages_queue = env.queue(MY_MESSAGES_BINDING_NAME)?;
20+
let raw_messages_queue = env.queue(RAW_MESSAGES_BINDING_NAME)?;
21+
22+
// Send a message with using a serializable struct
23+
my_messages_queue
24+
.send(MyType {
25+
foo: "Hello world".into(),
26+
bar: 1,
27+
})
28+
.await?;
29+
30+
// Send a batch of messages using some sort of iterator
31+
my_messages_queue
32+
.send_batch([
33+
// Use the MessageBuilder to set additional options
34+
MessageBuilder::new(MyType {
35+
foo: "Hello world".into(),
36+
bar: 2,
37+
})
38+
.delay_seconds(20)
39+
.build(),
40+
// Send a message with using a serializable struct
41+
MyType {
42+
foo: "Hello world".into(),
43+
bar: 4,
44+
}
45+
.into(),
46+
])
47+
.await?;
48+
49+
// Send a batch of messages using the BatchMessageBuilder
50+
my_messages_queue
51+
.send_batch(
52+
BatchMessageBuilder::new()
53+
.message(MyType {
54+
foo: "Hello world".into(),
55+
bar: 4,
56+
})
57+
.messages(vec![
58+
MyType {
59+
foo: "Hello world".into(),
60+
bar: 5,
61+
},
62+
MyType {
63+
foo: "Hello world".into(),
64+
bar: 6,
65+
},
66+
])
67+
.delay_seconds(10)
68+
.build(),
69+
)
70+
.await?;
71+
72+
// Send a raw JSValue
73+
raw_messages_queue
74+
.send_raw(
75+
// RawMessageBuilder has to be used as we should set content type of these raw messages
76+
RawMessageBuilder::new(JsValue::from_str("7"))
77+
.delay_seconds(30)
78+
.build_with_content_type(QueueContentType::Json),
79+
)
80+
.await?;
81+
82+
// Send a batch of raw JSValues using the BatchMessageBuilder
83+
raw_messages_queue
84+
.send_raw_batch(
85+
BatchMessageBuilder::new()
86+
.message(
87+
RawMessageBuilder::new(js_sys::Date::new_0().into())
88+
.build_with_content_type(QueueContentType::V8),
89+
)
90+
.message(
91+
RawMessageBuilder::new(JsValue::from_str("8"))
92+
.build_with_content_type(QueueContentType::Json),
93+
)
94+
.delay_seconds(10)
95+
.build(),
96+
)
97+
.await?;
98+
99+
// Send a batch of raw JsValues using some sort of iterator
100+
raw_messages_queue
101+
.send_raw_batch(vec![RawMessageBuilder::new(JsValue::from_str("9"))
102+
.delay_seconds(20)
103+
.build_with_content_type(QueueContentType::Text)])
104+
.await?;
105+
106+
Response::empty()
107+
}
108+
109+
// Consumes messages from `my_messages` queue and `raw_messages` queue
110+
#[event(queue)]
111+
pub async fn main(message_batch: MessageBatch<MyType>, _: Env, _: Context) -> Result<()> {
112+
match message_batch.queue().as_str() {
113+
MY_MESSAGES_QUEUE_NAME => {
114+
for message in message_batch.messages()? {
115+
console_log!(
116+
"Got message {:?}, with id {} and timestamp: {}",
117+
message.body(),
118+
message.id(),
119+
message.timestamp().to_string(),
120+
);
121+
if message.body().bar == 1 {
122+
message.retry_with_options(
123+
&QueueRetryOptionsBuilder::new()
124+
.with_delay_seconds(10)
125+
.build(),
126+
);
127+
} else {
128+
message.ack();
129+
}
130+
}
131+
}
132+
RAW_MESSAGES_QUEUE_NAME => {
133+
for message in message_batch.raw_iter() {
134+
console_log!(
135+
"Got raw message {:?}, with id {} and timestamp: {}",
136+
message.body(),
137+
message.id(),
138+
message.timestamp().to_string(),
139+
);
140+
}
141+
message_batch.ack_all();
142+
}
143+
_ => {
144+
console_error!("Unknown queue: {}", message_batch.queue());
145+
}
146+
}
147+
148+
Ok(())
149+
}

examples/queue/wrangler.toml

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
name = "queue-on-workers"
2+
main = "build/worker/shim.mjs"
3+
compatibility_date = "2024-03-26"
4+
5+
6+
[build]
7+
command = "cargo install --path ../../worker-build && worker-build --release"
8+
9+
[[queues.consumers]]
10+
queue = "mymessages"
11+
dead_letter_queue = "mymessagesdlq"
12+
13+
[[queues.consumers]]
14+
queue = "rawmessages"
15+
16+
17+
[[queues.producers]]
18+
queue = "mymessages"
19+
binding = "my_messages"
20+
21+
[[queues.producers]]
22+
queue = "rawmessages"
23+
binding = "raw_messages"

worker-macros/src/event.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -159,7 +159,7 @@ pub fn expand_macro(attr: TokenStream, item: TokenStream, http: bool) -> TokenSt
159159
pub async fn #wrapper_fn_ident(event: ::worker::worker_sys::MessageBatch, env: ::worker::Env, ctx: ::worker::worker_sys::Context) {
160160
// call the original fn
161161
let ctx = worker::Context::new(ctx);
162-
match #input_fn_ident(::worker::MessageBatch::new(event), env, ctx).await {
162+
match #input_fn_ident(::worker::MessageBatch::from(event), env, ctx).await {
163163
Ok(()) => {},
164164
Err(e) => {
165165
::worker::console_log!("{}", &e);

worker-sandbox/src/queue.rs

Lines changed: 30 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,9 @@ use serde::{Deserialize, Serialize};
22
use uuid::Uuid;
33

44
use super::{SomeSharedData, GLOBAL_QUEUE_STATE};
5-
use worker::{console_log, event, Context, Env, MessageBatch, Request, Response, Result};
5+
use worker::{
6+
console_log, event, Context, Env, MessageBatch, MessageExt, Request, Response, Result,
7+
};
68
#[derive(Serialize, Debug, Clone, Deserialize)]
79
pub struct QueueBody {
810
pub id: Uuid,
@@ -15,11 +17,11 @@ pub async fn queue(message_batch: MessageBatch<QueueBody>, _env: Env, _ctx: Cont
1517
for message in message_batch.messages()? {
1618
console_log!(
1719
"Received queue message {:?}, with id {} and timestamp: {}",
18-
message.body,
19-
message.id,
20-
message.timestamp.to_string()
20+
message.body(),
21+
message.id(),
22+
message.timestamp().to_string()
2123
);
22-
guard.push(message.body);
24+
guard.push(message.into_body());
2325
}
2426
Ok(())
2527
}
@@ -54,6 +56,29 @@ pub async fn handle_queue_send(req: Request, env: Env, _data: SomeSharedData) ->
5456
}
5557
}
5658

59+
#[worker::send]
60+
pub async fn handle_batch_send(mut req: Request, env: Env, _: SomeSharedData) -> Result<Response> {
61+
let messages: Vec<QueueBody> = match req.json().await {
62+
Ok(messages) => messages,
63+
Err(err) => {
64+
return Response::error(format!("Failed to parse request body: {err:?}"), 400);
65+
}
66+
};
67+
68+
let my_queue = match env.queue("my_queue") {
69+
Ok(queue) => queue,
70+
Err(err) => return Response::error(format!("Failed to get queue: {err:?}"), 500),
71+
};
72+
73+
match my_queue.send_batch(messages).await {
74+
Ok(()) => Response::ok("Message sent"),
75+
Err(err) => Response::error(
76+
format!("Failed to batch send message to queue: {err:?}"),
77+
500,
78+
),
79+
}
80+
}
81+
5782
pub async fn handle_queue(_req: Request, _env: Env, _data: SomeSharedData) -> Result<Response> {
5883
let guard = GLOBAL_QUEUE_STATE.lock().unwrap();
5984
let messages: Vec<QueueBody> = guard.clone();

worker-sandbox/src/router.rs

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -181,6 +181,10 @@ pub fn make_router(data: SomeSharedData, env: Env) -> axum::Router {
181181
get(handler!(service::handle_remote_by_path)),
182182
)
183183
.route("/queue/send/:id", post(handler!(queue::handle_queue_send)))
184+
.route(
185+
"/queue/send_batch",
186+
post(handler!(queue::handle_batch_send)),
187+
)
184188
.route("/queue", get(handler!(queue::handle_queue)))
185189
.route("/d1/prepared", get(handler!(d1::prepared_statement)))
186190
.route("/d1/batch", get(handler!(d1::batch)))
@@ -301,6 +305,7 @@ pub fn make_router<'a>(data: SomeSharedData) -> Router<'a, SomeSharedData> {
301305
)
302306
.get_async("/remote-by-path", handler!(service::handle_remote_by_path))
303307
.post_async("/queue/send/:id", handler!(queue::handle_queue_send))
308+
.post_async("/queue/send_batch", handler!(queue::handle_batch_send))
304309
.get_async("/queue", handler!(queue::handle_queue))
305310
.get_async("/d1/prepared", handler!(d1::prepared_statement))
306311
.get_async("/d1/batch", handler!(d1::batch))

0 commit comments

Comments
 (0)