Skip to content

Commit 67947b6

Browse files
yinli-systemsKevin-Li-2025Jefffrey
authored
Add any_value aggregate function (apache#23043)
## Which issue does this PR close? - Closes apache#22799. ## Rationale for this change `any_value` is a common aggregate in SQL engines for queries that need one representative non-null value from each group without imposing an ordering requirement. DataFusion currently has `first_value`, but that aggregate is order-sensitive, so exposing `any_value` gives users the intended arbitrary-value semantics directly. ## What changes are included in this PR? - Adds an `any_value(expression)` aggregate UDF and registers it with the default aggregate functions. - Reuses the existing trivial first-value accumulator with nulls ignored, so evaluation short-circuits after the first non-null value. - Marks the aggregate as order-insensitive and preserves the input field metadata/type in the return field. - Adds sqllogictest coverage for scalar, grouped, all-null, empty-input, and string return-type cases. ## Are these changes tested? Yes. I ran: ``` cargo fmt --all cargo test -p datafusion-functions-aggregate cargo test -p datafusion-sqllogictest --test sqllogictests -- aggregate_any_value.slt cargo clippy --all-targets --all-features -- -D warnings ``` ## Are there any user-facing changes? Yes. This adds a new SQL aggregate function, `any_value`. I used AI assistance to help inspect the codebase and run validation, and I reviewed the resulting implementation and tests. --------- Signed-off-by: Yin Li <kxl474@student.bham.ac.uk> Co-authored-by: Kevin-Li-2025 <2242139@qq.com> Co-authored-by: Jeffrey Vo <jeffrey.vo.australia@gmail.com>
1 parent 1734d4b commit 67947b6

4 files changed

Lines changed: 209 additions & 0 deletions

File tree

‎datafusion/functions-aggregate/src/any_value.rs‎

Lines changed: 125 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,125 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
//! Defines the ANY_VALUE aggregation.
19+
20+
use std::fmt::Debug;
21+
use std::hash::Hash;
22+
use std::sync::Arc;
23+
24+
use arrow::datatypes::{DataType, Field, FieldRef};
25+
use datafusion_common::{Result, not_impl_err};
26+
use datafusion_expr::function::{AccumulatorArgs, StateFieldsArgs};
27+
use datafusion_expr::utils::{AggregateOrderSensitivity, format_state_name};
28+
use datafusion_expr::{
29+
Accumulator, AggregateUDFImpl, Documentation, Signature, Volatility,
30+
};
31+
use datafusion_macros::user_doc;
32+
33+
use crate::first_last::TrivialFirstValueAccumulator;
34+
35+
make_udaf_expr_and_func!(
36+
AnyValue,
37+
any_value,
38+
expression,
39+
"Returns an arbitrary non-null value",
40+
any_value_udaf
41+
);
42+
43+
#[user_doc(
44+
doc_section(label = "General Functions"),
45+
description = "Returns an arbitrary non-null value from a group, or NULL if the group contains only NULL values.",
46+
syntax_example = "any_value(expression)",
47+
sql_example = r#"```sql
48+
> SELECT any_value(column_name) FROM table_name;
49+
+------------------------+
50+
| any_value(column_name) |
51+
+------------------------+
52+
| arbitrary_value |
53+
+------------------------+
54+
```"#,
55+
standard_argument(name = "expression",)
56+
)]
57+
#[derive(PartialEq, Eq, Hash, Debug)]
58+
pub struct AnyValue {
59+
signature: Signature,
60+
}
61+
62+
impl Default for AnyValue {
63+
fn default() -> Self {
64+
Self::new()
65+
}
66+
}
67+
68+
impl AnyValue {
69+
pub fn new() -> Self {
70+
Self {
71+
signature: Signature::any(1, Volatility::Immutable),
72+
}
73+
}
74+
}
75+
76+
impl AggregateUDFImpl for AnyValue {
77+
fn name(&self) -> &str {
78+
"any_value"
79+
}
80+
81+
fn signature(&self) -> &Signature {
82+
&self.signature
83+
}
84+
85+
fn return_type(&self, _arg_types: &[DataType]) -> Result<DataType> {
86+
not_impl_err!("Not called because return_field is implemented")
87+
}
88+
89+
fn return_field(&self, arg_fields: &[FieldRef]) -> Result<FieldRef> {
90+
Ok(Arc::new(
91+
Field::new(self.name(), arg_fields[0].data_type().clone(), true)
92+
.with_metadata(arg_fields[0].metadata().clone()),
93+
))
94+
}
95+
96+
fn accumulator(&self, acc_args: AccumulatorArgs) -> Result<Box<dyn Accumulator>> {
97+
TrivialFirstValueAccumulator::try_new(acc_args.return_field.data_type(), true)
98+
.map(|acc| Box::new(acc) as _)
99+
}
100+
101+
fn state_fields(&self, args: StateFieldsArgs) -> Result<Vec<FieldRef>> {
102+
Ok(vec![
103+
Field::new(
104+
format_state_name(args.name, "any_value"),
105+
args.return_type().clone(),
106+
true,
107+
)
108+
.into(),
109+
Field::new(
110+
format_state_name(args.name, "any_value_is_set"),
111+
DataType::Boolean,
112+
true,
113+
)
114+
.into(),
115+
])
116+
}
117+
118+
fn order_sensitivity(&self) -> AggregateOrderSensitivity {
119+
AggregateOrderSensitivity::Insensitive
120+
}
121+
122+
fn documentation(&self) -> Option<&Documentation> {
123+
self.doc()
124+
}
125+
}

‎datafusion/functions-aggregate/src/lib.rs‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,7 @@
6565
#[macro_use]
6666
pub mod macros;
6767

68+
pub mod any_value;
6869
pub mod approx_distinct;
6970
pub mod approx_median;
7071
pub mod approx_percentile_cont;
@@ -102,6 +103,7 @@ use std::sync::Arc;
102103

103104
/// Fluent-style API for creating `Expr`s
104105
pub mod expr_fn {
106+
pub use super::any_value::any_value;
105107
pub use super::approx_distinct::approx_distinct;
106108
pub use super::approx_median::approx_median;
107109
pub use super::approx_percentile_cont::approx_percentile_cont;
@@ -147,6 +149,7 @@ pub mod expr_fn {
147149
/// Returns all default aggregate functions
148150
pub fn all_default_aggregate_functions() -> Vec<Arc<AggregateUDF>> {
149151
vec![
152+
any_value::any_value_udaf(),
150153
array_agg::array_agg_udaf(),
151154
first_last::first_value_udaf(),
152155
first_last::last_value_udaf(),

‎datafusion/sqllogictest/test_files/aggregate_any_value.slt‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
18+
statement ok
19+
CREATE TABLE any_value_test AS VALUES
20+
(1, NULL, NULL),
21+
(1, 10, 'first'),
22+
(1, 20, 'second'),
23+
(2, NULL, NULL),
24+
(2, NULL, NULL),
25+
(3, 30, 'third');
26+
27+
query B
28+
SELECT any_value(column2) IN (10, 20) FROM any_value_test;
29+
----
30+
true
31+
32+
query IBB rowsort
33+
SELECT
34+
column1,
35+
any_value(column2) IN (10, 20, 30),
36+
any_value(column3) IN ('first', 'second', 'third')
37+
FROM any_value_test
38+
GROUP BY column1;
39+
----
40+
1 true true
41+
2 NULL NULL
42+
3 true true
43+
44+
query T
45+
SELECT arrow_typeof(any_value(column3)) FROM any_value_test;
46+
----
47+
Utf8
48+
49+
query I
50+
SELECT any_value(column2) FROM any_value_test WHERE false;
51+
----
52+
NULL
53+
54+
query I
55+
SELECT any_value(column2) FROM any_value_test WHERE column1 = 2;
56+
----
57+
NULL

‎docs/source/user-guide/sql/aggregate_functions.md‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -80,6 +80,7 @@ SELECT SUM(x) WITHIN GROUP (ORDER BY x) FROM t;
8080

8181
## General Functions
8282

83+
- [any_value](#any_value)
8384
- [array_agg](#array_agg)
8485
- [avg](#avg)
8586
- [bit_and](#bit_and)
@@ -105,6 +106,29 @@ SELECT SUM(x) WITHIN GROUP (ORDER BY x) FROM t;
105106
- [var_samp](#var_samp)
106107
- [var_sample](#var_sample)
107108

109+
### `any_value`
110+
111+
Returns an arbitrary non-null value from a group, or NULL if the group contains only NULL values.
112+
113+
```sql
114+
any_value(expression)
115+
```
116+
117+
#### Arguments
118+
119+
- **expression**: The expression to operate on. Can be a constant, column, or function, and any combination of operators.
120+
121+
#### Example
122+
123+
```sql
124+
> SELECT any_value(column_name) FROM table_name;
125+
+------------------------+
126+
| any_value(column_name) |
127+
+------------------------+
128+
| arbitrary_value |
129+
+------------------------+
130+
```
131+
108132
### `array_agg`
109133

110134
Returns an array created from the expression elements. If ordering is required, elements are inserted in the specified order.

0 commit comments

Comments
 (0)