fix select without where fields

Author: JingsongLi

paimon-python/pypaimon/cli/cli_table.py

@@ -63,24 +63,37 @@ def cmd_table_read(args):
     # Build read pipeline
     read_builder = table.new_read_builder()
 
-    # Apply projection (select columns) if specified
+    available_fields = set(field.name for field in table.table_schema.fields)
+
+    # Parse select and where options
     select_columns = args.select
+    where_clause = args.where
+    user_columns = None
+    extra_where_columns = []
+
     if select_columns:
         # Parse column names (comma-separated)
-        columns = [col.strip() for col in select_columns.split(',')]
-        
+        user_columns = [col.strip() for col in select_columns.split(',')]
+
         # Validate that all columns exist in the table schema
-        available_fields = set(field.name for field in table.table_schema.fields)
-        invalid_columns = [col for col in columns if col not in available_fields]
-        
+        invalid_columns = [col for col in user_columns if col not in available_fields]
         if invalid_columns:
             print(f"Error: Column(s) {invalid_columns} do not exist in table '{table_identifier}'.", file=sys.stderr)
             sys.exit(1)
-        
-        read_builder = read_builder.with_projection(columns)
+
+    # When both select and where are specified, ensure where-referenced fields
+    # are included in the projection so the filter can work correctly.
+    if user_columns and where_clause:
+        from pypaimon.cli.where_parser import extract_fields_from_where
+        where_fields = extract_fields_from_where(where_clause, available_fields)
+        user_column_set = set(user_columns)
+        extra_where_columns = [f for f in where_fields if f not in user_column_set]
+        projection_columns = user_columns + extra_where_columns
+        read_builder = read_builder.with_projection(projection_columns)
+    elif user_columns:
+        read_builder = read_builder.with_projection(user_columns)
 
     # Apply where filter if specified
-    where_clause = args.where
     if where_clause:
         from pypaimon.cli.where_parser import parse_where_clause
         try:
@@ -107,6 +120,11 @@ def cmd_table_read(args):
     df = read.to_pandas(splits)
     if limit and len(df) > limit:
         df = df.head(limit)
+
+    # Drop extra columns that were added only for where-clause filtering
+    if extra_where_columns:
+        df = df.drop(columns=extra_where_columns, errors='ignore')
+
     print(df.to_string(index=False))
 
 

paimon-python/pypaimon/cli/where_parser.py

@@ -48,6 +48,27 @@
 from pypaimon.schema.data_types import AtomicType, DataField
 
 
+def extract_fields_from_where(where_string: str, available_fields: set) -> set:
+    """Extract all field names referenced in a WHERE clause.
+
+    Args:
+        where_string: The WHERE clause string.
+        available_fields: Set of valid field names from the table schema.
+
+    Returns:
+        A set of field names referenced in the WHERE clause.
+    """
+    if not where_string or not where_string.strip():
+        return set()
+
+    tokens = _tokenize(where_string.strip())
+    referenced_fields = set()
+    for token in tokens:
+        if token in available_fields:
+            referenced_fields.add(token)
+    return referenced_fields
+
+
 def parse_where_clause(where_string: str, fields: List[DataField]) -> Optional[Predicate]:
     """Parse a SQL-like WHERE clause string into a Predicate.
 

paimon-python/pypaimon/tests/cli_table_test.py

@@ -1020,6 +1020,35 @@ def test_cli_table_read_with_where_and_select(self):
                 self.assertIn('Eve', output)
                 self.assertNotIn('Alice', output)
 
+    def test_cli_table_read_where_field_not_in_select(self):
+        """Test that where filter works even when the filtered field is not in select."""
+        with patch('sys.argv',
+                   ['paimon', '-c', self.config_file,
+                    'table', 'read', 'test_db.users',
+                    '--select', 'name,city',
+                    '--where', 'age > 30']):
+            with patch('sys.stdout', new_callable=StringIO) as mock_stdout:
+                try:
+                    main()
+                except SystemExit:
+                    pass
+
+                output = mock_stdout.getvalue()
+                # age > 30: Charlie(35,Guangzhou), Eve(32,Hangzhou)
+                # Filter should work even though 'age' is not in select
+                self.assertIn('Charlie', output)
+                self.assertIn('Eve', output)
+                self.assertIn('Guangzhou', output)
+                self.assertIn('Hangzhou', output)
+                # Excluded rows should not appear
+                self.assertNotIn('Alice', output)
+                self.assertNotIn('Bob', output)
+                self.assertNotIn('David', output)
+                # The 'age' column should NOT appear in output (it was only needed for filtering)
+                self.assertNotIn(' 35', output)
+                self.assertNotIn(' 32', output)
+                self.assertNotIn(' 25', output)
+
     def test_cli_table_read_with_invalid_where(self):
         """Test table read with invalid --where clause via CLI."""
         with patch('sys.argv',