lance-duckdb/src/include/lance_ffi.hpp at 1d163d0519013aa528afa075e9773f12d8f1f15c · lance-format/lance-duckdb · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
#pragma once

#include "duckdb/common/arrow/arrow.hpp"

#include <cstddef>
#include <cstdint>

extern "C" {
void *lance_open_dataset(const char *path);
void *lance_open_dataset_with_storage_options(const char *path,
                                              const char **option_keys,
                                              const char **option_values,
                                              size_t options_len);
const char *lance_dir_namespace_list_tables(const char *root,
                                            const char **option_keys,
                                            const char **option_values,
                                            size_t options_len);
int32_t lance_dir_namespace_drop_table(const char *root, const char *table_name,
                                       const char **option_keys,
                                       const char **option_values,
                                       size_t options_len);
void *lance_open_dataset_in_dir_namespace(
    const char *root, const char *table_name, const char **option_keys,
    const char **option_values, size_t options_len, const char **out_table_uri);
const char *lance_namespace_list_tables(const char *endpoint,
                                        const char *namespace_id,
                                        const char *bearer_token,
                                        const char *api_key,
                                        const char *delimiter);
void *lance_open_dataset_in_namespace(
    const char *endpoint, const char *table_id, const char *bearer_token,
    const char *api_key, const char *delimiter, const char **out_table_uri);
void lance_close_dataset(void *dataset);

void *lance_get_schema(void *dataset);
void lance_free_schema(void *schema);
int32_t lance_schema_to_arrow(void *schema, ArrowSchema *out_schema);

int32_t lance_stream_next(void *stream, void **out_batch);
void lance_close_stream(void *stream);

int32_t lance_last_error_code();
const char *lance_last_error_message();
void lance_free_string(const char *s);

int64_t lance_dataset_count_rows(void *dataset);
int32_t lance_dataset_delete(void *dataset, const uint8_t *filter_ir,
                             size_t filter_ir_len, int64_t *out_deleted_rows);

int32_t lance_dataset_add_columns(void *dataset,
                                  const ArrowSchema *new_columns_schema,
                                  const char **expressions,
                                  size_t expressions_len, uint32_t batch_size);
int32_t lance_dataset_drop_columns(void *dataset, const char **columns,
                                   size_t columns_len);
int32_t lance_dataset_alter_columns_rename(void *dataset, const char *path,
                                           const char *new_name);
int32_t lance_dataset_alter_columns_set_nullable(void *dataset,
                                                 const char *path,
                                                 uint8_t nullable);
int32_t lance_dataset_alter_columns_cast(void *dataset, const char *path,
                                         const ArrowSchema *new_type_schema);

int32_t lance_dataset_update_table_metadata(void *dataset, const char *key,
                                            const char *value);
int32_t lance_dataset_update_config(void *dataset, const char *key,
                                    const char *value);
int32_t lance_dataset_update_schema_metadata(void *dataset, const char *key,
                                             const char *value);
int32_t lance_dataset_update_field_metadata(void *dataset,
                                            const char *field_path,
                                            const char *key, const char *value);

int32_t lance_dataset_compact_files(void *dataset);
int32_t lance_dataset_cleanup_old_versions(void *dataset,
                                           int64_t older_than_seconds,
                                           uint8_t delete_unverified);

const char *lance_dataset_list_config(void *dataset);
const char *lance_dataset_list_table_metadata(void *dataset);
const char *lance_dataset_list_schema_metadata(void *dataset);
const char *lance_dataset_list_field_metadata(void *dataset,
                                              const char *field_path);
const char *lance_dataset_list_indices(void *dataset);
int32_t lance_dataset_create_scalar_index(void *dataset, const char *column,
                                          const char *index_name,
                                          uint8_t replace);

uint64_t *lance_dataset_list_fragments(void *dataset, size_t *out_len);
void lance_free_fragment_list(uint64_t *ptr, size_t len);
typedef struct LanceFieldStats {
  uint32_t field_id;
  uint64_t bytes_on_disk;
} LanceFieldStats;

typedef struct LanceFragmentStats {
  uint64_t fragment_id;
  int64_t num_rows;
  uint64_t bytes_on_disk;
} LanceFragmentStats;

LanceFragmentStats *lance_dataset_list_fragment_stats(void *dataset,
                                                      size_t *out_len);
void lance_free_fragment_stats_list(LanceFragmentStats *ptr, size_t len);

LanceFieldStats *lance_dataset_list_field_stats(void *dataset, size_t *out_len);
void lance_free_field_stats_list(LanceFieldStats *ptr, size_t len);
void *lance_create_fragment_stream_ir(void *dataset, uint64_t fragment_id,
                                      const char **columns, size_t columns_len,
                                      const uint8_t *filter_ir,
                                      size_t filter_ir_len);
void *lance_create_dataset_stream_ir(void *dataset, const char **columns,
                                     size_t columns_len,
                                     const uint8_t *filter_ir,
                                     size_t filter_ir_len, int64_t limit,
                                     int64_t offset);

void *lance_open_writer_with_storage_options(
    const char *path, const char *mode, const char **option_keys,
    const char **option_values, size_t options_len, uint64_t max_rows_per_file,
    uint64_t max_rows_per_group, uint64_t max_bytes_per_file,
    const ArrowSchema *schema);
void *lance_open_uncommitted_writer_with_storage_options(
    const char *path, const char *mode, const char **option_keys,
    const char **option_values, size_t options_len, uint64_t max_rows_per_file,
    uint64_t max_rows_per_group, uint64_t max_bytes_per_file,
    const ArrowSchema *schema);
int32_t lance_writer_write_batch(void *writer, ArrowArray *array);
int32_t lance_writer_finish(void *writer);
int32_t lance_writer_finish_uncommitted(void *writer, void **out_transaction);
void lance_close_writer(void *writer);

int32_t lance_commit_transaction_with_storage_options(
    const char *path, const char **option_keys, const char **option_values,
    size_t options_len, void *transaction);
void lance_free_transaction(void *transaction);

int32_t lance_overwrite_update_transaction_with_storage_options(
    const char *path, const char **option_keys, const char **option_values,
    size_t options_len, const char *predicate, const char **set_columns,
    const char **set_expressions, size_t set_len, uint64_t max_rows_per_file,
    uint64_t max_rows_per_group, uint64_t max_bytes_per_file,
    void **out_transaction, uint64_t *out_rows_updated);

const char *lance_explain_dataset_scan_ir(void *dataset, const char **columns,
                                          size_t columns_len,
                                          const uint8_t *filter_ir,
                                          size_t filter_ir_len, int64_t limit,
                                          int64_t offset, uint8_t verbose);

void *lance_get_knn_schema(void *dataset, const char *vector_column,
                           const float *query_values, size_t query_len,
                           uint64_t k, uint8_t prefilter, uint8_t use_index);
void *lance_create_knn_stream_ir(void *dataset, const char *vector_column,
                                 const float *query_values, size_t query_len,
                                 uint64_t k, const uint8_t *filter_ir,
                                 size_t filter_ir_len, uint8_t prefilter,
                                 uint8_t use_index);

const char *lance_explain_knn_scan_ir(void *dataset, const char *vector_column,
                                      const float *query_values,
                                      size_t query_len, uint64_t k,
                                      const uint8_t *filter_ir,
                                      size_t filter_ir_len, uint8_t prefilter,
                                      uint8_t use_index, uint8_t verbose);

void *lance_get_fts_schema(void *dataset, const char *text_column,
                           const char *query, uint64_t k, uint8_t prefilter);
void *lance_create_fts_stream_ir(void *dataset, const char *text_column,
                                 const char *query, uint64_t k,
                                 const uint8_t *filter_ir, size_t filter_ir_len,
                                 uint8_t prefilter);

void *lance_get_hybrid_schema(void *dataset);
void *lance_create_hybrid_stream_ir(void *dataset, const char *vector_column,
                                    const float *query_values, size_t query_len,
                                    const char *text_column,
                                    const char *text_query, uint64_t k,
                                    const uint8_t *filter_ir,
                                    size_t filter_ir_len, uint8_t prefilter,
                                    float alpha, uint32_t oversample_factor);

// Index DDL / metadata
int32_t lance_dataset_create_index(void *dataset, const char *index_name,
                                   const char **columns, size_t columns_len,
                                   const char *index_type,
                                   const char *params_json, uint8_t replace,
                                   uint8_t train);
int32_t lance_dataset_drop_index(void *dataset, const char *index_name);
int32_t lance_dataset_optimize_index(void *dataset, const char *index_name,
                                     uint8_t retrain);
void *lance_get_index_list_schema(void *dataset);
void *lance_create_index_list_stream(void *dataset);

void lance_free_batch(void *batch);
int32_t lance_batch_to_arrow(void *batch, ArrowArray *out_array,
                             ArrowSchema *out_schema);
}