From 662c6a0a58ac490a1cbad43fec03dbd3f29df734 Mon Sep 17 00:00:00 2001 From: Alex Petenchea Date: Fri, 28 Mar 2025 12:45:37 +0530 Subject: [PATCH 01/21] Adding more documentation --- docs/aql.rst | 152 ++++++++++++++++++++++++++++++++++++++++++++++++ docs/cursor.rst | 8 +++ docs/errors.rst | 15 ++++- docs/index.rst | 1 + 4 files changed, 175 insertions(+), 1 deletion(-) create mode 100644 docs/cursor.rst diff --git a/docs/aql.rst b/docs/aql.rst index 914c982..c5cc69b 100644 --- a/docs/aql.rst +++ b/docs/aql.rst @@ -8,3 +8,155 @@ operations such as creating or deleting :doc:`databases `, information, refer to `ArangoDB manual`_. .. _ArangoDB manual: https://docs.arangodb.com + +AQL Queries +=========== + +AQL queries are invoked from AQL API wrapper. Executing queries returns +:doc:`result cursors `. + +**Example:** + +.. code-block:: python + + from arangoasync import ArangoClient, AQLQueryKillError + from arangoasync.auth import Auth + + # Initialize the client for ArangoDB. + async with ArangoClient(hosts="http://localhost:8529") as client: + auth = Auth(username="root", password="passwd") + + # Connect to "test" database as root user. + db = await client.db("test", auth=auth) + + # Get the API wrapper for "students" collection. + students = db.collection("students") + + # Insert some test documents into "students" collection. + await students.insert_many([ + {"_key": "Abby", "age": 22}, + {"_key": "John", "age": 18}, + {"_key": "Mary", "age": 21} + ]) + + # Get the AQL API wrapper. + aql = db.aql + + # Retrieve the execution plan without running the query. + plan = await aql.explain("FOR doc IN students RETURN doc") + + # Validate the query without executing it. + validate = await aql.validate("FOR doc IN students RETURN doc") + + # Execute the query + cursor = await db.aql.execute( + "FOR doc IN students FILTER doc.age < @value RETURN doc", + bind_vars={"value": 19} + ) + + # Iterate through the result cursor + student_keys = [] + async for doc in cursor: + student_keys.append(doc) + + # List currently running queries. + queries = await aql.queries() + + # List any slow queries. + slow_queries = await aql.slow_queries() + + # Clear slow AQL queries if any. + await aql.clear_slow_queries() + + # Retrieve AQL query tracking properties. + await aql.tracking() + + # Configure AQL query tracking properties. + await aql.set_tracking( + max_slow_queries=10, + track_bind_vars=True, + track_slow_queries=True + ) + + # Kill a running query (this should fail due to invalid ID). + try: + await aql.kill("some_query_id") + except AQLQueryKillError as err: + assert err.http_code == 404 + +See :class:`arangoasync.aql.AQL` for API specification. + + +AQL User Functions +================== + +**AQL User Functions** are custom functions you define in Javascript to extend +AQL functionality. They are somewhat similar to SQL procedures. + +**Example:** + +.. code-block:: python + + from arangoasync import ArangoClient + from arangoasync.auth import Auth + + # Initialize the client for ArangoDB. + async with ArangoClient(hosts="http://localhost:8529") as client: + auth = Auth(username="root", password="passwd") + + # Connect to "test" database as root user. + db = await client.db("test", auth=auth) + + # Get the AQL API wrapper. + aql = db.aql + + # Create a new AQL user function. + await aql.create_function( + # Grouping by name prefix is supported. + name="functions::temperature::converter", + code="function (celsius) { return celsius * 1.8 + 32; }" + ) + + # List AQL user functions. + functions = await aql.functions() + + # Delete an existing AQL user function. + await aql.delete_function("functions::temperature::converter") + +See :class:`arangoasync.aql.AQL` for API specification. + + +AQL Query Cache +=============== + +**AQL Query Cache** is used to minimize redundant calculation of the same query +results. It is useful when read queries are issued frequently and write queries +are not. + +**Example:** + +.. code-block:: python + + from arangoasync import ArangoClient + from arangoasync.auth import Auth + + # Initialize the client for ArangoDB. + async with ArangoClient(hosts="http://localhost:8529") as client: + auth = Auth(username="root", password="passwd") + + # Connect to "test" database as root user. + db = await client.db("test", auth=auth) + + # Get the AQL API wrapper. + aql = db.aql + + # Retrieve AQL query cache properties. + await aql.cache.properties() + + # Configure AQL query cache properties + await aql.cache.configure(mode="demand", max_results=10000) + + # Clear results in AQL query cache. + await aql.cache.clear() + +See :class:`arangoasync.aql.AQLQueryCache` for API specification. diff --git a/docs/cursor.rst b/docs/cursor.rst new file mode 100644 index 0000000..8b746c6 --- /dev/null +++ b/docs/cursor.rst @@ -0,0 +1,8 @@ +Cursors +------- + +Many operations provided by python-arango-async (e.g. executing :doc:`aql` queries) +return result **cursors** to batch the network communication between ArangoDB +server and python-arango-async client. Each HTTP request from a cursor fetches the +next batch of results (usually documents). Depending on the query, the total +number of items in the result set may or may not be known in advance. diff --git a/docs/errors.rst b/docs/errors.rst index cba6d92..97d65a8 100644 --- a/docs/errors.rst +++ b/docs/errors.rst @@ -16,5 +16,18 @@ Client Errors ============= :class:`arangoasync.exceptions.ArangoClientError` exceptions originate from -python-arango client itself. They do not contain error codes nor HTTP request +python-arango-async client itself. They do not contain error codes nor HTTP request response details. + +**Example** + +.. testcode:: + + from arangoasync.exceptions import ArangoClientError, ArangoServerError + + try: + # Some operation that raises an error + except ArangoClientError: + # An error occurred on the client side + except ArangoServerError: + # An error occurred on the server side diff --git a/docs/index.rst b/docs/index.rst index 9e71989..76e8a44 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -63,6 +63,7 @@ Miscellaneous .. toctree:: :maxdepth: 1 + cursor errors errno From b25174f572559a4be2332de61bf95e3218d2f7cb Mon Sep 17 00:00:00 2001 From: Alex Petenchea Date: Fri, 28 Mar 2025 12:45:59 +0530 Subject: [PATCH 02/21] Minor edit --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 9e124b6..dc7b1ca 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ Python driver for [ArangoDB](https://www.arangodb.com), a scalable multi-model database natively supporting documents, graphs and search. -This is the _asyncio_ alternative of the officially supported [python-arango](https://github.com/arangodb/python-arango) +This is the _asyncio_ alternative of the [python-arango](https://github.com/arangodb/python-arango) driver. **Note: This project is still in active development, features might be added or removed.** From b4072a73cb140a5b4948b91274f6a1687834fb3e Mon Sep 17 00:00:00 2001 From: Alex Petenchea Date: Fri, 28 Mar 2025 16:05:25 +0530 Subject: [PATCH 03/21] No longer using testcode --- docs/errors.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/errors.rst b/docs/errors.rst index 97d65a8..855b152 100644 --- a/docs/errors.rst +++ b/docs/errors.rst @@ -21,7 +21,7 @@ response details. **Example** -.. testcode:: +.. code-block:: python from arangoasync.exceptions import ArangoClientError, ArangoServerError From 96126577a4d78adc255524c056653441ae9f9a07 Mon Sep 17 00:00:00 2001 From: Alex Petenchea Date: Tue, 22 Apr 2025 20:32:51 +0300 Subject: [PATCH 04/21] Completed transactions documentation --- README.md | 2 +- arangoasync/aql.py | 5 +++ arangoasync/collection.py | 27 ++++++++++++-- docs/aql.rst | 19 +++++++--- docs/collection.rst | 9 ++++- docs/database.rst | 2 ++ docs/document.rst | 30 ++++++++-------- docs/index.rst | 17 ++++----- docs/indexes.rst | 10 +++--- docs/transaction.rst | 76 +++++++++++++++++++++++++++++++++++++++ 10 files changed, 162 insertions(+), 35 deletions(-) diff --git a/README.md b/README.md index dc7b1ca..ea91bbf 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,7 @@ driver. ## Requirements - ArangoDB version 3.11+ -- Python version 3.9+ +- Python version 3.10+ ## Installation diff --git a/arangoasync/aql.py b/arangoasync/aql.py index c0e1b29..57d57e1 100644 --- a/arangoasync/aql.py +++ b/arangoasync/aql.py @@ -238,6 +238,11 @@ def name(self) -> str: """Return the name of the current database.""" return self._executor.db_name + @property + def context(self) -> str: + """Return the current API execution context.""" + return self._executor.context + @property def serializer(self) -> Serializer[Json]: """Return the serializer.""" diff --git a/arangoasync/collection.py b/arangoasync/collection.py index b6bb483..7079d8f 100644 --- a/arangoasync/collection.py +++ b/arangoasync/collection.py @@ -251,6 +251,15 @@ def name(self) -> str: """ return self._name + @property + def context(self) -> str: + """Return the context of the collection. + + Returns: + str: Context. + """ + return self._executor.context + @property def db_name(self) -> str: """Return the name of the current database. @@ -270,9 +279,17 @@ def deserializer(self) -> Deserializer[Json, Jsons]: """Return the deserializer.""" return self._executor.deserializer - async def indexes(self) -> Result[List[IndexProperties]]: + async def indexes( + self, + with_stats: Optional[bool], + with_hidden: Optional[bool], + ) -> Result[List[IndexProperties]]: """Fetch all index descriptions for the given collection. + Args: + with_stats (bool | None): Whether to include figures and estimates in the result. + with_hidden (bool | None): Whether to include hidden indexes in the result. + Returns: list: List of index properties. @@ -282,10 +299,16 @@ async def indexes(self) -> Result[List[IndexProperties]]: References: - `list-all-indexes-of-a-collection `__ """ # noqa: E501 + params: Params = dict(collection=self._name) + if with_stats is not None: + params["withStats"] = with_stats + if with_hidden is not None: + params["withHidden"] = with_hidden + request = Request( method=Method.GET, endpoint="/_api/index", - params=dict(collection=self._name), + params=params, ) def response_handler(resp: Response) -> List[IndexProperties]: diff --git a/docs/aql.rst b/docs/aql.rst index c5cc69b..69a9bf6 100644 --- a/docs/aql.rst +++ b/docs/aql.rst @@ -5,15 +5,15 @@ AQL to SQL for relational databases, but without the support for data definition operations such as creating or deleting :doc:`databases `, :doc:`collections ` or :doc:`indexes `. For more -information, refer to `ArangoDB manual`_. +information, refer to `ArangoDB Manual`_. -.. _ArangoDB manual: https://docs.arangodb.com +.. _ArangoDB Manual: https://docs.arangodb.com AQL Queries =========== -AQL queries are invoked from AQL API wrapper. Executing queries returns -:doc:`result cursors `. +AQL queries are invoked from AQL wrapper. Executing queries returns +:doc:`cursors `. **Example:** @@ -153,10 +153,19 @@ are not. # Retrieve AQL query cache properties. await aql.cache.properties() - # Configure AQL query cache properties + # Configure AQL query cache properties. await aql.cache.configure(mode="demand", max_results=10000) + # List results cache entries. + entries = await aql.cache.entries() + + # List plan cache entries. + plan_entries = await aql.cache.plan_entries() + # Clear results in AQL query cache. await aql.cache.clear() + # Clear results in AQL query plan cache. + await aql.cache.clear_plan() + See :class:`arangoasync.aql.AQLQueryCache` for API specification. diff --git a/docs/collection.rst b/docs/collection.rst index 42487f6..e6a846f 100644 --- a/docs/collection.rst +++ b/docs/collection.rst @@ -3,7 +3,12 @@ Collections A **collection** contains :doc:`documents `. It is uniquely identified by its name which must consist only of hyphen, underscore and alphanumeric -characters. +characters. There are three types of collections in python-arango: + +* **Standard Collection:** contains regular documents. +* **Vertex Collection:** contains vertex documents for graphs (not supported yet). +* **Edge Collection:** contains edge documents for graphs (not supported yet). + Here is an example showing how you can manage standard collections: @@ -40,3 +45,5 @@ Here is an example showing how you can manage standard collections: # Delete the collection. await db.delete_collection("students") + +See :class:`arangoasync.collection.StandardCollection` for API specification. diff --git a/docs/database.rst b/docs/database.rst index f510cb2..851cc9d 100644 --- a/docs/database.rst +++ b/docs/database.rst @@ -59,3 +59,5 @@ information. # Delete the database. Note that the new users will remain. await sys_db.delete_database("test") + +See :class:`arangoasync.client.ArangoClient` and :class:`arangoasync.database.StandardDatabase` for API specification. diff --git a/docs/document.rst b/docs/document.rst index 3398bf9..ff9121e 100644 --- a/docs/document.rst +++ b/docs/document.rst @@ -20,26 +20,26 @@ properties: to validate a document against its current revision. For more information on documents and associated terminologies, refer to -`ArangoDB manual`_. Here is an example of a valid document in "students" +`ArangoDB Manual`_. Here is an example of a valid document in "students" collection: -.. _ArangoDB manual: https://docs.arangodb.com +.. _ArangoDB Manual: https://docs.arangodb.com -.. testcode:: +.. code-block:: json { - '_id': 'students/bruce', - '_key': 'bruce', - '_rev': '_Wm3dzEi--_', - 'first_name': 'Bruce', - 'last_name': 'Wayne', - 'address': { - 'street' : '1007 Mountain Dr.', - 'city': 'Gotham', - 'state': 'NJ' + "_id": "students/bruce", + "_key": "bruce", + "_rev": "_Wm3dzEi--_", + "first_name": "Bruce", + "last_name": "Wayne", + "address": { + "street" : "1007 Mountain Dr.", + "city": "Gotham", + "state": "NJ" }, - 'is_rich': True, - 'friends': ['robin', 'gordon'] + "is_rich": true, + "friends": ["robin", "gordon"] } Standard documents are managed via collection API wrapper: @@ -129,3 +129,5 @@ Standard documents are managed via collection API wrapper: # Delete one or more matching documents. await students.delete_match({"first": "Emma"}) + +See :class:`arangoasync.database.StandardDatabase` and :class:`arangoasync.collection.StandardCollection` for API specification. diff --git a/docs/index.rst b/docs/index.rst index 76e8a44..dc8c716 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -5,7 +5,7 @@ python-arango-async ------------------- -Welcome to the documentation for **python-arango-async**, a Python driver for ArangoDB_. +Welcome to the documentation for python-arango-async_, a Python driver for ArangoDB_. **Note: This project is still in active development, features might be added or removed.** @@ -13,7 +13,7 @@ Requirements ============= - ArangoDB version 3.11+ -- Python version 3.9+ +- Python version 3.10+ Installation ============ @@ -25,7 +25,7 @@ Installation Contents ======== -Basics +**Basics** .. toctree:: :maxdepth: 1 @@ -37,28 +37,28 @@ Basics document aql -Specialized Features +**Specialized Features** .. toctree:: :maxdepth: 1 transaction -API Executions +**API Executions** .. toctree:: :maxdepth: 1 async -Administration +**Administration** .. toctree:: :maxdepth: 1 user -Miscellaneous +**Miscellaneous** .. toctree:: :maxdepth: 1 @@ -67,7 +67,7 @@ Miscellaneous errors errno -Development +**Development** .. toctree:: :maxdepth: 1 @@ -75,3 +75,4 @@ Development specs .. _ArangoDB: https://www.arangodb.com +.. _python-arango-async: https://github.com/arangodb/python-arango-async diff --git a/docs/indexes.rst b/docs/indexes.rst index e8ae208..911efaa 100644 --- a/docs/indexes.rst +++ b/docs/indexes.rst @@ -5,9 +5,9 @@ Indexes collection has a primary hash index on ``_key`` field by default. This index cannot be deleted or modified. Every edge collection has additional indexes on fields ``_from`` and ``_to``. For more information on indexes, refer to -`ArangoDB manual`_. +`ArangoDB Manual`_. -.. _ArangoDB manual: https://docs.arangodb.com +.. _ArangoDB Manual: https://docs.arangodb.com **Example:** @@ -30,11 +30,11 @@ on fields ``_from`` and ``_to``. For more information on indexes, refer to indexes = await cities.indexes() # Add a new persistent index on document fields "continent" and "country". - persistent_index = {"type": "persistent", "fields": ["continent", "country"], "unique": True} + # Indexes may be added with a name that can be referred to in AQL queries. persistent_index = await cities.add_index( type="persistent", fields=['continent', 'country'], - options={"unique": True} + options={"unique": True, "name": "continent_country_index"} ) # Add new fulltext indexes on fields "continent" and "country". @@ -49,3 +49,5 @@ on fields ``_from`` and ``_to``. For more information on indexes, refer to # Delete the last index from the collection. await cities.delete_index(index["id"]) + +See :class:`arangoasync.collection.StandardCollection` for API specification. diff --git a/docs/transaction.rst b/docs/transaction.rst index 225e226..e36738d 100644 --- a/docs/transaction.rst +++ b/docs/transaction.rst @@ -3,3 +3,79 @@ Transactions In **transactions**, requests to ArangoDB server are committed as a single, logical unit of work (ACID compliant). + +**Example:** + +.. code-block:: python + + from arangoasync import ArangoClient + from arangoasync.auth import Auth + + # Initialize the client for ArangoDB. + async with ArangoClient(hosts="http://localhost:8529") as client: + auth = Auth(username="root", password="passwd") + + # Connect to "test" database as root user. + db = await client.db("test", auth=auth) + + # Get the API wrapper for "students" collection. + students = db.collection("students") + + # Begin a transaction. Read and write collections must be declared ahead of + # time. This returns an instance of TransactionDatabase, database-level + # API wrapper tailored specifically for executing transactions. + txn_db = await db.begin_transaction(read=students.name, write=students.name) + + # The API wrapper is specific to a single transaction with a unique ID. + trx_id = txn_db.transaction_id + + # Child wrappers are also tailored only for the specific transaction. + txn_aql = txn_db.aql + txn_col = txn_db.collection("students") + + # API execution context is always set to "transaction". + assert txn_db.context == "transaction" + assert txn_aql.context == "transaction" + assert txn_col.context == "transaction" + + assert "_rev" in await txn_col.insert({"_key": "Abby"}) + assert "_rev" in await txn_col.insert({"_key": "John"}) + assert "_rev" in await txn_col.insert({"_key": "Mary"}) + + # Check the transaction status. + status = await txn_db.transaction_status() + + # Commit the transaction. + await txn_db.commit_transaction() + assert await students.has("Abby") + assert await students.has("John") + assert await students.has("Mary") + assert await students.count() == 3 + + # Begin another transaction. Note that the wrappers above are specific to + # the last transaction and cannot be reused. New ones must be created. + txn_db = await db.begin_transaction(read=students.name, write=students.name) + txn_col = txn_db.collection("students") + assert "_rev" in await txn_col.insert({"_key": "Kate"}) + assert "_rev" in await txn_col.insert({"_key": "Mike"}) + assert "_rev" in await txn_col.insert({"_key": "Lily"}) + assert await txn_col.count() == 6 + + # Abort the transaction + await txn_db.abort_transaction() + assert not await students.has("Kate") + assert not await students.has("Mike") + assert not await students.has("Lily") + assert await students.count() == 3 # transaction is aborted so txn_col cannot be used + + # Fetch an existing transaction. Useful if you have received a Transaction ID + # from an external system. + original_txn = await db.begin_transaction(write='students') + txn_col = original_txn.collection('students') + assert '_rev' in await txn_col.insert({'_key': 'Chip'}) + txn_db = db.fetch_transaction(original_txn.transaction_id) + txn_col = txn_db.collection('students') + assert '_rev' in await txn_col.insert({'_key': 'Alya'}) + await txn_db.abort_transaction() + +See :class:`arangoasync.database.TransactionDatabase` for API specification. From 7b7224d9cf39dc9d424ad5843ff8fac7c1e5eebe Mon Sep 17 00:00:00 2001 From: Alex Petenchea Date: Wed, 23 Apr 2025 20:31:44 +0300 Subject: [PATCH 05/21] Completed async documentation --- docs/async.rst | 148 +++++++++++++++++++++++++++++++++++++++++++- tests/test_async.py | 2 +- 2 files changed, 146 insertions(+), 4 deletions(-) diff --git a/docs/async.rst b/docs/async.rst index a47b131..3fe31ff 100644 --- a/docs/async.rst +++ b/docs/async.rst @@ -1,6 +1,148 @@ Async API Execution ------------------- -In **asynchronous API executions**, python-arango-async sends API requests to ArangoDB in -fire-and-forget style. The server processes the requests in the background, and -the results can be retrieved once available via `AsyncJob` objects. +In **asynchronous API executions**, the driver sends API requests to ArangoDB in +fire-and-forget style. The server processes them in the background, and +the results can be retrieved once available via :class:`arangoasync.job.AsyncJob` objects. + +**Example:** + +.. code-block:: python + + import time + from arangoasync import ArangoClient + from arangoasync.auth import Auth + from arangoasync.errno import HTTP_BAD_PARAMETER + from arangoasync.exceptions import ( + AQLQueryExecuteError, + AsyncJobCancelError, + AsyncJobClearError, + ) + + # Initialize the client for ArangoDB. + async with ArangoClient(hosts="http://localhost:8529") as client: + auth = Auth(username="root", password="passwd") + + # Connect to "test" database as root user. + db = await client.db("test", auth=auth) + + # Begin async execution. This returns an instance of AsyncDatabase, a + # database-level API wrapper tailored specifically for async execution. + async_db = db.begin_async_execution(return_result=True) + + # Child wrappers are also tailored for async execution. + async_aql = async_db.aql + async_col = async_db.collection("students") + + # API execution context is always set to "async". + assert async_db.context == "async" + assert async_aql.context == "async" + assert async_col.context == "async" + + # On API execution, AsyncJob objects are returned instead of results. + job1 = await async_col.insert({"_key": "Neal"}) + job2 = await async_col.insert({"_key": "Lily"}) + job3 = await async_aql.execute("RETURN 100000") + job4 = await async_aql.execute("INVALID QUERY") # Fails due to syntax error. + + # Retrieve the status of each async job. + for job in [job1, job2, job3, job4]: + # Job status can be "pending" or "done". + assert await job.status() in {"pending", "done"} + + # Let's wait until the jobs are finished. + while await job.status() != "done": + time.sleep(0.1) + + # Retrieve the results of successful jobs. + metadata = await job1.result() + assert metadata["_id"] == "students/Neal" + + metadata = await job2.result() + assert metadata["_id"] == "students/Lily" + + cursor = await job3.result() + assert await cursor.next() == 100000 + + # If a job fails, the exception is propagated up during result retrieval. + try: + result = await job4.result() + except AQLQueryExecuteError as err: + assert err.http_code == HTTP_BAD_PARAMETER + + # Cancel a job. Only pending jobs still in queue may be cancelled. + # Since job3 is done, there is nothing to cancel and an exception is raised. + try: + await job3.cancel() + except AsyncJobCancelError as err: + print(err.message) + + # Clear the result of a job from ArangoDB server to free up resources. + # Result of job4 was removed from the server automatically upon retrieval, + # so attempt to clear it raises an exception. + try: + await job4.clear() + except AsyncJobClearError as err: + print(err.message) + + # List the IDs of the first 100 async jobs completed. + jobs_done = await db.async_jobs(status="done", count=100) + + # List the IDs of the first 100 async jobs still pending. + jobs_pending = await db.async_jobs(status='pending', count=100) + + # Clear all async jobs still sitting on the server. + await db.clear_async_jobs() + +Cursors returned from async API wrappers will no longer send async requests when they fetch more results, but behave +like regular cursors instead. This makes sense, because the point of cursors is iteration, whereas async jobs are meant +for one-shot requests. However, the first result retrieval is still async, and only then the cursor is returned, making +async AQL requests effective for queries with a long execution time. + +**Example:** + +.. code-block:: python + + from arangoasync import ArangoClient + from arangoasync.auth import Auth + + # Initialize the client for ArangoDB. + async with ArangoClient(hosts="http://localhost:8529") as client: + auth = Auth(username="root", password="passwd") + + # Connect to "test" database as root user. + db = await client.db("test", auth=auth) + + # Get the API wrapper for "students" collection. + students = db.collection("students") + + # Insert some documents into the collection. + await students.insert_many([{"_key": "Neal"}, {"_key": "Lily"}]) + + # Begin async execution. + async_db = db.begin_async_execution(return_result=True) + + aql = async_db.aql + job = await aql.execute( + f"FOR d IN {students.name} SORT d._key RETURN d", + count=True, + batch_size=1, + ttl=1000, + ) + await job.wait() + + # Iterate through the cursor. + # Although the request to fetch the cursor is async, its underlying executor is no longer async. + # Next batches will be fetched in real-time. + doc_cnt = 0 + cursor = await job.result() + async with cursor as ctx: + async for _ in ctx: + doc_cnt += 1 + assert doc_cnt == 2 + +.. note:: + Be mindful of server-side memory capacity when issuing a large number of + async requests in small time interval. + +See :class:`arangoasync.database.AsyncDatabase` and :class:`arangoasync.job.AsyncJob` for API specification. diff --git a/tests/test_async.py b/tests/test_async.py index c4f7988..1bd3bda 100644 --- a/tests/test_async.py +++ b/tests/test_async.py @@ -126,7 +126,7 @@ async def test_async_cursor(db, doc_col, docs): ) await job.wait() - # Get the cursor. Bear in mind that its underlying executor is async. + # Get the cursor. Bear in mind that its underlying executor is no longer async. doc_cnt = 0 cursor = await job.result() async with cursor as ctx: From cc24a8de52e611581ce1ec2d9671da4845743886 Mon Sep 17 00:00:00 2001 From: Alex Petenchea Date: Thu, 24 Apr 2025 11:34:39 +0300 Subject: [PATCH 06/21] Completed user documentation --- docs/user.rst | 90 ++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 89 insertions(+), 1 deletion(-) diff --git a/docs/user.rst b/docs/user.rst index 015858c..c5184a5 100644 --- a/docs/user.rst +++ b/docs/user.rst @@ -1,5 +1,93 @@ Users and Permissions --------------------- -Python-arango provides operations for managing users and permissions. Most of +ArangoDB provides operations for managing users and permissions. Most of these operations can only be performed by admin users via ``_system`` database. + +.. code-block:: python + + from arangoasync import ArangoClient + from arangoasync.auth import Auth + from arangoasync.typings import UserInfo + + # Initialize the client for ArangoDB. + async with ArangoClient(hosts="http://localhost:8529") as client: + auth = Auth(username="root", password="passwd") + + # Connect to "_system" database as root user. + sys_db = await client.db("_system", auth=auth) + + # List all users. + users = await sys_db.users() + + johndoe = UserInfo( + user="johndoe@gmail.com", + password="first_password", + active=True, + extra={"team": "backend", "title": "engineer"} + ) + + # Create a new user. + await sys_db.create_user(johndoe) + + # Check if a user exists. + assert await sys_db.has_user(johndoe.user) is True + assert await sys_db.has_user("johndoe@gmail.com") is True + + # Retrieve details of a user. + user_info = await sys_db.user(johndoe.user) + assert user_info.user == "johndoe@gmail.com" + + # Update an existing user. + johndoe["password"] = "second_password" + await sys_db.update_user(johndoe) + + # Replace an existing user. + johndoe["password"] = "third_password" + await sys_db.replace_user(johndoe) + + # Retrieve user permissions for all databases and collections. + await sys_db.permissions(johndoe.user) + + # Retrieve user permission for "test" database. + perm = await sys_db.permission( + username="johndoe@gmail.com", + database="test" + ) + + # Retrieve user permission for "students" collection in "test" database. + perm = await sys_db.permission( + username="johndoe@gmail.com", + database="test", + collection="students" + ) + + # Update user permission for "test" database. + await sys_db.update_permission( + username="johndoe@gmail.com", + permission="rw", + database="test" + ) + + # Update user permission for "students" collection in "test" database. + await sys_db.update_permission( + username="johndoe@gmail.com", + permission="ro", + database="test", + collection="students" + ) + + # Reset user permission for "test" database. + await sys_db.reset_permission( + username="johndoe@gmail.com", + database="test" + ) + + # Reset user permission for "students" collection in "test" database. + await sys_db.reset_permission( + username="johndoe@gmail.com", + database="test", + collection="students" + ) + +See :class:`arangoasync.database.StandardDatabase` for API specification. From b2a2d7f3c529309979ffd04a83778a327e97a3e6 Mon Sep 17 00:00:00 2001 From: Alex Petenchea Date: Thu, 24 Apr 2025 11:41:27 +0300 Subject: [PATCH 07/21] Completed overview documentation --- docs/overview.rst | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/docs/overview.rst b/docs/overview.rst index ce3f45a..34ad714 100644 --- a/docs/overview.rst +++ b/docs/overview.rst @@ -38,3 +38,25 @@ Here is an example showing how **python-arango-async** client can be used: student_names = [] async for doc in cursor: student_names.append(doc["name"]) + +You may also use the client without a context manager, but you must ensure to close the client when done: + +.. code-block:: python + from arangoasync import ArangoClient + from arangoasync.auth import Auth + + client = ArangoClient(hosts="http://localhost:8529") + auth = Auth(username="root", password="passwd") + sys_db = await client.db("_system", auth=auth) + + # Create a new database named "test". + await sys_db.create_database("test") + + # Connect to "test" database as root user. + db = await client.db("test", auth=auth) + + # List all collections in the "test" database. + collections = await db.collections() + + # Close the client when done. + await client.close() From 2355c1bc27706f8ae10b2f6c5d329874d29d69fa Mon Sep 17 00:00:00 2001 From: Alex Petenchea Date: Thu, 24 Apr 2025 11:41:41 +0300 Subject: [PATCH 08/21] Completed overview documentation --- docs/overview.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/overview.rst b/docs/overview.rst index 34ad714..6f1f76a 100644 --- a/docs/overview.rst +++ b/docs/overview.rst @@ -42,6 +42,7 @@ Here is an example showing how **python-arango-async** client can be used: You may also use the client without a context manager, but you must ensure to close the client when done: .. code-block:: python + from arangoasync import ArangoClient from arangoasync.auth import Auth From 6c2da6d00d01d0f6d162799f5891461f7d474489 Mon Sep 17 00:00:00 2001 From: Alex Petenchea Date: Thu, 24 Apr 2025 12:37:35 +0300 Subject: [PATCH 09/21] Completed cursor documentation --- docs/cursor.rst | 213 +++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 211 insertions(+), 2 deletions(-) diff --git a/docs/cursor.rst b/docs/cursor.rst index 8b746c6..9d2d2bf 100644 --- a/docs/cursor.rst +++ b/docs/cursor.rst @@ -1,8 +1,217 @@ Cursors ------- -Many operations provided by python-arango-async (e.g. executing :doc:`aql` queries) +Many operations provided by the driver (e.g. executing :doc:`aql` queries) return result **cursors** to batch the network communication between ArangoDB -server and python-arango-async client. Each HTTP request from a cursor fetches the +server and the client. Each HTTP request from a cursor fetches the next batch of results (usually documents). Depending on the query, the total number of items in the result set may or may not be known in advance. + +**Example:** + +.. code-block:: python + + from arangoasync import ArangoClient + from arangoasync.auth import Auth + + # Initialize the client for ArangoDB. + async with ArangoClient(hosts="http://localhost:8529") as client: + auth = Auth(username="root", password="passwd") + + # Connect to "test" database as root user. + db = await client.db("test", auth=auth) + + # Set up some test data to query against. + await db.collection("students").insert_many([ + {"_key": "Abby", "age": 22}, + {"_key": "John", "age": 18}, + {"_key": "Mary", "age": 21}, + {"_key": "Suzy", "age": 23}, + {"_key": "Dave", "age": 20} + ]) + + # Execute an AQL query which returns a cursor object. + cursor = await db.aql.execute( + "FOR doc IN students FILTER doc.age > @val RETURN doc", + bind_vars={"val": 17}, + batch_size=2, + count=True + ) + + # Get the cursor ID. + cid = cursor.id + + # Get the items in the current batch. + batch = cursor.batch + + # Check if the current batch is empty. + is_empty = cursor.empty() + + # Get the total count of the result set. + cnt = cursor.count + + # Flag indicating if there are more to be fetched from server. + has_more = cursor.has_more + + # Flag indicating if the results are cached. + is_cached = cursor.cached + + # Get the cursor statistics. + stats = cursor.statistics + + # Get the performance profile. + profile = cursor.profile + + # Get any warnings produced from the query. + warnings = cursor.warnings + + # Return the next item from the cursor. If current batch is depleted, the + # next batch is fetched from the server automatically. + await cursor.next() + + # Return the next item from the cursor. If current batch is depleted, an + # exception is thrown. You need to fetch the next batch manually. + cursor.pop() + + # Fetch the next batch and add them to the cursor object. + await cursor.fetch() + + # Delete the cursor from the server. + await cursor.close() + +See :class:`arangoasync.cursor.Cursor` for API specification. + +Cursors can be used together with a context manager to ensure that the resources get freed up +when the cursor is no longer needed. Asynchronous iteration is also supported, allowing you to +iterate over the cursor results without blocking the event loop. + +**Example:** + +.. code-block:: python + + from arangoasync import ArangoClient + from arangoasync.auth import Auth + from arangoasync.exceptions import CursorCloseError + + # Initialize the client for ArangoDB. + async with ArangoClient(hosts="http://localhost:8529") as client: + auth = Auth(username="root", password="passwd") + + # Connect to "test" database as root user. + db = await client.db("test", auth=auth) + + # Set up some test data to query against. + await db.collection("students").insert_many([ + {"_key": "Abby", "age": 22}, + {"_key": "John", "age": 18}, + {"_key": "Mary", "age": 21}, + {"_key": "Suzy", "age": 23}, + {"_key": "Dave", "age": 20} + ]) + + # Execute an AQL query which returns a cursor object. + cursor = await db.aql.execute( + "FOR doc IN students FILTER doc.age > @val RETURN doc", + bind_vars={"val": 17}, + batch_size=2, + count=True + ) + + # Iterate over the cursor in an async context manager. + async with cursor as ctx: + async for student in ctx: + print(student) + + # The cursor is automatically closed when exiting the context manager. + try: + await cursor.close() + except CursorCloseError: + print(f"Cursor already closed!") + +If the fetched result batch is depleted while you are iterating over a cursor +(or while calling the method :func:`arangoasync.cursor.Cursor.next`), the driver +automatically sends an HTTP request to the server in order to fetch the next batch +(just-in-time style). To control exactly when the fetches occur, you can use +methods like :func:`arangoasync.cursor.Cursor.fetch` and :func:`arangoasync.cursor.Cursor.pop` +instead. + +**Example:** + +.. code-block:: python + + from arangoasync import ArangoClient + from arangoasync.auth import Auth + + # Initialize the client for ArangoDB. + async with ArangoClient(hosts="http://localhost:8529") as client: + auth = Auth(username="root", password="passwd") + + # Connect to "test" database as root user. + db = await client.db("test", auth=auth) + + # Set up some test data to query against. + await db.collection("students").insert_many([ + {"_key": "Abby", "age": 22}, + {"_key": "John", "age": 18}, + {"_key": "Mary", "age": 21} + ]) + + # You can manually fetch and pop for finer control. + cursor = await db.aql.execute("FOR doc IN students RETURN doc", batch_size=1) + while cursor.has_more: # Fetch until nothing is left on the server. + await cursor.fetch() + while not cursor.empty(): # Pop until nothing is left on the cursor. + student = cursor.pop() + print(student) + +You can use the `allow_retry` parameter of :func:`arangoasync.aql.AQL.execute` +to automatically retry the request if the cursor encountered any issues during +the previous fetch operation. Note that this feature causes the server to +cache the last batch. To allow re-fetching of the very last batch of the query, +the server cannot automatically delete the cursor. Once you have successfully +received the last batch, you should call :func:`arangoasync.cursor.Cursor.close`, +or use a context manager to ensure the cursor is closed properly. + +**Example:** + +.. code-block:: python + + from arangoasync import ArangoClient + from arangoasync.auth import Auth + from arangoasync.typings import QueryProperties + + # Initialize the client for ArangoDB. + async with ArangoClient(hosts="http://localhost:8529") as client: + auth = Auth(username="root", password="passwd") + + # Connect to "test" database as root user. + db = await client.db("test", auth=auth) + + # Set up some test data to query against. + await db.collection("students").insert_many([ + {"_key": "Abby", "age": 22}, + {"_key": "John", "age": 18}, + {"_key": "Mary", "age": 21} + ]) + + cursor = await db.aql.execute( + "FOR doc IN students RETURN doc", + batch_size=1, + options=QueryProperties(allow_retry=True) + ) + + while cursor.has_more: + try: + await cursor.fetch() + except ConnectionError: + # Retry the request. + continue + + while not cursor.empty(): + student = cursor.pop() + print(student) + + # Delete the cursor from the server. + await cursor.close() + +For more information about various query properties, see :class:`arangoasync.typings.QueryProperties`. From 220165999b83de0b320462a8a314d7d95b7f5d8c Mon Sep 17 00:00:00 2001 From: Alex Petenchea Date: Thu, 24 Apr 2025 12:40:17 +0300 Subject: [PATCH 10/21] Completed errno documentation --- docs/errno.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/errno.rst b/docs/errno.rst index f4ee457..0a64e49 100644 --- a/docs/errno.rst +++ b/docs/errno.rst @@ -1,7 +1,7 @@ Error Codes ----------- -Python-Arango-Async provides ArangoDB error code constants for convenience. +ArangoDB error code constants are provided for convenience. **Example** @@ -14,6 +14,6 @@ Python-Arango-Async provides ArangoDB error code constants for convenience. assert errno.DOCUMENT_REV_BAD == 1239 assert errno.DOCUMENT_NOT_FOUND == 1202 -For more information, refer to `ArangoDB manual`_. +For more information, refer to the `ArangoDB Manual`_. -.. _ArangoDB manual: https://www.arangodb.com/docs/stable/appendix-error-codes.html +.. _ArangoDB Manual: https://www.arangodb.com/docs/stable/appendix-error-codes.html From 619c5c4cdf5eaff0118b9baecad710466a62599a Mon Sep 17 00:00:00 2001 From: Alex Petenchea Date: Thu, 24 Apr 2025 20:16:29 +0300 Subject: [PATCH 11/21] Completed errors documentation --- arangoasync/collection.py | 2 + docs/errno.rst | 5 +- docs/errors.rst | 104 ++++++++++++++++++++++++++++++++++---- docs/specs.rst | 3 -- 4 files changed, 101 insertions(+), 13 deletions(-) diff --git a/arangoasync/collection.py b/arangoasync/collection.py index 7079d8f..551c2c9 100644 --- a/arangoasync/collection.py +++ b/arangoasync/collection.py @@ -587,6 +587,7 @@ async def get( Raises: DocumentRevisionError: If the revision is incorrect. DocumentGetError: If retrieval fails. + DocumentParseError: If the document is malformed. References: - `get-a-document `__ @@ -730,6 +731,7 @@ async def insert( Raises: DocumentInsertError: If insertion fails. + DocumentParseError: If the document is malformed. References: - `create-a-document `__ diff --git a/docs/errno.rst b/docs/errno.rst index 0a64e49..06011fd 100644 --- a/docs/errno.rst +++ b/docs/errno.rst @@ -5,7 +5,7 @@ ArangoDB error code constants are provided for convenience. **Example** -.. testcode:: +.. code-block:: python from arangoasync import errno @@ -14,6 +14,9 @@ ArangoDB error code constants are provided for convenience. assert errno.DOCUMENT_REV_BAD == 1239 assert errno.DOCUMENT_NOT_FOUND == 1202 +You can see the full list of error codes in the `errno.py`_ file. + For more information, refer to the `ArangoDB Manual`_. .. _ArangoDB Manual: https://www.arangodb.com/docs/stable/appendix-error-codes.html +.. _errno.py: https://github.com/arangodb/python-arango-async/blob/main/arangoasync/errno.py diff --git a/docs/errors.rst b/docs/errors.rst index 855b152..87036f0 100644 --- a/docs/errors.rst +++ b/docs/errors.rst @@ -5,6 +5,20 @@ All python-arango exceptions inherit :class:`arangoasync.exceptions.ArangoError` which splits into subclasses :class:`arangoasync.exceptions.ArangoServerError` and :class:`arangoasync.exceptions.ArangoClientError`. +**Example** + +.. code-block:: python + + from arangoasync.exceptions import ArangoClientError, ArangoServerError + + try: + # Some operation that raises an error + except ArangoClientError: + # An error occurred on the client side + except ArangoServerError: + # An error occurred on the server side + + Server Errors ============= @@ -12,22 +26,94 @@ Server Errors HTTP responses coming from ArangoDB. Each exception object contains the error message, error code and HTTP request response details. +**Example:** + +.. code-block:: python + + from arangoasync import ArangoClient, ArangoServerError, DocumentInsertError + from arangoasync.auth import Auth + + # Initialize the client for ArangoDB. + async with ArangoClient(hosts="http://localhost:8529") as client: + auth = Auth(username="root", password="passwd") + + # Connect to "test" database as root user. + db = await client.db("test", auth=auth) + + # Get the API wrapper for "students" collection. + students = db.collection("students") + + try: + await students.insert({"_key": "John"}) + await students.insert({"_key": "John"}) # duplicate key error + except DocumentInsertError as err: + assert isinstance(err, ArangoServerError) + assert err.source == "server" + + msg = err.message # Exception message usually from ArangoDB + err_msg = err.error_message # Raw error message from ArangoDB + code = err.error_code # Error code from ArangoDB + url = err.url # URL (API endpoint) + method = err.http_method # HTTP method (e.g. "POST") + headers = err.http_headers # Response headers + http_code = err.http_code # Status code (e.g. 200) + + # You can inspect the ArangoDB response directly. + response = err.response + method = response.method # HTTP method + headers = response.headers # Response headers + url = response.url # Full request URL + success = response.is_success # Set to True if HTTP code is 2XX + raw_body = response.raw_body # Raw string response body + status_txt = response.status_text # Status text (e.g "OK") + status_code = response.status_code # Status code (e.g. 200) + err_code = response.error_code # Error code from ArangoDB + + # You can also inspect the request sent to ArangoDB. + request = err.request + method = request.method # HTTP method + endpoint = request.endpoint # API endpoint starting with "/_api" + headers = request.headers # Request headers + params = request.params # URL parameters + data = request.data # Request payload + Client Errors ============= :class:`arangoasync.exceptions.ArangoClientError` exceptions originate from -python-arango-async client itself. They do not contain error codes nor HTTP request +driver client itself. They do not contain error codes nor HTTP request response details. -**Example** +**Example:** .. code-block:: python - from arangoasync.exceptions import ArangoClientError, ArangoServerError + from arangoasync import ArangoClient, ArangoClientError, DocumentParseError + from arangoasync.auth import Auth - try: - # Some operation that raises an error - except ArangoClientError: - # An error occurred on the client side - except ArangoServerError: - # An error occurred on the server side + # Initialize the client for ArangoDB. + async with ArangoClient(hosts="http://localhost:8529") as client: + auth = Auth(username="root", password="passwd") + + # Connect to "test" database as root user. + db = await client.db("test", auth=auth) + + # Get the API wrapper for "students" collection. + students = db.collection("students") + + try: + await students.get({"_id": "invalid_id"}) # malformed document + except DocumentParseError as err: + assert isinstance(err, ArangoClientError) + assert err.source == "client" + + # Only the error message is set. + print(err.message) + +Exceptions +========== + +Below are all exceptions. + +.. automodule:: arangoasync.exceptions + :members: diff --git a/docs/specs.rst b/docs/specs.rst index 2de6ae9..b063754 100644 --- a/docs/specs.rst +++ b/docs/specs.rst @@ -34,9 +34,6 @@ python-arango-async. .. automodule:: arangoasync.connection :members: -.. automodule:: arangoasync.exceptions - :members: - .. automodule:: arangoasync.http :members: From 9490914fa9d8010349b2f27c6b28c29ba948e23d Mon Sep 17 00:00:00 2001 From: Alex Petenchea Date: Thu, 24 Apr 2025 20:45:43 +0300 Subject: [PATCH 12/21] Completed compression documentation --- docs/compression.rst | 56 ++++++++++++++++++++++++++++++++++++++++++++ docs/index.rst | 1 + 2 files changed, 57 insertions(+) create mode 100644 docs/compression.rst diff --git a/docs/compression.rst b/docs/compression.rst new file mode 100644 index 0000000..114f83e --- /dev/null +++ b/docs/compression.rst @@ -0,0 +1,56 @@ +Compression +------------ + +The :class:`arangoasync.client.ArangoClient` lets you define the preferred compression policy for request and responses. By default +compression is disabled. You can change this by passing the `compression` parameter when creating the client. You may use +:class:`arangoasync.compression.DefaultCompressionManager` or a custom subclass of :class:`arangoasync.compression.CompressionManager`. + +.. code-block:: python + + from arangoasync import ArangoClient + from arangoasync.compression import DefaultCompressionManager + + client = ArangoClient( + hosts="http://localhost:8529", + compression=DefaultCompressionManager(), + ) + +Furthermore, you can customize the request compression policy by defining the minimum size of the request body that +should be compressed and the desired compression level. Or, in order to explicitly disable compression, you can set the +threshold parameter to -1. + +.. code-block:: python + + from arangoasync import ArangoClient + from arangoasync.compression import DefaultCompressionManager + + # Disable request compression. + client1 = ArangoClient( + hosts="http://localhost:8529", + compression=DefaultCompressionManager(threshold=-1), + ) + + # Enable request compression with a minimum size of 2 KB and a compression level of 8. + client2 = ArangoClient( + hosts="http://localhost:8529", + compression=DefaultCompressionManager(threshold=2048, level=8), + ) + +You can set the `accept` parameter in order to inform the server that the client prefers compressed responses (in the form +of an *Accept-Encoding* header). By default the `DefaultCompressionManager` is configured to accept responses compressed using +the *deflate* algorithm. Note that the server may or may not honor this preference, depending on how it is +configured. This can be controlled by setting the `--http.compress-response-threshold` option to a value greater than 0 +when starting the ArangoDB server. + +.. code-block:: python + + from arangoasync import ArangoClient + from arangoasync.compression import AcceptEncoding, DefaultCompressionManager + + # Accept compressed responses explicitly. + client = ArangoClient( + hosts="http://localhost:8529", + compression=DefaultCompressionManager(accept=AcceptEncoding.DEFLATE), + ) + +See the :class:`arangoasync.compression.CompressionManager` class for more details on how to customize the compression policy. diff --git a/docs/index.rst b/docs/index.rst index dc8c716..2ff092f 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -64,6 +64,7 @@ Contents :maxdepth: 1 cursor + compression errors errno From 1b4cf80b332b339f49820487b380e18c150c432b Mon Sep 17 00:00:00 2001 From: Alex Petenchea Date: Thu, 24 Apr 2025 21:05:21 +0300 Subject: [PATCH 13/21] Completed logging documentation --- arangoasync/connection.py | 3 +++ docs/index.rst | 1 + docs/logging.rst | 30 ++++++++++++++++++++++++++++++ 3 files changed, 34 insertions(+) create mode 100644 docs/logging.rst diff --git a/arangoasync/connection.py b/arangoasync/connection.py index cac1b01..f404248 100644 --- a/arangoasync/connection.py +++ b/arangoasync/connection.py @@ -177,6 +177,9 @@ async def process_request(self, request: Request) -> Response: host_index = self._host_resolver.get_host_index() for tries in range(self._host_resolver.max_tries): try: + logger.debug( + f"Sending request to host {host_index} ({tries}): {request}" + ) resp = await self._http_client.send_request( self._sessions[host_index], request ) diff --git a/docs/index.rst b/docs/index.rst index 2ff092f..1819b25 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -67,6 +67,7 @@ Contents compression errors errno + logging **Development** diff --git a/docs/logging.rst b/docs/logging.rst new file mode 100644 index 0000000..bd7eeb3 --- /dev/null +++ b/docs/logging.rst @@ -0,0 +1,30 @@ +Logging +------- + +If if helps to debug your application, you can enable logging to see all the requests sent by the driver to the ArangoDB server. + +.. code-block:: python + + import logging + from arangoasync import ArangoClient + from arangoasync.auth import Auth + from arangoasync.logger import logger + + # Set up logging + logging.basicConfig(level=logging.DEBUG) + logger.setLevel(level=logging.DEBUG) + + # Initialize the client for ArangoDB. + async with ArangoClient(hosts="http://localhost:8529") as client: + auth = Auth(username="root", password="passwd") + + # Connect to "test" database as root user. + db = await client.db("test", auth=auth) + + # Get the API wrapper for "students" collection. + students = db.collection("students") + + # Insert a document into the collection. + await students.insert({"name": "John Doe", "age": 25}) + +The insert generates a log message similar to: `DEBUG:arangoasync:Sending request to host 0 (0): `. From 09ee6ca4c8a71ecc5ec9776bee3f6a88d10de5f9 Mon Sep 17 00:00:00 2001 From: Alex Petenchea Date: Thu, 24 Apr 2025 21:33:09 +0300 Subject: [PATCH 14/21] Completed helpers documentation --- docs/helpers.rst | 86 ++++++++++++++++++++++++++++++++++++++++++++++++ docs/index.rst | 1 + docs/specs.rst | 3 -- 3 files changed, 87 insertions(+), 3 deletions(-) create mode 100644 docs/helpers.rst diff --git a/docs/helpers.rst b/docs/helpers.rst new file mode 100644 index 0000000..0f9061b --- /dev/null +++ b/docs/helpers.rst @@ -0,0 +1,86 @@ +Helper Types +------------ + +The driver comes with a set of helper types and wrappers to make it easier to work with the ArangoDB API. These are +designed to behave like dictionaries, but with some additional features and methods. See the :class:`arangoasync.typings.JsonWrapper` class for more details. + +**Example:** + +.. code-block:: python + + from arangoasync import ArangoClient + from arangoasync.auth import Auth + from arangoasync.typings import QueryProperties + + # Initialize the client for ArangoDB. + async with ArangoClient(hosts="http://localhost:8529") as client: + auth = Auth(username="root", password="passwd") + + # Connect to "test" database as root user. + db = await client.db("test", auth=auth) + + properties = QueryProperties( + allow_dirty_reads=True, + allow_retry=False, + fail_on_warning=True, + fill_block_cache=False, + full_count=True, + intermediate_commit_count=1000, + intermediate_commit_size=1048576, + max_dnf_condition_members=10, + max_nodes_per_callstack=100, + max_number_of_plans=5, + max_runtime=60.0, + max_transaction_size=10485760, + max_warning_count=10, + optimizer={"rules": ["-all", "+use-indexes"]}, + profile=1, + satellite_sync_wait=10.0, + skip_inaccessible_collections=True, + spill_over_threshold_memory_usage=10485760, + spill_over_threshold_num_rows=100000, + stream=True, + use_plan_cache=True, + ) + + # The types are fully serializable. + print(properties) + + await db.aql.execute( + "FOR doc IN students RETURN doc", + batch_size=1, + options=properties, + ) + +You can easily customize the data representation using formatters. By default, keys are in the format used by the ArangoDB +API, but you can change them to snake_case if you prefer. See :func:`arangoasync.typings.JsonWrapper.format` for more details. + +**Example:** + +.. code-block:: python + + from arangoasync.typings import Json, UserInfo + + data = { + "user": "john", + "password": "secret", + "active": True, + "extra": {"role": "admin"}, + } + user_info = UserInfo(**data) + + def uppercase_formatter(data: Json) -> Json: + result: Json = {} + for key, value in data.items(): + result[key.upper()] = value + return result + + print(user_info.format(uppercase_formatter)) + +Helpers +======= + +Below are all the available helpers. + +.. automodule:: arangoasync.typings + :members: diff --git a/docs/index.rst b/docs/index.rst index 1819b25..9be1875 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -68,6 +68,7 @@ Contents errors errno logging + helpers **Development** diff --git a/docs/specs.rst b/docs/specs.rst index b063754..dc92bd9 100644 --- a/docs/specs.rst +++ b/docs/specs.rst @@ -46,8 +46,5 @@ python-arango-async. .. automodule:: arangoasync.response :members: -.. automodule:: arangoasync.typings - :members: - .. automodule:: arangoasync.result :members: From 6ab77cccbf2b40f93164b20892d409618f8b18b5 Mon Sep 17 00:00:00 2001 From: Alex Petenchea Date: Thu, 24 Apr 2025 21:38:34 +0300 Subject: [PATCH 15/21] Fixing test and lint --- arangoasync/collection.py | 4 ++-- arangoasync/database.py | 4 ---- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/arangoasync/collection.py b/arangoasync/collection.py index 551c2c9..3b4e5a9 100644 --- a/arangoasync/collection.py +++ b/arangoasync/collection.py @@ -281,8 +281,8 @@ def deserializer(self) -> Deserializer[Json, Jsons]: async def indexes( self, - with_stats: Optional[bool], - with_hidden: Optional[bool], + with_stats: Optional[bool] = None, + with_hidden: Optional[bool] = None, ) -> Result[List[IndexProperties]]: """Fetch all index descriptions for the given collection. diff --git a/arangoasync/database.py b/arangoasync/database.py index 3022cc4..e1200df 100644 --- a/arangoasync/database.py +++ b/arangoasync/database.py @@ -596,7 +596,6 @@ async def create_collection( ) def response_handler(resp: Response) -> StandardCollection[T, U, V]: - nonlocal doc_serializer, doc_deserializer if not resp.is_success: raise CollectionCreateError(resp, request) if doc_serializer is None: @@ -648,7 +647,6 @@ async def delete_collection( ) def response_handler(resp: Response) -> bool: - nonlocal ignore_missing if resp.is_success: return True if resp.status_code == HTTP_NOT_FOUND and ignore_missing: @@ -1001,7 +999,6 @@ async def update_permission( ) def response_handler(resp: Response) -> bool: - nonlocal ignore_failure if resp.is_success: return True if ignore_failure: @@ -1046,7 +1043,6 @@ async def reset_permission( ) def response_handler(resp: Response) -> bool: - nonlocal ignore_failure if resp.is_success: return True if ignore_failure: From c03acae24fb4c4d94d41a4b17bf43ab3b1660f6b Mon Sep 17 00:00:00 2001 From: Alex Petenchea Date: Thu, 8 May 2025 10:45:46 +0000 Subject: [PATCH 16/21] Added authentication documentation --- docs/authentication.rst | 116 ++++++++++++++++++++++++++++++++++++++++ docs/index.rst | 1 + tests/test_client.py | 18 +++---- 3 files changed, 126 insertions(+), 9 deletions(-) create mode 100644 docs/authentication.rst diff --git a/docs/authentication.rst b/docs/authentication.rst new file mode 100644 index 0000000..275408c --- /dev/null +++ b/docs/authentication.rst @@ -0,0 +1,116 @@ +Authentication +-------------- + +Two HTTP authentication methods are supported out of the box: +- basic username and password authentication +- JSON Web Tokens (JWT) + +Basic Authentication +==================== + +This is the default authentication method. + +**Example:** + +.. code-block:: python + + from arangoasync import ArangoClient + from arangoasync.auth import Auth + + # Initialize the client for ArangoDB. + async with ArangoClient(hosts="http://localhost:8529") as client: + auth = Auth( + username="root", + password="passwd", + encoding="utf-8" # Encoding for the password, default is utf-8. + ) + + # Connect to "test" database as root user. + db = await client.db( + "test", # database name + auth_method="basic", # use basic authentication (default) + auth=auth, # authentication details + verify=True, # verify the connection (optional) + ) + +JSON Web Tokens (JWT) +===================== + +You can obtain the JWT token from the use server using username and password. +Upon expiration, the token gets refreshed automatically and requests are retried. +The client and server clocks must be synchronized for the automatic refresh +to work correctly. + +**Example:** + +.. code-block:: python + + from arangoasync import ArangoClient + from arangoasync.auth import Auth + + # Initialize the client for ArangoDB. + async with ArangoClient(hosts="http://localhost:8529") as client: + auth = Auth(username="root", password="passwd") + + # Successful authentication with auth only + db = await client.db( + "test", + auth_method="jwt", + auth=auth, + verify=True, + ) + + # Now you have the token on hand. + token = db.connection.token + + # You can use the token directly. + db = await client.db("test", auth_method="jwt", token=token, verify=True) + + # In order to allow the token to be automatically refreshed, you should use both auth and token. + db = await client.db( + "test", + auth_method="jwt", + auth=auth, + token=token, + verify=True, + ) + + # Force a token refresh. + await db.connection.refresh_token() + new_token = db.connection.token + + # Log in with the first token. + db2 = await client.db( + "test", + auth_method="jwt", + token=token, + verify=True, + ) + + # You can manually set tokens. + db2.connection.token = new_token + await db2.connection.ping() + + +If you configured a superuser token, you don't need to provide any credentials. + +**Example:** + +.. code-block:: python + + from arangoasync import ArangoClient + from arangoasync.auth import JwtToken + + # Initialize the client for ArangoDB. + async with ArangoClient(hosts="http://localhost:8529") as client: + + # Generate a JWT token for authentication. You must know the "secret". + token = JwtToken.generate_token("secret") + + # Superuser authentication, no need for the auth parameter. + db = await client.db( + "test", + auth_method="superuser", + token=token, + verify=True + ) diff --git a/docs/index.rst b/docs/index.rst index 9be1875..ebbf791 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -64,6 +64,7 @@ Contents :maxdepth: 1 cursor + authentication compression errors errno diff --git a/tests/test_client.py b/tests/test_client.py index 718d307..6210412 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -102,15 +102,15 @@ async def test_client_jwt_auth(url, sys_db_name, basic_auth_root): async with ArangoClient(hosts=url) as client: await client.db(sys_db_name, auth_method="jwt", token=token, verify=True) - # successful authentication with both - async with ArangoClient(hosts=url) as client: - await client.db( - sys_db_name, - auth_method="jwt", - auth=basic_auth_root, - token=token, - verify=True, - ) + # successful authentication with both + async with ArangoClient(hosts=url) as client: + await client.db( + sys_db_name, + auth_method="jwt", + auth=basic_auth_root, + token=token, + verify=True, + ) # auth and token missing async with ArangoClient(hosts=url) as client: From de8c702f865cc1f8af315e09c782d350f73d4600 Mon Sep 17 00:00:00 2001 From: Alex Petenchea Date: Thu, 8 May 2025 10:49:49 +0000 Subject: [PATCH 17/21] Minor fixes --- docs/authentication.rst | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/docs/authentication.rst b/docs/authentication.rst index 275408c..b7dff45 100644 --- a/docs/authentication.rst +++ b/docs/authentication.rst @@ -2,8 +2,9 @@ Authentication -------------- Two HTTP authentication methods are supported out of the box: -- basic username and password authentication -- JSON Web Tokens (JWT) + +1. Basic username and password authentication +2. JSON Web Tokens (JWT) Basic Authentication ==================== @@ -27,10 +28,10 @@ This is the default authentication method. # Connect to "test" database as root user. db = await client.db( - "test", # database name + "test", # database name auth_method="basic", # use basic authentication (default) - auth=auth, # authentication details - verify=True, # verify the connection (optional) + auth=auth, # authentication details + verify=True, # verify the connection (optional) ) JSON Web Tokens (JWT) From b152d7f80021734e8cec1209b209d5ae6aea17ed Mon Sep 17 00:00:00 2001 From: Alex Petenchea Date: Thu, 8 May 2025 16:30:55 +0000 Subject: [PATCH 18/21] TLS docs --- docs/certificates.rst | 110 ++++++++++++++++++++++++++++++++++++++++++ docs/http.rst | 4 ++ docs/index.rst | 2 + 3 files changed, 116 insertions(+) create mode 100644 docs/certificates.rst create mode 100644 docs/http.rst diff --git a/docs/certificates.rst b/docs/certificates.rst new file mode 100644 index 0000000..c0665fa --- /dev/null +++ b/docs/certificates.rst @@ -0,0 +1,110 @@ +TLS +--- + +When you need fine-grained control over TLS settings, you build a Python +:class:`ssl.SSLContext` and hand it to the :class:`arangoasync.http.DefaultHTTPClient` class. +Here are the most common patterns. + + +Basic client-side HTTPS with default settings +============================================= + +Create a “secure by default” client context. This will verify server certificates against your +OS trust store and check hostnames. + +**Example:** + +.. code-block:: python + + from arangoasync import ArangoClient + from arangoasync.auth import Auth + from arangoasync.http import DefaultHTTPClient + import ssl + + # Create a default client context. + ssl_ctx = ssl.create_default_context(purpose=ssl.Purpose.SERVER_AUTH) + http_client = DefaultHTTPClient(ssl_context=ssl_ctx) + + # Initialize the client for ArangoDB. + client = ArangoClient( + hosts="https://localhost:8529", + http_client=http_client, + ) + +Custom CA bundle +================ + +If you have a custom CA file, this allows you to trust the private CA. + +**Example:** + +.. code-block:: python + + from arangoasync import ArangoClient + from arangoasync.auth import Auth + from arangoasync.http import DefaultHTTPClient + import ssl + + # Use a custom CA bundle. + ssl_ctx = ssl.create_default_context(cafile="path/to/ca.pem") + http_client = DefaultHTTPClient(ssl_context=ssl_ctx) + + # Initialize the client for ArangoDB. + client = ArangoClient( + hosts="https://localhost:8529", + http_client=http_client, + ) + +Disabling certificate verification +================================== + +If you want to disable *all* certification checks (not recommended), create an unverified +context. + +**Example:** + +.. code-block:: python + + from arangoasync import ArangoClient + from arangoasync.auth import Auth + from arangoasync.http import DefaultHTTPClient + import ssl + + # Disable certificate verification. + ssl_ctx = ssl._create_unverified_context() + http_client = DefaultHTTPClient(ssl_context=ssl_ctx) + + # Initialize the client for ArangoDB. + client = ArangoClient( + hosts="https://localhost:8529", + http_client=http_client, + ) + +Use a client certificate chain +============================== + +**Example:** + +.. code-block:: python + + from arangoasync import ArangoClient + from arangoasync.auth import Auth + from arangoasync.http import DefaultHTTPClient + import ssl + + # Load a certificate chain. + ssl_ctx = ssl.create_default_context(cafile="path/to/ca.pem") + ssl_ctx.load_cert_chain(certfile="path/to/cert.pem", keyfile="path/to/key.pem") + http_client = DefaultHTTPClient(ssl_context=ssl_ctx) + + # Initialize the client for ArangoDB. + client = ArangoClient( + hosts="https://localhost:8529", + http_client=http_client, + ) + +.. note:: + For best performance, re-use one SSLContext across many requests/sessions to amortize handshake cost. + +If you want to have fine-grained control over the HTTP connection, you should define +your HTTP client as described in the :ref:`HTTP` section. diff --git a/docs/http.rst b/docs/http.rst new file mode 100644 index 0000000..ff374c9 --- /dev/null +++ b/docs/http.rst @@ -0,0 +1,4 @@ +.. _HTTP: + +HTTP +---- diff --git a/docs/index.rst b/docs/index.rst index ebbf791..4649173 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -65,6 +65,8 @@ Contents cursor authentication + http + certificates compression errors errno From 88abb3e5ec71818b85e1196d53a9d2db7d8fb58d Mon Sep 17 00:00:00 2001 From: Alex Petenchea Date: Thu, 8 May 2025 18:13:46 +0000 Subject: [PATCH 19/21] HTTP docs --- arangoasync/client.py | 4 +- arangoasync/http.py | 22 +++++++ docs/http.rst | 132 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 157 insertions(+), 1 deletion(-) diff --git a/arangoasync/client.py b/arangoasync/client.py index 1b1159f..235cfae 100644 --- a/arangoasync/client.py +++ b/arangoasync/client.py @@ -139,7 +139,9 @@ def version(self) -> str: async def close(self) -> None: """Close HTTP sessions.""" - await asyncio.gather(*(session.close() for session in self._sessions)) + await asyncio.gather( + *(self._http_client.close_session(session) for session in self._sessions) + ) async def db( self, diff --git a/arangoasync/http.py b/arangoasync/http.py index 02b88da..7fb4724 100644 --- a/arangoasync/http.py +++ b/arangoasync/http.py @@ -33,6 +33,8 @@ class HTTPClient(ABC): # pragma: no cover class MyCustomHTTPClient(HTTPClient): def create_session(self, host): pass + async def close_session(self, session): + pass async def send_request(self, session, request): pass """ @@ -52,6 +54,18 @@ def create_session(self, host: str) -> Any: """ raise NotImplementedError + @abstractmethod + async def close_session(self, session: Any) -> None: + """Close the session. + + Note: + This method must be overridden by the user. + + Args: + session (Any): Client session object. + """ + raise NotImplementedError + @abstractmethod async def send_request( self, @@ -129,6 +143,14 @@ def create_session(self, host: str) -> ClientSession: read_bufsize=self._read_bufsize, ) + async def close_session(self, session: ClientSession) -> None: + """Close the session. + + Args: + session (Any): Client session object. + """ + await session.close() + async def send_request( self, session: ClientSession, diff --git a/docs/http.rst b/docs/http.rst index ff374c9..53a5480 100644 --- a/docs/http.rst +++ b/docs/http.rst @@ -2,3 +2,135 @@ HTTP ---- + +You can define your own HTTP client for sending requests to +ArangoDB server. The default implementation uses the aiohttp_ library. + +Your HTTP client must inherit :class:`arangoasync.http.HTTPClient` and implement the +following abstract methods: + +* :func:`arangoasync.http.HTTPClient.create_session` +* :func:`arangoasync.http.HTTPClient.close_session` +* :func:`arangoasync.http.HTTPClient.send_request` + +Let's take for example, the default implementation of :class:`arangoasync.http.AioHTTPClient`: + +* The **create_session** method returns a :class:`aiohttp.ClientSession` instance per + connected host (coordinator). The session objects are stored in the client. +* The **close_session** method performs the necessary cleanup for a :class:`aiohttp.ClientSession` instance. + This is usually called only by the client. +* The **send_request** method must uses the session to send an HTTP request, and + returns a fully populated instance of :class:`arangoasync.response.Response`. + +**Example:** + +Suppose you're working on a project that uses httpx_ as a dependency and you want your +own HTTP client implementation on top of :class:`httpx.AsyncClient`. Your ``HttpxHTTPClient`` +class might look something like this: + +.. code-block:: python + + import httpx + import ssl + from typing import Any, Optional + from arangoasync.exceptions import ClientConnectionError + from arangoasync.http import HTTPClient + from arangoasync.request import Request + from arangoasync.response import Response + + class HttpxHTTPClient(HTTPClient): + """HTTP client implementation on top of httpx.AsyncClient. + + Args: + limits (httpx.Limits | None): Connection pool limits.n + timeout (httpx.Timeout | float | None): Request timeout settings. + ssl_context (ssl.SSLContext | bool): SSL validation mode. + `True` (default) uses httpx’s default validation (system CAs). + `False` disables SSL checks. + Or pass a custom `ssl.SSLContext`. + """ + + def __init__( + self, + limits: Optional[httpx.Limits] = None, + timeout: Optional[httpx.Timeout | float] = None, + ssl_context: bool | ssl.SSLContext = True, + ) -> None: + self._limits = limits or httpx.Limits( + max_connections=100, + max_keepalive_connections=20 + ) + self._timeout = timeout or httpx.Timeout(300.0, connect=60.0) + if ssl_context is True: + self._verify: bool | ssl.SSLContext = True + elif ssl_context is False: + self._verify = False + else: + self._verify = ssl_context + + def create_session(self, host: str) -> httpx.AsyncClient: + return httpx.AsyncClient( + base_url=host, + limits=self._limits, + timeout=self._timeout, + verify=self._verify, + ) + + async def close_session(self, session: httpx.AsyncClient) -> None: + await session.aclose() + + async def send_request( + self, + session: httpx.AsyncClient, + request: Request, + ) -> Response: + auth: Any = None + if request.auth is not None: + auth = httpx.BasicAuth( + username=request.auth.username, + password=request.auth.password, + ) + + try: + resp = await session.request( + method=request.method.name, + url=request.endpoint, + headers=request.normalized_headers(), + params=request.normalized_params(), + content=request.data, + auth=auth, + ) + raw_body = resp.content + return Response( + method=request.method, + url=str(resp.url), + headers=resp.headers, + status_code=resp.status_code, + status_text=resp.reason_phrase, + raw_body=raw_body, + ) + except httpx.HTTPError as e: + raise ClientConnectionError(str(e)) from e + +Then you would inject your client as follows: + +.. code-block:: python + + from arangoasync import ArangoClient + from arangoasync.auth import Auth + + # Initialize the client for ArangoDB. + async with ArangoClient( + hosts="http://localhost:8529", + http_client=HttpxHTTPClient(), + ) as client: + auth = Auth(username="root", password="passwd") + + # Connect to "test" database as root user. + db = await client.db("test", auth=auth, verify=True) + + # List all collections. + cols = await db.collections() + +.. _aiohttp: https://docs.aiohttp.org/en/stable/ +.. _httpx: https://www.python-httpx.org/ From 260fd4a0c3d230a40079f0edebdd2f4eea5dde24 Mon Sep 17 00:00:00 2001 From: Alex Petenchea Date: Fri, 9 May 2025 15:35:22 +0000 Subject: [PATCH 20/21] Serialization docs --- docs/index.rst | 1 + docs/serialization.rst | 181 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 182 insertions(+) create mode 100644 docs/serialization.rst diff --git a/docs/index.rst b/docs/index.rst index 4649173..46601d5 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -68,6 +68,7 @@ Contents http certificates compression + serialization errors errno logging diff --git a/docs/serialization.rst b/docs/serialization.rst new file mode 100644 index 0000000..532ebf7 --- /dev/null +++ b/docs/serialization.rst @@ -0,0 +1,181 @@ +Serialization +------------- + +There are two serialization mechanisms employed by the driver: + +* JSON serialization/deserialization +* Document serialization/deserialization + +All serializers must inherit from the :class:`arangoasync.serialization.Serializer` class. They must +implement a :func:`arangoasync.serialization.Serializer.dumps` method can handle both +single objects and sequences. + +Deserializers mush inherit from the :class:`arangoasync.serialization.Deserializer` class. These have +two methods, :func:`arangoasync.serialization.Deserializer.loads` and :func:`arangoasync.serialization.Deserializer.loads_many`, +which must handle loading of a single document and multiple documents, respectively. + +JSON +==== + +Usually there's no need to implement your own JSON serializer/deserializer, but such an +implementation could look like the following. + +**Example:** + +.. code-block:: python + + import json + from typing import Sequence, cast + from arangoasync.collection import StandardCollection + from arangoasync.database import StandardDatabase + from arangoasync.exceptions import DeserializationError, SerializationError + from arangoasync.serialization import Serializer, Deserializer + from arangoasync.typings import Json, Jsons + + + class CustomJsonSerializer(Serializer[Json]): + def dumps(self, data: Json | Sequence[str | Json]) -> str: + try: + return json.dumps(data, separators=(",", ":")) + except Exception as e: + raise SerializationError("Failed to serialize data to JSON.") from e + + + class CustomJsonDeserializer(Deserializer[Json, Jsons]): + def loads(self, data: bytes) -> Json: + try: + return json.loads(data) # type: ignore[no-any-return] + except Exception as e: + raise DeserializationError("Failed to deserialize data from JSON.") from e + + def loads_many(self, data: bytes) -> Jsons: + return self.loads(data) # type: ignore[return-value] + +You would then use the custom serializer/deserializer when creating a client: + +.. code-block:: python + + from arangoasync import ArangoClient + from arangoasync.auth import Auth + + # Initialize the client for ArangoDB. + async with ArangoClient( + hosts="http://localhost:8529", + serializer=CustomJsonSerializer(), + deserializer=CustomJsonDeserializer(), + ) as client: + auth = Auth(username="root", password="passwd") + + # Connect to "test" database as root user. + test = await client.db("test", auth=auth) + +Documents +========= + +By default, the JSON serializer/deserializer is used for documents too, but you can provide your own +document serializer and deserializer for fine-grained control over the format of a collection. Say +that you are modeling your students data using Pydantic_. You want to be able to insert documents +of a certain type, and also be able to read them back. More so, you would like to get multiple documents +back using one of the formats provided by pandas_. + +**Example:** + +.. code-block:: python + + import json + import pandas as pd + import pydantic + import pydantic_core + from typing import Sequence, cast + from arangoasync import ArangoClient + from arangoasync.auth import Auth + from arangoasync.collection import StandardCollection + from arangoasync.database import StandardDatabase + from arangoasync.exceptions import DeserializationError, SerializationError + from arangoasync.serialization import Serializer, Deserializer + from arangoasync.typings import Json, Jsons + + + class Student(pydantic.BaseModel): + name: str + age: int + + + class StudentSerializer(Serializer[Student]): + def dumps(self, data: Student | Sequence[Student | str]) -> str: + try: + if isinstance(data, Student): + return data.model_dump_json() + else: + # You are required to support both str and Student types. + serialized_data = [] + for student in data: + if isinstance(student, str): + serialized_data.append(student) + else: + serialized_data.append(student.model_dump()) + return json.dumps(serialized_data, separators=(",", ":")) + except Exception as e: + raise SerializationError("Failed to serialize data.") from e + + + class StudentDeserializer(Deserializer[Student, pd.DataFrame]): + def loads(self, data: bytes) -> Student: + # Load a single document. + try: + return Student.model_validate(pydantic_core.from_json(data)) + except Exception as e: + raise DeserializationError("Failed to deserialize data.") from e + + def loads_many(self, data: bytes) -> pd.DataFrame: + # Load multiple documents. + return pd.DataFrame(json.loads(data)) + +You would then use the custom serializer/deserializer when working with collections: + +**Example:** + +.. code-block:: python + + async def main(): + # Initialize the client for ArangoDB. + async with ArangoClient( + hosts="http://localhost:8529", + serializer=CustomJsonSerializer(), + deserializer=CustomJsonDeserializer(), + ) as client: + auth = Auth(username="root", password="passwd") + + # Connect to "test" database as root user. + db: StandardDatabase = await client.db("test", auth=auth, verify=True) + + # Populate the "students" collection. + col = cast( + StandardCollection[Student, Student, pd.DataFrame], + db.collection( + "students", + doc_serializer=StudentSerializer(), + doc_deserializer=StudentDeserializer()), + ) + + # Insert one document. + doc = cast(Json, await col.insert(Student(name="John Doe", age=20))) + + # Insert multiple documents. + docs = cast(Jsons, await col.insert_many([ + Student(name="Jane Doe", age=22), + Student(name="Alice Smith", age=19), + Student(name="Bob Johnson", age=21), + ])) + + # Get one document. + john = await col.get(doc) + assert type(john) == Student + + # Get multiple documents. + keys = [doc["_key"] for doc in docs] + students = await col.get_many(keys) + assert type(students) == pd.DataFrame + +.. _Pydantic: https://docs.pydantic.dev/latest/ +.. _pandas: https://pandas.pydata.org/ From f5c1c50df17ba84faeca4cf4a3d3235ae1f38888 Mon Sep 17 00:00:00 2001 From: Alex Petenchea Date: Fri, 9 May 2025 18:55:06 +0000 Subject: [PATCH 21/21] Migration docs --- docs/helpers.rst | 2 + docs/index.rst | 1 + docs/migration.rst | 94 ++++++++++++++++++++++++++++++++++++++++++ docs/serialization.rst | 2 + 4 files changed, 99 insertions(+) create mode 100644 docs/migration.rst diff --git a/docs/helpers.rst b/docs/helpers.rst index 0f9061b..e16fe0c 100644 --- a/docs/helpers.rst +++ b/docs/helpers.rst @@ -1,3 +1,5 @@ +.. _Helpers: + Helper Types ------------ diff --git a/docs/index.rst b/docs/index.rst index 46601d5..3252629 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -73,6 +73,7 @@ Contents errno logging helpers + migration **Development** diff --git a/docs/migration.rst b/docs/migration.rst new file mode 100644 index 0000000..f26e7d6 --- /dev/null +++ b/docs/migration.rst @@ -0,0 +1,94 @@ +Coming from python-arango +------------------------- + +Generally, migrating from `python-arango`_ should be a smooth transition. For the most part, the API is similar, +but there are a few things to note._ + +Helpers +======= + +The current driver comes with :ref:`Helpers`, because we want to: + +1. Facilitate better type hinting and auto-completion in IDEs. +2. Ensure an easier 1-to-1 mapping of the ArangoDB API. + +For example, coming from the synchronous driver, creating a new user looks like this: + +.. code-block:: python + + sys_db.create_user( + username="johndoe@gmail.com", + password="first_password", + active=True, + extra={"team": "backend", "title": "engineer"} + ) + +In the asynchronous driver, it looks like this: + +.. code-block:: python + + from arangoasync.typings import UserInfo + + user_info = UserInfo( + username="johndoe@gmail.com", + password="first_password", + active=True, + extra={"team": "backend", "title": "engineer"} + ) + await sys_db.create_user(user_info) + +CamelCase vs. snake_case +======================== + +Upon returning results, for the most part, the synchronous driver mostly tries to stick to snake case. Unfortunately, +this is not always consistent. + +.. code-block:: python + + status = db.status() + assert "host" in status + assert "operation_mode" in status + +The asynchronous driver, however, tries to stick to a simple rule: + +* If the API returns a camel case key, it will be returned as is. +* Parameters passed from client to server use the snake case equivalent of the camel case keys required by the API + (e.g. `userName` becomes `user_name`). This is done to ensure PEP8 compatibility. + +.. code-block:: python + + from arangoasync.typings import ServerStatusInformation + + status: ServerStatusInformation = await db.status() + assert "host" in status + assert "operationMode" in status + print(status.host) + print(status.operation_mode) + +You can use the :func:`arangoasync.typings.JsonWrapper.format` method to gain more control over the formatting of +keys. + +Serialization +============= + +Check out the :ref:`Serialization` section to learn more about how to implement your own serializer/deserializer. The +current driver makes use of generic types and allows for a higher degree of customization. + +Mixing sync and async +===================== + +Sometimes you may need to mix the two. This is not recommended, but it takes time to migrate everything. If you need to +do this, you can use the :func:`asyncio.to_thread` function to run a synchronous function in separate thread, without +compromising the async event loop. + +.. code-block:: python + + # Use a python-arango synchronous client + sys_db = await asyncio.to_thread( + client.db, + "_system", + username="root", + password="passwd" + ) + +.. _python-arango: https://docs.python-arango.com diff --git a/docs/serialization.rst b/docs/serialization.rst index 532ebf7..1866ee3 100644 --- a/docs/serialization.rst +++ b/docs/serialization.rst @@ -1,3 +1,5 @@ +.. _Serialization: + Serialization -------------