diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml new file mode 100644 index 0000000..02cd719 --- /dev/null +++ b/.github/FUNDING.yml @@ -0,0 +1 @@ +github: [londogard] diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..3c69169 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,61 @@ +name: CI + +on: + push: + pull_request: + +jobs: + test-and-build: + runs-on: ubuntu-latest + + steps: + - name: Check out repository + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Set up uv + uses: astral-sh/setup-uv@v4 + + - name: Install dependencies + run: uv sync --group dev + + - name: Reject macOS junk files + run: | + if git ls-files | grep -E '(^__MACOSX/|(^|/)\.DS_Store$|(^|/)\._)'; then + echo "Remove macOS archive/junk files before merging." + exit 1 + fi + + - name: Run non-integration tests + run: uv run pytest tests -m "not integration" + + - name: Smoke-test CLI help + run: uv run fluxel --help + + - name: Build package + run: uv build + + s3-integration: + runs-on: ubuntu-latest + + steps: + - name: Check out repository + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Set up uv + uses: astral-sh/setup-uv@v4 + + - name: Install dependencies + run: uv sync --group dev + + - name: Run S3 integration tests against MiniStack + run: bash scripts/run_s3_integration.sh diff --git a/.gitignore b/.gitignore index 68bc17f..5fc46bd 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,9 @@ __pycache__/ *.py[cod] *$py.class +.DS_Store +._* +__MACOSX/ # C extensions *.so diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..b9fec50 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,27 @@ +# Changelog + +All notable changes to this project will be documented in this file. + +The format is based on Keep a Changelog, and Fluxel currently tracks changes before its first public beta release. + +## Unreleased + +### Added + +- Streaming S3 import with metadata identity mode and repeatable path filters. +- Manifest sidecar indexes for exact-path and prefix-based lookups. +- Staged add support for arbitrary local files, directories, S3 objects, and S3 prefixes. +- Real S3 integration coverage for remote repository flows. +- AGPL-3.0-or-later licensing, attribution notice, and funding metadata for the first public beta. + +### Changed + +- CLI examples and tests now prefer `--repo` repository selection semantics. +- Client-local state writes now use atomic replace semantics for HEAD and staging payloads. +- Manifest index prefix iteration now streams rows instead of materializing full result sets. + +### Fixed + +- Manifest parsing now validates entry shape, digests, and metadata-only invariants with line-aware errors. +- CLI commands now return clean `... error:` messages for common validation, filesystem, and object-storage failures instead of raw tracebacks. +- Corrupt S3-hosted manifest lines now surface actionable validation errors. \ No newline at end of file diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..b9e1b0e --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,49 @@ +# Contributing + +## Development Setup + +Fluxel uses `uv` for dependency management and local commands. + +```bash +uv sync --group dev +uv run fluxel --help +``` + +## Local Validation + +Run the full local suite: + +```bash +uv run pytest tests +``` + +Run only the real S3 integration tests against Ministack: + +```bash +export FLUXEL_MINISTACK_ENDPOINT=http://127.0.0.1:9000 +export FLUXEL_MINISTACK_ACCESS_KEY=ministack +export FLUXEL_MINISTACK_SECRET_KEY=ministack123 +export FLUXEL_MINISTACK_REGION=us-east-1 + +uv run pytest tests/test_s3_integration.py -m integration +``` + +If `FLUXEL_MINISTACK_ENDPOINT` is unset or unreachable, the integration tests skip automatically. + +## Project Expectations + +- Fluxel is licensed under AGPL-3.0-or-later; preserve the license and notice files in redistributions. +- Prefer Python type hints by default. +- Keep Fluxel client-first: no server, daemon, or central database. +- Keep canonical blob storage simple and immutable. +- Metadata-only operations must not read blob payloads. +- Prefer stream-safe, O(1)-memory patterns when working with manifests and large imports. +- Use Blake3 for Fluxel content and identity hashing. + +## Pull Requests + +- Keep changes focused and explain user-facing behavior changes clearly. +- Add or update tests for every functional change. +- Update [README.md](README.md) and [CHANGELOG.md](CHANGELOG.md) when the CLI, packaging, or documented workflows change. +- Keep [NOTICE](NOTICE) aligned with project attribution and sponsorship guidance. +- Prefer `--repo` in docs and examples for repository selection. \ No newline at end of file diff --git a/ISSUES.MD b/ISSUES.MD index 6be8cb3..da19e61 100644 --- a/ISSUES.MD +++ b/ISSUES.MD @@ -87,7 +87,7 @@ Implement `fluxel log` using commit parent links only. - Optional JSON output mode. **Acceptance Criteria** -- `fluxel log --root main` prints deterministic history. +- `fluxel log --repo main` prints deterministic history. - No blob reads occur during history traversal. - Unit tests cover empty/single/multi-commit chains. diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..be3f7b2 --- /dev/null +++ b/LICENSE @@ -0,0 +1,661 @@ + GNU AFFERO GENERAL PUBLIC LICENSE + Version 3, 19 November 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU Affero General Public License is a free, copyleft license for +software and other kinds of works, specifically designed to ensure +cooperation with the community in the case of network server software. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +our General Public Licenses are intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + Developers that use our General Public Licenses protect your rights +with two steps: (1) assert copyright on the software, and (2) offer +you this License which gives you legal permission to copy, distribute +and/or modify the software. + + A secondary benefit of defending all users' freedom is that +improvements made in alternate versions of the program, if they +receive widespread use, become available for other developers to +incorporate. Many developers of free software are heartened and +encouraged by the resulting cooperation. However, in the case of +software used on network servers, this result may fail to come about. +The GNU General Public License permits making a modified version and +letting the public access it on a server without ever releasing its +source code to the public. + + The GNU Affero General Public License is designed specifically to +ensure that, in such cases, the modified source code becomes available +to the community. It requires the operator of a network server to +provide the source code of the modified version running there to the +users of that server. Therefore, public use of a modified version, on +a publicly accessible server, gives the public access to the source +code of the modified version. + + An older license, called the Affero General Public License and +published by Affero, was designed to accomplish similar goals. This is +a different license, not a version of the Affero GPL, but Affero has +released a new version of the Affero GPL which permits relicensing under +this license. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU Affero General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Remote Network Interaction; Use with the GNU General Public License. + + Notwithstanding any other provision of this License, if you modify the +Program, your modified version must prominently offer all users +interacting with it remotely through a computer network (if your version +supports such interaction) an opportunity to receive the Corresponding +Source of your version by providing access to the Corresponding Source +from a network server at no charge, through some standard or customary +means of facilitating copying of software. This Corresponding Source +shall include the Corresponding Source for any work covered by version 3 +of the GNU General Public License that is incorporated pursuant to the +following paragraph. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the work with which it is combined will remain governed by version +3 of the GNU General Public License. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU Affero General Public License from time to time. Such new versions +will be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU Affero General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU Affero General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU Affero General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + + If your software can interact with users remotely through a computer +network, you should also make sure that it provides a way for users to +get its source. For example, if your program is a web application, its +interface could display a "Source" link that leads users to an archive +of the code. There are many ways you could offer source, and different +solutions will be better for different programs; see section 13 for the +specific requirements. + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU AGPL, see +. diff --git a/NOTICE b/NOTICE new file mode 100644 index 0000000..1e772e5 --- /dev/null +++ b/NOTICE @@ -0,0 +1,10 @@ +Fluxel +Copyright (C) 2026 Hampus Londögård + +Fluxel is distributed under the GNU Affero General Public License v3.0 or later. +Please preserve this notice, the LICENSE file, and the project copyright notices +when redistributing Fluxel or modified versions of it. + +Organizations that depend on Fluxel are encouraged to sponsor ongoing +maintenance and development: +https://github.com/sponsors/londogard \ No newline at end of file diff --git a/README.md b/README.md index 93ae5d7..9c3b442 100644 --- a/README.md +++ b/README.md @@ -32,21 +32,27 @@ Fluxel is intentionally in MVP mode. ### Implemented - Commit snapshots over a dataset root (`fluxel commit`). +- Repository URI support via `--repo ` and `open_repository(...)`. +- Streaming S3 imports (`fluxel import`) with `blake3` or metadata identity modes. - Branch-scoped staging workflow (`fluxel add`, `fluxel rm`, `fluxel status`, `fluxel commit --staged`). +- Incremental ingress paths that preserve existing manifest entries while adding only new content metadata/blobs. - Commit identity modes: `blake3` (default) and `meta` (`hash(path+size)`). - Verify command to promote metadata-only entries to canonical blobs (`fluxel verify`). - Zero-copy branch pointers (`fluxel branch`). +- Fast-forward-only branch merge (`fluxel merge`). - Metadata-only diff between refs (`fluxel diff`). +- Metadata-only manifest mutations for committed refs (`fluxel rm -m ...`, `fluxel mv -m ... ...`). - Disposable analytical index from manifest (`fluxel index build/query/drop`, DuckDB + optional Parquet export). - `fsspec` provider for `fluxel://` URI reads. - Local + S3 storage backend abstractions available in code. ### Not Fully Wired Yet -- Repository commands still use local-path flow as primary execution path. - No remote sync CLI (`push/pull/fetch`) yet. - No `log/status/list/checkout` CLI surface yet. - No `s5cmd` command-list generation path for bulk transfer yet. +- S3 branch locking now recovers expired stale lock objects automatically, but there is still no operator-facing lock inspection or cleanup command. +- S3 branch locking now recovers expired stale lock objects automatically, but there is still no operator-facing lock inspection or cleanup command. ## Technical Stack @@ -63,33 +69,60 @@ uv sync uv run fluxel --help ``` +## License And Support + +Fluxel is licensed under the GNU Affero General Public License v3.0 or later. + +- The license keeps copyright and license notices attached to redistributed copies. +- Modified networked deployments must make their corresponding source available under the AGPL terms. +- That gives companies a practical reason to fund maintenance if they depend on Fluxel while keeping the project genuinely open source. + +If your company uses Fluxel, sponsor ongoing maintenance at . + ## Quickstart ```bash mkdir -p /tmp/fluxel-demo echo "hello" > /tmp/fluxel-demo/a.txt -uv run fluxel commit --root /tmp/fluxel-demo -m "initial" -uv run fluxel commit --root /tmp/fluxel-demo -m "fast metadata snapshot" --identity meta -uv run fluxel verify --root /tmp/fluxel-demo --ref main +uv run fluxel commit --repo /tmp/fluxel-demo -m "initial" +uv run fluxel commit --repo /tmp/fluxel-demo -m "fast metadata snapshot" --identity meta +uv run fluxel import --repo /tmp/fluxel-demo s3://my-bucket/bootstrap -m "bootstrap import" +uv run fluxel import --repo /tmp/fluxel-demo s3://my-bucket/bootstrap -m "metadata import" --identity meta +uv run fluxel import --repo /tmp/fluxel-demo s3://my-bucket/bootstrap -m "jpg subset" --path "**/*.jpg" --path root.csv +uv run fluxel import --repo /tmp/fluxel-demo s3://my-bucket/incremental -m "add new import batch" +uv run fluxel verify --repo /tmp/fluxel-demo --ref main # branch-scoped staged flow -uv run fluxel branch --root /tmp/fluxel-demo feature -uv run fluxel add --root /tmp/fluxel-demo --ref feature data/new.csv -uv run fluxel status --root /tmp/fluxel-demo --ref feature -uv run fluxel commit --root /tmp/fluxel-demo --ref feature --staged -m "feature updates" +uv run fluxel branch --repo /tmp/fluxel-demo feature +uv run fluxel add --repo /tmp/fluxel-demo --ref feature data/new.csv +uv run fluxel add --repo /tmp/fluxel-demo --ref feature --as imports/raw.csv /tmp/outside-repo/raw.csv +uv run fluxel add --repo /tmp/fluxel-demo --ref feature --as imports/bundle /tmp/outside-repo/bundle +uv run fluxel add --repo /tmp/fluxel-demo --ref feature --identity meta --as imports/bootstrap.csv s3://my-bucket/bootstrap.csv +uv run fluxel add --repo /tmp/fluxel-demo --ref feature --identity meta --as imports/bootstrap s3://my-bucket/bootstrap +uv run fluxel status --repo /tmp/fluxel-demo --ref feature +uv run fluxel commit --repo /tmp/fluxel-demo --ref feature --staged -m "feature updates" +uv run fluxel merge --repo /tmp/fluxel-demo feature main echo "hello v2" > /tmp/fluxel-demo/a.txt -uv run fluxel commit --root /tmp/fluxel-demo -m "update" - -uv run fluxel branch --root /tmp/fluxel-demo experiment -uv run fluxel diff --root /tmp/fluxel-demo +uv run fluxel commit --repo /tmp/fluxel-demo -m "update" + +uv run fluxel branch --repo /tmp/fluxel-demo experiment +uv run fluxel diff --repo /tmp/fluxel-demo +uv run fluxel rm --repo /tmp/fluxel-demo old-prefix -m "remove old files" +uv run fluxel mv --repo /tmp/fluxel-demo raw/images curated/images -m "rename image prefix" + +# remote repo metadata operations from the current working tree +uv run fluxel branch --repo s3://my-bucket/datasets/demo feature +uv run fluxel commit --repo s3://my-bucket/datasets/demo -m "snapshot current working tree" +uv run fluxel rm --repo s3://my-bucket/datasets/demo obsolete -m "drop obsolete paths" +uv run fluxel mv --repo s3://my-bucket/datasets/demo bootstrap final -m "rename imported prefix" ``` ## Analytical Index (Derived, Disposable) ```bash -uv run fluxel index build --root /tmp/fluxel-demo --ref main --parquet +uv run fluxel index build --repo /tmp/fluxel-demo --ref main --parquet uv run fluxel index query --db /path/to/.duckdb --sql "SELECT COUNT(*) FROM files" uv run fluxel index drop --db /path/to/.duckdb ``` @@ -133,9 +166,9 @@ This is useful for large bootstrap imports where strong content verification can `fluxel verify` promotes metadata-only (`--identity meta`) manifest entries into canonical `blake3` blob-backed entries: ```bash -uv run fluxel verify --root /tmp/fluxel-demo --ref main -uv run fluxel verify --root /tmp/fluxel-demo --ref main --path images --path logs/2026 -uv run fluxel verify --root /tmp/fluxel-demo --ref main --dry-run +uv run fluxel verify --repo /tmp/fluxel-demo --ref main +uv run fluxel verify --repo /tmp/fluxel-demo --ref main --path images --path logs/2026 +uv run fluxel verify --repo /tmp/fluxel-demo --ref main --dry-run ``` - Verifies all entries by default (or selected path prefixes with `--path`). @@ -143,6 +176,118 @@ uv run fluxel verify --root /tmp/fluxel-demo --ref main --dry-run - Reads bytes from each entry's `source_uri`, computes Blake3, and stores canonical blob content. - Writes a new commit only when at least one entry is promoted. +## Incremental Ingress + +Fluxel's efficient content-ingress paths are: + +```bash +uv run fluxel add --repo /tmp/fluxel-demo local/new.csv +uv run fluxel add --repo /tmp/fluxel-demo --as imports/new.csv /tmp/random/new.csv +uv run fluxel add --repo /tmp/fluxel-demo --as imports/new-batch /tmp/random/new-batch +uv run fluxel add --repo /tmp/fluxel-demo --identity meta --as imports/bootstrap.csv s3://my-bucket/bootstrap.csv +uv run fluxel add --repo /tmp/fluxel-demo --identity meta --as imports/bootstrap s3://my-bucket/bootstrap +uv run fluxel commit --repo /tmp/fluxel-demo --staged -m "add one file" +uv run fluxel import --repo /tmp/fluxel-demo s3://my-bucket/incremental -m "merge imported batch" +uv run fluxel verify --repo /tmp/fluxel-demo --ref main --path images --path root.txt +``` + +- `add` + `commit --staged` preserves the current branch manifest and reads bytes only for staged additions. +- `add` accepts repo-relative files, arbitrary local files, local directories, single S3 objects, and S3 prefixes; `--as` maps a single file/object to one logical path or remaps a directory/prefix under a destination prefix. +- `import` merges imported S3 entries into the current branch manifest instead of replacing the snapshot. +- `verify` reads bytes only for selected metadata-only entries that still need canonical blobs. +- Existing manifest entries are preserved without re-uploading unchanged blob content. + +## S3 Integration Tests + +Fluxel includes `integration`-marked tests for real S3-compatible behavior. The preferred target is Ministack. + +For the standard local workflow, run a single command from the repository root: + +```bash +bash scripts/run_s3_integration.sh +``` + +That script starts a temporary Ministack container on `127.0.0.1:4566`, waits for the health endpoint, resets emulator state, runs `tests/test_s3_integration.py`, and cleans up the container when the test run finishes. + +If you prefer task-runner aliases, the repo also provides: + +```bash +make test-s3-integration +``` + +GitHub Actions runs the same script in the dedicated S3 integration job. + +Start Ministack locally: + +```bash +docker run --rm -p 4566:4566 nahuelnucera/ministack +``` + +If you also want MiniStack features that launch real sidecar containers such as RDS, ECS, or Docker-backed Lambda runtimes, mount the Docker socket: + +```bash +docker run --rm -p 4566:4566 -v /var/run/docker.sock:/var/run/docker.sock nahuelnucera/ministack +``` + +Verify the emulator is ready: + +```bash +curl http://127.0.0.1:4566/_ministack/health +``` + +Then set these environment variables before running the suite: + +```bash +export FLUXEL_MINISTACK_ENDPOINT=http://127.0.0.1:4566 +export FLUXEL_MINISTACK_ACCESS_KEY=test +export FLUXEL_MINISTACK_SECRET_KEY=test +export FLUXEL_MINISTACK_REGION=us-east-1 +``` + +Fluxel's integration fixture already uses path-style boto3 S3 addressing, so no extra S3 client flags are needed. + +Then run: + +```bash +uv run pytest tests/test_s3_integration.py -m integration +``` + +If `FLUXEL_MINISTACK_ENDPOINT` is unset or the endpoint is unreachable, the integration tests skip automatically. + +To wipe the local emulator state between runs without restarting the container: + +```bash +curl -X POST http://127.0.0.1:4566/_ministack/reset +``` + +## Merge Command + +`fluxel merge` updates a target branch by fast-forward only: + +```bash +uv run fluxel merge --repo /tmp/fluxel-demo feature main +``` + +- The source ref can be a branch or commit. +- The target ref must be a branch. +- The merge succeeds only when the target branch head is an ancestor of the source ref. +- Non-fast-forward merges are rejected. + +## Metadata-Only Remove And Move + +`fluxel rm` and `fluxel mv` can mutate committed refs directly by writing a new manifest and commit: + +```bash +uv run fluxel rm --repo /tmp/fluxel-demo logs/2025 -m "remove old logs" +uv run fluxel mv --repo /tmp/fluxel-demo incoming/images curated/images -m "rename prefix" +uv run fluxel rm --repo s3://my-bucket/datasets/demo temp -m "drop temp data" +``` + +- These operations read manifest metadata only; they do not download unchanged blob payloads. +- `rm` accepts file paths or path prefixes and removes all matching logical entries. +- `mv` accepts a file path or prefix and rewrites matching logical paths in the manifest. +- Existing staged behavior remains available: `fluxel rm` without `-m/--message` still stages removals for `fluxel commit --staged`. + ## Repository Layout Fluxel creates `.fluxel/` under each dataset root: @@ -165,4 +310,4 @@ Run test suite: ```bash uv run pytest tests -``` \ No newline at end of file +``` diff --git a/ROADMAP.md b/ROADMAP.md new file mode 100644 index 0000000..51f4045 --- /dev/null +++ b/ROADMAP.md @@ -0,0 +1,275 @@ +# Fluxel Roadmap + +## Product Direction + +Fluxel should remain client-driven while supporting a shared repository stored in object storage. + +Target operating modes: + +- Local repository backend for development, testing, and small datasets. +- S3 repository backend for shared collaborative datasets. + +The CLI and repository model should stay unified across both modes. The user should interact with one repository concept, with the backend selected by repository URI or path. + +Recommended long-term shape: + +- Shared remote repository state in S3. +- Local client state for staging, current branch preference, caches, and temp files. +- Metadata-first operations for branching, diffing, merging, removing, and renaming. +- Content reads only when a command truly needs object bytes. + +## Decisions + +### Keep Both Local And S3 + +Keep both backends. + +- Local remains useful for tests, development, offline workflows, and small datasets. +- S3 becomes the primary collaboration backend. +- The user-facing abstraction should be one repository concept, not two separate products. + +### Keep Both Identity Modes + +Keep both `blake3` and `meta`. + +- `blake3` stays the default and durable canonical mode. +- `meta` stays the cheap bootstrap/import mode. +- `verify` remains the promotion path from metadata-only entries to canonical blob-backed entries. + +To reduce complexity, `meta` should primarily be used for import/bootstrap flows rather than expanded into every workflow. + +## Architectural Goal + +Move Fluxel from a local-working-tree-first design to a manifest-first repository design. + +For large datasets, operations such as these must not require downloading or rewriting the full dataset: + +- add 3 files +- remove 5 files +- rename `image.jpg` to `image.jpeg` +- merge a fast-forward branch +- bulk path rewrites + +These should become metadata transformations over manifests and refs, plus selective uploads only for genuinely new content. + +## Phase 1: Repository Store Abstraction + +Goal: separate shared repository data from local client state. + +Introduce a repository store abstraction that owns: + +- blobs +- manifests +- commits +- refs + +Operations needed: + +- read commit object +- write commit object +- stream manifest reads +- stream manifest writes +- read branch ref +- compare-and-set branch ref +- read blob bytes +- write blob if missing +- check object existence +- retrieve backend version token or etag where available + +Deliverables: + +- `LocalRepositoryStore` +- `S3RepositoryStore` +- refactor repository code to stop reading and writing repository state directly with `Path` + +Acceptance criteria: + +- Existing local tests still pass. +- Repository mutations no longer depend on direct local `.fluxel` path writes. + +## Phase 2: Local Client State + +Goal: keep user-specific mutable state local even when the repo is remote. + +Keep these local only: + +- active branch preference +- staging state +- temp files +- caches + +Important decision: + +- Do not keep shared remote `HEAD`. +- Shared truth is branch refs under `refs/heads/*`. +- Current branch selection is a client preference. + +Deliverables: + +- local client-state abstraction +- branch preference handling decoupled from shared repo refs +- local staging store independent of repository backend + +Acceptance criteria: + +- Multiple users can share the same S3-backed repo without clobbering each other's active branch preference. + +## Phase 3: Safe Ref Updates On S3 + +Goal: make commits and merges safe for concurrent clients. + +Implement optimistic concurrency for branch refs: + +- read current ref value plus version token or etag +- update ref only if the token still matches +- fail clearly on write conflicts + +Deliverables: + +- compare-and-set branch updates in the repository store +- conflict errors surfaced clearly from commit and merge flows + +Unimplemented note: + +- stale-lock recovery is still missing if a client dies mid-update and leaves behind a branch lock object + +Acceptance criteria: + +- Concurrent updates6 7 do not silently overwrite each other. +- Fast-forward merge remains safe under concurrent clients. + +## Phase 4: Repo URI Support + +Goal: unify local and remote repositories behind one CLI and Python API. + +Introduce a repository argument that can be either: + +- local path +- `s3://bucket/prefix` + +Examples: + +```bash +uv run fluxel --repo /tmp/demo commit -m "local commit" +uv run fluxel --repo s3://my-bucket/datasets/demo branch feature +``` + +Deliverables: + +- repository URI parsing +- backend selection based on repo location +- migration of existing commands to repository-aware `--repo` semantics + +Acceptance criteria: + +- Same command set works for both local and S3-backed repositories. + +## Phase 5: Remote-Native Metadata Operations + +Goal: support metadata changes without local checkout. + +Prioritized commands: + +- `branch` +- `diff` +- `merge` +- `rm` +- `mv` or rename + +These operations should: + +- load manifest metadata only +- transform logical paths or membership +- write a new manifest and commit +- update the target branch ref + +They should not: + +- download unchanged blobs +- reupload unchanged blobs +- depend on a full local working tree + +Acceptance criteria: + +- Removing and renaming paths in a large S3-backed repo can complete without reading full object payloads. + +## Phase 6: Content Ingress Paths + +Goal: support adding new content efficiently without forcing full dataset materialization. + +Supported ingress modes: + +- local file add +- S3 import +- verify selected metadata-only entries into canonical blobs + +Rules: + +- only read bytes for newly added or verified objects +- preserve existing manifest entries without rewriting blob content + +Acceptance criteria: + +- adding a small number of files to a large repo only uploads those new files and metadata + +## Phase 7: S3-Native Integration Tests + +Goal: validate shared-object-store behavior against a real S3-compatible service. + +Preferred test target: + +- Ministack + +Coverage: + +- branch creation +- commit updates +- fast-forward merge +- metadata import +- selective verify +- metadata-only remove and rename +- optimistic concurrency conflicts + +Testing strategy: + +- keep current fake-client tests as fast unit coverage +- add integration-marked tests for real object storage behavior using Ministack + +Acceptance criteria: + +- Core repo flows succeed against a real S3-compatible API, not just mocked boto calls. + +## Phase 8: Optional Path Ergonomics + +Goal: improve URI and path handling without weakening backend correctness. + +`cloudpathlib.AnyPath` may be used as a convenience layer for path or URI ergonomics, but it should not replace the repository store abstraction. + +Reason: + +- Fluxel still needs explicit backend semantics for optimistic locking, conditional writes, etags, and streaming control. + +Recommendation: + +- keep `StorageBackend` or `RepositoryStore` as the core contract +- optionally use `AnyPath` at the edges for parsing and convenience + +## Initial Execution Order + +Recommended implementation order: + +1. Repository store abstraction +2. Local client state split +3. Safe S3 ref updates +4. Repo URI support +5. Remote-native `rm` +6. Remote-native `mv` +7. Ministack integration tests + +## Explicit Non-Goals For Now + +- write-capable `fsspec` interface +- non-fast-forward merge support +- server or daemon architecture +- central database +- remote sync commands as a substitute for repository-native S3 support \ No newline at end of file diff --git a/__MACOSX/src/._.DS_Store b/__MACOSX/src/._.DS_Store deleted file mode 100644 index a5b28df..0000000 Binary files a/__MACOSX/src/._.DS_Store and /dev/null differ diff --git a/__MACOSX/src/fluxel/._.DS_Store b/__MACOSX/src/fluxel/._.DS_Store deleted file mode 100644 index a5b28df..0000000 Binary files a/__MACOSX/src/fluxel/._.DS_Store and /dev/null differ diff --git a/__MACOSX/src/fluxel/.___pycache__ b/__MACOSX/src/fluxel/.___pycache__ deleted file mode 100755 index d053f9a..0000000 Binary files a/__MACOSX/src/fluxel/.___pycache__ and /dev/null differ diff --git a/__MACOSX/src/fluxel/core/.___pycache__ b/__MACOSX/src/fluxel/core/.___pycache__ deleted file mode 100755 index d053f9a..0000000 Binary files a/__MACOSX/src/fluxel/core/.___pycache__ and /dev/null differ diff --git a/__MACOSX/tests/.___pycache__ b/__MACOSX/tests/.___pycache__ deleted file mode 100755 index d053f9a..0000000 Binary files a/__MACOSX/tests/.___pycache__ and /dev/null differ diff --git a/pyproject.toml b/pyproject.toml index 06424c7..0d55344 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,12 +1,31 @@ [project] name = "fluxel" version = "0.1.0" -description = "Add your description here" +description = "A client-first, object-storage-first data versioning engine" readme = "README.md" authors = [ { name = "Hampus Londögård", email = "hampus.londogard@verisure.com" }, ] +license = { file = "LICENSE" } requires-python = ">=3.11" +keywords = [ + "data-versioning", + "dataset-versioning", + "object-storage", + "s3", + "serverless", +] +classifiers = [ + "Development Status :: 4 - Beta", + "Intended Audience :: Developers", + "Intended Audience :: Science/Research", + "Operating System :: OS Independent", + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Topic :: Software Development :: Version Control", +] dependencies = [ "blake3>=1.0.8", "boto3>=1.42.57", @@ -15,6 +34,13 @@ dependencies = [ "simple-parsing>=0.1.7", ] +[project.urls] +Repository = "https://github.com/londogard/strand" +Documentation = "https://github.com/londogard/strand#readme" +Issues = "https://github.com/londogard/strand/issues" +Changelog = "https://github.com/londogard/strand/blob/main/CHANGELOG.md" +Funding = "https://github.com/sponsors/londogard" + [project.scripts] fluxel = "fluxel:main" @@ -24,3 +50,8 @@ build-backend = "hatchling.build" [dependency-groups] dev = ["pytest>=9.0.2"] + +[tool.pytest.ini_options] +markers = [ + "integration: tests that require a real S3-compatible service such as Ministack", +] diff --git a/scripts/run_s3_integration.sh b/scripts/run_s3_integration.sh new file mode 100644 index 0000000..7d530cd --- /dev/null +++ b/scripts/run_s3_integration.sh @@ -0,0 +1,91 @@ +#!/usr/bin/env bash + +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +MINISTACK_IMAGE="${FLUXEL_MINISTACK_IMAGE:-nahuelnucera/ministack}" +MINISTACK_HOST="${FLUXEL_MINISTACK_HOST:-127.0.0.1}" +MINISTACK_PORT="${FLUXEL_MINISTACK_PORT:-4566}" +DEFAULT_ENDPOINT="http://${MINISTACK_HOST}:${MINISTACK_PORT}" +MANAGE_CONTAINER="${FLUXEL_MINISTACK_MANAGE_CONTAINER:-auto}" + +export FLUXEL_MINISTACK_ENDPOINT="${FLUXEL_MINISTACK_ENDPOINT:-$DEFAULT_ENDPOINT}" +export FLUXEL_MINISTACK_ACCESS_KEY="${FLUXEL_MINISTACK_ACCESS_KEY:-test}" +export FLUXEL_MINISTACK_SECRET_KEY="${FLUXEL_MINISTACK_SECRET_KEY:-test}" +export FLUXEL_MINISTACK_REGION="${FLUXEL_MINISTACK_REGION:-us-east-1}" + +HEALTH_URL="${FLUXEL_MINISTACK_ENDPOINT%/}/_ministack/health" +RESET_URL="${FLUXEL_MINISTACK_ENDPOINT%/}/_ministack/reset" + +container_started=0 +container_name="fluxel-ministack-${USER:-user}-$$" + +cleanup() { + if [[ "$container_started" -eq 1 ]]; then + docker rm -f "$container_name" >/dev/null 2>&1 || true + fi +} + +print_docker_daemon_help() { + case "$(uname -s)" in + Darwin) + echo "docker daemon is not running. Start Docker Desktop, Colima, or another local Docker runtime, then rerun this script." >&2 + ;; + Linux) + echo "docker daemon is not running. Start Docker Engine or point docker at a running daemon, then rerun this script." >&2 + ;; + *) + echo "docker daemon is not running. Start your local Docker runtime, then rerun this script." >&2 + ;; + esac +} + +wait_for_ministack() { + local attempt + + for attempt in $(seq 1 30); do + if curl -fsS "$HEALTH_URL" >/dev/null; then + return 0 + fi + sleep 1 + done + + echo "MiniStack did not become healthy at $HEALTH_URL" >&2 + return 1 +} + +start_ministack_if_needed() { + if curl -fsS "$HEALTH_URL" >/dev/null 2>&1; then + return 0 + fi + + if [[ "$MANAGE_CONTAINER" == "0" ]]; then + return 0 + fi + + if [[ "$MANAGE_CONTAINER" == "auto" && "$FLUXEL_MINISTACK_ENDPOINT" != "$DEFAULT_ENDPOINT" ]]; then + return 0 + fi + + if ! command -v docker >/dev/null 2>&1; then + echo "docker is required to start MiniStack locally" >&2 + return 1 + fi + + if ! docker info >/dev/null 2>&1; then + print_docker_daemon_help + return 1 + fi + + docker run -d --rm -p "${MINISTACK_PORT}:4566" --name "$container_name" "$MINISTACK_IMAGE" >/dev/null + container_started=1 +} + +trap cleanup EXIT + +start_ministack_if_needed +wait_for_ministack +curl -fsS -X POST "$RESET_URL" >/dev/null 2>&1 || true + +cd "$ROOT_DIR" +uv run pytest tests/test_s3_integration.py -v -s -m integration "$@" \ No newline at end of file diff --git a/src/.DS_Store b/src/.DS_Store deleted file mode 100644 index 9adee68..0000000 Binary files a/src/.DS_Store and /dev/null differ diff --git a/src/fluxel/.DS_Store b/src/fluxel/.DS_Store deleted file mode 100644 index ec48ac9..0000000 Binary files a/src/fluxel/.DS_Store and /dev/null differ diff --git a/src/fluxel/__init__.py b/src/fluxel/__init__.py index dd78801..9d00326 100644 --- a/src/fluxel/__init__.py +++ b/src/fluxel/__init__.py @@ -2,21 +2,31 @@ from .core import ( DEFAULT_CHUNK_SIZE, AnalyticalIndexPaths, + LocalClientState, CommitObject, DiffEntry, FluxelFileSystem, FluxelLayout, FluxelRepository, FluxelURI, + BranchRefState, + MoveResult, + open_repository, LocalStorageBackend, + LocalRepositoryStore, ManifestEntry, ManifestReader, ManifestWriter, OptimisticLockError, + RepositoryStore, + RefConflictError, + RemoveResult, + S3RepositoryStore, S3StorageBackend, StorageBackend, StageChange, StageStatus, + MergeResult, VerifyResult, add, blake3_digest_file, @@ -28,8 +38,12 @@ commit, diff, drop_analytical_index, + import_s3, initialize_fluxel_layout, + merge, + move, query_analytical_index, + remove, rm, status, verify, @@ -39,20 +53,30 @@ __all__ = [ "DEFAULT_CHUNK_SIZE", "AnalyticalIndexPaths", + "LocalClientState", "CommitObject", "DiffEntry", "FluxelFileSystem", "FluxelLayout", "FluxelRepository", "FluxelURI", + "BranchRefState", + "MoveResult", + "open_repository", + "RefConflictError", + "RemoveResult", "StageChange", "StageStatus", + "MergeResult", "VerifyResult", + "LocalRepositoryStore", "LocalStorageBackend", "ManifestEntry", "ManifestReader", "ManifestWriter", "OptimisticLockError", + "RepositoryStore", + "S3RepositoryStore", "S3StorageBackend", "StorageBackend", "blake3_digest_file", @@ -66,8 +90,12 @@ "commit", "diff", "drop_analytical_index", + "import_s3", "initialize_fluxel_layout", + "merge", + "move", "query_analytical_index", + "remove", "rm", "status", "verify", diff --git a/src/fluxel/cli.py b/src/fluxel/cli.py index d640654..549f740 100644 --- a/src/fluxel/cli.py +++ b/src/fluxel/cli.py @@ -3,19 +3,25 @@ import json import sys from dataclasses import dataclass -from pathlib import Path from typing import Literal +from botocore.exceptions import BotoCoreError, ClientError from simple_parsing import ArgumentParser from simple_parsing.helpers import field, flag, subparsers from .core import ( - FluxelRepository, + RefConflictError, add, + branch, build_analytical_index, commit, + diff, drop_analytical_index, + import_s3, + merge, + move, query_analytical_index, + remove, rm, status, verify, @@ -23,13 +29,28 @@ IdentityMode = Literal["blake3", "meta"] +HANDLED_CLI_ERRORS = ( + BotoCoreError, + ClientError, + FileNotFoundError, + OSError, + PermissionError, + RefConflictError, + ValueError, +) @dataclass class CommitArgs: - identity: IdentityMode = "blake3" # Identity strategy: full-content blake3 or metadata hash(path+size)" message: str = field(alias=["-m", "--message"], help="Commit message") - root: str = "." # "Dataset root path" + identity: IdentityMode = ( + "blake3" # Identity strategy: full-content blake3 or metadata hash(path+size)" + ) + root: str = field( + default=".", + alias="--repo", + help="Repository path or URI", + ) staged: bool = flag( False, help="Commit only staged changes for a branch", @@ -37,11 +58,45 @@ class CommitArgs: ref: str | None = None # Branch ref to update (defaults to current branch) +@dataclass +class ImportArgs: + source: str = field( + positional=True, help="S3 URI to import, e.g. s3://bucket/prefix" + ) + message: str = field(alias=["-m", "--message"], help="Commit message") + identity: IdentityMode = ( + "blake3" # Identity strategy: full-content blake3 or metadata hash(path+size)" + ) + path_patterns: list[str] = field( + default_factory=list, + alias="--path", + action="append", + help="Optional relative path/glob filter (repeatable)", + ) + root: str = field( + default=".", + alias="--repo", + help="Repository path or URI", + ) + ref: str | None = None # Branch ref to update (defaults to current branch) + + @dataclass class AddArgs: + paths: list[str] = field( + positional=True, nargs="+", help="Files, directories, or S3 paths to stage" + ) identity: IdentityMode = "blake3" # Identity strategy for staged additions - paths: list[str] = field(positional=True, nargs="+", help="Paths to stage") - root: str = field(default=".", help="Dataset root path") + destination_path: str | None = field( + default=None, + alias="--as", + help="Logical destination path for a single staged source", + ) + root: str = field( + default=".", + alias="--repo", + help="Repository path or URI", + ) ref: str | None = field( default=None, help="Branch ref for staging (defaults to current branch)", @@ -51,16 +106,52 @@ class AddArgs: @dataclass class RmArgs: paths: list[str] = field(positional=True, nargs="+", help="Paths to remove") - root: str = field(default=".", help="Dataset root path") + message: str | None = field( + default=None, + alias=["-m", "--message"], + help="Commit message for a metadata-only removal", + ) + root: str = field( + default=".", + alias="--repo", + help="Repository path or URI", + ) ref: str | None = field( default=None, help="Branch ref for staging (defaults to current branch)", ) + staged: bool = flag( + False, + help="Stage removals instead of writing a metadata-only commit", + ) + + +@dataclass +class MoveArgs: + source_path: str = field(positional=True, help="Path or prefix to rename") + destination_path: str = field( + positional=True, + help="Destination path or prefix", + ) + message: str = field(alias=["-m", "--message"], help="Commit message") + root: str = field( + default=".", + alias="--repo", + help="Repository path or URI", + ) + ref: str | None = field( + default=None, + help="Branch ref to update (defaults to current branch)", + ) @dataclass class StatusArgs: - root: str = field(default=".", help="Dataset root path") + root: str = field( + default=".", + alias="--repo", + help="Repository path or URI", + ) ref: str | None = field( default=None, help="Branch ref for staging (defaults to current branch)", @@ -70,19 +161,42 @@ class StatusArgs: @dataclass class BranchArgs: name: str = field(positional=True, help="Branch name") - root: str = field(default=".", help="Dataset root path") + root: str = field( + default=".", + alias="--repo", + help="Repository path or URI", + ) @dataclass class DiffArgs: from_ref: str = field(positional=True, help="Source ref (branch or commit)") to_ref: str = field(positional=True, help="Target ref (branch or commit)") - root: str = "." # Dataset root path + root: str = field( + default=".", + alias="--repo", + help="Repository path or URI", + ) + + +@dataclass +class MergeArgs: + source_ref: str = field(positional=True, help="Ref to merge from") + target_ref: str = field(positional=True, help="Branch ref to fast-forward") + root: str = field( + default=".", + alias="--repo", + help="Repository path or URI", + ) @dataclass class VerifyArgs: - root: str = "." # Dataset root path + root: str = field( + default=".", + alias="--repo", + help="Repository path or URI", + ) ref: str = "main" # Branch ref to verify path: list[str] = field( default_factory=list, @@ -99,7 +213,11 @@ class VerifyArgs: @dataclass class IndexBuildArgs: - root: str = "." # Dataset root path + root: str = field( + default=".", + alias="--repo", + help="Repository path or URI", + ) ref: str = "main" # Ref to index output_dir: str | None = field( default=None, @@ -138,21 +256,27 @@ class IndexArgs: class FluxelCLI: command: ( CommitArgs + | ImportArgs | AddArgs | RmArgs + | MoveArgs | StatusArgs | BranchArgs | DiffArgs + | MergeArgs | VerifyArgs | IndexArgs ) = subparsers( { "commit": CommitArgs, + "import": ImportArgs, "add": AddArgs, "rm": RmArgs, + "mv": MoveArgs, "status": StatusArgs, "branch": BranchArgs, "diff": DiffArgs, + "merge": MergeArgs, "verify": VerifyArgs, "index": IndexArgs, } @@ -173,120 +297,255 @@ def _stage_payload(stage: object) -> dict[str, object]: } -def run_cli(argv: list[str] | None = None) -> int: - parser = build_parser() - args = parser.parse_args(argv).cli - command = args.command +def _flatten_option_values(values: list[object]) -> list[str]: + flattened: list[str] = [] + for value in values: + # `simple_parsing` may surface repeated list-valued flags as nested lists + # (e.g. [['**/*.jpg'], ['root.txt']]) for this dataclass field shape. + if isinstance(value, list): + flattened.extend(str(item) for item in value) + continue + flattened.append(str(value)) + return flattened - if isinstance(command, CommitArgs): - commit_id = commit( - Path(command.root), - command.message, - identity_mode=command.identity, - staged=command.staged, - ref=command.ref, - ) - print(commit_id) - return 0 +def _command_name(command: object) -> str: + if isinstance(command, CommitArgs): + return "commit" + if isinstance(command, ImportArgs): + return "import" if isinstance(command, AddArgs): - stage = add( - root=Path(command.root), - paths=command.paths, - ref=command.ref, - identity_mode=command.identity, - ) - print(json.dumps(_stage_payload(stage), indent=2)) - return 0 - + return "add" if isinstance(command, RmArgs): - stage = rm( - root=Path(command.root), - paths=command.paths, - ref=command.ref, - ) - print(json.dumps(_stage_payload(stage), indent=2)) - return 0 - + return "rm" + if isinstance(command, MoveArgs): + return "mv" if isinstance(command, StatusArgs): - stage = status( - root=Path(command.root), - ref=command.ref, - ) - print(json.dumps(_stage_payload(stage), indent=2)) - return 0 - + return "status" if isinstance(command, BranchArgs): - branch_path = FluxelRepository(Path(command.root)).branch(command.name) - print(str(branch_path)) - return 0 - + return "branch" if isinstance(command, DiffArgs): - changes = FluxelRepository(Path(command.root)).diff( - command.from_ref, - command.to_ref, - ) - payload = [ - { - "path": change.path, - "change": change.change, - "before_hash": change.before_hash, - "after_hash": change.after_hash, - "before_size": change.before_size, - "after_size": change.after_size, - } - for change in changes - ] - print(json.dumps(payload, indent=2)) - return 0 - + return "diff" + if isinstance(command, MergeArgs): + return "merge" if isinstance(command, VerifyArgs): - result = verify( - root=Path(command.root), - ref=command.ref, - path_prefixes=command.path, - dry_run=command.dry_run, - ) - payload = { - "commit_id": result.commit_id, - "verified_entries": result.verified_entries, - "candidate_entries": result.candidate_entries, - "total_entries": result.total_entries, - "created_commit": result.created_commit, - "dry_run": result.dry_run, - } - print(json.dumps(payload, indent=2)) - return 0 - + return "verify" if isinstance(command, IndexArgs): - index_command = command.command - if isinstance(index_command, IndexBuildArgs): - paths = build_analytical_index( - root=Path(index_command.root), - ref=index_command.ref, - output_dir=index_command.output_dir, - export_parquet=index_command.parquet, + if isinstance(command.command, IndexBuildArgs): + return "index build" + if isinstance(command.command, IndexQueryArgs): + return "index query" + if isinstance(command.command, IndexDropArgs): + return "index drop" + return "index" + return "fluxel" + + +def run_cli(argv: list[str] | None = None) -> int: + argv = argv or sys.argv[1:] + parser = build_parser() + args = parser.parse_args(argv).cli + command = args.command + command_name = _command_name(command) + + try: + if isinstance(command, CommitArgs): + commit_id = commit( + command.root, + command.message, + identity_mode=command.identity, + staged=command.staged, + ref=command.ref, ) - result = { - "database_path": str(paths.database_path), - "parquet_path": str(paths.parquet_path) if paths.parquet_path else None, - } - print(json.dumps(result, indent=2)) + print(commit_id) return 0 - if isinstance(index_command, IndexQueryArgs): - rows = query_analytical_index(index_command.db, index_command.sql) - print(json.dumps(rows)) + if isinstance(command, ImportArgs): + commit_id = import_s3( + command.root, + command.source, + command.message, + identity_mode=command.identity, + path_patterns=_flatten_option_values(command.path_patterns), + ref=command.ref, + ) + print(commit_id) return 0 - if isinstance(index_command, IndexDropArgs): - drop_analytical_index(index_command.db) - print("ok") + if isinstance(command, AddArgs): + stage = add( + root=command.root, + paths=command.paths, + ref=command.ref, + identity_mode=command.identity, + destination_path=command.destination_path, + ) + print(json.dumps(_stage_payload(stage), indent=2)) + return 0 + + if isinstance(command, RmArgs): + if command.message is not None and command.staged: + print( + "rm error: cannot combine --message with --staged", + file=sys.stderr, + ) + return 2 + if command.message is not None: + result = remove( + root=command.root, + paths=command.paths, + message=command.message, + ref=command.ref, + ) + print( + json.dumps( + { + "ref": result.ref, + "commit_id": result.commit_id, + "removed_paths": result.removed_paths, + }, + indent=2, + ) + ) + return 0 + + stage = rm(root=command.root, paths=command.paths, ref=command.ref) + print(json.dumps(_stage_payload(stage), indent=2)) return 0 - parser.error("Unsupported command") - return 2 + if isinstance(command, MoveArgs): + result = move( + root=command.root, + source_path=command.source_path, + destination_path=command.destination_path, + message=command.message, + ref=command.ref, + ) + print( + json.dumps( + { + "ref": result.ref, + "commit_id": result.commit_id, + "source_path": result.source_path, + "destination_path": result.destination_path, + "moved_paths": result.moved_paths, + }, + indent=2, + ) + ) + return 0 + + if isinstance(command, StatusArgs): + stage = status( + root=command.root, + ref=command.ref, + ) + print(json.dumps(_stage_payload(stage), indent=2)) + return 0 + + if isinstance(command, BranchArgs): + branch_path = branch(command.root, command.name) + print(str(branch_path)) + return 0 + if isinstance(command, DiffArgs): + changes = diff( + command.root, + command.from_ref, + command.to_ref, + ) + payload = [ + { + "path": change.path, + "change": change.change, + "before_hash": change.before_hash, + "after_hash": change.after_hash, + "before_size": change.before_size, + "after_size": change.after_size, + } + for change in changes + ] + print(json.dumps(payload, indent=2)) + return 0 + + if isinstance(command, MergeArgs): + result = merge( + root=command.root, + source_ref=command.source_ref, + target_ref=command.target_ref, + ) + print( + json.dumps( + { + "source_ref": result.source_ref, + "target_ref": result.target_ref, + "commit_id": result.commit_id, + "updated": result.updated, + }, + indent=2, + ) + ) + return 0 + + if isinstance(command, VerifyArgs): + result = verify( + root=command.root, + ref=command.ref, + path_prefixes=_flatten_option_values(command.path), + dry_run=command.dry_run, + ) + payload = { + "commit_id": result.commit_id, + "verified_entries": result.verified_entries, + "candidate_entries": result.candidate_entries, + "total_entries": result.total_entries, + "created_commit": result.created_commit, + "dry_run": result.dry_run, + } + print(json.dumps(payload, indent=2)) + return 0 -def main() -> None: - raise SystemExit(run_cli(sys.argv[1:])) + if isinstance(command, IndexArgs): + index_command = command.command + if isinstance(index_command, IndexBuildArgs): + paths = build_analytical_index( + root=index_command.root, + ref=index_command.ref, + output_dir=index_command.output_dir, + export_parquet=index_command.parquet, + ) + print( + json.dumps( + { + "database_path": str(paths.database_path), + "parquet_path": ( + str(paths.parquet_path) + if paths.parquet_path is not None + else None + ), + }, + indent=2, + ) + ) + return 0 + if isinstance(index_command, IndexQueryArgs): + rows = query_analytical_index(index_command.db, index_command.sql) + print(json.dumps(rows, indent=2)) + return 0 + if isinstance(index_command, IndexDropArgs): + drop_analytical_index(index_command.db) + print("ok") + return 0 + except HANDLED_CLI_ERRORS as error: + print(f"{command_name} error: {error}", file=sys.stderr) + return 2 + + raise AssertionError(f"Unsupported command type: {type(command).__name__}") + + +def main(argv: list[str] | None = None) -> int: + return run_cli(argv) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/fluxel/core/SPEC.md b/src/fluxel/core/SPEC.md new file mode 100644 index 0000000..55bc553 --- /dev/null +++ b/src/fluxel/core/SPEC.md @@ -0,0 +1,49 @@ +# Spec (Human Authored) + + +## `flx add` + +We add files using two (2) different modes: + +1. `meta` which is a pure metadata transaction, we simply store location of file with a hash of name + size +2. `hash` which is a git-like transaction where we store the file as `hash[:2]/hash[2:]` + - We're using `blake3` to calculate the hash + +The action of adding also needs to store a Index. + +We chose actively not to store the actions transpired (e.g. `mv`, `add`, `rm`, ..) + +The `add` action is local until commited. + +### Logic + +O(1) and simply appends to a list of operations. + +## `flx rm` + +Removes, like `add` very much. + +### Logic + +O(1) simply appends to a list of operations. + + +## `flx commit` + +Commits and saves all the added (or removed/moved) files to a set branch. + +## `flx branch` + +Branch is checked out on S3-level directly. + +## `flx merge` + +Merge is done on S3-level. + +## `flx diff` + +Diff should be quite fast, but is allowed to use analytical index (that's built on-demand). + +## `flx index` + +Build an analytical index on-demand that's using DuckDB. \ No newline at end of file diff --git a/src/fluxel/core/__init__.py b/src/fluxel/core/__init__.py index 1429031..1c2ec48 100644 --- a/src/fluxel/core/__init__.py +++ b/src/fluxel/core/__init__.py @@ -1,3 +1,4 @@ +from .client_state import LocalClientState from .filesystem import FluxelFileSystem, FluxelURI from .hashing import DEFAULT_CHUNK_SIZE, blake3_digest_file, blake3_digest_stream from .index import ( @@ -14,10 +15,21 @@ build_manifest_entries, walk_files, ) +from .repository_store import ( + BranchRefState, + LocalRepositoryStore, + RepositoryStore, + S3RepositoryStore, +) from .repository import ( CommitObject, DiffEntry, FluxelRepository, + MergeResult, + MoveResult, + open_repository, + RefConflictError, + RemoveResult, StageChange, StageStatus, VerifyResult, @@ -25,6 +37,10 @@ branch, commit, diff, + import_s3, + merge, + move, + remove, rm, status, verify, @@ -32,27 +48,41 @@ from .storage import ( LocalStorageBackend, OptimisticLockError, + S3ObjectMetadata, S3StorageBackend, StorageBackend, + iter_s3_objects, + open_source_uri, + parse_s3_uri, ) __all__ = [ "DEFAULT_CHUNK_SIZE", "AnalyticalIndexPaths", + "LocalClientState", "CommitObject", "DiffEntry", "FluxelFileSystem", "FluxelRepository", + "MergeResult", + "MoveResult", + "open_repository", + "RefConflictError", + "RemoveResult", "StageChange", "StageStatus", "VerifyResult", "FluxelURI", "FluxelLayout", + "BranchRefState", + "LocalRepositoryStore", "LocalStorageBackend", "ManifestEntry", "ManifestReader", "ManifestWriter", "OptimisticLockError", + "RepositoryStore", + "S3RepositoryStore", "S3StorageBackend", "StorageBackend", "blake3_digest_file", @@ -64,11 +94,19 @@ "branch", "commit", "diff", + "import_s3", + "merge", + "move", + "remove", "rm", "status", "verify", "drop_analytical_index", "initialize_fluxel_layout", "query_analytical_index", + "S3ObjectMetadata", "walk_files", + "iter_s3_objects", + "open_source_uri", + "parse_s3_uri", ] diff --git a/src/fluxel/core/client_state.py b/src/fluxel/core/client_state.py new file mode 100644 index 0000000..449379c --- /dev/null +++ b/src/fluxel/core/client_state.py @@ -0,0 +1,111 @@ +from __future__ import annotations + +import json +from dataclasses import dataclass +from pathlib import Path +from tempfile import NamedTemporaryFile + + +HEAD_FILE = "HEAD" + + +@dataclass(frozen=True) +class LocalBranchSnapshot: + branch: str + commit_id: str | None + version_token: str | None + + +class LocalClientState: + def __init__(self, root: str | Path) -> None: + self.root = Path(root).resolve() + self.fluxel_dir = self.root / ".fluxel" + self.refs_dir = self.fluxel_dir / "refs" + self.branch_state_dir = self.refs_dir / "heads" + self.staging_dir = self.fluxel_dir / "staging" + self.head_path = self.refs_dir / HEAD_FILE + + for path in ( + self.fluxel_dir, + self.refs_dir, + self.branch_state_dir, + self.staging_dir, + ): + path.mkdir(parents=True, exist_ok=True) + + def ensure_current_branch(self, default_branch: str) -> None: + if not self.head_path.exists(): + self.set_current_branch(default_branch) + + def current_branch(self) -> str: + content = self.head_path.read_text(encoding="utf-8").strip() + if not content.startswith("refs/heads/"): + raise ValueError("HEAD must be a symbolic ref under refs/heads/") + return content.split("refs/heads/", maxsplit=1)[1] + + def set_current_branch(self, branch: str) -> None: + self._atomic_write_text(self.head_path, f"refs/heads/{branch}\n") + + def read_staging_payload(self, branch: str) -> str | None: + stage_path = self.stage_path(branch) + if not stage_path.exists(): + return None + return stage_path.read_text(encoding="utf-8") + + def write_staging_payload(self, branch: str, payload: str | None) -> None: + stage_path = self.stage_path(branch) + if payload is None: + stage_path.unlink(missing_ok=True) + return + self._atomic_write_text(stage_path, payload) + + def read_branch_snapshot(self, branch: str) -> LocalBranchSnapshot | None: + snapshot_path = self.branch_snapshot_path(branch) + if not snapshot_path.exists(): + return None + payload = json.loads(snapshot_path.read_text(encoding="utf-8")) + return LocalBranchSnapshot( + branch=branch, + commit_id=payload.get("commit_id"), + version_token=payload.get("version_token"), + ) + + def write_branch_snapshot( + self, + branch: str, + *, + commit_id: str | None, + version_token: str | None, + ) -> None: + payload = json.dumps( + { + "commit_id": commit_id, + "version_token": version_token, + }, + sort_keys=True, + ) + self._atomic_write_text(self.branch_snapshot_path(branch), f"{payload}\n") + + def branch_snapshot_path(self, branch: str) -> Path: + return self.branch_state_dir / f"{branch}.json" + + def stage_path(self, branch: str) -> Path: + return self.staging_dir / f"{branch}.json" + + def _atomic_write_text(self, path: Path, payload: str) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + with NamedTemporaryFile( + mode="w", + encoding="utf-8", + dir=path.parent, + prefix=f".{path.name}.", + suffix=".tmp", + delete=False, + ) as temp: + temp_path = Path(temp.name) + temp.write(payload) + + try: + temp_path.replace(path) + finally: + temp_path.unlink(missing_ok=True) diff --git a/src/fluxel/core/filesystem.py b/src/fluxel/core/filesystem.py index 9b060da..cf3a71d 100644 --- a/src/fluxel/core/filesystem.py +++ b/src/fluxel/core/filesystem.py @@ -15,9 +15,9 @@ import fsspec from fsspec.spec import AbstractFileSystem -from .layout import blob_relpath from .manifest import ManifestEntry from .repository import FluxelRepository +from .storage import open_source_uri @dataclass(frozen=True) @@ -41,6 +41,7 @@ def __init__( self.dataset_roots = { name: Path(root).resolve() for name, root in (dataset_roots or {}).items() } + self._repositories: dict[Path, FluxelRepository] = {} @classmethod def _strip_protocol(cls, path: str) -> str: @@ -59,10 +60,10 @@ def _open( ) resolved = self._resolve_entry(path) if resolved.entry.blob_hash: - blob_path = self._blob_path(resolved.root, resolved.entry.blob_hash) - return io.BytesIO(blob_path.read_bytes()) + repo = self._repository(resolved.root) + return io.BytesIO(repo.read_blob(resolved.entry.blob_hash)) if resolved.entry.source_uri: - return fsspec.open(resolved.entry.source_uri, mode="rb").open() + return _SourceURIFile(open_source_uri(resolved.entry.source_uri)) raise FileNotFoundError( "Entry has no canonical blob hash and no readable source URI" ) @@ -93,19 +94,18 @@ def ls( ) -> list[dict[str, object]] | list[str]: uri = self._parse_uri(path, allow_empty_path=True) root = self._dataset_root(uri.dataset) - repo = FluxelRepository(root) - entries = repo.resolve_entries(uri.ref, include_staging=uri.include_staging) + repo = self._repository(root) normalized_prefix = uri.logical_path.strip("/") if normalized_prefix == "*": normalized_prefix = "" + entries = repo.resolve_entries_for_prefix( + uri.ref, + normalized_prefix, + include_staging=uri.include_staging, + ) results: list[dict[str, object] | str] = [] for entry in entries.values(): - if normalized_prefix and not ( - entry.path == normalized_prefix - or entry.path.startswith(f"{normalized_prefix}/") - ): - continue as_uri = f"fluxel://{uri.dataset}@{uri.ref}/{entry.path}" if detail: results.append( @@ -124,14 +124,25 @@ def ls( def _resolve_entry(self, path: str) -> "ResolvedEntry": uri = self._parse_uri(path) root = self._dataset_root(uri.dataset) - repo = FluxelRepository(root) + repo = self._repository(root) commit_id = repo.resolve_ref(uri.ref) - entries = repo.resolve_entries(uri.ref, include_staging=uri.include_staging) - entry = entries.get(uri.logical_path) + entry = repo.resolve_entry( + uri.ref, + uri.logical_path, + include_staging=uri.include_staging, + commit_id=commit_id, + ) if entry is None: raise FileNotFoundError(path) return ResolvedEntry(uri=uri, root=root, commit_id=commit_id, entry=entry) + def _repository(self, root: Path) -> FluxelRepository: + repo = self._repositories.get(root) + if repo is None: + repo = FluxelRepository(root) + self._repositories[root] = repo + return repo + def _dataset_root(self, dataset: str) -> Path: if dataset in self.dataset_roots: return self.dataset_roots[dataset] @@ -170,9 +181,6 @@ def _parse_uri(self, path: str, *, allow_empty_path: bool = False) -> FluxelURI: include_staging=include_staging, ) - def _blob_path(self, dataset_root: Path, content_hash: str) -> Path: - return dataset_root / ".fluxel" / "blobs" / blob_relpath(content_hash) - @dataclass(frozen=True) class ResolvedEntry: @@ -182,4 +190,36 @@ class ResolvedEntry: entry: ManifestEntry +class _SourceURIFile(io.IOBase): + def __init__(self, context_manager: object) -> None: + self._context_manager = context_manager + self._handle = context_manager.__enter__() + + def read(self, size: int = -1) -> bytes: + return self._handle.read(size) + + def readable(self) -> bool: + return True + + def seekable(self) -> bool: + return bool(getattr(self._handle, "seekable", lambda: False)()) + + def seek(self, offset: int, whence: int = io.SEEK_SET) -> int: + return self._handle.seek(offset, whence) + + def tell(self) -> int: + return self._handle.tell() + + def close(self) -> None: + if self.closed: + return + try: + close = getattr(self._handle, "close", None) + if callable(close): + close() + finally: + self._context_manager.__exit__(None, None, None) + super().close() + + fsspec.register_implementation("fluxel", FluxelFileSystem) diff --git a/src/fluxel/core/index.py b/src/fluxel/core/index.py index 7871b25..25ec0f3 100644 --- a/src/fluxel/core/index.py +++ b/src/fluxel/core/index.py @@ -7,12 +7,14 @@ from __future__ import annotations +import csv from dataclasses import dataclass from pathlib import Path +from tempfile import NamedTemporaryFile import duckdb -from .repository import FluxelRepository +from .repository import open_repository @dataclass(frozen=True) @@ -28,38 +30,60 @@ def build_analytical_index( output_dir: str | Path | None = None, export_parquet: bool = False, ) -> AnalyticalIndexPaths: - root_path = Path(root).resolve() - repo = FluxelRepository(root_path) + repo = open_repository(root) commit_id = repo.resolve_ref(ref) commit = repo.read_commit(commit_id) - manifest_path = root_path / ".fluxel" / "manifests" / f"{commit.manifest}.jsonl" - if not manifest_path.exists(): - raise FileNotFoundError(f"Manifest not found for ref '{ref}': {manifest_path}") - index_root = Path(output_dir).resolve() if output_dir else (root_path / ".fluxel" / "index") + index_root = ( + Path(output_dir).resolve() + if output_dir + else (repo.client_state.fluxel_dir / "index") + ) index_root.mkdir(parents=True, exist_ok=True) db_path = index_root / f"{commit_id}.duckdb" - conn = duckdb.connect(str(db_path)) + with NamedTemporaryFile( + mode="w", suffix=".csv", delete=False, encoding="utf-8", newline="" + ) as temp: + manifest_path = Path(temp.name) try: - conn.execute( - """ - CREATE OR REPLACE TABLE files AS - SELECT - path::VARCHAR AS path, - hash::VARCHAR AS hash, - size::BIGINT AS size, - mtime_ns::BIGINT AS mtime_ns, - ?::VARCHAR AS commit_id, - ?::VARCHAR AS branch - FROM read_json_auto(?, format='newline_delimited') - """, - [commit_id, commit.branch, str(manifest_path)], - ) - conn.execute("CREATE INDEX IF NOT EXISTS idx_files_path ON files(path)") - conn.execute("CREATE INDEX IF NOT EXISTS idx_files_size ON files(size)") + with manifest_path.open("w", encoding="utf-8", newline="") as handle: + writer = csv.writer(handle) + writer.writerow(["path", "hash", "size", "mtime_ns", "commit_id", "branch"]) + for entry in repo.store.iter_manifest_entries(commit.manifest): + writer.writerow( + [ + entry.path, + entry.hash, + entry.size, + entry.mtime_ns, + commit_id, + commit.branch, + ] + ) + + conn = duckdb.connect(str(db_path)) + try: + conn.execute( + """ + CREATE OR REPLACE TABLE files AS + SELECT + path::VARCHAR AS path, + hash::VARCHAR AS hash, + size::BIGINT AS size, + mtime_ns::BIGINT AS mtime_ns, + commit_id::VARCHAR AS commit_id, + branch::VARCHAR AS branch + FROM read_csv_auto(?, header=true) + """, + [str(manifest_path)], + ) + conn.execute("CREATE INDEX IF NOT EXISTS idx_files_path ON files(path)") + conn.execute("CREATE INDEX IF NOT EXISTS idx_files_size ON files(size)") + finally: + conn.close() finally: - conn.close() + manifest_path.unlink(missing_ok=True) parquet_path: Path | None = None if export_parquet: @@ -73,7 +97,9 @@ def build_analytical_index( return AnalyticalIndexPaths(database_path=db_path, parquet_path=parquet_path) -def query_analytical_index(database_path: str | Path, query: str) -> list[tuple[object, ...]]: +def query_analytical_index( + database_path: str | Path, query: str +) -> list[tuple[object, ...]]: db_path = Path(database_path).resolve() if not db_path.exists(): raise FileNotFoundError(f"Index database not found: {db_path}") diff --git a/src/fluxel/core/layout.py b/src/fluxel/core/layout.py index 869c130..48028c6 100644 --- a/src/fluxel/core/layout.py +++ b/src/fluxel/core/layout.py @@ -22,37 +22,41 @@ class FluxelLayout: refs_dir: Path heads_dir: Path + @classmethod + def initialize(cls, root: str | Path) -> "FluxelLayout": + root_path = Path(root).resolve() + fluxel_dir = root_path / ".fluxel" + blobs_dir = fluxel_dir / "blobs" + commits_dir = fluxel_dir / "commits" + manifests_dir = fluxel_dir / "manifests" + staging_dir = fluxel_dir / "staging" + refs_dir = fluxel_dir / "refs" + heads_dir = refs_dir / "heads" + + for path in ( + blobs_dir, + commits_dir, + manifests_dir, + staging_dir, + refs_dir, + heads_dir, + ): + path.mkdir(parents=True, exist_ok=True) + + return cls( + root=root_path, + fluxel_dir=fluxel_dir, + blobs_dir=blobs_dir, + commits_dir=commits_dir, + manifests_dir=manifests_dir, + staging_dir=staging_dir, + refs_dir=refs_dir, + heads_dir=heads_dir, + ) + def initialize_fluxel_layout(root: str | Path) -> FluxelLayout: - root_path = Path(root).resolve() - fluxel_dir = root_path / ".fluxel" - blobs_dir = fluxel_dir / "blobs" - commits_dir = fluxel_dir / "commits" - manifests_dir = fluxel_dir / "manifests" - staging_dir = fluxel_dir / "staging" - refs_dir = fluxel_dir / "refs" - heads_dir = refs_dir / "heads" - - for path in ( - blobs_dir, - commits_dir, - manifests_dir, - staging_dir, - refs_dir, - heads_dir, - ): - path.mkdir(parents=True, exist_ok=True) - - return FluxelLayout( - root=root_path, - fluxel_dir=fluxel_dir, - blobs_dir=blobs_dir, - commits_dir=commits_dir, - manifests_dir=manifests_dir, - staging_dir=staging_dir, - refs_dir=refs_dir, - heads_dir=heads_dir, - ) + return FluxelLayout.initialize(root) def blob_relpath(content_hash: str) -> Path: diff --git a/src/fluxel/core/manifest.py b/src/fluxel/core/manifest.py index 6830da2..5b34bcc 100644 --- a/src/fluxel/core/manifest.py +++ b/src/fluxel/core/manifest.py @@ -8,14 +8,49 @@ from __future__ import annotations import json -import os -from dataclasses import asdict, dataclass +from dataclasses import dataclass +from json import JSONDecodeError from pathlib import Path +from pathlib import PurePosixPath from typing import Callable, Iterable, Iterator from .hashing import blake3_digest_file +SUPPORTED_IDENTITY_MODES = frozenset({"blake3", "meta"}) +_BLAKE3_HEX_LENGTH = 64 +_HEX_DIGITS = frozenset("0123456789abcdef") +_BLOB_BACKED_MANIFEST_TAG = "b" +_META_ONLY_MANIFEST_TAG = "m" + + +def _is_hex_digest(value: str) -> bool: + normalized = value.lower() + return len(normalized) == _BLAKE3_HEX_LENGTH and all( + character in _HEX_DIGITS for character in normalized + ) + + +def _validate_manifest_path(path: str) -> None: + if not path: + raise ValueError("Manifest entry path cannot be empty") + if path.startswith("/") or path.endswith("/"): + raise ValueError("Manifest entry path must be a normalized relative path") + if "\\" in path or "//" in path: + raise ValueError("Manifest entry path must use normalized POSIX separators") + + parts = PurePosixPath(path).parts + if not parts or any(part in {"", ".", ".."} for part in parts): + raise ValueError("Manifest entry path must be a normalized relative path") + + +def _validate_hex_digest(value: str, *, field_name: str) -> None: + if not _is_hex_digest(value): + raise ValueError( + f"Manifest entry {field_name} must be a 64-character hex digest" + ) + + @dataclass(frozen=True) class ManifestEntry: path: str @@ -27,8 +62,52 @@ class ManifestEntry: blob_hash: str | None = None source_uri: str | None = None + def __post_init__(self) -> None: + if self.identity_value is None: + object.__setattr__(self, "identity_value", self.hash) + + _validate_manifest_path(self.path) + _validate_hex_digest(self.hash, field_name="hash") + + identity_value = self.identity_value + if identity_value is None: + raise ValueError("Manifest entry identity_value cannot be empty") + _validate_hex_digest(identity_value, field_name="identity_value") + + if self.identity_mode not in SUPPORTED_IDENTITY_MODES: + supported_modes = ", ".join(sorted(SUPPORTED_IDENTITY_MODES)) + raise ValueError( + f"Manifest entry identity_mode must be one of: {supported_modes}" + ) + if self.size < 0: + raise ValueError("Manifest entry size cannot be negative") + if self.mtime_ns < 0: + raise ValueError("Manifest entry mtime_ns cannot be negative") + if identity_value != self.hash: + raise ValueError("Manifest entry identity_value must match hash") + + if self.blob_hash is not None: + _validate_hex_digest(self.blob_hash, field_name="blob_hash") + + if self.source_uri is not None and not self.source_uri.strip(): + raise ValueError("Manifest entry source_uri cannot be empty") + + if self.identity_mode == "meta": + if self.blob_hash is not None: + raise ValueError( + "Metadata-only manifest entries cannot include blob_hash" + ) + if self.source_uri is None: + raise ValueError( + "Metadata-only manifest entries must include source_uri" + ) + elif self.blob_hash is not None and self.blob_hash != self.hash: + raise ValueError("Blob-backed manifest entries must keep blob_hash aligned") + @staticmethod def from_dict(data: dict[str, object]) -> "ManifestEntry": + if not isinstance(data, dict): + raise ValueError("Manifest entry payload must be an object") hash_value = str(data.get("hash") or data.get("identity_value") or "") if not hash_value: raise ValueError("Manifest entry must include hash or identity_value") @@ -38,11 +117,25 @@ def from_dict(data: dict[str, object]) -> "ManifestEntry": if blob_hash is None and "blob_hash" not in data and identity_mode == "blake3": blob_hash = hash_value source_uri = data.get("source_uri") + + try: + path = str(data["path"]) + size = int(data["size"]) + mtime_ns = int(data["mtime_ns"]) + except KeyError as error: + raise ValueError( + f"Manifest entry is missing required field: {error.args[0]}" + ) from error + except (TypeError, ValueError) as error: + raise ValueError( + "Manifest entry size and mtime_ns must be integers" + ) from error + return ManifestEntry( - path=str(data["path"]), + path=path, hash=hash_value, - size=int(data["size"]), - mtime_ns=int(data["mtime_ns"]), + size=size, + mtime_ns=mtime_ns, identity_mode=identity_mode, identity_value=( str(identity_value) if identity_value is not None else hash_value @@ -52,6 +145,86 @@ def from_dict(data: dict[str, object]) -> "ManifestEntry": ) +def serialize_manifest_entry(entry: ManifestEntry) -> str: + if entry.identity_mode == "blake3": + payload: list[object] = [ + _BLOB_BACKED_MANIFEST_TAG, + entry.path, + entry.hash, + entry.size, + entry.mtime_ns, + ] + elif entry.identity_mode == "meta": + payload = [ + _META_ONLY_MANIFEST_TAG, + entry.path, + entry.hash, + entry.size, + entry.mtime_ns, + entry.source_uri, + ] + else: + supported_modes = ", ".join(sorted(SUPPORTED_IDENTITY_MODES)) + raise ValueError( + f"Manifest entry identity_mode must be one of: {supported_modes}" + ) + return json.dumps(payload, separators=(",", ":")) + + +def deserialize_manifest_entry(payload_text: str) -> ManifestEntry: + try: + payload = _load_manifest_payload(payload_text) + except JSONDecodeError as error: + raise ValueError("Corrupt manifest entry payload") from error + + return _manifest_entry_from_payload(payload) + + +def manifest_entry_path(payload_text: str) -> str: + try: + payload = _load_manifest_payload(payload_text) + except JSONDecodeError as error: + raise ValueError("Corrupt manifest entry payload") from error + if not isinstance(payload, list) or len(payload) < 2: + raise ValueError("Manifest entry payload must be a JSON array") + return str(payload[1]) + + +def _load_manifest_payload(payload_text: str) -> object: + return json.loads(payload_text) + + +def _manifest_entry_from_payload(payload: object) -> ManifestEntry: + + if not isinstance(payload, list): + raise ValueError("Manifest entry payload must be a JSON array") + if len(payload) == 5 and payload[0] == _BLOB_BACKED_MANIFEST_TAG: + _, path, hash_value, size, mtime_ns = payload + return ManifestEntry( + path=str(path), + hash=str(hash_value), + size=int(size), + mtime_ns=int(mtime_ns), + identity_mode="blake3", + identity_value=str(hash_value), + blob_hash=str(hash_value), + source_uri=None, + ) + if len(payload) == 6 and payload[0] == _META_ONLY_MANIFEST_TAG: + _, path, hash_value, size, mtime_ns, source_uri = payload + return ManifestEntry( + path=str(path), + hash=str(hash_value), + size=int(size), + mtime_ns=int(mtime_ns), + identity_mode="meta", + identity_value=str(hash_value), + blob_hash=None, + source_uri=str(source_uri), + ) + raise ValueError("Manifest entry payload has an unsupported shape") + + class ManifestWriter: def __init__(self, manifest_path: str | Path) -> None: self.manifest_path = Path(manifest_path) @@ -61,7 +234,7 @@ def write_entries(self, entries: Iterable[ManifestEntry]) -> int: written = 0 with self.manifest_path.open("w", encoding="utf-8") as handle: for entry in entries: - handle.write(json.dumps(asdict(entry), separators=(",", ":"))) + handle.write(serialize_manifest_entry(entry)) handle.write("\n") written += 1 return written @@ -87,11 +260,22 @@ def iter_entries(self) -> Iterator[ManifestEntry]: if not self.manifest_path.exists(): return with self.manifest_path.open("r", encoding="utf-8") as handle: - for line in handle: + for line_number, line in enumerate(handle, start=1): line = line.strip() if not line: continue - yield ManifestEntry.from_dict(json.loads(line)) + try: + payload = _load_manifest_payload(line) + except JSONDecodeError as error: + raise ValueError( + f"Corrupt manifest JSON at line {line_number} in {self.manifest_path}" + ) from error + try: + yield _manifest_entry_from_payload(payload) + except ValueError as error: + raise ValueError( + f"Invalid manifest entry at line {line_number} in {self.manifest_path}: {error}" + ) from error def get_entry(self, logical_path: str) -> ManifestEntry | None: match: ManifestEntry | None = None @@ -127,7 +311,15 @@ def build_manifest_entries( def walk_files(root: str | Path) -> Iterator[Path]: root_path = Path(root).resolve() - for dirpath, dirnames, filenames in os.walk(root_path): - dirnames[:] = [name for name in dirnames if name != ".fluxel"] - for filename in filenames: - yield Path(dirpath) / filename + + def iter_dir(path: Path) -> Iterator[Path]: + for child in sorted(path.iterdir(), key=lambda item: item.name): + if child.name == ".fluxel": + continue + if child.is_dir(): + yield from iter_dir(child) + continue + if child.is_file(): + yield child + + yield from iter_dir(root_path) diff --git a/src/fluxel/core/manifest_index.py b/src/fluxel/core/manifest_index.py new file mode 100644 index 0000000..427753a --- /dev/null +++ b/src/fluxel/core/manifest_index.py @@ -0,0 +1,174 @@ +from __future__ import annotations + +from bisect import bisect_right +from dataclasses import dataclass +import json +from pathlib import Path +from typing import Callable, Iterator + +from .manifest import manifest_entry_path + + +DEFAULT_INDEX_BLOCK_ENTRY_COUNT = 4096 + + +@dataclass(frozen=True) +class ManifestIndexBlock: + first_path: str + offset: int + + +@dataclass(frozen=True) +class ManifestIndex: + manifest_size: int + block_entry_count: int + blocks: tuple[ManifestIndexBlock, ...] + + +def build_manifest_index( + manifest_path: str | Path, + index_path: str | Path, + *, + block_entry_count: int = DEFAULT_INDEX_BLOCK_ENTRY_COUNT, +) -> None: + manifest = Path(manifest_path) + index = Path(index_path) + index.parent.mkdir(parents=True, exist_ok=True) + index.unlink(missing_ok=True) + if block_entry_count <= 0: + raise ValueError("block_entry_count must be positive") + + blocks: list[ManifestIndexBlock] = [] + previous_path: str | None = None + entry_count = 0 + current_offset = 0 + with manifest.open("rb") as handle: + for raw_line in handle: + stripped = raw_line.strip() + if stripped: + entry_json = stripped.decode("utf-8") + path = manifest_entry_path(entry_json) + if previous_path is not None and path <= previous_path: + raise ValueError( + "Manifest entries must be sorted by path to build an index" + ) + if entry_count % block_entry_count == 0: + blocks.append( + ManifestIndexBlock(first_path=path, offset=current_offset) + ) + previous_path = path + entry_count += 1 + current_offset += len(raw_line) + + payload = { + "manifest_size": current_offset, + "block_entry_count": block_entry_count, + "blocks": [[block.first_path, block.offset] for block in blocks], + } + index.write_text(json.dumps(payload, separators=(",", ":")), encoding="utf-8") + + +def lookup_manifest_index_entry_json( + index_path: str | Path, + logical_path: str, + *, + read_range: Callable[[int, int], bytes], +) -> str | None: + index = load_manifest_index(index_path) + block_index = _lookup_block_index(index, logical_path) + if block_index is None: + return None + + for entry_json, path in _iter_block_entries(index, block_index, read_range): + if path == logical_path: + return entry_json + if path > logical_path: + return None + return None + + +def iter_manifest_index_entry_jsons( + index_path: str | Path, + logical_prefix: str | None = None, + *, + read_range: Callable[[int, int], bytes], +) -> Iterator[str]: + index = load_manifest_index(index_path) + if not index.blocks: + return + + if not logical_prefix: + for block_index in range(len(index.blocks)): + for entry_json, _ in _iter_block_entries(index, block_index, read_range): + yield entry_json + return + + normalized_prefix = logical_prefix.strip("/") + descendant_prefix = f"{normalized_prefix.rstrip('/')}/" + prefix_upper_bound = _prefix_upper_bound(descendant_prefix) + block_index = _lookup_block_index(index, normalized_prefix) + if block_index is None: + block_index = 0 + + for current_index in range(block_index, len(index.blocks)): + first_path = index.blocks[current_index].first_path + if first_path >= prefix_upper_bound and first_path != normalized_prefix: + return + + for entry_json, path in _iter_block_entries(index, current_index, read_range): + if path == normalized_prefix or ( + descendant_prefix <= path < prefix_upper_bound + ): + yield entry_json + continue + if path >= prefix_upper_bound and path != normalized_prefix: + return + + +def load_manifest_index(index_path: str | Path) -> ManifestIndex: + payload = json.loads(Path(index_path).read_text(encoding="utf-8")) + blocks = tuple( + ManifestIndexBlock(first_path=str(first_path), offset=int(offset)) + for first_path, offset in payload.get("blocks", []) + ) + return ManifestIndex( + manifest_size=int(payload.get("manifest_size", 0)), + block_entry_count=int( + payload.get("block_entry_count", DEFAULT_INDEX_BLOCK_ENTRY_COUNT) + ), + blocks=blocks, + ) + + +def _prefix_upper_bound(prefix: str) -> str: + return f"{prefix}\U0010ffff" + + +def _lookup_block_index(index: ManifestIndex, logical_path: str) -> int | None: + if not index.blocks: + return None + first_paths = [block.first_path for block in index.blocks] + block_index = bisect_right(first_paths, logical_path) - 1 + if block_index < 0: + return 0 + return block_index + + +def _iter_block_entries( + index: ManifestIndex, + block_index: int, + read_range: Callable[[int, int], bytes], +) -> Iterator[tuple[str, str]]: + start_offset = index.blocks[block_index].offset + end_offset = ( + index.blocks[block_index + 1].offset + if block_index + 1 < len(index.blocks) + else index.manifest_size + ) + payload = read_range(start_offset, end_offset) + for raw_line in payload.splitlines(): + stripped = raw_line.strip() + if not stripped: + continue + entry_json = stripped.decode("utf-8") + yield entry_json, manifest_entry_path(entry_json) diff --git a/src/fluxel/core/repository.py b/src/fluxel/core/repository.py index 9d3c713..d918e26 100644 --- a/src/fluxel/core/repository.py +++ b/src/fluxel/core/repository.py @@ -11,18 +11,37 @@ from dataclasses import asdict, dataclass from datetime import datetime, timezone from pathlib import Path +from pathlib import PurePosixPath from tempfile import NamedTemporaryFile from typing import Iterator, Literal from blake3 import blake3 -import fsspec +from .client_state import LocalClientState from .hashing import DEFAULT_CHUNK_SIZE, blake3_digest_file -from .layout import FluxelLayout, blob_relpath, initialize_fluxel_layout -from .manifest import ManifestEntry, ManifestReader, ManifestWriter, walk_files - - -HEAD_FILE = "HEAD" +from .layout import initialize_fluxel_layout +from .manifest import ManifestEntry, ManifestWriter, walk_files +from .repository_store import ( + BranchRefState, + LocalRepositoryStore, + RepositoryStore, + S3RepositoryStore, + build_manifest_index_file, +) +from .repository_support import ( + matches_any_logical_path, + matches_import_patterns, + matches_logical_path, + metadata_identity, + move_logical_path, + normalize_import_patterns, + normalize_logical_paths, + normalize_repository_path, + normalize_s3_import_path, + relocate_manifest_entry, +) +from .storage import iter_s3_objects, open_source_uri, parse_s3_uri +from .storage import describe_source_uri @dataclass(frozen=True) @@ -55,22 +74,49 @@ class VerifyResult: dry_run: bool +@dataclass(frozen=True) +class MergeResult: + source_ref: str + target_ref: str + commit_id: str + updated: bool + + +@dataclass(frozen=True) +class RemoveResult: + ref: str + commit_id: str + removed_paths: list[str] + + +@dataclass(frozen=True) +class MoveResult: + ref: str + commit_id: str + source_path: str + destination_path: str + moved_paths: list[str] + + @dataclass(frozen=True) class StageChange: path: str action: str identity_mode: str | None = None + source_uri: str | None = None @staticmethod def from_dict(data: dict[str, object]) -> "StageChange": action = str(data["action"]) identity_mode_raw = data.get("identity_mode") + source_uri_raw = data.get("source_uri") return StageChange( path=str(data["path"]), action=action, identity_mode=( str(identity_mode_raw) if identity_mode_raw is not None else None ), + source_uri=str(source_uri_raw) if source_uri_raw is not None else None, ) @@ -81,68 +127,146 @@ class StageStatus: removed: list[str] +class RefConflictError(RuntimeError): + def __init__( + self, + *, + branch: str, + operation: str, + expected_commit_id: str | None, + current_commit_id: str | None, + ) -> None: + expected = expected_commit_id or "" + current = current_commit_id or "" + super().__init__( + f"Branch update conflict for '{branch}' during {operation}: expected {expected}, found {current}" + ) + self.branch = branch + self.operation = operation + self.expected_commit_id = expected_commit_id + self.current_commit_id = current_commit_id + + class FluxelRepository: - def __init__(self, root: str | Path) -> None: + def __init__( + self, + root: str | Path, + *, + store: RepositoryStore | None = None, + client_state: LocalClientState | None = None, + ) -> None: self.layout = initialize_fluxel_layout(root) + self.store = store or LocalRepositoryStore(self.layout.root) + self.client_state = client_state or LocalClientState(self.layout.root) + self._resolved_ref_cache: dict[str, BranchRefState] = {} + self._commit_cache: dict[str, CommitObject] = {} self._ensure_head(default_branch="main") @property def root(self) -> Path: return self.layout.root - def _head_path(self) -> Path: - return self.layout.refs_dir / HEAD_FILE - def _branch_path(self, branch_name: str) -> Path: - return self.layout.heads_dir / branch_name + return self.store.branch_path(branch_name) def _ensure_head(self, default_branch: str) -> None: - head_path = self._head_path() - if not head_path.exists(): - head_path.write_text(f"refs/heads/{default_branch}\n", encoding="utf-8") - default_ref = self._branch_path(default_branch) - if not default_ref.exists(): - default_ref.parent.mkdir(parents=True, exist_ok=True) - default_ref.write_text("", encoding="utf-8") - - def _read_head_ref(self) -> str: - content = self._head_path().read_text(encoding="utf-8").strip() - if not content.startswith("refs/heads/"): - raise ValueError("HEAD must be a symbolic ref under refs/heads/") - return content + self.client_state.ensure_current_branch(default_branch) + if self.store.read_branch_ref(default_branch) is None: + self.store.write_branch_ref(default_branch, None) def current_branch(self) -> str: - return self._read_head_ref().split("refs/heads/", maxsplit=1)[1] + return self.client_state.current_branch() + + def set_current_branch(self, branch: str) -> None: + self._ensure_branch_exists(branch) + self.client_state.set_current_branch(branch) def head_commit(self) -> str | None: - branch_path = self.layout.fluxel_dir / self._read_head_ref() - if not branch_path.exists(): + branch_ref = self.store.read_branch_ref(self.current_branch()) + if branch_ref is None: return None - commit_id = branch_path.read_text(encoding="utf-8").strip() - return commit_id or None + return branch_ref.commit_id def resolve_ref(self, branch_or_commit: str) -> str: - maybe_branch = self._branch_path(branch_or_commit) - if maybe_branch.exists(): - commit_id = maybe_branch.read_text(encoding="utf-8").strip() - if not commit_id: + cached_branch_ref = self._resolved_ref_cache.get(branch_or_commit) + if cached_branch_ref is not None: + current_token = self.store.version_token("ref", branch_or_commit) + if current_token == cached_branch_ref.version_token: + if not cached_branch_ref.commit_id: + raise ValueError(f"Branch has no commits: {branch_or_commit}") + return cached_branch_ref.commit_id + + branch_ref = self.store.read_branch_ref(branch_or_commit) + if branch_ref is not None: + self._resolved_ref_cache[branch_or_commit] = branch_ref + if not branch_ref.commit_id: raise ValueError(f"Branch has no commits: {branch_or_commit}") - return commit_id - commit_path = self.layout.commits_dir / f"{branch_or_commit}.json" - if commit_path.exists(): + return branch_ref.commit_id + if self.store.object_exists("commit", branch_or_commit): return branch_or_commit raise ValueError(f"Unknown branch or commit: {branch_or_commit}") def branch(self, name: str) -> Path: if not name or "/" in name or name.startswith("."): raise ValueError("Invalid branch name") - target = self._branch_path(name) - if target.exists(): + if self.store.read_branch_ref(name) is not None: raise ValueError(f"Branch already exists: {name}") head_commit = self.head_commit() or "" - target.parent.mkdir(parents=True, exist_ok=True) - target.write_text(f"{head_commit}\n" if head_commit else "", encoding="utf-8") - return target + if not self.store.compare_and_set_branch_ref( + name, + head_commit or None, + expected_version_token=None, + expected_commit_id=None, + ): + raise ValueError(f"Branch already exists: {name}") + created_state = self.store.read_branch_ref(name) + if created_state is not None: + self.client_state.write_branch_snapshot( + name, + commit_id=created_state.commit_id, + version_token=created_state.version_token, + ) + self._resolved_ref_cache.pop(name, None) + return self._branch_path(name) + + def merge(self, source_ref: str, target_ref: str) -> MergeResult: + if not source_ref: + raise ValueError("Source ref cannot be empty") + if not target_ref: + raise ValueError("Target ref cannot be empty") + + target_branch_state = self._require_branch_state(target_ref) + source_commit = self.resolve_ref(source_ref) + target_commit = self.resolve_ref(target_ref) + + if source_commit == target_commit: + return MergeResult( + source_ref=source_ref, + target_ref=target_ref, + commit_id=target_commit, + updated=False, + ) + + if not self._is_ancestor( + ancestor_commit=target_commit, descendant_commit=source_commit + ): + raise ValueError( + f"Cannot fast-forward {target_ref} to {source_ref}: target is not an ancestor" + ) + + self._update_branch_ref( + branch=target_ref, + commit_id=source_commit, + expected_version_token=target_branch_state.version_token, + expected_commit_id=target_branch_state.commit_id, + operation="merge", + ) + return MergeResult( + source_ref=source_ref, + target_ref=target_ref, + commit_id=source_commit, + updated=True, + ) def commit( self, @@ -157,28 +281,76 @@ def commit( if identity_mode not in {"blake3", "meta"}: raise ValueError("identity_mode must be one of: blake3, meta") branch = ref or self.current_branch() - self._ensure_branch_exists(branch) + branch_state = self._require_branch_state(branch) if staged: - return self._commit_staged(message=message, branch=branch) + return self._commit_staged( + message=message, + branch_state=branch_state, + ) temp_manifest = self._write_temp_manifest( self._materialize_blobs_and_entries(identity_mode=identity_mode) ) manifest_hash = blake3_digest_file(temp_manifest) - manifest_target = self.layout.manifests_dir / f"{manifest_hash}.jsonl" - manifest_target.parent.mkdir(parents=True, exist_ok=True) - if not manifest_target.exists(): - temp_manifest.replace(manifest_target) + self._persist_manifest(temp_manifest, manifest_hash) + + parent_commit = branch_state.commit_id + return self._write_commit_object( + branch=branch, + message=message, + parent_commit=parent_commit, + manifest_hash=manifest_hash, + expected_version_token=branch_state.version_token, + operation="commit", + ) + + def import_s3( + self, + source_uri: str, + message: str, + identity_mode: Literal["blake3", "meta"] = "blake3", + *, + path_patterns: list[str] | None = None, + ref: str | None = None, + ) -> str: + if not message.strip(): + raise ValueError("Commit message cannot be empty") + if identity_mode not in {"blake3", "meta"}: + raise ValueError("identity_mode must be one of: blake3, meta") + branch = ref or self.current_branch() + branch_state = self._require_branch_state(branch) + parent_commit = branch_state.commit_id + + index: dict[str, ManifestEntry] + if parent_commit: + parent = self.read_commit(parent_commit) + index = self._manifest_index(parent.manifest) else: - temp_manifest.unlink(missing_ok=True) + index = {} + + for entry in self._materialize_s3_entries( + source_uri=source_uri, + identity_mode=identity_mode, + path_patterns=path_patterns, + ): + index[entry.path] = entry + + def iter_entries() -> Iterator[ManifestEntry]: + for path in sorted(index): + yield index[path] + + temp_manifest = self._write_temp_manifest(iter_entries()) + manifest_hash = blake3_digest_file(temp_manifest) + self._persist_manifest(temp_manifest, manifest_hash) - parent_commit = self._branch_head_commit(branch) return self._write_commit_object( branch=branch, message=message, parent_commit=parent_commit, manifest_hash=manifest_hash, + expected_version_token=branch_state.version_token, + operation="commit", ) def add( @@ -187,22 +359,26 @@ def add( *, ref: str | None = None, identity_mode: str = "blake3", + destination_path: str | None = None, ) -> StageStatus: if identity_mode not in {"blake3", "meta"}: raise ValueError("identity_mode must be one of: blake3, meta") branch = ref or self.current_branch() self._ensure_branch_exists(branch) staged = self._load_stage(branch) - for path in paths: - normalized = self._normalize_stage_path(path) - source = self.root / normalized - if not source.exists() or not source.is_file(): - raise FileNotFoundError(f"Cannot stage missing file: {normalized}") - staged[normalized] = StageChange( - path=normalized, - action="add", - identity_mode=identity_mode, - ) + if destination_path is not None and len(paths) != 1: + raise ValueError("--as can only be used when staging exactly one source") + for raw_source in paths: + for logical_path, source_uri in self._expand_stage_source( + raw_source, + destination_path=destination_path, + ): + staged[logical_path] = StageChange( + path=logical_path, + action="add", + identity_mode=identity_mode, + source_uri=source_uri, + ) self._save_stage(branch, staged) return self.status(ref=branch) @@ -211,11 +387,139 @@ def rm(self, paths: list[str], *, ref: str | None = None) -> StageStatus: self._ensure_branch_exists(branch) staged = self._load_stage(branch) for path in paths: - normalized = self._normalize_stage_path(path) + normalized = normalize_repository_path(path) staged[normalized] = StageChange(path=normalized, action="remove") self._save_stage(branch, staged) return self.status(ref=branch) + def remove_paths( + self, + paths: list[str], + message: str, + *, + ref: str | None = None, + ) -> RemoveResult: + if not message.strip(): + raise ValueError("Commit message cannot be empty") + branch = ref or self.current_branch() + branch_state = self._require_branch_state(branch) + base_commit = self._require_commit_for_metadata_mutation(branch_state.branch) + normalized_paths = normalize_logical_paths(paths) + removed_paths: set[str] = set() + + def iter_entries() -> Iterator[ManifestEntry]: + nonlocal removed_paths + for entry in self.store.iter_manifest_entries(base_commit.manifest): + if matches_any_logical_path(entry.path, normalized_paths): + removed_paths.add(entry.path) + continue + yield entry + + temp_manifest = self._write_temp_manifest(iter_entries()) + if not removed_paths: + temp_manifest.unlink(missing_ok=True) + missing = ", ".join(normalized_paths) + raise FileNotFoundError(f"Path not found in branch '{branch}': {missing}") + + manifest_hash = blake3_digest_file(temp_manifest) + self._persist_manifest(temp_manifest, manifest_hash) + commit_id = self._write_commit_object( + branch=branch, + message=message, + parent_commit=branch_state.commit_id, + manifest_hash=manifest_hash, + expected_version_token=branch_state.version_token, + operation="rm", + ) + return RemoveResult( + ref=branch, + commit_id=commit_id, + removed_paths=sorted(removed_paths), + ) + + def move( + self, + source_path: str, + destination_path: str, + message: str, + *, + ref: str | None = None, + ) -> MoveResult: + if not message.strip(): + raise ValueError("Commit message cannot be empty") + branch = ref or self.current_branch() + branch_state = self._require_branch_state(branch) + base_commit = self._require_commit_for_metadata_mutation(branch_state.branch) + source = normalize_repository_path(source_path) + destination = normalize_repository_path(destination_path) + + if source == destination: + raise ValueError("Source and destination paths must differ") + if destination.startswith(f"{source}/"): + raise ValueError("Cannot move a path into itself") + + existing_paths: set[str] = set() + source_paths: list[str] = [] + moved_paths: list[str] = [] + path_map: dict[str, str] = {} + + for entry in self.store.iter_manifest_entries(base_commit.manifest): + existing_paths.add(entry.path) + if not matches_logical_path(entry.path, source): + continue + moved_path = move_logical_path( + entry.path, + source_path=source, + destination_path=destination, + ) + source_paths.append(entry.path) + moved_paths.append(moved_path) + path_map[entry.path] = moved_path + + if not path_map: + raise FileNotFoundError(f"Path not found in branch '{branch}': {source}") + if len(set(moved_paths)) != len(moved_paths): + raise ValueError("Move would create duplicate logical paths") + + source_path_set = set(source_paths) + for moved_path in moved_paths: + if moved_path in existing_paths and moved_path not in source_path_set: + raise ValueError( + f"Destination already exists in branch '{branch}': {moved_path}" + ) + + updated_entries: dict[str, ManifestEntry] = {} + for entry in self.store.iter_manifest_entries(base_commit.manifest): + moved_path = path_map.get(entry.path) + if moved_path is None: + updated_entries[entry.path] = entry + continue + relocated_entry = relocate_manifest_entry(entry, moved_path) + updated_entries[relocated_entry.path] = relocated_entry + + def iter_entries() -> Iterator[ManifestEntry]: + for path in sorted(updated_entries): + yield updated_entries[path] + + temp_manifest = self._write_temp_manifest(iter_entries()) + manifest_hash = blake3_digest_file(temp_manifest) + self._persist_manifest(temp_manifest, manifest_hash) + commit_id = self._write_commit_object( + branch=branch, + message=message, + parent_commit=branch_state.commit_id, + manifest_hash=manifest_hash, + expected_version_token=branch_state.version_token, + operation="mv", + ) + return MoveResult( + ref=branch, + commit_id=commit_id, + source_path=source, + destination_path=destination, + moved_paths=sorted(moved_paths), + ) + def status(self, *, ref: str | None = None) -> StageStatus: branch = ref or self.current_branch() self._ensure_branch_exists(branch) @@ -229,11 +533,14 @@ def status(self, *, ref: str | None = None) -> StageStatus: return StageStatus(ref=branch, added=added, removed=removed) def read_commit(self, commit_id: str) -> CommitObject: - commit_path = self.layout.commits_dir / f"{commit_id}.json" - if not commit_path.exists(): + cached_commit = self._commit_cache.get(commit_id) + if cached_commit is not None: + return cached_commit + commit_payload = self.store.read_commit_bytes(commit_id) + if commit_payload is None: raise ValueError(f"Unknown commit: {commit_id}") - data = json.loads(commit_path.read_text(encoding="utf-8")) - return CommitObject( + data = json.loads(commit_payload.decode("utf-8")) + commit = CommitObject( id=str(data["id"]), message=str(data["message"]), manifest=str(data["manifest"]), @@ -241,6 +548,8 @@ def read_commit(self, commit_id: str) -> CommitObject: created_at=str(data["created_at"]), branch=str(data["branch"]), ) + self._commit_cache[commit_id] = commit + return commit def diff(self, from_ref: str, to_ref: str) -> list[DiffEntry]: from_commit = self.read_commit(self.resolve_ref(from_ref)) @@ -304,13 +613,10 @@ def verify( *, dry_run: bool = False, ) -> VerifyResult: - branch_path = self._branch_path(ref) - if not branch_path.exists(): - raise ValueError("verify currently supports branch refs only") + branch_state = self._require_branch_state(ref) base_commit_id = self.resolve_ref(ref) base_commit = self.read_commit(base_commit_id) - manifest_path = self.layout.manifests_dir / f"{base_commit.manifest}.jsonl" normalized_prefixes = [ prefix.strip("/") for prefix in (path_prefixes or []) if prefix.strip("/") ] @@ -329,8 +635,7 @@ def should_verify(entry_path: str) -> bool: def iter_verified_entries() -> Iterator[ManifestEntry]: nonlocal verified_entries, candidate_entries, total_entries - reader = ManifestReader(manifest_path) - for entry in reader.iter_entries(): + for entry in self.store.iter_manifest_entries(base_commit.manifest): total_entries += 1 if not should_verify(entry.path): yield entry @@ -372,12 +677,7 @@ def iter_verified_entries() -> Iterator[ManifestEntry]: ) manifest_hash = blake3_digest_file(temp_manifest) - manifest_target = self.layout.manifests_dir / f"{manifest_hash}.jsonl" - manifest_target.parent.mkdir(parents=True, exist_ok=True) - if not manifest_target.exists(): - temp_manifest.replace(manifest_target) - else: - temp_manifest.unlink(missing_ok=True) + self._persist_manifest(temp_manifest, manifest_hash) commit_body = { "message": f"verify {ref}", @@ -391,13 +691,21 @@ def iter_verified_entries() -> Iterator[ManifestEntry]: ).encode("utf-8") commit_id = blake3(canonical).hexdigest() commit_object = CommitObject(id=commit_id, **commit_body) + self._commit_cache[commit_id] = commit_object - commit_path = self.layout.commits_dir / f"{commit_id}.json" - commit_path.write_text( - json.dumps(asdict(commit_object), indent=2, sort_keys=True) + "\n", - encoding="utf-8", + self.store.write_commit_bytes( + commit_id, + (json.dumps(asdict(commit_object), indent=2, sort_keys=True) + "\n").encode( + "utf-8" + ), + ) + self._update_branch_ref( + branch=ref, + commit_id=commit_id, + expected_version_token=branch_state.version_token, + expected_commit_id=branch_state.commit_id, + operation="verify", ) - self._branch_path(ref).write_text(f"{commit_id}\n", encoding="utf-8") return VerifyResult( commit_id=commit_id, verified_entries=verified_entries, @@ -408,9 +716,8 @@ def iter_verified_entries() -> Iterator[ManifestEntry]: ) def _manifest_index(self, manifest_hash: str) -> dict[str, ManifestEntry]: - manifest_path = self.layout.manifests_dir / f"{manifest_hash}.jsonl" index: dict[str, ManifestEntry] = {} - for entry in ManifestReader(manifest_path).iter_entries(): + for entry in self.store.iter_manifest_entries(manifest_hash): index[entry.path] = entry return index @@ -425,13 +732,82 @@ def resolve_entries( index.pop(change.path, None) continue if change.action == "add": - index[change.path] = self._entry_from_working_path( - change.path, + index[change.path] = self._entry_from_stage_change( + change, change.identity_mode or "blake3", store_blob=False, ) return index + def resolve_entries_for_prefix( + self, + ref: str, + logical_prefix: str, + *, + include_staging: bool = False, + commit_id: str | None = None, + ) -> dict[str, ManifestEntry]: + normalized_prefix = logical_prefix.strip("/") + resolved_commit_id = commit_id or self.resolve_ref(ref) + commit = self.read_commit(resolved_commit_id) + + if not normalized_prefix: + index = self._manifest_index(commit.manifest) + else: + index = { + entry.path: entry + for entry in self.store.iter_manifest_entries_for_prefix( + commit.manifest, + normalized_prefix, + ) + } + + if include_staging: + for change in self._load_stage(ref).values(): + if normalized_prefix and not _matches_logical_prefix( + change.path, + normalized_prefix, + ): + continue + if change.action == "remove": + index.pop(change.path, None) + continue + if change.action == "add": + index[change.path] = self._entry_from_stage_change( + change, + change.identity_mode or "blake3", + store_blob=False, + ) + return index + + def resolve_entry( + self, + ref: str, + logical_path: str, + *, + include_staging: bool = False, + commit_id: str | None = None, + ) -> ManifestEntry | None: + normalized_path = logical_path.strip("/") + if not normalized_path: + return None + + if include_staging: + change = self._load_stage(ref).get(normalized_path) + if change is not None: + if change.action == "remove": + return None + if change.action == "add": + return self._entry_from_stage_change( + change, + change.identity_mode or "blake3", + store_blob=False, + ) + + resolved_commit_id = commit_id or self.resolve_ref(ref) + commit = self.read_commit(resolved_commit_id) + return self.store.lookup_manifest_entry(commit.manifest, normalized_path) + def _materialize_blobs_and_entries( self, *, identity_mode: str ) -> Iterator[ManifestEntry]: @@ -444,7 +820,7 @@ def _materialize_blobs_and_entries( blob_hash = identity_value self._store_blob(file_path, blob_hash) else: - identity_value = self._metadata_identity(relative_path, stat.st_size) + identity_value = metadata_identity(relative_path, stat.st_size) blob_hash = None yield ManifestEntry( path=relative_path, @@ -457,6 +833,45 @@ def _materialize_blobs_and_entries( source_uri=source_uri, ) + def _materialize_s3_entries( + self, + *, + source_uri: str, + identity_mode: str, + path_patterns: list[str] | None = None, + ) -> Iterator[ManifestEntry]: + _, prefix = parse_s3_uri(source_uri) + normalized_prefix = prefix.strip("/") + normalized_patterns = normalize_import_patterns(path_patterns) + for obj in iter_s3_objects(source_uri): + relative_path = normalize_s3_import_path( + key=obj.key, + prefix=normalized_prefix, + size=obj.size, + ) + if relative_path is None: + continue + if not matches_import_patterns(relative_path, normalized_patterns): + continue + if identity_mode == "blake3": + identity_value = self._store_blob_from_source_uri(obj.source_uri) + blob_hash = identity_value + elif identity_mode == "meta": + identity_value = metadata_identity(relative_path, obj.size) + blob_hash = None + else: + raise ValueError("identity_mode must be one of: blake3, meta") + yield ManifestEntry( + path=relative_path, + hash=identity_value, + size=obj.size, + mtime_ns=obj.mtime_ns, + identity_mode=identity_mode, + identity_value=identity_value, + blob_hash=blob_hash, + source_uri=obj.source_uri, + ) + def _entry_from_working_path( self, relative_path: str, @@ -477,7 +892,7 @@ def _entry_from_working_path( else: blob_hash = None elif identity_mode == "meta": - identity_value = self._metadata_identity(relative_path, stat.st_size) + identity_value = metadata_identity(relative_path, stat.st_size) blob_hash = None else: raise ValueError("identity_mode must be one of: blake3, meta") @@ -492,26 +907,203 @@ def _entry_from_working_path( source_uri=source_uri, ) - def _metadata_identity(self, relative_path: str, size: int) -> str: - payload = f"{relative_path}\n{size}".encode("utf-8") - return blake3(payload).hexdigest() + def _entry_from_stage_change( + self, + change: StageChange, + identity_mode: str, + *, + store_blob: bool, + ) -> ManifestEntry: + if change.source_uri is not None: + return self._entry_from_source_uri( + logical_path=change.path, + source_uri=change.source_uri, + identity_mode=identity_mode, + store_blob=store_blob, + ) + return self._entry_from_working_path( + change.path, + identity_mode, + store_blob=store_blob, + ) - def _stage_path(self, branch: str) -> Path: - return self.layout.staging_dir / f"{branch}.json" + def _entry_from_source_uri( + self, + *, + logical_path: str, + source_uri: str, + identity_mode: str, + store_blob: bool, + ) -> ManifestEntry: + metadata = describe_source_uri(source_uri) + if identity_mode == "blake3": + identity_value = self._store_blob_from_source_uri(source_uri) + blob_hash = identity_value if store_blob else None + elif identity_mode == "meta": + identity_value = metadata_identity(logical_path, metadata.size) + blob_hash = None + else: + raise ValueError("identity_mode must be one of: blake3, meta") + return ManifestEntry( + path=logical_path, + hash=identity_value, + size=metadata.size, + mtime_ns=metadata.mtime_ns, + identity_mode=identity_mode, + identity_value=identity_value, + blob_hash=blob_hash, + source_uri=metadata.source_uri, + ) - def _normalize_stage_path(self, path: str) -> str: - normalized = path.strip().strip("/") - if not normalized: - raise ValueError("Path cannot be empty") - if normalized.startswith("../") or "/../" in normalized or normalized == "..": - raise ValueError("Path cannot traverse outside repository root") - return normalized + def _expand_stage_source( + self, + raw_source: str, + *, + destination_path: str | None, + ) -> list[tuple[str, str]]: + if raw_source.startswith("s3://"): + return self._expand_s3_stage_source( + raw_source, + destination_path=destination_path, + ) + + return self._expand_local_stage_source( + raw_source, + destination_path=destination_path, + ) + + def _expand_local_stage_source( + self, + raw_source: str, + *, + destination_path: str | None, + ) -> list[tuple[str, str]]: + raw_path = Path(raw_source).expanduser() + source_path = ( + raw_path.resolve() + if raw_path.is_absolute() + else (self.root / raw_path).resolve() + ) + if not source_path.exists(): + raise FileNotFoundError(f"Cannot stage missing path: {raw_source}") + + if source_path.is_file(): + logical_path = self._single_local_logical_path( + source_path, + destination_path=destination_path, + ) + return [(logical_path, source_path.as_uri())] + + if not source_path.is_dir(): + raise FileNotFoundError(f"Cannot stage unsupported path: {raw_source}") + + repo_relative_dir: str | None + try: + repo_relative_dir = normalize_repository_path( + source_path.relative_to(self.root).as_posix() + ) + except ValueError: + repo_relative_dir = None + + destination_prefix = ( + normalize_repository_path(destination_path) + if destination_path is not None + else repo_relative_dir or normalize_repository_path(source_path.name) + ) + + staged_entries: list[tuple[str, str]] = [] + for file_path in walk_files(source_path): + relative_suffix = file_path.relative_to(source_path).as_posix() + logical_path = normalize_repository_path( + PurePosixPath(destination_prefix, relative_suffix).as_posix() + ) + staged_entries.append((logical_path, file_path.as_uri())) + + if not staged_entries: + raise FileNotFoundError(f"Cannot stage empty directory: {raw_source}") + return staged_entries + + def _single_local_logical_path( + self, + source_path: Path, + *, + destination_path: str | None, + ) -> str: + if destination_path is not None: + return normalize_repository_path(destination_path) + try: + return normalize_repository_path( + source_path.relative_to(self.root).as_posix() + ) + except ValueError: + return normalize_repository_path(source_path.name) + + def _expand_s3_stage_source( + self, + raw_source: str, + *, + destination_path: str | None, + ) -> list[tuple[str, str]]: + bucket, key = parse_s3_uri(raw_source) + normalized_key = key.strip("/") + if not normalized_key: + raise ValueError(f"S3 source cannot be bucket root: {raw_source}") + + objects = list(iter_s3_objects(raw_source)) + if not objects: + raise FileNotFoundError(f"Cannot stage missing S3 path: {raw_source}") + + exact_object_uri = f"s3://{bucket}/{normalized_key}" + exact_object_matches = [ + obj for obj in objects if obj.source_uri == exact_object_uri + ] + is_single_object = len(exact_object_matches) == 1 and len(objects) == 1 + + if is_single_object: + logical_path = ( + normalize_repository_path(destination_path) + if destination_path is not None + else normalize_repository_path(PurePosixPath(normalized_key).name) + ) + return [(logical_path, exact_object_uri)] + + prefix = normalized_key + destination_prefix = ( + normalize_repository_path(destination_path) + if destination_path is not None + else normalize_repository_path(PurePosixPath(prefix).name) + ) + + staged_entries: list[tuple[str, str]] = [] + for obj in objects: + relative_path = normalize_s3_import_path( + key=obj.key, + prefix=prefix, + size=obj.size, + ) + if relative_path is None: + continue + logical_path = normalize_repository_path( + PurePosixPath(destination_prefix, relative_path).as_posix() + ) + staged_entries.append((logical_path, obj.source_uri)) + + if not staged_entries and exact_object_matches: + logical_path = ( + normalize_repository_path(destination_path) + if destination_path is not None + else normalize_repository_path(PurePosixPath(normalized_key).name) + ) + return [(logical_path, exact_object_uri)] + if not staged_entries: + raise FileNotFoundError(f"Cannot stage empty S3 prefix: {raw_source}") + return staged_entries def _load_stage(self, branch: str) -> dict[str, StageChange]: - stage_path = self._stage_path(branch) - if not stage_path.exists(): + stage_payload = self.client_state.read_staging_payload(branch) + if stage_payload is None: return {} - raw = json.loads(stage_path.read_text(encoding="utf-8")) + raw = json.loads(stage_payload) if not isinstance(raw, list): raise ValueError("Stage file must contain a list") staged: dict[str, StageChange] = {} @@ -523,32 +1115,66 @@ def _load_stage(self, branch: str) -> dict[str, StageChange]: return staged def _save_stage(self, branch: str, staged: dict[str, StageChange]) -> None: - stage_path = self._stage_path(branch) if not staged: - stage_path.unlink(missing_ok=True) + self.client_state.write_staging_payload(branch, None) return payload = [ { "path": change.path, "action": change.action, "identity_mode": change.identity_mode, + "source_uri": change.source_uri, } for change in sorted(staged.values(), key=lambda item: item.path) ] - stage_path.parent.mkdir(parents=True, exist_ok=True) - stage_path.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") + self.client_state.write_staging_payload( + branch, + json.dumps(payload, indent=2) + "\n", + ) def _ensure_branch_exists(self, branch: str) -> None: - branch_path = self._branch_path(branch) - if not branch_path.exists(): + self._require_branch_state(branch) + + def _require_branch_state(self, branch: str) -> BranchRefState: + cached_state = self.client_state.read_branch_snapshot(branch) + if cached_state is not None: + return BranchRefState( + branch=branch, + commit_id=cached_state.commit_id, + version_token=cached_state.version_token, + ) + + branch_state = self.store.read_branch_ref(branch) + if branch_state is None: raise ValueError(f"Unknown branch: {branch}") + self.client_state.write_branch_snapshot( + branch, + commit_id=branch_state.commit_id, + version_token=branch_state.version_token, + ) + return branch_state + + def _require_commit_for_metadata_mutation(self, branch: str) -> CommitObject: + branch_state = self._require_branch_state(branch) + if not branch_state.commit_id: + raise FileNotFoundError( + f"Branch '{branch}' has no committed manifest to mutate" + ) + return self.read_commit(branch_state.commit_id) def _branch_head_commit(self, branch: str) -> str | None: - branch_path = self._branch_path(branch) - if not branch_path.exists(): + branch_ref = self.store.read_branch_ref(branch) + if branch_ref is None: return None - value = branch_path.read_text(encoding="utf-8").strip() - return value or None + return branch_ref.commit_id + + def _is_ancestor(self, *, ancestor_commit: str, descendant_commit: str) -> bool: + current_commit: str | None = descendant_commit + while current_commit: + if current_commit == ancestor_commit: + return True + current_commit = self.read_commit(current_commit).parent + return False def _write_commit_object( self, @@ -557,6 +1183,8 @@ def _write_commit_object( message: str, parent_commit: str | None, manifest_hash: str, + expected_version_token: str | None, + operation: str, ) -> str: commit_body = { "message": message, @@ -570,20 +1198,28 @@ def _write_commit_object( ).encode("utf-8") commit_id = blake3(canonical).hexdigest() commit_object = CommitObject(id=commit_id, **commit_body) - commit_path = self.layout.commits_dir / f"{commit_id}.json" - commit_path.write_text( - json.dumps(asdict(commit_object), indent=2, sort_keys=True) + "\n", - encoding="utf-8", + self._commit_cache[commit_id] = commit_object + self.store.write_commit_bytes( + commit_id, + (json.dumps(asdict(commit_object), indent=2, sort_keys=True) + "\n").encode( + "utf-8" + ), + ) + self._update_branch_ref( + branch=branch, + commit_id=commit_id, + expected_version_token=expected_version_token, + expected_commit_id=parent_commit, + operation=operation, ) - self._branch_path(branch).write_text(f"{commit_id}\n", encoding="utf-8") return commit_id - def _commit_staged(self, *, message: str, branch: str) -> str: - staged = self._load_stage(branch) + def _commit_staged(self, *, message: str, branch_state: BranchRefState) -> str: + staged = self._load_stage(branch_state.branch) if not staged: - raise ValueError(f"No staged changes for branch: {branch}") + raise ValueError(f"No staged changes for branch: {branch_state.branch}") - parent_commit = self._branch_head_commit(branch) + parent_commit = branch_state.commit_id index: dict[str, ManifestEntry] if parent_commit: parent = self.read_commit(parent_commit) @@ -596,8 +1232,8 @@ def _commit_staged(self, *, message: str, branch: str) -> str: index.pop(change.path, None) continue if change.action == "add": - index[change.path] = self._entry_from_working_path( - change.path, + index[change.path] = self._entry_from_stage_change( + change, change.identity_mode or "blake3", store_blob=True, ) @@ -610,40 +1246,27 @@ def iter_entries() -> Iterator[ManifestEntry]: temp_manifest = self._write_temp_manifest(iter_entries()) manifest_hash = blake3_digest_file(temp_manifest) - manifest_target = self.layout.manifests_dir / f"{manifest_hash}.jsonl" - manifest_target.parent.mkdir(parents=True, exist_ok=True) - if not manifest_target.exists(): - temp_manifest.replace(manifest_target) - else: - temp_manifest.unlink(missing_ok=True) + self._persist_manifest(temp_manifest, manifest_hash) commit_id = self._write_commit_object( - branch=branch, + branch=branch_state.branch, message=message, parent_commit=parent_commit, manifest_hash=manifest_hash, + expected_version_token=branch_state.version_token, + operation="commit", ) - self._save_stage(branch, {}) + self._save_stage(branch_state.branch, {}) return commit_id def _store_blob(self, source_file: Path, content_hash: str) -> None: - rel = blob_relpath(content_hash) - target = self.layout.blobs_dir / rel - if target.exists(): - return - target.parent.mkdir(parents=True, exist_ok=True) - with source_file.open("rb") as src, target.open("wb") as dst: - while True: - chunk = src.read(DEFAULT_CHUNK_SIZE) - if not chunk: - break - dst.write(chunk) + self.store.write_blob_file(content_hash, source_file, if_missing=True) def _store_blob_from_source_uri(self, source_uri: str) -> str: with NamedTemporaryFile(mode="wb", delete=False) as temp: temp_path = Path(temp.name) hasher = blake3() - with fsspec.open(source_uri, mode="rb").open() as source: + with open_source_uri(source_uri) as source: while True: chunk = source.read(DEFAULT_CHUNK_SIZE) if not chunk: @@ -651,14 +1274,78 @@ def _store_blob_from_source_uri(self, source_uri: str) -> str: hasher.update(chunk) temp.write(chunk) digest = hasher.hexdigest() - target = self.layout.blobs_dir / blob_relpath(digest) - if target.exists(): + if self.store.object_exists("blob", digest): temp_path.unlink(missing_ok=True) return digest - target.parent.mkdir(parents=True, exist_ok=True) - temp_path.replace(target) + self.store.write_blob_file(digest, temp_path, if_missing=True) + temp_path.unlink(missing_ok=True) return digest + def read_blob(self, blob_hash: str) -> bytes: + return self.store.read_blob_bytes(blob_hash) + + def _persist_manifest(self, temp_manifest: Path, manifest_hash: str) -> None: + temp_index = self._write_temp_manifest_index(temp_manifest) + try: + if self.store.object_exists("manifest", manifest_hash): + temp_manifest.unlink(missing_ok=True) + else: + self.store.write_manifest_file( + manifest_hash, temp_manifest, if_missing=True + ) + + if self.store.object_exists("manifest-index", manifest_hash): + temp_index.unlink(missing_ok=True) + else: + self.store.write_manifest_index_file( + manifest_hash, + temp_index, + if_missing=True, + ) + finally: + temp_manifest.unlink(missing_ok=True) + temp_index.unlink(missing_ok=True) + + def _update_branch_ref( + self, + *, + branch: str, + commit_id: str | None, + expected_version_token: str | None, + expected_commit_id: str | None, + operation: str, + ) -> None: + updated = self.store.compare_and_set_branch_ref( + branch, + commit_id, + expected_version_token=expected_version_token, + expected_commit_id=expected_commit_id, + ) + if updated: + self._resolved_ref_cache.pop(branch, None) + current_state = self.store.read_branch_ref(branch) + if current_state is not None: + self._resolved_ref_cache[branch] = current_state + self.client_state.write_branch_snapshot( + branch, + commit_id=current_state.commit_id, + version_token=current_state.version_token, + ) + return + current_state = self.store.read_branch_ref(branch) + if current_state is not None: + self.client_state.write_branch_snapshot( + branch, + commit_id=current_state.commit_id, + version_token=current_state.version_token, + ) + raise RefConflictError( + branch=branch, + operation=operation, + expected_commit_id=expected_commit_id, + current_commit_id=current_state.commit_id if current_state else None, + ) + def _write_temp_manifest(self, entries: Iterator[ManifestEntry]) -> Path: with NamedTemporaryFile( mode="w", suffix=".jsonl", delete=False, encoding="utf-8" @@ -668,6 +1355,58 @@ def _write_temp_manifest(self, entries: Iterator[ManifestEntry]) -> Path: writer.write_entries(entries) return temp_path + def _write_temp_manifest_index(self, manifest_path: Path) -> Path: + return build_manifest_index_file(manifest_path) + + +def _default_remote_client_root(worktree_root: Path, repo_uri: str) -> Path: + repo_id = blake3(repo_uri.encode("utf-8")).hexdigest()[:16] + return worktree_root / ".fluxel" / "clients" / repo_id + + +def _matches_logical_prefix(path: str, logical_prefix: str) -> bool: + return path == logical_prefix or path.startswith(f"{logical_prefix}/") + + +def _matches_logical_prefix(path: str, logical_prefix: str) -> bool: + return path == logical_prefix or path.startswith(f"{logical_prefix}/") + + +def open_repository( + root: str | Path, + *, + worktree: str | Path | None = None, + client_root: str | Path | None = None, + s3_client: object | None = None, +) -> FluxelRepository: + if isinstance(root, str) and root.startswith("s3://"): + bucket, prefix = parse_s3_uri(root) + worktree_root = Path(worktree or ".").resolve() + resolved_client_root = ( + Path(client_root).resolve() + if client_root + else _default_remote_client_root(worktree_root, root) + ) + return FluxelRepository( + worktree_root, + store=S3RepositoryStore( + bucket, + prefix, + client=s3_client, + branch_root=worktree_root / ".fluxel" / "refs" / "heads", + ), + client_state=LocalClientState(resolved_client_root), + ) + + repo_root = Path(root).resolve() + worktree_root = Path(worktree).resolve() if worktree else repo_root + resolved_client_root = Path(client_root).resolve() if client_root else repo_root + return FluxelRepository( + worktree_root, + store=LocalRepositoryStore(repo_root), + client_state=LocalClientState(resolved_client_root), + ) + def commit( root: str | Path, @@ -677,7 +1416,7 @@ def commit( staged: bool = False, ref: str | None = None, ) -> str: - return FluxelRepository(root).commit( + return open_repository(root).commit( message, identity_mode=identity_mode, staged=staged, @@ -685,8 +1424,30 @@ def commit( ) +def import_s3( + root: str | Path, + source_uri: str, + message: str, + identity_mode: str = "blake3", + *, + path_patterns: list[str] | None = None, + ref: str | None = None, +) -> str: + return open_repository(root).import_s3( + source_uri, + message, + identity_mode=identity_mode, + path_patterns=path_patterns, + ref=ref, + ) + + def branch(root: str | Path, name: str) -> Path: - return FluxelRepository(root).branch(name) + return open_repository(root).branch(name) + + +def merge(root: str | Path, source_ref: str, target_ref: str) -> MergeResult: + return open_repository(root).merge(source_ref, target_ref) def add( @@ -695,20 +1456,52 @@ def add( *, ref: str | None = None, identity_mode: str = "blake3", + destination_path: str | None = None, ) -> StageStatus: - return FluxelRepository(root).add(paths, ref=ref, identity_mode=identity_mode) + return open_repository(root).add( + paths, + ref=ref, + identity_mode=identity_mode, + destination_path=destination_path, + ) def rm(root: str | Path, paths: list[str], *, ref: str | None = None) -> StageStatus: - return FluxelRepository(root).rm(paths, ref=ref) + return open_repository(root).rm(paths, ref=ref) + + +def remove( + root: str | Path, + paths: list[str], + message: str, + *, + ref: str | None = None, +) -> RemoveResult: + return open_repository(root).remove_paths(paths, message, ref=ref) + + +def move( + root: str | Path, + source_path: str, + destination_path: str, + message: str, + *, + ref: str | None = None, +) -> MoveResult: + return open_repository(root).move( + source_path, + destination_path, + message, + ref=ref, + ) def status(root: str | Path, *, ref: str | None = None) -> StageStatus: - return FluxelRepository(root).status(ref=ref) + return open_repository(root).status(ref=ref) def diff(root: str | Path, from_ref: str, to_ref: str) -> list[DiffEntry]: - return FluxelRepository(root).diff(from_ref=from_ref, to_ref=to_ref) + return open_repository(root).diff(from_ref=from_ref, to_ref=to_ref) def verify( @@ -718,6 +1511,6 @@ def verify( *, dry_run: bool = False, ) -> VerifyResult: - return FluxelRepository(root).verify( + return open_repository(root).verify( ref=ref, path_prefixes=path_prefixes, dry_run=dry_run ) diff --git a/src/fluxel/core/repository_store.py b/src/fluxel/core/repository_store.py new file mode 100644 index 0000000..103a583 --- /dev/null +++ b/src/fluxel/core/repository_store.py @@ -0,0 +1,745 @@ +from __future__ import annotations + +import json +from dataclasses import dataclass +from datetime import datetime, timedelta, timezone +from pathlib import Path +from tempfile import NamedTemporaryFile +from uuid import uuid4 +from typing import BinaryIO, Iterator, Literal, Protocol + +import boto3 +from botocore.exceptions import ClientError + +from .layout import blob_relpath, initialize_fluxel_layout +from .manifest import ManifestEntry, ManifestReader, deserialize_manifest_entry +from .manifest_index import ( + build_manifest_index, + iter_manifest_index_entry_jsons, + lookup_manifest_index_entry_json, +) +from .storage import OptimisticLockError + + +RepositoryObjectKind = Literal["blob", "commit", "manifest", "manifest-index", "ref"] +DEFAULT_BRANCH_LOCK_TIMEOUT_SECONDS = 30 + + +@dataclass(frozen=True) +class BranchRefState: + branch: str + commit_id: str | None + version_token: str | None + + +@dataclass(frozen=True) +class BranchLockState: + token: str + expires_at: datetime | None + last_modified: datetime | None + + +class RepositoryStore(Protocol): + def read_commit_bytes(self, commit_id: str) -> bytes | None: ... + + def write_commit_bytes( + self, + commit_id: str, + payload: bytes, + *, + if_missing: bool = False, + ) -> None: ... + + def iter_manifest_entries(self, manifest_hash: str) -> Iterator[ManifestEntry]: ... + + def write_manifest_file( + self, + manifest_hash: str, + source_path: str | Path, + *, + if_missing: bool = False, + ) -> None: ... + + def write_manifest_index_file( + self, + manifest_hash: str, + source_path: str | Path, + *, + if_missing: bool = False, + ) -> None: ... + + def branch_path(self, branch: str) -> Path: ... + + def read_branch_ref(self, branch: str) -> BranchRefState | None: ... + + def write_branch_ref(self, branch: str, commit_id: str | None) -> None: ... + + def compare_and_set_branch_ref( + self, + branch: str, + commit_id: str | None, + *, + expected_version_token: str | None, + expected_commit_id: str | None = None, + ) -> bool: ... + + def read_blob_bytes(self, blob_hash: str) -> bytes: ... + + def lookup_manifest_entry( + self, manifest_hash: str, logical_path: str + ) -> ManifestEntry | None: ... + + def iter_manifest_entries_for_prefix( + self, manifest_hash: str, logical_prefix: str + ) -> Iterator[ManifestEntry]: ... + + def write_blob_file( + self, + blob_hash: str, + source_path: str | Path, + *, + if_missing: bool = True, + ) -> None: ... + + def object_exists(self, kind: RepositoryObjectKind, object_id: str) -> bool: ... + + def version_token( + self, kind: RepositoryObjectKind, object_id: str + ) -> str | None: ... + + +class LocalRepositoryStore(RepositoryStore): + def __init__(self, root: str | Path) -> None: + self.layout = initialize_fluxel_layout(root) + + def read_commit_bytes(self, commit_id: str) -> bytes | None: + commit_path = self.commit_path(commit_id) + if not commit_path.exists(): + return None + return commit_path.read_bytes() + + def write_commit_bytes( + self, + commit_id: str, + payload: bytes, + *, + if_missing: bool = False, + ) -> None: + commit_path = self.commit_path(commit_id) + commit_path.parent.mkdir(parents=True, exist_ok=True) + if if_missing and commit_path.exists(): + raise OptimisticLockError(f"Commit already exists: {commit_id}") + commit_path.write_bytes(payload) + + def iter_manifest_entries(self, manifest_hash: str) -> Iterator[ManifestEntry]: + yield from ManifestReader(self.manifest_path(manifest_hash)).iter_entries() + + def write_manifest_file( + self, + manifest_hash: str, + source_path: str | Path, + *, + if_missing: bool = False, + ) -> None: + manifest_path = self.manifest_path(manifest_hash) + source = Path(source_path) + manifest_path.parent.mkdir(parents=True, exist_ok=True) + if if_missing and manifest_path.exists(): + raise OptimisticLockError(f"Manifest already exists: {manifest_hash}") + source.replace(manifest_path) + + def write_manifest_index_file( + self, + manifest_hash: str, + source_path: str | Path, + *, + if_missing: bool = False, + ) -> None: + index_path = self.manifest_index_path(manifest_hash) + source = Path(source_path) + index_path.parent.mkdir(parents=True, exist_ok=True) + if if_missing and index_path.exists(): + raise OptimisticLockError(f"Manifest index already exists: {manifest_hash}") + source.replace(index_path) + + def read_branch_ref(self, branch: str) -> BranchRefState | None: + branch_path = self.branch_path(branch) + if not branch_path.exists(): + return None + commit_id = branch_path.read_text(encoding="utf-8").strip() or None + return BranchRefState( + branch=branch, + commit_id=commit_id, + version_token=self.version_token("ref", branch), + ) + + def write_branch_ref(self, branch: str, commit_id: str | None) -> None: + branch_path = self.branch_path(branch) + branch_path.parent.mkdir(parents=True, exist_ok=True) + payload = f"{commit_id}\n" if commit_id else "" + branch_path.write_text(payload, encoding="utf-8") + + def compare_and_set_branch_ref( + self, + branch: str, + commit_id: str | None, + *, + expected_version_token: str | None, + expected_commit_id: str | None = None, + ) -> bool: + current_token = self.version_token("ref", branch) + if current_token != expected_version_token: + return False + current_state = self.read_branch_ref(branch) + current_commit_id = current_state.commit_id if current_state else None + if current_commit_id != expected_commit_id: + return False + self.write_branch_ref(branch, commit_id) + return True + + def read_blob_bytes(self, blob_hash: str) -> bytes: + return self.blob_path(blob_hash).read_bytes() + + def lookup_manifest_entry( + self, manifest_hash: str, logical_path: str + ) -> ManifestEntry | None: + index_path = self.manifest_index_path(manifest_hash) + manifest_path = self.manifest_path(manifest_hash) + entry_json = lookup_manifest_index_entry_json( + index_path, + logical_path, + read_range=lambda start, end: self._read_manifest_range( + manifest_path, + start, + end, + ), + ) + if entry_json is None: + return None + return deserialize_manifest_entry(entry_json) + + def iter_manifest_entries_for_prefix( + self, manifest_hash: str, logical_prefix: str + ) -> Iterator[ManifestEntry]: + normalized_prefix = logical_prefix.strip("/") + index_path = self.manifest_index_path(manifest_hash) + manifest_path = self.manifest_path(manifest_hash) + for entry_json in iter_manifest_index_entry_jsons( + index_path, + normalized_prefix or None, + read_range=lambda start, end: self._read_manifest_range( + manifest_path, + start, + end, + ), + ): + yield deserialize_manifest_entry(entry_json) + + def write_blob_file( + self, + blob_hash: str, + source_path: str | Path, + *, + if_missing: bool = True, + ) -> None: + blob_path = self.blob_path(blob_hash) + if if_missing and blob_path.exists(): + return + blob_path.parent.mkdir(parents=True, exist_ok=True) + with Path(source_path).open("rb") as src, blob_path.open("wb") as dst: + while chunk := src.read(1024 * 1024): + dst.write(chunk) + + def object_exists(self, kind: RepositoryObjectKind, object_id: str) -> bool: + return self._path_for(kind, object_id).exists() + + def version_token(self, kind: RepositoryObjectKind, object_id: str) -> str | None: + path = self._path_for(kind, object_id) + if not path.exists(): + return None + stat = path.stat() + return f"{stat.st_mtime_ns}-{stat.st_size}" + + def blob_path(self, blob_hash: str) -> Path: + return self.layout.blobs_dir / blob_relpath(blob_hash) + + def commit_path(self, commit_id: str) -> Path: + return self.layout.commits_dir / f"{commit_id}.json" + + def manifest_path(self, manifest_hash: str) -> Path: + return self.layout.manifests_dir / f"{manifest_hash}.jsonl" + + def manifest_index_path(self, manifest_hash: str) -> Path: + return self.layout.manifests_dir / f"{manifest_hash}.idx" + + def branch_path(self, branch: str) -> Path: + return self.layout.heads_dir / branch + + def _path_for(self, kind: RepositoryObjectKind, object_id: str) -> Path: + if kind == "blob": + return self.blob_path(object_id) + if kind == "commit": + return self.commit_path(object_id) + if kind == "manifest": + return self.manifest_path(object_id) + if kind == "manifest-index": + return self.manifest_index_path(object_id) + return self.branch_path(object_id) + + def _read_manifest_range(self, manifest_path: Path, start: int, end: int) -> bytes: + if end <= start: + return b"" + with manifest_path.open("rb") as handle: + handle.seek(start) + return handle.read(end - start) + + +class S3RepositoryStore(RepositoryStore): + def __init__( + self, + bucket: str, + prefix: str = "", + *, + client: object | None = None, + branch_root: str | Path | None = None, + lock_timeout_seconds: int = DEFAULT_BRANCH_LOCK_TIMEOUT_SECONDS, + ) -> None: + self.bucket = bucket + self.prefix = prefix.strip("/") + self.client = client or boto3.client("s3") + self.branch_root = Path(branch_root).resolve() if branch_root else None + self.lock_timeout_seconds = max(1, lock_timeout_seconds) + self._manifest_index_cache: dict[str, Path] = {} + + def read_commit_bytes(self, commit_id: str) -> bytes | None: + try: + response = self.client.get_object( + Bucket=self.bucket, + Key=self._key("commit", commit_id), + ) + except ClientError as error: + if self._missing(error): + return None + raise + return response["Body"].read() + + def write_commit_bytes( + self, + commit_id: str, + payload: bytes, + *, + if_missing: bool = False, + ) -> None: + kwargs = { + "Bucket": self.bucket, + "Key": self._key("commit", commit_id), + "Body": payload, + } + if if_missing: + kwargs["IfNoneMatch"] = "*" + try: + self.client.put_object(**kwargs) + except ClientError as error: + if if_missing and self._precondition_failed(error): + raise OptimisticLockError( + f"Commit already exists: {commit_id}" + ) from error + raise + + def iter_manifest_entries(self, manifest_hash: str) -> Iterator[ManifestEntry]: + try: + response = self.client.get_object( + Bucket=self.bucket, + Key=self._key("manifest", manifest_hash), + ) + except ClientError as error: + if self._missing(error): + return + raise + body = response["Body"] + try: + manifest_uri = f"s3://{self.bucket}/{self._key('manifest', manifest_hash)}" + for line_number, raw_line in enumerate(body.iter_lines(), start=1): + line = raw_line.decode("utf-8").strip() + if not line: + continue + try: + yield deserialize_manifest_entry(line) + except ValueError as error: + raise ValueError( + f"Invalid manifest entry at line {line_number} in {manifest_uri}: {error}" + ) from error + finally: + body.close() + + def write_manifest_file( + self, + manifest_hash: str, + source_path: str | Path, + *, + if_missing: bool = False, + ) -> None: + with Path(source_path).open("rb") as handle: + self._put_stream( + key=self._key("manifest", manifest_hash), + body=handle, + if_missing=if_missing, + error_message=f"Manifest already exists: {manifest_hash}", + ) + + def write_manifest_index_file( + self, + manifest_hash: str, + source_path: str | Path, + *, + if_missing: bool = False, + ) -> None: + with Path(source_path).open("rb") as handle: + self._put_stream( + key=self._key("manifest-index", manifest_hash), + body=handle, + if_missing=if_missing, + error_message=f"Manifest index already exists: {manifest_hash}", + ) + + def read_branch_ref(self, branch: str) -> BranchRefState | None: + key = self._key("ref", branch) + try: + response = self.client.get_object(Bucket=self.bucket, Key=key) + except ClientError as error: + if self._missing(error): + return None + raise + commit_id = response["Body"].read().decode("utf-8").strip() or None + return BranchRefState( + branch=branch, + commit_id=commit_id, + version_token=response.get("ETag", "").strip('"') or None, + ) + + def branch_path(self, branch: str) -> Path: + if self.branch_root is not None: + return self.branch_root / branch + return Path(".fluxel") / "refs" / "heads" / branch + + def write_branch_ref(self, branch: str, commit_id: str | None) -> None: + payload = f"{commit_id}\n".encode("utf-8") if commit_id else b"" + self.client.put_object( + Bucket=self.bucket, + Key=self._key("ref", branch), + Body=payload, + ) + + def compare_and_set_branch_ref( + self, + branch: str, + commit_id: str | None, + *, + expected_version_token: str | None, + expected_commit_id: str | None = None, + ) -> bool: + lock_token = str(uuid4()) + if not self._acquire_branch_lock(branch, lock_token): + return False + try: + current = self.read_branch_ref(branch) + current_version = current.version_token if current else None + if current_version != expected_version_token: + return False + current_commit_id = current.commit_id if current else None + if current_commit_id != expected_commit_id: + return False + self.write_branch_ref(branch, commit_id) + return True + finally: + self._release_branch_lock(branch, lock_token) + + def read_blob_bytes(self, blob_hash: str) -> bytes: + response = self.client.get_object( + Bucket=self.bucket, + Key=self._key("blob", blob_hash), + ) + return response["Body"].read() + + def lookup_manifest_entry( + self, manifest_hash: str, logical_path: str + ) -> ManifestEntry | None: + index_path = self._cached_manifest_index_path(manifest_hash) + if index_path is None: + raise FileNotFoundError(f"Missing manifest index for: {manifest_hash}") + entry_json = lookup_manifest_index_entry_json( + index_path, + logical_path, + read_range=lambda start, end: self._read_remote_manifest_range( + manifest_hash, + start, + end, + ), + ) + if entry_json is None: + return None + return deserialize_manifest_entry(entry_json) + + def iter_manifest_entries_for_prefix( + self, manifest_hash: str, logical_prefix: str + ) -> Iterator[ManifestEntry]: + normalized_prefix = logical_prefix.strip("/") + index_path = self._cached_manifest_index_path(manifest_hash) + if index_path is None: + raise FileNotFoundError(f"Missing manifest index for: {manifest_hash}") + + for entry_json in iter_manifest_index_entry_jsons( + index_path, + normalized_prefix or None, + read_range=lambda start, end: self._read_remote_manifest_range( + manifest_hash, + start, + end, + ), + ): + yield deserialize_manifest_entry(entry_json) + + def write_blob_file( + self, + blob_hash: str, + source_path: str | Path, + *, + if_missing: bool = True, + ) -> None: + with Path(source_path).open("rb") as handle: + self._put_stream( + key=self._key("blob", blob_hash), + body=handle, + if_missing=if_missing, + error_message=f"Blob already exists: {blob_hash}", + ) + + def object_exists(self, kind: RepositoryObjectKind, object_id: str) -> bool: + try: + self.client.head_object(Bucket=self.bucket, Key=self._key(kind, object_id)) + return True + except ClientError as error: + if self._missing(error): + return False + raise + + def version_token(self, kind: RepositoryObjectKind, object_id: str) -> str | None: + try: + response = self.client.head_object( + Bucket=self.bucket, + Key=self._key(kind, object_id), + ) + except ClientError as error: + if self._missing(error): + return None + raise + return response.get("ETag", "").strip('"') or None + + def _key(self, kind: RepositoryObjectKind, object_id: str) -> str: + relative_path = self._relative_path(kind, object_id) + if self.prefix: + return f"{self.prefix}/{relative_path}" + return relative_path + + def _relative_path(self, kind: RepositoryObjectKind, object_id: str) -> str: + if kind == "blob": + return f"blobs/{blob_relpath(object_id).as_posix()}" + if kind == "commit": + return f"commits/{object_id}.json" + if kind == "manifest": + return f"manifests/{object_id}.jsonl" + if kind == "manifest-index": + return f"manifests/{object_id}.idx" + return f"refs/heads/{object_id}" + + def _cached_manifest_index_path(self, manifest_hash: str) -> Path | None: + cached_path = self._manifest_index_cache.get(manifest_hash) + if cached_path is not None and cached_path.exists(): + return cached_path + + try: + response = self.client.get_object( + Bucket=self.bucket, + Key=self._key("manifest-index", manifest_hash), + ) + except ClientError as error: + if self._missing(error): + return None + raise + + with NamedTemporaryFile(mode="wb", suffix=".idx", delete=False) as temp: + temp_path = Path(temp.name) + temp.write(response["Body"].read()) + self._manifest_index_cache[manifest_hash] = temp_path + return temp_path + + def _read_remote_manifest_range( + self, manifest_hash: str, start: int, end: int + ) -> bytes: + if end <= start: + return b"" + response = self.client.get_object( + Bucket=self.bucket, + Key=self._key("manifest", manifest_hash), + Range=f"bytes={start}-{end - 1}", + ) + body = response["Body"] + try: + return body.read() + finally: + body.close() + + def _lock_key(self, branch: str) -> str: + suffix = f"locks/refs/heads/{branch}.lock" + if self.prefix: + return f"{self.prefix}/{suffix}" + return suffix + + def _acquire_branch_lock(self, branch: str, token: str) -> bool: + if self._try_acquire_branch_lock(branch, token): + return True + + current_lock = self._read_branch_lock(branch) + if current_lock is None or not self._is_stale_branch_lock(current_lock): + return False + + self._release_branch_lock(branch, current_lock.token) + return self._try_acquire_branch_lock(branch, token) + + def _try_acquire_branch_lock(self, branch: str, token: str) -> bool: + expires_at = datetime.now(timezone.utc) + timedelta( + seconds=self.lock_timeout_seconds + ) + payload = json.dumps( + { + "token": token, + "expires_at": expires_at.isoformat(), + }, + sort_keys=True, + ).encode("utf-8") + try: + self.client.put_object( + Bucket=self.bucket, + Key=self._lock_key(branch), + Body=payload, + IfNoneMatch="*", + ) + except ClientError as error: + if self._precondition_failed(error): + return False + raise + return True + + def _read_branch_lock(self, branch: str) -> BranchLockState | None: + try: + response = self.client.get_object( + Bucket=self.bucket, + Key=self._lock_key(branch), + ) + except ClientError as error: + if self._missing(error): + return None + raise + + body = response["Body"] + try: + raw_payload = body.read().decode("utf-8") + finally: + body.close() + + last_modified_raw = response.get("LastModified") + last_modified: datetime | None = None + if isinstance(last_modified_raw, datetime): + if last_modified_raw.tzinfo is None: + last_modified = last_modified_raw.replace(tzinfo=timezone.utc) + else: + last_modified = last_modified_raw.astimezone(timezone.utc) + + try: + payload = json.loads(raw_payload) + except json.JSONDecodeError: + payload = None + + if isinstance(payload, dict): + token = str(payload.get("token") or "").strip() + expires_at_raw = payload.get("expires_at") + expires_at: datetime | None = None + if isinstance(expires_at_raw, str) and expires_at_raw: + expires_at = datetime.fromisoformat(expires_at_raw) + if expires_at.tzinfo is None: + expires_at = expires_at.replace(tzinfo=timezone.utc) + else: + expires_at = expires_at.astimezone(timezone.utc) + if token: + return BranchLockState( + token=token, + expires_at=expires_at, + last_modified=last_modified, + ) + + token = raw_payload.strip() + if not token: + return None + return BranchLockState( + token=token, + expires_at=None, + last_modified=last_modified, + ) + + def _is_stale_branch_lock(self, lock_state: BranchLockState) -> bool: + now = datetime.now(timezone.utc) + if lock_state.expires_at is not None: + return lock_state.expires_at <= now + if lock_state.last_modified is None: + return False + return ( + lock_state.last_modified + timedelta(seconds=self.lock_timeout_seconds) + <= now + ) + + def _release_branch_lock(self, branch: str, token: str) -> None: + current_lock = self._read_branch_lock(branch) + if current_lock is None or current_lock.token != token: + return + self.client.delete_object(Bucket=self.bucket, Key=self._lock_key(branch)) + + def _put_stream( + self, + *, + key: str, + body: BinaryIO, + if_missing: bool, + error_message: str, + ) -> None: + kwargs = { + "Bucket": self.bucket, + "Key": key, + "Body": body, + } + if if_missing: + kwargs["IfNoneMatch"] = "*" + try: + self.client.put_object(**kwargs) + except ClientError as error: + if if_missing and self._precondition_failed(error): + raise OptimisticLockError(error_message) from error + raise + + def _missing(self, error: ClientError) -> bool: + code = error.response.get("Error", {}).get("Code", "") + return code in {"404", "NoSuchKey", "NotFound"} + + def _precondition_failed(self, error: ClientError) -> bool: + code = error.response.get("Error", {}).get("Code", "") + return code in {"PreconditionFailed", "412"} + + +def build_manifest_index_file( + manifest_path: str | Path, + *, + suffix: str = ".idx", +) -> Path: + with NamedTemporaryFile(mode="wb", suffix=suffix, delete=False) as temp: + index_path = Path(temp.name) + build_manifest_index(manifest_path, index_path) + return index_path diff --git a/src/fluxel/core/repository_support.py b/src/fluxel/core/repository_support.py new file mode 100644 index 0000000..521ad22 --- /dev/null +++ b/src/fluxel/core/repository_support.py @@ -0,0 +1,125 @@ +from __future__ import annotations + +from dataclasses import replace +from pathlib import PurePosixPath + +from blake3 import blake3 + +from .manifest import ManifestEntry + + +def metadata_identity(relative_path: str, size: int) -> str: + payload = f"{relative_path}\n{size}".encode("utf-8") + return blake3(payload).hexdigest() + + +def normalize_repository_path(path: str) -> str: + normalized = path.strip().strip("/") + if not normalized: + raise ValueError("Path cannot be empty") + if normalized.startswith("../") or "/../" in normalized or normalized == "..": + raise ValueError("Path cannot traverse outside repository root") + return normalized + + +def normalize_logical_paths(paths: list[str]) -> list[str]: + normalized: list[str] = [] + seen: set[str] = set() + for path in paths: + value = normalize_repository_path(path) + if value in seen: + continue + seen.add(value) + normalized.append(value) + return normalized + + +def matches_logical_path(entry_path: str, logical_path: str) -> bool: + return entry_path == logical_path or entry_path.startswith(f"{logical_path}/") + + +def matches_any_logical_path(entry_path: str, logical_paths: list[str]) -> bool: + return any( + matches_logical_path(entry_path, logical_path) for logical_path in logical_paths + ) + + +def move_logical_path( + entry_path: str, + *, + source_path: str, + destination_path: str, +) -> str: + if entry_path == source_path: + return destination_path + suffix = entry_path[len(source_path) :] + return f"{destination_path}{suffix}" + + +def relocate_manifest_entry( + entry: ManifestEntry, + destination_path: str, +) -> ManifestEntry: + if entry.identity_mode == "meta": + identity_value = metadata_identity(destination_path, entry.size) + return replace( + entry, + path=destination_path, + hash=identity_value, + identity_value=identity_value, + ) + return replace(entry, path=destination_path) + + +def normalize_s3_import_path( + *, + key: str, + prefix: str, + size: int, +) -> str | None: + if key.endswith("/") and size == 0: + return None + normalized_key = key.strip("/") + if not normalized_key: + return None + normalized_prefix = prefix.strip("/") + if normalized_prefix: + if normalized_key == normalized_prefix: + relative_path = normalized_key.rsplit("/", maxsplit=1)[-1] + elif normalized_key.startswith(f"{normalized_prefix}/"): + relative_path = normalized_key[len(normalized_prefix) + 1 :] + else: + raise ValueError(f"S3 key '{key}' is outside import prefix '{prefix}'") + else: + relative_path = normalized_key + return normalize_repository_path(relative_path) + + +def normalize_import_patterns(path_patterns: list[str] | None) -> list[str]: + patterns: list[str] = [] + for pattern in path_patterns or []: + normalized = pattern.strip().strip("/") + if not normalized: + raise ValueError("Import path filter cannot be empty") + if normalized.startswith("../") or "/../" in normalized or normalized == "..": + raise ValueError( + "Import path filter cannot traverse outside repository root" + ) + patterns.append(normalized) + return patterns + + +def matches_import_patterns(relative_path: str, path_patterns: list[str]) -> bool: + if not path_patterns: + return True + path = PurePosixPath(relative_path) + return any(match_import_pattern(path, pattern) for pattern in path_patterns) + + +def match_import_pattern(path: PurePosixPath, pattern: str) -> bool: + if path.match(pattern): + return True + if pattern.startswith("**/"): + pattern_suffix = pattern[len("**/") :] + return len(path.parts) == 1 and path.match(pattern_suffix) + return False diff --git a/src/fluxel/core/storage.py b/src/fluxel/core/storage.py index 63d2213..4e7eb2b 100644 --- a/src/fluxel/core/storage.py +++ b/src/fluxel/core/storage.py @@ -7,10 +7,15 @@ from __future__ import annotations +from contextlib import contextmanager +from dataclasses import dataclass +from datetime import datetime, timezone from pathlib import Path -from typing import Iterator, Protocol +from typing import BinaryIO, Iterator, Protocol +from urllib.parse import unquote, urlparse import boto3 +import fsspec from botocore.exceptions import ClientError @@ -38,6 +43,25 @@ def iter_keys(self, prefix: str = "") -> Iterator[str]: ... def etag(self, relative_path: str) -> str | None: ... +@dataclass(frozen=True) +class S3ObjectMetadata: + bucket: str + key: str + size: int + mtime_ns: int + + @property + def source_uri(self) -> str: + return f"s3://{self.bucket}/{self.key}" + + +@dataclass(frozen=True) +class SourceObjectMetadata: + source_uri: str + size: int + mtime_ns: int + + class LocalStorageBackend: def __init__(self, root: str | Path) -> None: self.root = Path(root).resolve() @@ -100,7 +124,9 @@ def _key(self, relative_path: str) -> str: return relative_path.lstrip("/") def read_bytes(self, relative_path: str) -> bytes: - response = self.client.get_object(Bucket=self.bucket, Key=self._key(relative_path)) + response = self.client.get_object( + Bucket=self.bucket, Key=self._key(relative_path) + ) return response["Body"].read() def write_bytes( @@ -122,7 +148,9 @@ def write_bytes( except ClientError as error: code = error.response.get("Error", {}).get("Code", "") if if_none_match and code in {"PreconditionFailed", "412"}: - raise OptimisticLockError(f"Path already exists: {relative_path}") from error + raise OptimisticLockError( + f"Path already exists: {relative_path}" + ) from error raise def exists(self, relative_path: str) -> bool: @@ -150,10 +178,99 @@ def iter_keys(self, prefix: str = "") -> Iterator[str]: def etag(self, relative_path: str) -> str | None: try: - response = self.client.head_object(Bucket=self.bucket, Key=self._key(relative_path)) + response = self.client.head_object( + Bucket=self.bucket, Key=self._key(relative_path) + ) except ClientError as error: code = error.response.get("Error", {}).get("Code", "") if code in {"404", "NoSuchKey", "NotFound"}: return None raise return response.get("ETag", "").strip('"') or None + + +def parse_s3_uri(uri: str) -> tuple[str, str]: + parsed = urlparse(uri) + if parsed.scheme != "s3" or not parsed.netloc: + raise ValueError(f"Invalid S3 URI: {uri}") + return parsed.netloc, parsed.path.lstrip("/") + + +def _mtime_ns(value: object) -> int: + if isinstance(value, datetime): + timestamp = value + if timestamp.tzinfo is None: + timestamp = timestamp.replace(tzinfo=timezone.utc) + return int(timestamp.timestamp() * 1_000_000_000) + return 0 + + +def iter_s3_objects( + source_uri: str, + *, + client: object | None = None, +) -> Iterator[S3ObjectMetadata]: + bucket, prefix = parse_s3_uri(source_uri) + s3_client = client or boto3.client("s3") + paginator = s3_client.get_paginator("list_objects_v2") + for page in paginator.paginate(Bucket=bucket, Prefix=prefix): + for obj in page.get("Contents", []): + key = str(obj["Key"]) + yield S3ObjectMetadata( + bucket=bucket, + key=key, + size=int(obj.get("Size", 0)), + mtime_ns=_mtime_ns(obj.get("LastModified")), + ) + + +def describe_source_uri( + source_uri: str, + *, + client: object | None = None, +) -> SourceObjectMetadata: + if source_uri.startswith("s3://"): + bucket, key = parse_s3_uri(source_uri) + if not key or key.endswith("/"): + raise ValueError(f"S3 source must be an object, not a prefix: {source_uri}") + s3_client = client or boto3.client("s3") + response = s3_client.head_object(Bucket=bucket, Key=key) + return SourceObjectMetadata( + source_uri=source_uri, + size=int(response.get("ContentLength", 0)), + mtime_ns=_mtime_ns(response.get("LastModified")), + ) + + parsed = urlparse(source_uri) + if parsed.scheme == "file": + path = Path(unquote(parsed.path)).resolve() + else: + path = Path(source_uri).expanduser().resolve() + if not path.exists() or not path.is_file(): + raise FileNotFoundError(f"Cannot stage missing file: {path}") + stat = path.stat() + return SourceObjectMetadata( + source_uri=path.as_uri(), + size=stat.st_size, + mtime_ns=stat.st_mtime_ns, + ) + + +@contextmanager +def open_source_uri( + source_uri: str, + *, + client: object | None = None, +) -> Iterator[BinaryIO]: + if source_uri.startswith("s3://"): + bucket, key = parse_s3_uri(source_uri) + s3_client = client or boto3.client("s3") + response = s3_client.get_object(Bucket=bucket, Key=key) + body = response["Body"] + try: + yield body + finally: + body.close() + return + with fsspec.open(source_uri, mode="rb").open() as handle: + yield handle diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..354da32 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,226 @@ +from __future__ import annotations + +import io +import os +from datetime import datetime, timezone +from uuid import uuid4 + +import boto3 +import pytest +from botocore.config import Config +from botocore.exceptions import BotoCoreError, ClientError, EndpointConnectionError + + +class FakeStreamingBody: + def __init__(self, payload: bytes) -> None: + self._buffer = io.BytesIO(payload) + + def read(self, size: int = -1) -> bytes: + return self._buffer.read(size) + + def iter_lines(self) -> list[bytes]: + return self._buffer.getvalue().splitlines() + + def close(self) -> None: + self._buffer.close() + + +class FakeS3Paginator: + def __init__(self, objects: dict[str, dict[str, object]]) -> None: + self._objects = objects + + def paginate(self, *, Bucket: str, Prefix: str) -> list[dict[str, object]]: + contents = [] + for key, metadata in sorted(self._objects.items()): + if not key.startswith(Prefix): + continue + contents.append( + { + "Key": key, + "Size": len(metadata["Body"]), + "LastModified": metadata["LastModified"], + } + ) + return [{"Contents": contents}] + + +class FakeS3Client: + def __init__(self, objects: dict[str, dict[str, object]]) -> None: + self._objects = objects + self.fixed_etag: str | None = None + + def get_paginator(self, operation_name: str) -> FakeS3Paginator: + assert operation_name == "list_objects_v2" + return FakeS3Paginator(self._objects) + + def get_object( + self, + *, + Bucket: str, + Key: str, + Range: str | None = None, + ) -> dict[str, object]: + assert Bucket == "demo-bucket" + metadata = self._objects.get(Key) + if metadata is None: + raise self._client_error("NoSuchKey") + payload = metadata["Body"] + if Range is not None: + prefix, byte_range = Range.split("=", maxsplit=1) + assert prefix == "bytes" + start_raw, end_raw = byte_range.split("-", maxsplit=1) + start = int(start_raw) + end = int(end_raw) + payload = payload[start : end + 1] + return { + "Body": FakeStreamingBody(payload), + "ETag": metadata["ETag"], + "LastModified": metadata["LastModified"], + } + + def put_object( + self, + *, + Bucket: str, + Key: str, + Body: object, + IfNoneMatch: str | None = None, + ) -> dict[str, object]: + assert Bucket == "demo-bucket" + if IfNoneMatch == "*" and Key in self._objects: + raise self._client_error("PreconditionFailed") + + payload = Body.read() if hasattr(Body, "read") else Body + if not isinstance(payload, bytes): + payload = bytes(payload) + self._objects[Key] = { + "Body": payload, + "LastModified": datetime.now(timezone.utc), + "ETag": self._etag(payload), + } + return {"ETag": self._objects[Key]["ETag"]} + + def head_object(self, *, Bucket: str, Key: str) -> dict[str, object]: + assert Bucket == "demo-bucket" + metadata = self._objects.get(Key) + if metadata is None: + raise self._client_error("404") + return { + "ContentLength": len(metadata["Body"]), + "ETag": metadata["ETag"], + "LastModified": metadata["LastModified"], + } + + def delete_object(self, *, Bucket: str, Key: str) -> dict[str, object]: + assert Bucket == "demo-bucket" + self._objects.pop(Key, None) + return {} + + def _etag(self, payload: bytes) -> str: + if self.fixed_etag is not None: + return self.fixed_etag + return f'"{len(payload):x}-{sum(payload):x}"' + + def _client_error(self, code: str) -> ClientError: + return ClientError({"Error": {"Code": code, "Message": code}}, "fake_s3") + + +@pytest.fixture +def fake_s3_installer(monkeypatch: pytest.MonkeyPatch): + def install(objects: dict[str, bytes]) -> FakeS3Client: + client = FakeS3Client( + { + key: { + "Body": payload, + "LastModified": datetime(2026, 1, 2, tzinfo=timezone.utc), + "ETag": f'"{len(payload):x}-{sum(payload):x}"', + } + for key, payload in objects.items() + } + ) + monkeypatch.setattr( + "fluxel.core.storage.boto3.client", lambda service_name: client + ) + monkeypatch.setattr( + "fluxel.core.repository_store.boto3.client", + lambda service_name: client, + ) + return client + + return install + + +def _integration_config() -> dict[str, str]: + endpoint = os.getenv("FLUXEL_MINISTACK_ENDPOINT") + if not endpoint: + pytest.skip("Set FLUXEL_MINISTACK_ENDPOINT to run S3 integration tests") + + access_key = os.getenv("FLUXEL_MINISTACK_ACCESS_KEY") or os.getenv( + "AWS_ACCESS_KEY_ID" + ) + secret_key = os.getenv("FLUXEL_MINISTACK_SECRET_KEY") or os.getenv( + "AWS_SECRET_ACCESS_KEY" + ) + if not access_key or not secret_key: + pytest.skip( + "Set FLUXEL_MINISTACK_ACCESS_KEY/SECRET_KEY or AWS_ACCESS_KEY_ID/AWS_SECRET_ACCESS_KEY" + ) + + return { + "endpoint": endpoint, + "access_key": access_key, + "secret_key": secret_key, + "region": os.getenv("FLUXEL_MINISTACK_REGION", "us-east-1"), + } + + +@pytest.fixture +def ministack_client(monkeypatch: pytest.MonkeyPatch): + config = _integration_config() + client = boto3.client( + "s3", + endpoint_url=config["endpoint"], + aws_access_key_id=config["access_key"], + aws_secret_access_key=config["secret_key"], + region_name=config["region"], + config=Config(s3={"addressing_style": "path"}), + ) + try: + client.list_buckets() + except (EndpointConnectionError, BotoCoreError, ClientError) as error: + pytest.skip(f"S3 integration endpoint unavailable: {error}") + + monkeypatch.setattr("fluxel.core.storage.boto3.client", lambda service_name: client) + monkeypatch.setattr( + "fluxel.core.repository_store.boto3.client", lambda service_name: client + ) + return client + + +@pytest.fixture +def s3_repo_root(ministack_client) -> str: + bucket = f"fluxel-it-{uuid4().hex[:20]}" + prefix = f"repos/{uuid4().hex}" + ministack_client.create_bucket(Bucket=bucket) + try: + yield f"s3://{bucket}/{prefix}" + finally: + continuation_token: str | None = None + while True: + kwargs = {"Bucket": bucket} + if continuation_token: + kwargs["ContinuationToken"] = continuation_token + response = ministack_client.list_objects_v2(**kwargs) + contents = response.get("Contents", []) + if contents: + ministack_client.delete_objects( + Bucket=bucket, + Delete={ + "Objects": [{"Key": item["Key"]} for item in contents], + "Quiet": True, + }, + ) + if not response.get("IsTruncated"): + break + continuation_token = response.get("NextContinuationToken") + ministack_client.delete_bucket(Bucket=bucket) diff --git a/tests/test_cli.py b/tests/test_cli.py index 48155c1..c2a35ce 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1,6 +1,8 @@ from __future__ import annotations import json +import shutil +from datetime import datetime, timezone from pathlib import Path from fluxel.core import FluxelFileSystem @@ -8,23 +10,129 @@ from fluxel.core import ManifestReader +def test_cli_supports_command_local_repo_flag_for_s3_repositories( + tmp_path: Path, capsys, monkeypatch, fake_s3_installer +) -> None: + client = fake_s3_installer({}) + monkeypatch.chdir(tmp_path) + repo_uri = "s3://demo-bucket/repos/demo" + + (tmp_path / "a.txt").write_text("one") + assert run_cli(["commit", "--repo", repo_uri, "-m", "initial"]) == 0 + commit_a = capsys.readouterr().out.strip() + assert len(commit_a) == 64 + + assert run_cli(["branch", "--repo", repo_uri, "feature"]) == 0 + branch_out = capsys.readouterr().out.strip() + assert branch_out.endswith("/.fluxel/refs/heads/feature") + + (tmp_path / "a.txt").write_text("two") + assert run_cli(["commit", "--repo", repo_uri, "-m", "update"]) == 0 + commit_b = capsys.readouterr().out.strip() + assert len(commit_b) == 64 + assert commit_a != commit_b + + assert run_cli(["diff", "--repo", repo_uri, commit_a, commit_b]) == 0 + diff_payload = json.loads(capsys.readouterr().out) + assert [entry["path"] for entry in diff_payload] == ["a.txt"] + assert diff_payload[0]["change"] == "modified" + + assert run_cli(["index", "build", "--repo", repo_uri]) == 0 + build_payload = json.loads(capsys.readouterr().out) + db_path = Path(build_payload["database_path"]) + assert db_path.exists() + assert "clients" in db_path.as_posix() + + assert ( + run_cli( + ["index", "query", "--db", str(db_path), "--sql", "SELECT path FROM files"] + ) + == 0 + ) + query_payload = json.loads(capsys.readouterr().out) + assert query_payload == [["a.txt"]] + + assert not list((tmp_path / ".fluxel" / "commits").glob("*.json")) + assert f"repos/demo/commits/{commit_b}.json" in client._objects + assert any(key.startswith("repos/demo/manifests/") for key in client._objects) + assert any( + key.startswith("repos/demo/manifests/") and key.endswith(".idx") + for key in client._objects + ) + assert "repos/demo/refs/heads/main" in client._objects + + +def test_cli_metadata_only_rm_and_mv_work_for_s3_repositories( + tmp_path: Path, capsys, monkeypatch, fake_s3_installer +) -> None: + client = fake_s3_installer({}) + monkeypatch.chdir(tmp_path) + repo_uri = "s3://demo-bucket/repos/demo" + + (tmp_path / "a.txt").write_text("one") + (tmp_path / "dir").mkdir() + (tmp_path / "dir" / "b.txt").write_text("two") + + assert run_cli(["commit", "--repo", repo_uri, "-m", "initial"]) == 0 + initial_commit = capsys.readouterr().out.strip() + assert initial_commit + + (tmp_path / "a.txt").unlink() + shutil.rmtree(tmp_path / "dir") + + assert ( + run_cli( + [ + "mv", + "--repo", + repo_uri, + "a.txt", + "renamed.txt", + "-m", + "rename a", + ] + ) + == 0 + ) + mv_payload = json.loads(capsys.readouterr().out) + assert mv_payload["moved_paths"] == ["renamed.txt"] + moved_commit = mv_payload["commit_id"] + + assert run_cli(["rm", "--repo", repo_uri, "dir", "-m", "remove dir"]) == 0 + rm_payload = json.loads(capsys.readouterr().out) + assert rm_payload["removed_paths"] == ["dir/b.txt"] + removed_commit = rm_payload["commit_id"] + assert removed_commit != moved_commit + + assert run_cli(["diff", "--repo", repo_uri, initial_commit, removed_commit]) == 0 + diff_payload = json.loads(capsys.readouterr().out) + assert [(entry["path"], entry["change"]) for entry in diff_payload] == [ + ("a.txt", "removed"), + ("dir/b.txt", "removed"), + ("renamed.txt", "added"), + ] + + assert not list((tmp_path / ".fluxel" / "commits").glob("*.json")) + assert f"repos/demo/commits/{removed_commit}.json" in client._objects + + def test_cli_commit_branch_and_diff(tmp_path: Path, capsys) -> None: (tmp_path / "a.txt").write_text("one") - assert run_cli(["commit", "--root", str(tmp_path), "-m", "initial"]) == 0 + assert run_cli(["commit", "--repo", str(tmp_path), "-m", "initial"]) == 0 commit_a = capsys.readouterr().out.strip() assert len(commit_a) == 64 (tmp_path / "a.txt").write_text("two") - assert run_cli(["commit", "--root", str(tmp_path), "-m", "update"]) == 0 + assert run_cli(["commit", "--repo", str(tmp_path), "-m", "update"]) == 0 commit_b = capsys.readouterr().out.strip() assert len(commit_b) == 64 - assert run_cli(["branch", "--root", str(tmp_path), "exp"]) == 0 + assert run_cli(["branch", "--repo", str(tmp_path), "exp"]) == 0 branch_out = capsys.readouterr().out.strip() assert branch_out.endswith("/.fluxel/refs/heads/exp") - assert run_cli(["diff", "--root", str(tmp_path), commit_a, commit_b]) == 0 + assert run_cli(["diff", "--repo", str(tmp_path), commit_a, commit_b]) == 0 diff_payload = json.loads(capsys.readouterr().out) assert diff_payload == [ { @@ -43,10 +151,10 @@ def test_cli_index_build_query_drop(tmp_path: Path, capsys) -> None: (tmp_path / "x.jpg").write_bytes(b"img") (tmp_path / "y.txt").write_text("text") - assert run_cli(["commit", "--root", str(tmp_path), "-m", "seed"]) == 0 + assert run_cli(["commit", "--repo", str(tmp_path), "-m", "seed"]) == 0 capsys.readouterr() - assert run_cli(["index", "build", "--root", str(tmp_path), "--parquet"]) == 0 + assert run_cli(["index", "build", "--repo", str(tmp_path), "--parquet"]) == 0 build_payload = json.loads(capsys.readouterr().out) db_path = Path(build_payload["database_path"]) parquet_path = Path(build_payload["parquet_path"]) @@ -79,7 +187,7 @@ def test_cli_commit_with_metadata_identity_mode(tmp_path: Path, capsys) -> None: (tmp_path / "a.txt").write_text("payload") assert ( - run_cli(["commit", "--root", str(tmp_path), "-m", "meta", "--identity", "meta"]) + run_cli(["commit", "--repo", str(tmp_path), "-m", "meta", "--identity", "meta"]) == 0 ) capsys.readouterr() @@ -95,17 +203,65 @@ def test_cli_commit_with_metadata_identity_mode(tmp_path: Path, capsys) -> None: assert entry.source_uri is not None +def test_cli_add_reports_missing_source_cleanly(tmp_path: Path, capsys) -> None: + repo_root = tmp_path / "repo" + repo_root.mkdir() + + assert run_cli(["add", "--repo", str(repo_root), "missing.txt"]) == 2 + stderr = capsys.readouterr().err + assert "add error: Cannot stage missing path: missing.txt" in stderr + + +def test_cli_import_rejects_invalid_path_filter_cleanly(tmp_path: Path, capsys) -> None: + assert ( + run_cli( + [ + "import", + "--repo", + str(tmp_path), + "s3://demo-bucket/bootstrap", + "-m", + "filtered import", + "--path", + "../bad", + ] + ) + == 2 + ) + stderr = capsys.readouterr().err + assert ( + "import error: Import path filter cannot traverse outside repository root" + in stderr + ) + + +def test_cli_verify_reports_missing_source_cleanly(tmp_path: Path, capsys) -> None: + (tmp_path / "a.txt").write_text("payload") + assert ( + run_cli(["commit", "--repo", str(tmp_path), "-m", "meta", "--identity", "meta"]) + == 0 + ) + capsys.readouterr() + + (tmp_path / "a.txt").unlink() + + assert run_cli(["verify", "--repo", str(tmp_path), "--ref", "main"]) == 2 + stderr = capsys.readouterr().err + assert "verify error:" in stderr + assert "a.txt" in stderr + + def test_cli_verify_promotes_metadata_entries(tmp_path: Path, capsys) -> None: (tmp_path / "a.txt").write_text("payload") assert ( - run_cli(["commit", "--root", str(tmp_path), "-m", "meta", "--identity", "meta"]) + run_cli(["commit", "--repo", str(tmp_path), "-m", "meta", "--identity", "meta"]) == 0 ) first_commit = capsys.readouterr().out.strip() assert first_commit - assert run_cli(["verify", "--root", str(tmp_path), "--ref", "main"]) == 0 + assert run_cli(["verify", "--repo", str(tmp_path), "--ref", "main"]) == 0 verify_payload = json.loads(capsys.readouterr().out) assert verify_payload["created_commit"] is True assert verify_payload["verified_entries"] == 1 @@ -123,7 +279,7 @@ def test_cli_verify_promotes_metadata_entries(tmp_path: Path, capsys) -> None: assert latest_entries[0].identity_mode == "blake3" assert latest_entries[0].blob_hash is not None - assert run_cli(["verify", "--root", str(tmp_path), "--ref", "main"]) == 0 + assert run_cli(["verify", "--repo", str(tmp_path), "--ref", "main"]) == 0 verify_again_payload = json.loads(capsys.readouterr().out) assert verify_again_payload["created_commit"] is False assert verify_again_payload["verified_entries"] == 0 @@ -133,13 +289,13 @@ def test_cli_verify_dry_run_reports_without_changes(tmp_path: Path, capsys) -> N (tmp_path / "a.txt").write_text("payload") assert ( - run_cli(["commit", "--root", str(tmp_path), "-m", "meta", "--identity", "meta"]) + run_cli(["commit", "--repo", str(tmp_path), "-m", "meta", "--identity", "meta"]) == 0 ) first_commit = capsys.readouterr().out.strip() assert ( - run_cli(["verify", "--root", str(tmp_path), "--ref", "main", "--dry-run"]) == 0 + run_cli(["verify", "--repo", str(tmp_path), "--ref", "main", "--dry-run"]) == 0 ) payload = json.loads(capsys.readouterr().out) assert payload["dry_run"] is True @@ -151,11 +307,11 @@ def test_cli_verify_dry_run_reports_without_changes(tmp_path: Path, capsys) -> N def test_cli_staging_commit_is_branch_scoped(tmp_path: Path, capsys) -> None: (tmp_path / "shared.txt").write_text("base") - assert run_cli(["commit", "--root", str(tmp_path), "-m", "base"]) == 0 + assert run_cli(["commit", "--repo", str(tmp_path), "-m", "base"]) == 0 base_commit = capsys.readouterr().out.strip() assert base_commit - assert run_cli(["branch", "--root", str(tmp_path), "feature"]) == 0 + assert run_cli(["branch", "--repo", str(tmp_path), "feature"]) == 0 capsys.readouterr() (tmp_path / "feature.txt").write_text("feature") @@ -163,7 +319,7 @@ def test_cli_staging_commit_is_branch_scoped(tmp_path: Path, capsys) -> None: run_cli( [ "add", - "--root", + "--repo", str(tmp_path), "--ref", "feature", @@ -191,7 +347,7 @@ def test_cli_staging_commit_is_branch_scoped(tmp_path: Path, capsys) -> None: run_cli( [ "commit", - "--root", + "--repo", str(tmp_path), "--ref", "feature", @@ -205,12 +361,12 @@ def test_cli_staging_commit_is_branch_scoped(tmp_path: Path, capsys) -> None: feature_commit = capsys.readouterr().out.strip() assert feature_commit and feature_commit != base_commit - assert run_cli(["status", "--root", str(tmp_path), "--ref", "feature"]) == 0 + assert run_cli(["status", "--repo", str(tmp_path), "--ref", "feature"]) == 0 status_payload = json.loads(capsys.readouterr().out) assert status_payload["added"] == [] assert status_payload["removed"] == [] - assert run_cli(["diff", "--root", str(tmp_path), "main", "feature"]) == 0 + assert run_cli(["diff", "--repo", str(tmp_path), "main", "feature"]) == 0 diff_payload = json.loads(capsys.readouterr().out) assert diff_payload == [ { @@ -222,3 +378,881 @@ def test_cli_staging_commit_is_branch_scoped(tmp_path: Path, capsys) -> None: "after_size": 7, } ] + + +def test_cli_add_supports_arbitrary_local_source_with_logical_destination( + tmp_path: Path, capsys +) -> None: + repo_root = tmp_path / "repo" + repo_root.mkdir() + external_file = tmp_path / "external.txt" + external_file.write_text("external payload") + + assert ( + run_cli( + [ + "add", + "--repo", + str(repo_root), + "--as", + "imports/external.txt", + str(external_file), + ] + ) + == 0 + ) + add_payload = json.loads(capsys.readouterr().out) + assert add_payload["added"] == ["imports/external.txt"] + + assert ( + run_cli( + [ + "commit", + "--repo", + str(repo_root), + "--staged", + "-m", + "ingest external", + ] + ) + == 0 + ) + commit_id = capsys.readouterr().out.strip() + assert len(commit_id) == 64 + + commit_payload = json.loads( + (repo_root / ".fluxel" / "commits" / f"{commit_id}.json").read_text() + ) + manifest_path = ( + repo_root / ".fluxel" / "manifests" / f"{commit_payload['manifest']}.jsonl" + ) + entries = list(ManifestReader(manifest_path).iter_entries()) + assert [entry.path for entry in entries] == ["imports/external.txt"] + assert entries[0].source_uri is None + + +def test_cli_add_supports_s3_source_with_staged_read_and_logical_destination( + tmp_path: Path, capsys, fake_s3_installer +) -> None: + fake_s3_installer({"incoming/source.txt": b"remote payload"}) + repo_root = tmp_path / "repo" + repo_root.mkdir() + + (repo_root / "base.txt").write_text("base") + assert run_cli(["commit", "--repo", str(repo_root), "-m", "base"]) == 0 + capsys.readouterr() + + assert ( + run_cli( + [ + "add", + "--repo", + str(repo_root), + "--identity", + "meta", + "--as", + "imports/source.txt", + "s3://demo-bucket/incoming/source.txt", + ] + ) + == 0 + ) + add_payload = json.loads(capsys.readouterr().out) + assert add_payload["added"] == ["imports/source.txt"] + + fs = FluxelFileSystem(dataset_roots={"demo": repo_root}) + with fs.open("fluxel://demo@main+staged/imports/source.txt", "rb") as handle: + assert handle.read() == b"remote payload" + + assert ( + run_cli( + [ + "commit", + "--repo", + str(repo_root), + "--staged", + "-m", + "ingest remote", + ] + ) + == 0 + ) + commit_id = capsys.readouterr().out.strip() + assert len(commit_id) == 64 + + commit_payload = json.loads( + (repo_root / ".fluxel" / "commits" / f"{commit_id}.json").read_text() + ) + manifest_path = ( + repo_root / ".fluxel" / "manifests" / f"{commit_payload['manifest']}.jsonl" + ) + entries = list(ManifestReader(manifest_path).iter_entries()) + assert {entry.path for entry in entries} == {"base.txt", "imports/source.txt"} + imported_entry = next( + entry for entry in entries if entry.path == "imports/source.txt" + ) + assert imported_entry.identity_mode == "meta" + assert imported_entry.blob_hash is None + assert imported_entry.source_uri == "s3://demo-bucket/incoming/source.txt" + + +def test_cli_add_supports_local_directory_source_with_destination_prefix( + tmp_path: Path, capsys +) -> None: + repo_root = tmp_path / "repo" + repo_root.mkdir() + external_dir = tmp_path / "bundle" + (external_dir / "nested").mkdir(parents=True) + (external_dir / "a.txt").write_text("alpha") + (external_dir / "nested" / "b.txt").write_text("beta") + + assert ( + run_cli( + [ + "add", + "--repo", + str(repo_root), + "--as", + "imports/bundle", + str(external_dir), + ] + ) + == 0 + ) + add_payload = json.loads(capsys.readouterr().out) + assert add_payload["added"] == [ + "imports/bundle/a.txt", + "imports/bundle/nested/b.txt", + ] + + assert ( + run_cli( + [ + "commit", + "--repo", + str(repo_root), + "--staged", + "-m", + "ingest bundle", + ] + ) + == 0 + ) + commit_id = capsys.readouterr().out.strip() + commit_payload = json.loads( + (repo_root / ".fluxel" / "commits" / f"{commit_id}.json").read_text() + ) + manifest_path = ( + repo_root / ".fluxel" / "manifests" / f"{commit_payload['manifest']}.jsonl" + ) + assert [entry.path for entry in ManifestReader(manifest_path).iter_entries()] == [ + "imports/bundle/a.txt", + "imports/bundle/nested/b.txt", + ] + + +def test_cli_add_supports_s3_prefix_with_destination_prefix( + tmp_path: Path, capsys, fake_s3_installer +) -> None: + fake_s3_installer( + { + "incoming/batch/a.txt": b"alpha", + "incoming/batch/nested/b.txt": b"beta", + } + ) + repo_root = tmp_path / "repo" + repo_root.mkdir() + (repo_root / "base.txt").write_text("base") + assert run_cli(["commit", "--repo", str(repo_root), "-m", "base"]) == 0 + capsys.readouterr() + + assert ( + run_cli( + [ + "add", + "--repo", + str(repo_root), + "--identity", + "meta", + "--as", + "imports/batch", + "s3://demo-bucket/incoming/batch", + ] + ) + == 0 + ) + add_payload = json.loads(capsys.readouterr().out) + assert add_payload["added"] == [ + "imports/batch/a.txt", + "imports/batch/nested/b.txt", + ] + + fs = FluxelFileSystem(dataset_roots={"demo": repo_root}) + with fs.open( + "fluxel://demo@main+staged/imports/batch/nested/b.txt", "rb" + ) as handle: + assert handle.read() == b"beta" + + assert ( + run_cli( + [ + "commit", + "--repo", + str(repo_root), + "--staged", + "-m", + "ingest batch", + ] + ) + == 0 + ) + capsys.readouterr() + + +def test_cli_remote_staged_add_preserves_existing_entries_and_uploads_one_blob( + tmp_path: Path, capsys, monkeypatch, fake_s3_installer +) -> None: + client = fake_s3_installer({}) + monkeypatch.chdir(tmp_path) + repo_uri = "s3://demo-bucket/repos/demo" + + (tmp_path / "a.txt").write_text("alpha") + assert run_cli(["commit", "--repo", repo_uri, "-m", "initial"]) == 0 + first_commit = capsys.readouterr().out.strip() + assert len(first_commit) == 64 + + initial_blob_keys = { + key for key in client._objects if key.startswith("repos/demo/blobs/") + } + assert len(initial_blob_keys) == 1 + + (tmp_path / "a.txt").unlink() + (tmp_path / "b.txt").write_text("beta") + + assert run_cli(["add", "--repo", repo_uri, "b.txt"]) == 0 + add_payload = json.loads(capsys.readouterr().out) + assert add_payload["added"] == ["b.txt"] + + assert run_cli(["commit", "--repo", repo_uri, "--staged", "-m", "add b"]) == 0 + second_commit = capsys.readouterr().out.strip() + assert len(second_commit) == 64 + assert second_commit != first_commit + + new_blob_keys = { + key for key in client._objects if key.startswith("repos/demo/blobs/") + } + assert len(new_blob_keys - initial_blob_keys) == 1 + + assert run_cli(["diff", "--repo", repo_uri, first_commit, second_commit]) == 0 + diff_payload = json.loads(capsys.readouterr().out) + assert diff_payload == [ + { + "path": "b.txt", + "change": "added", + "before_hash": None, + "after_hash": diff_payload[0]["after_hash"], + "before_size": None, + "after_size": 4, + } + ] + + +def test_cli_remote_commit_recovers_from_stale_branch_lock( + tmp_path: Path, capsys, monkeypatch, fake_s3_installer +) -> None: + client = fake_s3_installer({}) + monkeypatch.chdir(tmp_path) + repo_uri = "s3://demo-bucket/repos/demo" + lock_key = "repos/demo/locks/refs/heads/main.lock" + client._objects[lock_key] = { + "Body": b"legacy-stale-lock", + "LastModified": datetime(2025, 1, 1, tzinfo=timezone.utc), + "ETag": '"11-11"', + } + + (tmp_path / "a.txt").write_text("alpha") + assert run_cli(["commit", "--repo", repo_uri, "-m", "initial"]) == 0 + commit_id = capsys.readouterr().out.strip() + + assert len(commit_id) == 64 + assert lock_key not in client._objects + assert "repos/demo/refs/heads/main" in client._objects + + +def test_cli_merge_fast_forwards_target_branch(tmp_path: Path, capsys) -> None: + (tmp_path / "shared.txt").write_text("base") + assert run_cli(["commit", "--repo", str(tmp_path), "-m", "base"]) == 0 + base_commit = capsys.readouterr().out.strip() + assert base_commit + + assert run_cli(["branch", "--repo", str(tmp_path), "feature"]) == 0 + capsys.readouterr() + + (tmp_path / "feature.txt").write_text("feature") + assert ( + run_cli( + [ + "add", + "--repo", + str(tmp_path), + "--ref", + "feature", + "feature.txt", + ] + ) + == 0 + ) + capsys.readouterr() + + assert ( + run_cli( + [ + "commit", + "--repo", + str(tmp_path), + "--ref", + "feature", + "--staged", + "-m", + "feature commit", + ] + ) + == 0 + ) + feature_commit = capsys.readouterr().out.strip() + assert feature_commit and feature_commit != base_commit + + assert run_cli(["merge", "--repo", str(tmp_path), "feature", "main"]) == 0 + merge_payload = json.loads(capsys.readouterr().out) + assert merge_payload == { + "source_ref": "feature", + "target_ref": "main", + "commit_id": feature_commit, + "updated": True, + } + + assert run_cli(["diff", "--repo", str(tmp_path), "main", "feature"]) == 0 + assert json.loads(capsys.readouterr().out) == [] + + +def test_cli_merge_rejects_non_fast_forward(tmp_path: Path, capsys) -> None: + (tmp_path / "shared.txt").write_text("base") + assert run_cli(["commit", "--repo", str(tmp_path), "-m", "base"]) == 0 + capsys.readouterr() + + assert run_cli(["branch", "--repo", str(tmp_path), "feature"]) == 0 + capsys.readouterr() + + (tmp_path / "main.txt").write_text("main") + assert run_cli(["commit", "--repo", str(tmp_path), "-m", "main commit"]) == 0 + main_commit = capsys.readouterr().out.strip() + assert main_commit + + (tmp_path / "feature.txt").write_text("feature") + assert ( + run_cli( + [ + "add", + "--repo", + str(tmp_path), + "--ref", + "feature", + "feature.txt", + ] + ) + == 0 + ) + capsys.readouterr() + + assert ( + run_cli( + [ + "commit", + "--repo", + str(tmp_path), + "--ref", + "feature", + "--staged", + "-m", + "feature commit", + ] + ) + == 0 + ) + feature_commit = capsys.readouterr().out.strip() + assert feature_commit and feature_commit != main_commit + + assert run_cli(["merge", "--repo", str(tmp_path), "feature", "main"]) == 2 + stderr = capsys.readouterr().err + assert "merge error: Cannot fast-forward" in stderr + + +def test_cli_import_s3_writes_manifest_and_blobs( + tmp_path: Path, capsys, fake_s3_installer +) -> None: + fake_s3_installer( + { + "bootstrap/a.txt": b"alpha", + "bootstrap/nested/b.txt": b"beta", + "bootstrap/": b"", + }, + ) + + assert ( + run_cli( + [ + "import", + "--repo", + str(tmp_path), + "s3://demo-bucket/bootstrap", + "-m", + "bootstrap", + ] + ) + == 0 + ) + commit_id = capsys.readouterr().out.strip() + assert len(commit_id) == 64 + + manifest_paths = sorted((tmp_path / ".fluxel" / "manifests").glob("*.jsonl")) + assert len(manifest_paths) == 1 + entries = list(ManifestReader(manifest_paths[0]).iter_entries()) + assert [entry.path for entry in entries] == ["a.txt", "nested/b.txt"] + assert all(entry.identity_mode == "blake3" for entry in entries) + assert all(entry.blob_hash for entry in entries) + assert all(entry.source_uri is None for entry in entries) + for entry in entries: + blob_path = ( + tmp_path / ".fluxel" / "blobs" / entry.blob_hash[:2] / entry.blob_hash[2:] + ) + assert blob_path.exists() + + +def test_cli_import_s3_preserves_existing_manifest_entries( + tmp_path: Path, capsys, fake_s3_installer +) -> None: + fake_s3_installer( + { + "bootstrap/a.txt": b"alpha", + "incremental/b.txt": b"beta", + }, + ) + + assert ( + run_cli( + [ + "import", + "--repo", + str(tmp_path), + "s3://demo-bucket/bootstrap", + "-m", + "bootstrap", + ] + ) + == 0 + ) + first_commit = capsys.readouterr().out.strip() + assert len(first_commit) == 64 + + assert ( + run_cli( + [ + "import", + "--repo", + str(tmp_path), + "s3://demo-bucket/incremental", + "-m", + "incremental", + ] + ) + == 0 + ) + second_commit = capsys.readouterr().out.strip() + assert len(second_commit) == 64 + assert second_commit != first_commit + + commit_payload = json.loads( + (tmp_path / ".fluxel" / "commits" / f"{second_commit}.json").read_text() + ) + manifest_path = ( + tmp_path / ".fluxel" / "manifests" / f"{commit_payload['manifest']}.jsonl" + ) + entries = list(ManifestReader(manifest_path).iter_entries()) + + assert [entry.path for entry in entries] == ["a.txt", "b.txt"] + assert all(entry.source_uri is None for entry in entries) + assert ( + sum(1 for path in (tmp_path / ".fluxel" / "blobs").rglob("*") if path.is_file()) + == 2 + ) + + +def test_cli_import_s3_metadata_entries_can_be_read_and_verified( + tmp_path: Path, capsys, fake_s3_installer +) -> None: + fake_s3_installer( + { + "imports/a.txt": b"alpha", + "imports/nested/b.txt": b"beta", + }, + ) + + assert ( + run_cli( + [ + "import", + "--repo", + str(tmp_path), + "s3://demo-bucket/imports", + "-m", + "metadata import", + "--identity", + "meta", + ] + ) + == 0 + ) + first_commit = capsys.readouterr().out.strip() + assert len(first_commit) == 64 + assert not any((tmp_path / ".fluxel" / "blobs").rglob("*")) + + manifest_path = next((tmp_path / ".fluxel" / "manifests").glob("*.jsonl")) + entries = list(ManifestReader(manifest_path).iter_entries()) + assert [entry.identity_mode for entry in entries] == ["meta", "meta"] + assert [entry.blob_hash for entry in entries] == [None, None] + + fs = FluxelFileSystem(dataset_roots={"demo": tmp_path}) + with fs.open("fluxel://demo@main/nested/b.txt", "rb") as handle: + assert handle.read() == b"beta" + + assert run_cli(["verify", "--repo", str(tmp_path), "--ref", "main"]) == 0 + verify_payload = json.loads(capsys.readouterr().out) + assert verify_payload["created_commit"] is True + assert verify_payload["verified_entries"] == 2 + assert verify_payload["commit_id"] != first_commit + + +def test_cli_import_s3_supports_repeated_path_filters( + tmp_path: Path, capsys, fake_s3_installer +) -> None: + fake_s3_installer( + { + "gallery/root.jpg": b"root-jpg", + "gallery/root.txt": b"root-txt", + "gallery/nested/photo.jpg": b"nested-jpg", + "gallery/nested/notes.txt": b"nested-txt", + }, + ) + + assert ( + run_cli( + [ + "import", + "--repo", + str(tmp_path), + "s3://demo-bucket/gallery", + "-m", + "filtered import", + "--path", + "**/*.jpg", + "--path", + "root.txt", + ] + ) + == 0 + ) + commit_id = capsys.readouterr().out.strip() + assert len(commit_id) == 64 + + manifest_path = next((tmp_path / ".fluxel" / "manifests").glob("*.jsonl")) + entries = list(ManifestReader(manifest_path).iter_entries()) + assert [entry.path for entry in entries] == [ + "nested/photo.jpg", + "root.jpg", + "root.txt", + ] + + +def test_cli_import_s3_path_star_imports_all_entries( + tmp_path: Path, capsys, fake_s3_installer +) -> None: + fake_s3_installer( + { + "all/a.txt": b"a", + "all/nested/b.jpg": b"b", + }, + ) + + assert ( + run_cli( + [ + "import", + "--repo", + str(tmp_path), + "s3://demo-bucket/all", + "-m", + "all entries", + "--path", + "*", + ] + ) + == 0 + ) + capsys.readouterr() + + manifest_path = next((tmp_path / ".fluxel" / "manifests").glob("*.jsonl")) + entries = list(ManifestReader(manifest_path).iter_entries()) + assert [entry.path for entry in entries] == ["a.txt", "nested/b.jpg"] + + +def test_cli_dataset_can_mix_s3_meta_local_blake3_and_verified_s3_entries( + tmp_path: Path, capsys, fake_s3_installer +) -> None: + fake_s3_installer( + { + "dataset/root.txt": b"root-from-s3", + "dataset/images/cat.jpg": b"cat-image", + "dataset/images/dog.jpg": b"dog-image", + "dataset/docs/readme.md": b"ignored", + }, + ) + + assert ( + run_cli( + [ + "import", + "--repo", + str(tmp_path), + "s3://demo-bucket/dataset", + "-m", + "bootstrap metadata import", + "--identity", + "meta", + "--path", + "root.txt", + "--path", + "**/*.jpg", + ] + ) + == 0 + ) + first_commit = capsys.readouterr().out.strip() + assert len(first_commit) == 64 + assert (tmp_path / ".fluxel").exists() + + (tmp_path / "local").mkdir() + (tmp_path / "local" / "one.txt").write_text("one") + (tmp_path / "local" / "two.txt").write_text("two") + + assert ( + run_cli( + [ + "add", + "--repo", + str(tmp_path), + "--identity", + "blake3", + "local/one.txt", + "local/two.txt", + ] + ) + == 0 + ) + add_payload = json.loads(capsys.readouterr().out) + assert add_payload["added"] == ["local/one.txt", "local/two.txt"] + + assert ( + run_cli( + [ + "commit", + "--repo", + str(tmp_path), + "--staged", + "-m", + "add local blake3 files", + ] + ) + == 0 + ) + second_commit = capsys.readouterr().out.strip() + assert len(second_commit) == 64 + assert second_commit != first_commit + + assert ( + run_cli( + [ + "verify", + "--repo", + str(tmp_path), + "--ref", + "main", + "--path", + "root.txt", + "--path", + "images/cat.jpg", + ] + ) + == 0 + ) + verify_payload = json.loads(capsys.readouterr().out) + assert verify_payload["created_commit"] is True + assert verify_payload["verified_entries"] == 2 + + commit_payload = json.loads( + ( + tmp_path / ".fluxel" / "commits" / f"{verify_payload['commit_id']}.json" + ).read_text() + ) + manifest_path = ( + tmp_path / ".fluxel" / "manifests" / f"{commit_payload['manifest']}.jsonl" + ) + entries = { + entry.path: entry for entry in ManifestReader(manifest_path).iter_entries() + } + + assert sorted(entries) == [ + "images/cat.jpg", + "images/dog.jpg", + "local/one.txt", + "local/two.txt", + "root.txt", + ] + + assert entries["root.txt"].identity_mode == "blake3" + assert entries["root.txt"].blob_hash is not None + assert entries["images/cat.jpg"].identity_mode == "blake3" + assert entries["images/cat.jpg"].blob_hash is not None + assert entries["images/dog.jpg"].identity_mode == "meta" + assert entries["images/dog.jpg"].blob_hash is None + assert entries["local/one.txt"].identity_mode == "blake3" + assert entries["local/one.txt"].blob_hash is not None + assert entries["local/two.txt"].identity_mode == "blake3" + assert entries["local/two.txt"].blob_hash is not None + + assert run_cli(["index", "build", "--repo", str(tmp_path)]) == 0 + build_payload = json.loads(capsys.readouterr().out) + db_path = Path(build_payload["database_path"]) + assert db_path.exists() + + assert ( + run_cli( + [ + "index", + "query", + "--db", + str(db_path), + "--sql", + "SELECT path FROM files ORDER BY path", + ] + ) + == 0 + ) + query_payload = json.loads(capsys.readouterr().out) + assert query_payload == [ + ["images/cat.jpg"], + ["images/dog.jpg"], + ["local/one.txt"], + ["local/two.txt"], + ["root.txt"], + ] + + +def test_cli_meta_import_branch_removal_and_fast_forward_merge( + tmp_path: Path, capsys, fake_s3_installer +) -> None: + fake_s3_installer( + { + "images/image0.jpg": b"image-0", + "images/image1.jpg": b"image-1", + "images/nested/photo0.jpg": b"photo-0", + "images/nested/photo1.jpg": b"photo-1", + "images/notes.txt": b"ignore-me", + }, + ) + + assert ( + run_cli( + [ + "import", + "--repo", + str(tmp_path), + "s3://demo-bucket/images", + "-m", + "import jpg metadata", + "--identity", + "meta", + "--path", + "**/*.jpg", + ] + ) + == 0 + ) + main_commit = capsys.readouterr().out.strip() + assert len(main_commit) == 64 + + assert run_cli(["branch", "--repo", str(tmp_path), "feature"]) == 0 + capsys.readouterr() + + assert ( + run_cli( + [ + "rm", + "--repo", + str(tmp_path), + "--ref", + "feature", + "image0.jpg", + "nested/photo0.jpg", + ] + ) + == 0 + ) + rm_payload = json.loads(capsys.readouterr().out) + assert rm_payload["removed"] == ["image0.jpg", "nested/photo0.jpg"] + + assert ( + run_cli( + [ + "commit", + "--repo", + str(tmp_path), + "--ref", + "feature", + "--staged", + "-m", + "remove zero-suffixed jpgs", + ] + ) + == 0 + ) + feature_commit = capsys.readouterr().out.strip() + assert len(feature_commit) == 64 + assert feature_commit != main_commit + + fs = FluxelFileSystem(dataset_roots={"demo": tmp_path}) + main_listing = sorted(fs.ls("fluxel://demo@main/*", detail=False)) + assert main_listing == [ + "fluxel://demo@main/image0.jpg", + "fluxel://demo@main/image1.jpg", + "fluxel://demo@main/nested/photo0.jpg", + "fluxel://demo@main/nested/photo1.jpg", + ] + + feature_listing = sorted(fs.ls("fluxel://demo@feature/*", detail=False)) + assert feature_listing == [ + "fluxel://demo@feature/image1.jpg", + "fluxel://demo@feature/nested/photo1.jpg", + ] + + main_ref_path = tmp_path / ".fluxel" / "refs" / "heads" / "main" + main_ref_path.write_text(f"{feature_commit}\n", encoding="utf-8") + + merged_main_listing = sorted(fs.ls("fluxel://demo@main/*", detail=False)) + assert merged_main_listing == [ + "fluxel://demo@main/image1.jpg", + "fluxel://demo@main/nested/photo1.jpg", + ] + + assert run_cli(["diff", "--repo", str(tmp_path), "main", "feature"]) == 0 + diff_payload = json.loads(capsys.readouterr().out) + assert diff_payload == [] diff --git a/tests/test_mandatory_validation.py b/tests/test_mandatory_validation.py index df25df8..1b5770a 100644 --- a/tests/test_mandatory_validation.py +++ b/tests/test_mandatory_validation.py @@ -3,18 +3,51 @@ import tracemalloc from pathlib import Path +from blake3 import blake3 +import pytest + from fluxel.core import ( FluxelFileSystem, FluxelRepository, + LocalClientState, + LocalRepositoryStore, ManifestEntry, ManifestReader, ManifestWriter, + RefConflictError, build_analytical_index, drop_analytical_index, + open_repository, query_analytical_index, ) +class ConflictOnceLocalRepositoryStore(LocalRepositoryStore): + def __init__(self, root: str | Path, *, conflict_commit_id: str) -> None: + super().__init__(root) + self.conflict_commit_id = conflict_commit_id + self.conflict_next_ref_update = False + + def compare_and_set_branch_ref( + self, + branch: str, + commit_id: str | None, + *, + expected_version_token: str | None, + expected_commit_id: str | None = None, + ) -> bool: + if self.conflict_next_ref_update: + self.conflict_next_ref_update = False + self.write_branch_ref(branch, self.conflict_commit_id) + return False + return super().compare_and_set_branch_ref( + branch, + commit_id, + expected_version_token=expected_version_token, + expected_commit_id=expected_commit_id, + ) + + def test_metadata_only_diff_does_not_read_blobs(tmp_path: Path, monkeypatch) -> None: (tmp_path / "a.txt").write_text("alpha") repo = FluxelRepository(tmp_path) @@ -43,6 +76,68 @@ def tracking_read_bytes(path: Path) -> bytes: assert touched_blob_reads == [] +def test_metadata_only_remove_does_not_read_blobs(tmp_path: Path, monkeypatch) -> None: + (tmp_path / "a.txt").write_text("alpha") + (tmp_path / "b.txt").write_text("beta") + repo = FluxelRepository(tmp_path) + repo.commit("initial") + + touched_blob_reads: list[Path] = [] + original_read_bytes = Path.read_bytes + + def tracking_read_bytes(path: Path) -> bytes: + path_obj = Path(path) + if ".fluxel" in path_obj.parts and "blobs" in path_obj.parts: + touched_blob_reads.append(path_obj) + return original_read_bytes(path_obj) + + monkeypatch.setattr(Path, "read_bytes", tracking_read_bytes) + + result = repo.remove_paths(["a.txt"], "remove a") + + assert result.removed_paths == ["a.txt"] + assert set(repo.resolve_entries("main")) == {"b.txt"} + assert touched_blob_reads == [] + + +def test_metadata_only_move_updates_meta_identity_without_blob_read( + tmp_path: Path, monkeypatch +) -> None: + nested = tmp_path / "dir" + nested.mkdir() + (nested / "file.txt").write_text("payload") + repo = FluxelRepository(tmp_path) + repo.commit("metadata only", identity_mode="meta") + before_entry = repo.resolve_entries("main")["dir/file.txt"] + + touched_blob_reads: list[Path] = [] + original_read_bytes = Path.read_bytes + + def tracking_read_bytes(path: Path) -> bytes: + path_obj = Path(path) + if ".fluxel" in path_obj.parts and "blobs" in path_obj.parts: + touched_blob_reads.append(path_obj) + return original_read_bytes(path_obj) + + monkeypatch.setattr(Path, "read_bytes", tracking_read_bytes) + + result = repo.move("dir", "archive", "rename prefix") + after_entries = repo.resolve_entries("main") + moved_entry = after_entries["archive/file.txt"] + expected_identity = blake3( + f"archive/file.txt\n{before_entry.size}".encode("utf-8") + ).hexdigest() + + assert result.moved_paths == ["archive/file.txt"] + assert "dir/file.txt" not in after_entries + assert moved_entry.identity_mode == "meta" + assert moved_entry.hash == expected_identity + assert moved_entry.identity_value == expected_identity + assert moved_entry.blob_hash is None + assert moved_entry.source_uri == before_entry.source_uri + assert touched_blob_reads == [] + + def test_memory_safe_manifesting_100k_entries(tmp_path: Path) -> None: entry_count = 100_000 manifest_path = tmp_path / ".fluxel" / "manifests" / "large.jsonl" @@ -71,6 +166,90 @@ def entries(): assert sum(1 for _ in ManifestReader(manifest_path).iter_entries()) == entry_count +def test_manifest_entry_validation_rejects_invalid_payloads() -> None: + invalid_payloads = [ + ( + { + "path": "../escape.txt", + "hash": "a" * 64, + "size": 1, + "mtime_ns": 1, + }, + "normalized relative path", + ), + ( + { + "path": "valid.txt", + "hash": "g" * 64, + "size": 1, + "mtime_ns": 1, + }, + "64-character hex digest", + ), + ( + { + "path": "valid.txt", + "hash": "a" * 64, + "size": -1, + "mtime_ns": 1, + }, + "size cannot be negative", + ), + ( + { + "path": "meta.txt", + "hash": "a" * 64, + "size": 1, + "mtime_ns": 1, + "identity_mode": "meta", + }, + "must include source_uri", + ), + ] + + for payload, message in invalid_payloads: + with pytest.raises(ValueError, match=message): + ManifestEntry.from_dict(payload) + + +def test_manifest_reader_reports_corrupt_json_with_line_context( + tmp_path: Path, +) -> None: + manifest_path = tmp_path / "broken.jsonl" + manifest_path.write_text( + '["b","ok.txt","' + ("a" * 64) + '",1,1]\n' '{"path": invalid json}\n', + encoding="utf-8", + ) + + with pytest.raises(ValueError, match=r"Corrupt manifest JSON at line 2"): + list(ManifestReader(manifest_path).iter_entries()) + + +def test_local_client_state_writes_use_atomic_replace( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + state = LocalClientState(tmp_path) + replaced_targets: list[Path] = [] + original_replace = Path.replace + + def tracking_replace(path: Path, target: str | Path) -> Path: + replaced_targets.append(Path(target)) + return original_replace(path, target) + + monkeypatch.setattr(Path, "replace", tracking_replace) + + state.set_current_branch("feature") + state.write_staging_payload("feature", '[{"path":"a.txt","action":"add"}]\n') + + assert state.current_branch() == "feature" + assert ( + state.read_staging_payload("feature") == '[{"path":"a.txt","action":"add"}]\n' + ) + assert state.head_path in replaced_targets + assert state.stage_path("feature") in replaced_targets + assert not list(state.fluxel_dir.rglob("*.tmp")) + + def test_uri_routing_reads_expected_blob_bytes(tmp_path: Path) -> None: dataset_root = tmp_path / "my_data" dataset_root.mkdir(parents=True) @@ -84,6 +263,264 @@ def test_uri_routing_reads_expected_blob_bytes(tmp_path: Path) -> None: assert handle.read() == b"col\n123\n" +def test_exact_lookup_uses_manifest_sidecar_without_full_scan(tmp_path: Path) -> None: + (tmp_path / "a.txt").write_text("alpha") + repo = FluxelRepository(tmp_path) + repo.commit("initial") + + manifest_indexes = list((tmp_path / ".fluxel" / "manifests").glob("*.idx")) + assert len(manifest_indexes) == 1 + + original_iter_manifest_entries = repo.store.iter_manifest_entries + + def fail_iter_manifest_entries(manifest_hash: str): + raise AssertionError(f"unexpected full manifest scan for {manifest_hash}") + + repo.store.iter_manifest_entries = fail_iter_manifest_entries # type: ignore[method-assign] + try: + entry = repo.resolve_entry("main", "a.txt") + finally: + repo.store.iter_manifest_entries = original_iter_manifest_entries # type: ignore[method-assign] + + assert entry is not None + assert entry.path == "a.txt" + + +def test_uri_routing_uses_manifest_sidecar_for_point_reads(tmp_path: Path) -> None: + dataset_root = tmp_path / "sidecar_data" + dataset_root.mkdir(parents=True) + (dataset_root / "test.csv").write_text("value\n7\n") + + repo = FluxelRepository(dataset_root) + repo.commit("add file") + + fs = FluxelFileSystem(dataset_roots={"sidecar_data": dataset_root}) + cached_repo = fs._repository(dataset_root) + original_iter_manifest_entries = cached_repo.store.iter_manifest_entries + + def fail_iter_manifest_entries(manifest_hash: str): + raise AssertionError(f"unexpected full manifest scan for {manifest_hash}") + + cached_repo.store.iter_manifest_entries = fail_iter_manifest_entries # type: ignore[method-assign] + try: + with fs.open("fluxel://sidecar_data@main/test.csv", "rb") as handle: + assert handle.read() == b"value\n7\n" + finally: + cached_repo.store.iter_manifest_entries = original_iter_manifest_entries # type: ignore[method-assign] + + +def test_uri_listing_uses_manifest_sidecar_for_prefix_reads(tmp_path: Path) -> None: + dataset_root = tmp_path / "listing_data" + dataset_root.mkdir(parents=True) + (dataset_root / "logs").mkdir() + (dataset_root / "logs" / "a.txt").write_text("a") + (dataset_root / "logs" / "b.txt").write_text("b") + (dataset_root / "other.txt").write_text("c") + + repo = FluxelRepository(dataset_root) + repo.commit("add files") + + fs = FluxelFileSystem(dataset_roots={"listing_data": dataset_root}) + cached_repo = fs._repository(dataset_root) + original_iter_manifest_entries = cached_repo.store.iter_manifest_entries + + def fail_iter_manifest_entries(manifest_hash: str): + raise AssertionError(f"unexpected full manifest scan for {manifest_hash}") + + cached_repo.store.iter_manifest_entries = fail_iter_manifest_entries # type: ignore[method-assign] + try: + paths = fs.ls("fluxel://listing_data@main/logs", detail=False) + finally: + cached_repo.store.iter_manifest_entries = original_iter_manifest_entries # type: ignore[method-assign] + + assert paths == [ + "fluxel://listing_data@main/logs/a.txt", + "fluxel://listing_data@main/logs/b.txt", + ] + + +def test_repeated_exact_lookup_reuses_cached_commit(tmp_path: Path) -> None: + (tmp_path / "a.txt").write_text("alpha") + repo = FluxelRepository(tmp_path) + repo.commit("initial") + + first = repo.resolve_entry("main", "a.txt") + assert first is not None + + original_read_commit_bytes = repo.store.read_commit_bytes + + def fail_read_commit_bytes(commit_id: str): + raise AssertionError(f"unexpected commit reread for {commit_id}") + + repo.store.read_commit_bytes = fail_read_commit_bytes # type: ignore[method-assign] + try: + second = repo.resolve_entry("main", "a.txt") + finally: + repo.store.read_commit_bytes = original_read_commit_bytes # type: ignore[method-assign] + + assert second is not None + assert second.path == "a.txt" + + +def test_remote_exact_lookup_uses_manifest_sidecar( + tmp_path: Path, fake_s3_installer +) -> None: + client = fake_s3_installer({}) + worktree = tmp_path / "worktree" + client_root = tmp_path / "client" + worktree.mkdir(parents=True) + client_root.mkdir(parents=True) + (worktree / "remote.txt").write_text("payload") + + repo = open_repository( + "s3://demo-bucket/repos/demo", + worktree=worktree, + client_root=client_root, + s3_client=client, + ) + repo.commit("initial") + + assert any( + key.startswith("repos/demo/manifests/") and key.endswith(".idx") + for key in client._objects + ) + + original_iter_manifest_entries = repo.store.iter_manifest_entries + + def fail_iter_manifest_entries(manifest_hash: str): + raise AssertionError(f"unexpected full manifest scan for {manifest_hash}") + + repo.store.iter_manifest_entries = fail_iter_manifest_entries # type: ignore[method-assign] + try: + entry = repo.resolve_entry("main", "remote.txt") + finally: + repo.store.iter_manifest_entries = original_iter_manifest_entries # type: ignore[method-assign] + + assert entry is not None + assert entry.path == "remote.txt" + + +def test_remote_prefix_listing_uses_manifest_sidecar( + tmp_path: Path, fake_s3_installer +) -> None: + client = fake_s3_installer({}) + worktree = tmp_path / "worktree-list" + client_root = tmp_path / "client-list" + worktree.mkdir(parents=True) + client_root.mkdir(parents=True) + (worktree / "logs").mkdir() + (worktree / "logs" / "a.txt").write_text("a") + (worktree / "logs" / "b.txt").write_text("b") + (worktree / "other.txt").write_text("c") + + repo = open_repository( + "s3://demo-bucket/repos/demo-prefix", + worktree=worktree, + client_root=client_root, + s3_client=client, + ) + repo.commit("initial") + + original_iter_manifest_entries = repo.store.iter_manifest_entries + + def fail_iter_manifest_entries(manifest_hash: str): + raise AssertionError(f"unexpected full manifest scan for {manifest_hash}") + + repo.store.iter_manifest_entries = fail_iter_manifest_entries # type: ignore[method-assign] + try: + entries = repo.resolve_entries_for_prefix("main", "logs") + finally: + repo.store.iter_manifest_entries = original_iter_manifest_entries # type: ignore[method-assign] + + assert sorted(entries) == ["logs/a.txt", "logs/b.txt"] + + +def test_s3_compare_and_set_checks_expected_commit_id_under_lock( + tmp_path: Path, fake_s3_installer +) -> None: + client = fake_s3_installer({}) + client.fixed_etag = '"static-etag"' + + worktree = tmp_path / "store-worktree" + client_root = tmp_path / "store-client" + worktree.mkdir(parents=True) + client_root.mkdir(parents=True) + + repo = open_repository( + "s3://demo-bucket/repos/conflict-store", + worktree=worktree, + client_root=client_root, + s3_client=client, + ) + store = repo.store + + store.write_branch_ref("main", "base") + base_state = store.read_branch_ref("main") + assert base_state is not None + assert base_state.commit_id == "base" + + store.write_branch_ref("main", "winning") + + updated = store.compare_and_set_branch_ref( + "main", + "stale", + expected_version_token=base_state.version_token, + expected_commit_id=base_state.commit_id, + ) + + assert updated is False + current_state = store.read_branch_ref("main") + assert current_state is not None + assert current_state.commit_id == "winning" + + +def test_remote_repo_commit_detects_conflict_even_if_s3_etag_is_unchanged( + tmp_path: Path, fake_s3_installer +) -> None: + client = fake_s3_installer({}) + client.fixed_etag = '"static-etag"' + + client_a_worktree = tmp_path / "client-a-worktree" + client_b_worktree = tmp_path / "client-b-worktree" + client_a_root = tmp_path / "client-a-state" + client_b_root = tmp_path / "client-b-state" + for path in ( + client_a_worktree, + client_b_worktree, + client_a_root, + client_b_root, + ): + path.mkdir(parents=True) + + (client_a_worktree / "data.txt").write_text("alpha") + repo_a = open_repository( + "s3://demo-bucket/repos/conflict-repo", + worktree=client_a_worktree, + client_root=client_a_root, + s3_client=client, + ) + base_commit = repo_a.commit("base") + + (client_b_worktree / "data.txt").write_text("alpha") + repo_b = open_repository( + "s3://demo-bucket/repos/conflict-repo", + worktree=client_b_worktree, + client_root=client_b_root, + s3_client=client, + ) + (client_b_worktree / "data.txt").write_text("gamma") + winning_commit = repo_b.commit("winning update") + + (client_a_worktree / "data.txt").write_text("beta") + with pytest.raises(RefConflictError) as error_info: + repo_a.commit("stale update") + + error = error_info.value + assert error.operation == "commit" + assert error.expected_commit_id == base_commit + assert error.current_commit_id == winning_commit + + def test_disposable_analytical_index(tmp_path: Path) -> None: (tmp_path / "a.jpg").write_bytes(b"x" * 10) (tmp_path / "b.txt").write_text("hello") @@ -122,3 +559,164 @@ def test_uri_routing_supports_metadata_identity_entries(tmp_path: Path) -> None: fs = FluxelFileSystem(dataset_roots={"meta_data": dataset_root}) with fs.open("fluxel://meta_data@main/test.csv", "rb") as handle: assert handle.read() == b"value\n42\n" + + +def test_repository_mutations_can_use_separate_local_store(tmp_path: Path) -> None: + dataset_root = tmp_path / "dataset" + store_root = tmp_path / "repo-state" + dataset_root.mkdir(parents=True) + store_root.mkdir(parents=True) + (dataset_root / "sample.txt").write_text("payload") + + repo = FluxelRepository( + dataset_root, + store=LocalRepositoryStore(store_root), + ) + commit_id = repo.commit("initial") + + assert commit_id + assert list((dataset_root / ".fluxel" / "commits").glob("*.json")) == [] + assert list((dataset_root / ".fluxel" / "manifests").glob("*.jsonl")) == [] + assert not any((dataset_root / ".fluxel" / "blobs").rglob("*")) + + assert list((store_root / ".fluxel" / "commits").glob("*.json")) + manifest_paths = list((store_root / ".fluxel" / "manifests").glob("*.jsonl")) + assert manifest_paths + assert any((store_root / ".fluxel" / "blobs").rglob("*")) + assert ( + store_root / ".fluxel" / "refs" / "heads" / "main" + ).read_text().strip() == commit_id + + manifest_entries = list(ManifestReader(manifest_paths[0]).iter_entries()) + assert [entry.path for entry in manifest_entries] == ["sample.txt"] + + +def test_current_branch_preference_is_local_per_client(tmp_path: Path) -> None: + dataset_root = tmp_path / "dataset" + store_root = tmp_path / "repo-state" + client_a_root = tmp_path / "client-a" + client_b_root = tmp_path / "client-b" + dataset_root.mkdir(parents=True) + store_root.mkdir(parents=True) + client_a_root.mkdir(parents=True) + client_b_root.mkdir(parents=True) + (dataset_root / "shared.txt").write_text("base") + + repo_a = FluxelRepository( + dataset_root, + store=LocalRepositoryStore(store_root), + client_state=LocalClientState(client_a_root), + ) + repo_b = FluxelRepository( + dataset_root, + store=LocalRepositoryStore(store_root), + client_state=LocalClientState(client_b_root), + ) + + repo_a.commit("base") + repo_a.branch("feature") + repo_a.set_current_branch("feature") + repo_b.set_current_branch("main") + + assert repo_a.current_branch() == "feature" + assert repo_b.current_branch() == "main" + assert ( + client_a_root / ".fluxel" / "refs" / "HEAD" + ).read_text().strip() == "refs/heads/feature" + assert ( + client_b_root / ".fluxel" / "refs" / "HEAD" + ).read_text().strip() == "refs/heads/main" + assert not (store_root / ".fluxel" / "refs" / "HEAD").exists() + + +def test_staging_state_is_local_per_client(tmp_path: Path) -> None: + dataset_root = tmp_path / "dataset" + store_root = tmp_path / "repo-state" + client_a_root = tmp_path / "client-a" + client_b_root = tmp_path / "client-b" + dataset_root.mkdir(parents=True) + store_root.mkdir(parents=True) + client_a_root.mkdir(parents=True) + client_b_root.mkdir(parents=True) + (dataset_root / "shared.txt").write_text("base") + (dataset_root / "feature.txt").write_text("feature") + + repo_a = FluxelRepository( + dataset_root, + store=LocalRepositoryStore(store_root), + client_state=LocalClientState(client_a_root), + ) + repo_b = FluxelRepository( + dataset_root, + store=LocalRepositoryStore(store_root), + client_state=LocalClientState(client_b_root), + ) + + repo_a.commit("base") + repo_a.branch("feature") + repo_a.set_current_branch("feature") + repo_b.set_current_branch("feature") + + repo_a.add(["feature.txt"]) + + assert repo_a.status().added == ["feature.txt"] + assert repo_b.status().added == [] + assert (client_a_root / ".fluxel" / "staging" / "feature.json").exists() + assert not (client_b_root / ".fluxel" / "staging" / "feature.json").exists() + assert not (store_root / ".fluxel" / "staging" / "feature.json").exists() + + +def test_commit_fails_clearly_on_branch_update_conflict(tmp_path: Path) -> None: + conflict_commit_id = "f" * 64 + store = ConflictOnceLocalRepositoryStore( + tmp_path / "repo-state", + conflict_commit_id=conflict_commit_id, + ) + (tmp_path / "data.txt").write_text("alpha") + + repo = FluxelRepository(tmp_path, store=store) + base_commit = repo.commit("base") + + (tmp_path / "data.txt").write_text("beta") + store.conflict_next_ref_update = True + + try: + repo.commit("update") + except RefConflictError as error: + assert str(error) == ( + f"Branch update conflict for 'main' during commit: expected {base_commit}, found {conflict_commit_id}" + ) + else: + raise AssertionError("Expected RefConflictError") + + assert store.read_branch_ref("main") is not None + assert store.read_branch_ref("main").commit_id == conflict_commit_id + + +def test_merge_fails_clearly_on_branch_update_conflict(tmp_path: Path) -> None: + conflict_commit_id = "e" * 64 + store = ConflictOnceLocalRepositoryStore( + tmp_path / "repo-state", + conflict_commit_id=conflict_commit_id, + ) + (tmp_path / "shared.txt").write_text("base") + + repo = FluxelRepository(tmp_path, store=store) + base_commit = repo.commit("base") + repo.branch("feature") + + (tmp_path / "feature.txt").write_text("feature") + repo.add(["feature.txt"], ref="feature") + feature_commit = repo.commit("feature commit", staged=True, ref="feature") + assert feature_commit != base_commit + + store.conflict_next_ref_update = True + + try: + repo.merge("feature", "main") + except RefConflictError as error: + assert str(error) == ( + f"Branch update conflict for 'main' during merge: expected {base_commit}, found {conflict_commit_id}" + ) + else: + raise AssertionError("Expected RefConflictError") diff --git a/tests/test_s3_integration.py b/tests/test_s3_integration.py new file mode 100644 index 0000000..156bde7 --- /dev/null +++ b/tests/test_s3_integration.py @@ -0,0 +1,332 @@ +from __future__ import annotations + +import time +from pathlib import Path +from uuid import uuid4 + +import pytest + +from fluxel.core import FluxelRepository, RefConflictError, open_repository + + +pytestmark = pytest.mark.integration + + +def _open_remote_repo( + repo_uri: str, + *, + worktree: Path, + client_root: Path, + s3_client: object, +) -> FluxelRepository: + return open_repository( + repo_uri, + worktree=worktree, + client_root=client_root, + s3_client=s3_client, + ) + + +def test_s3_integration_branch_commit_and_fast_forward_merge( + tmp_path: Path, + ministack_client, + s3_repo_root: str, +) -> None: + main_worktree = tmp_path / "main-worktree" + feature_worktree = tmp_path / "feature-worktree" + main_client = tmp_path / "main-client" + feature_client = tmp_path / "feature-client" + for path in (main_worktree, feature_worktree, main_client, feature_client): + path.mkdir(parents=True) + + (main_worktree / "shared.txt").write_text("base") + repo_main = _open_remote_repo( + s3_repo_root, + worktree=main_worktree, + client_root=main_client, + s3_client=ministack_client, + ) + base_commit = repo_main.commit("base") + assert base_commit + + repo_main.branch("feature") + + (feature_worktree / "shared.txt").write_text("base") + (feature_worktree / "feature.txt").write_text("feature") + repo_feature = _open_remote_repo( + s3_repo_root, + worktree=feature_worktree, + client_root=feature_client, + s3_client=ministack_client, + ) + repo_feature.set_current_branch("feature") + repo_feature.add(["feature.txt"]) + feature_commit = repo_feature.commit("feature commit", staged=True) + + merge_result = repo_main.merge("feature", "main") + assert merge_result.updated is True + assert merge_result.commit_id == feature_commit + + changes = repo_main.diff(base_commit, "main") + assert [(entry.path, entry.change) for entry in changes] == [ + ("feature.txt", "added") + ] + + +def test_s3_integration_metadata_import_verify_remove_and_move( + tmp_path: Path, + ministack_client, + s3_repo_root: str, +) -> None: + source_bucket = s3_repo_root.split("//", maxsplit=1)[1].split("/", maxsplit=1)[0] + source_prefix = f"imports/{uuid4().hex}" + worktree = tmp_path / "client-worktree" + client_root = tmp_path / "client-state" + worktree.mkdir(parents=True) + client_root.mkdir(parents=True) + + ministack_client.put_object( + Bucket=source_bucket, + Key=f"{source_prefix}/root.txt", + Body=b"root", + ) + ministack_client.put_object( + Bucket=source_bucket, + Key=f"{source_prefix}/images/cat.jpg", + Body=b"cat", + ) + ministack_client.put_object( + Bucket=source_bucket, + Key=f"{source_prefix}/images/dog.jpg", + Body=b"dog", + ) + ministack_client.put_object( + Bucket=source_bucket, + Key=f"{source_prefix}/docs/readme.md", + Body=b"readme", + ) + + repo = _open_remote_repo( + s3_repo_root, + worktree=worktree, + client_root=client_root, + s3_client=ministack_client, + ) + source_uri = f"s3://{source_bucket}/{source_prefix}" + imported_commit = repo.import_s3( + source_uri, + "metadata import", + identity_mode="meta", + path_patterns=["root.txt", "**/*.jpg"], + ) + assert imported_commit + + imported_entries = repo.resolve_entries("main") + assert sorted(imported_entries) == ["images/cat.jpg", "images/dog.jpg", "root.txt"] + assert all(entry.blob_hash is None for entry in imported_entries.values()) + + verify_result = repo.verify( + ref="main", + path_prefixes=["root.txt", "images/cat.jpg"], + ) + assert verify_result.created_commit is True + assert verify_result.verified_entries == 2 + + verified_entries = repo.resolve_entries("main") + assert verified_entries["root.txt"].blob_hash is not None + assert verified_entries["images/cat.jpg"].blob_hash is not None + assert verified_entries["images/dog.jpg"].blob_hash is None + + move_result = repo.move("images", "photos", "rename image prefix") + assert move_result.moved_paths == ["photos/cat.jpg", "photos/dog.jpg"] + + remove_result = repo.remove_paths(["root.txt"], "remove root") + assert remove_result.removed_paths == ["root.txt"] + + final_entries = repo.resolve_entries("main") + assert sorted(final_entries) == ["photos/cat.jpg", "photos/dog.jpg"] + assert final_entries["photos/cat.jpg"].blob_hash is not None + assert final_entries["photos/dog.jpg"].blob_hash is None + + +def test_s3_integration_reports_optimistic_concurrency_conflicts( + tmp_path: Path, + ministack_client, + s3_repo_root: str, +) -> None: + client_a_worktree = tmp_path / "client-a-worktree" + client_b_worktree = tmp_path / "client-b-worktree" + client_a_root = tmp_path / "client-a-state" + client_b_root = tmp_path / "client-b-state" + for path in ( + client_a_worktree, + client_b_worktree, + client_a_root, + client_b_root, + ): + path.mkdir(parents=True) + + (client_a_worktree / "data.txt").write_text("alpha") + repo_a = _open_remote_repo( + s3_repo_root, + worktree=client_a_worktree, + client_root=client_a_root, + s3_client=ministack_client, + ) + base_commit = repo_a.commit("base") + assert base_commit + + (client_a_worktree / "data.txt").write_text("beta") + + (client_b_worktree / "data.txt").write_text("alpha") + repo_b = _open_remote_repo( + s3_repo_root, + worktree=client_b_worktree, + client_root=client_b_root, + s3_client=ministack_client, + ) + (client_b_worktree / "data.txt").write_text("gamma") + winning_commit = repo_b.commit("winning update") + assert winning_commit != base_commit + + with pytest.raises(RefConflictError) as error_info: + repo_a.commit("stale update") + + error = error_info.value + assert error.operation == "commit" + assert error.expected_commit_id == base_commit + assert error.current_commit_id == winning_commit + + +def test_s3_integration_million_file_scale( + tmp_path: Path, + ministack_client, + s3_repo_root: str, + caplog, +) -> None: + """Test manifest index performance at 1M files with timing measurements. + + Validates: + - Specific file lookup is O(log B + 1) + - Prefix listing with 200 matches is O(log B + 200) + - Bulk downloads use manifest cache effectively + + Logs timing measurements for performance regression detection. + """ + worktree = tmp_path / "worktree" + client_root = tmp_path / "client-state" + worktree.mkdir(parents=True) + client_root.mkdir(parents=True) + + # 1. Generate 1M files locally (much faster than S3 API calls) + print("\n[SCALE TEST] Generating 1M files locally...") + gen_start = time.perf_counter() + (worktree / "images" / "cats").mkdir(parents=True, exist_ok=True) + (worktree / "images" / "dogs").mkdir(parents=True, exist_ok=True) + (worktree / "logs").mkdir(parents=True, exist_ok=True) + (worktree / "other").mkdir(parents=True, exist_ok=True) + cat_content = f"cat".encode() + dog_content = f"dog".encode() + data_content = f"data".encode() + + for i in range(1_000_000): + # Distribute: images/cats/* (200), images/dogs/* (300), other/* (999_500) + if i < 200: + path = worktree / "images" / "cats" / f"cat_{i:06d}.jpg" + # content = cat_content + elif i < 500: + path = worktree / "images" / "dogs" / f"dog_{i:06d}.jpg" + # content = dog_content + else: + # Alternate between logs and other for realistic distribution + category = "logs" if (i % 2) == 0 else "other" + path = worktree / category / f"file_{i:07d}.bin" + # content = data_content + + path.touch() + + gen_time = time.perf_counter() - gen_start + print( + f"[SCALE TEST] Generated 1M files in {gen_time:.2f}s ({1_000_000/gen_time:.0f} files/sec)" + ) + + # 2. Commit to Fluxel (creates manifest + index from local files) + print("[SCALE TEST] Committing 1M files to Fluxel...") + repo = _open_remote_repo( + s3_repo_root, + worktree=worktree, + client_root=client_root, + s3_client=ministack_client, + ) + commit_start = time.perf_counter() + commit_id = repo.commit("1M file snapshot (meta)", identity_mode="meta") + commit_time = time.perf_counter() - commit_start + assert commit_id + print( + f"[SCALE TEST] Commit + manifest build in {commit_time:.2f}s ({1_000_000/commit_time:.0f} files/sec)" + ) + + # 3. Test: Specific file lookup (should be O(log B + 1)) + print("[SCALE TEST] Testing specific file lookup...") + lookup_start = time.perf_counter() + cat_50 = repo.resolve_entry("main", "images/cats/cat_000050.jpg") + lookup_time = time.perf_counter() - lookup_start + assert cat_50 is not None + assert cat_50.size == 0 # touch + print(f"[SCALE TEST] Single file lookup: {lookup_time*1000:.3f}ms") + + # 4. Test: Prefix listing (should be O(log B + 200)) + print("[SCALE TEST] Testing prefix listing (images/cats/*)...") + listing_start = time.perf_counter() + cats = repo.resolve_entries_for_prefix("main", "images/cats") + listing_time = time.perf_counter() - listing_start + assert len(cats) == 200 + print(f"[SCALE TEST] Prefix listing 200 files: {listing_time*1000:.3f}ms") + + # 5. Test: Bulk metadata access (simulating download planning) + print("[SCALE TEST] Bulk metadata access (200 files)...") + bulk_start = time.perf_counter() + bulk_data = [] + for path, entry in cats.items(): + # Simulate metadata-only access (no blob reads) + bulk_data.append((path, entry.size, entry.identity_mode)) + bulk_time = time.perf_counter() - bulk_start + assert len(bulk_data) == 200 + assert all(t[1] == 0 for t in bulk_data) # All have size + print(f"[SCALE TEST] Bulk metadata access (200): {bulk_time*1000:.3f}ms") + + # 6. Test: Cached prefix listing (should be faster) + print("[SCALE TEST] Testing cached prefix listing...") + cached_start = time.perf_counter() + cats_again = repo.resolve_entries_for_prefix("main", "images/cats") + cached_time = time.perf_counter() - cached_start + assert len(cats_again) == 200 + print(f"[SCALE TEST] Cached prefix listing: {cached_time*1000:.3f}ms") + + # 7. Summary and assertions + print("\n[SCALE TEST] Performance Summary:") + print( + f" Generate 1M files: {gen_time:.2f}s ({1_000_000/gen_time:.0f} files/sec)" + ) + print( + f" Commit + manifest: {commit_time:.2f}s ({1_000_000/commit_time:.0f} files/sec)" + ) + print(f" Single lookup: {lookup_time*1000:.3f}ms") + print(f" Prefix list (200): {listing_time*1000:.3f}ms") + print(f" Bulk metadata (200): {bulk_time*1000:.3f}ms") + print(f" Cached prefix list: {cached_time*1000:.3f}ms") + + # Verify manifest index is working: specific lookup should be fast (< 100ms) + assert ( + lookup_time < 0.1 + ), f"Single lookup took {lookup_time*1000:.3f}ms (expected < 100ms)" + + # Prefix listing should also be fast (< 200ms for 200 matches) + assert ( + listing_time < 0.2 + ), f"Prefix listing took {listing_time*1000:.3f}ms (expected < 200ms)" + + # Cached listing should be noticeably faster than initial + assert ( + cached_time <= listing_time or cached_time < 0.1 + ), "Cached listing should be <= initial listing time"