Skip to content

Add similar fastmigrate system #24

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
ncoop57 wants to merge 5 commits into
base: main
Choose a base branch
from
Open

Add similar fastmigrate system #24

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
2354ecd
add migrations
ncoop57 Jan 12, 2026
d8843e9
Add migration example files
ncoop57 Jan 12, 2026
68d8b9c
update readme
ncoop57 Jan 12, 2026
10bcd8e
add missing import
ncoop57 Jan 12, 2026
dc57f0f
fix version getter
ncoop57 Jan 12, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
310 changes: 307 additions & 3 deletions 00_core.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,12 +33,13 @@
"from pathlib import Path\n",
"from typing import Any, Optional, Union, Iterable, Generator, List, Tuple, Dict, get_args\n",
"\n",
"import sqlalchemy as sa\n",
"from sqlalchemy.orm import Session\n",
"from fastcore.utils import *\n",
"from fastcore.test import test_fail, test_eq\n",
"from fastcore.xtras import dataclass_src\n",
"from itertools import starmap\n"
"from itertools import starmap\n",
"\n",
"import sqlalchemy as sa, subprocess"
]
},
{
Expand Down Expand Up @@ -927,6 +928,7 @@
" ignore=True,\n",
" transform=False,\n",
" strict=False,\n",
" constraints=None,\n",
"):\n",
" \"Get a table object, creating in DB if needed\"\n",
" pk = listify(pk)\n",
Expand All @@ -944,7 +946,8 @@
" )\n",
" for o in fields(cls)\n",
" ]\n",
" tbl = sa.Table(name, self.meta, *cols, extend_existing=True)\n",
" constraints = [sa.CheckConstraint(c) for c in (constraints or [])]\n",
" tbl = sa.Table(name, self.meta, *cols, *constraints, extend_existing=True)\n",
" res = DBTable(tbl, self, cls)\n",
" tbl.cls = cls\n",
" self._tables[name] = res\n",
Expand Down Expand Up @@ -2467,6 +2470,307 @@
"assert 'student' not in db.t"
]
},
{
"cell_type": "markdown",
"id": "98351f6b",
"metadata": {},
"source": [
"## Migrations"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "2bfcbe50",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"todo, user"
]
},
"execution_count": null,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"db.t"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "85b4bfd6",
"metadata": {},
"outputs": [],
"source": [
"#| export\n",
"class Meta: id: int; version: int = 0"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "8229af4b",
"metadata": {},
"outputs": [],
"source": [
"#| export\n",
"@patch\n",
"def _add_meta(self: Database):\n",
" \"Create _meta table if it doesn't exist\"\n",
" meta = self.create(Meta, name='_meta', pk='id', constraints=['id = 1'])\n",
" meta.upsert({'id': 1, 'version': 0})"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "de86dc3e",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"_meta, todo, user"
]
},
"execution_count": null,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"db._add_meta(); db.t"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "d06cfbf2",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"CREATE TABLE _meta (\n",
"\tid INTEGER, \n",
"\tversion INTEGER, \n",
"\tPRIMARY KEY (id), \n",
"\tCHECK (id = 1)\n",
")\n"
]
}
],
"source": [
"print(db['_meta'].schema)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "0c465c94",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"[Meta(id=1, version=0)]"
]
},
"execution_count": null,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"db['_meta']()"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "76e56b1c",
"metadata": {},
"outputs": [],
"source": [
"#| export\n",
"def _get_version(self):\n",
" try:\n",
" res = self['_meta'].selectone('id=1')\n",
" return res['version'] if isinstance(res, dict) else res.version\n",
" except: return 0\n",
"\n",
"def _set_version(self, v): self['_meta'].update_where({'version': v}, 'id = 1')\n",
"\n",
"Database.version = property(_get_version, _set_version)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "78c4dadf",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"0"
]
},
"execution_count": null,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"db.version"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "589d553c",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"1"
]
},
"execution_count": null,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"db.version = 1; db.version"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "bca7f6e2",
"metadata": {},
"outputs": [],
"source": [
"#| export\n",
"def _get_migrations(mdir):\n",
" scripts = {}\n",
" for p in Path(mdir).iterdir():\n",
" if m := re.match(r'^(\\d+)-', p.name): scripts[int(m.group(1))] = p\n",
" return sorted(scripts.items())"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "2951298e",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"[(1, Path('migrations/1-add_priority_to_todo.sql'))]"
]
},
"execution_count": null,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"mdir = Path('./migrations')\n",
"_get_migrations(mdir)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "c5fc3863",
"metadata": {},
"outputs": [],
"source": [
"#| export\n",
"@patch\n",
"def migrate(self:Database, mdir):\n",
" if '_meta' not in self.t: self._add_meta()\n",
" cver = self.version\n",
" for v, p in _get_migrations(mdir)[self.version:]:\n",
" try:\n",
" if p.suffix == '.sql': self.execute(sa.text(p.read_text()))\n",
" elif p.suffix == '.py':\n",
" subprocess.run([sys.executable, p, self.conn_str], check=True)\n",
" self.version = v\n",
" self.conn.commit()\n",
" print(f\"Applied migration {v}: {p.name}\")\n",
" except Exception as e:\n",
" self.conn.rollback()\n",
" raise e\n",
" self.conn.commit()\n",
" cls_map = {nm: tbl.cls for nm, tbl in self._tables.items() if tbl.cls}\n",
" self._tables.clear()\n",
" self.meta.clear()\n",
" self.meta.reflect(bind=self.engine)\n",
" for tbl in self.t: tbl.dataclass()"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "1cf4a4e1",
"metadata": {},
"outputs": [],
"source": [
"db.version = 0"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "c069f2cf",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Applied migration 1: 1-add_priority_to_todo.sql\n"
]
}
],
"source": [
"db.migrate(mdir)\n",
"test_eq(db.version, 1)\n",
"assert 'priority' in db.t.todo.c"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "e12e42a1",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"[Todo(title='get it done', name='rlt', id=2, done=False, details='', priority=0),\n",
" Todo(title='get it done', name='rlt', id=4, done=False, details='', priority=0)]"
]
},
"execution_count": null,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"db.t.todo()"
]
},
{
"cell_type": "markdown",
"id": "1eda1cc8",
Expand Down
Loading