Skip to content
Open
Show file tree
Hide file tree
Changes from 65 commits
Commits
Show all changes
102 commits
Select commit Hold shift + click to select a range
5bd3372
Add Spyglass quickstart scripts and documentation
edeno Sep 27, 2025
ca4e571
Refactor config validation and update SystemInfo mutability
edeno Sep 27, 2025
e810f97
Improve Docker error messages in quickstart script
edeno Sep 27, 2025
580afe9
Expand fallback config for Spyglass quickstart
edeno Sep 27, 2025
faa513d
Refactor config creation to use SpyglassConfig directly
edeno Sep 27, 2025
fe1f1bf
Fix database setup when no strategy is selected
edeno Sep 27, 2025
37121e4
Add interactive installation type selection to quickstart
edeno Sep 27, 2025
37dd736
Add environment name selection to quickstart script
edeno Sep 27, 2025
f9b3bab
Install additional deps only if environment updated
edeno Sep 27, 2025
8f4631c
Refactor quickstart.py for clarity and error handling
edeno Sep 27, 2025
0bd6148
Refactor Docker and environment commands in quickstart
edeno Sep 27, 2025
74b0b82
Update quickstart.py
edeno Sep 27, 2025
3a6782d
Improve validation and messaging in quickstart script
edeno Sep 27, 2025
eb46f79
Refactor quickstart database and config setup logic
edeno Sep 27, 2025
f585d37
Remove quickstart.sh setup script
edeno Sep 27, 2025
aafae3c
Refactor quickstart.py for modular orchestration
edeno Sep 27, 2025
64fc378
Update warnings filter and clean up imports in scripts
edeno Sep 27, 2025
776774f
Improve environment setup and diagnostics in quickstart
edeno Sep 27, 2025
a32d6a4
Improve database setup and validation UX
edeno Sep 27, 2025
36b5178
Filter out false-positive conda error messages in stderr
edeno Sep 27, 2025
4c8e4a4
Improve directory handling and config in SpyglassConfig
edeno Sep 27, 2025
9ac86df
Add minimal environment file and update quickstart logic
edeno Sep 27, 2025
4e36964
Expand DataJoint config file search locations
edeno Sep 27, 2025
b4f8434
Improve database connection result message
edeno Sep 27, 2025
b94899d
Simplify kachery_zone config environment lookup
edeno Sep 27, 2025
056325c
Add support for auto-yes and custom DB port in quickstart
edeno Sep 27, 2025
5d78e92
Add --config-file option to specify DataJoint config path
edeno Sep 27, 2025
f89970c
Add explicit Spyglass settings initialization and auto-init control
edeno Sep 27, 2025
fd5097a
Add repo root as fallback for config file lookup
edeno Sep 27, 2025
3120e91
Improve quickstart: port check, error context, auto-yes
edeno Sep 27, 2025
65ec4aa
Refactor shared utilities into common.py
edeno Sep 27, 2025
20c991e
Improve error handling and input validation in scripts
edeno Sep 27, 2025
d0279d5
Add type annotations and docstrings to quickstart and validator scripts
edeno Sep 27, 2025
79f0d28
Refactor color handling and progress output in scripts
edeno Sep 27, 2025
58c6dd5
Add detailed docstrings and fix directory creation
edeno Sep 27, 2025
5fae72c
Fix typos and update docstrings in settings.py
edeno Sep 27, 2025
c882f99
Refactor error handling and minor logic in scripts
edeno Sep 27, 2025
334e718
Use dj.config for database connection info
edeno Sep 27, 2025
58c4a1e
Add Rich-enhanced Spyglass scripts and documentation
edeno Sep 27, 2025
8cb7cf3
Remove Rich-enhanced quickstart and validation scripts
edeno Sep 27, 2025
c7bc043
Revise and simplify QUICKSTART.md instructions
edeno Sep 27, 2025
b36cc7f
Delete demo_rich.py
edeno Sep 27, 2025
b7f2666
Update package manager recommendation in QUICKSTART.md
edeno Sep 27, 2025
9c16b7c
Improve non-interactive input handling and config creation
edeno Sep 28, 2025
25552be
Simplify Spyglass settings initialization logic
edeno Sep 28, 2025
14c5257
Add unit tests for quickstart.py architecture
edeno Sep 28, 2025
9922e44
Add walkthrough docs for quickstart and validation scripts
edeno Sep 28, 2025
d7595e3
Add modular core, utils, and UX packages for setup
edeno Sep 28, 2025
4df0b3d
Refactor system requirements checks in quickstart
edeno Sep 28, 2025
c393556
Refactor install type selection to avoid duplicate menu
edeno Sep 28, 2025
b4beaee
Add interactive environment name selection to quickstart
edeno Sep 28, 2025
9fa8811
Improve environment name prompt with color and UI methods
edeno Sep 28, 2025
e1be1b7
Refactor Docker database setup with pure functions
edeno Sep 28, 2025
39e5953
Enhance user input validation and feedback in quickstart
edeno Sep 28, 2025
a4df4a0
Add persona-based onboarding to quickstart script
edeno Sep 29, 2025
505a13d
Add structured error recovery guidance to setup scripts
edeno Sep 29, 2025
0edf34f
Refactor quickstart.py and add comprehensive tests
edeno Sep 29, 2025
0114bdf
Update README with quickstart installation instructions
edeno Sep 29, 2025
33718ac
Fix severity level for missing DataJoint core dependency
edeno Sep 29, 2025
c9826fe
Address PR review comments: clarify path resolution comment
edeno Sep 29, 2025
d4fd310
Refactor and reformat scripts for consistency and readability
edeno Sep 29, 2025
00a2cbd
Update database connection errors to Severity.ERROR
edeno Sep 29, 2025
0db4a02
Refactor long lines for readability in settings.py
edeno Sep 29, 2025
8375e22
Improve disk space check and TLS logic in setup scripts
edeno Sep 29, 2025
d3d12c9
Add config file existence check and test_mode to SpyglassConfig
edeno Sep 29, 2025
3e45d0d
Remove tests (will replace later)
edeno Sep 29, 2025
24dd1a7
Add installer, validation, and Docker utilities
edeno Oct 1, 2025
5f70148
Improve installation docs and database setup UX
edeno Oct 1, 2025
b7ed983
Add hostname validation to install script
edeno Oct 1, 2025
a6e9798
Add port availability checks for database setup
edeno Oct 1, 2025
0a8486a
Improve install and validation script messaging
edeno Oct 1, 2025
e8f99e5
Remove Spyglass setup scripts and related modules
edeno Oct 1, 2025
39d1b54
Format long print statements for readability
edeno Oct 1, 2025
32ed3b2
Add Docker Compose support for database setup
edeno Oct 2, 2025
a710e67
Refactor installer and validation scripts for clarity
edeno Oct 3, 2025
2fd03aa
Update .gitignore
edeno Oct 3, 2025
c283631
Reorder imports in validate.py and docker.py
edeno Oct 3, 2025
e1a4742
Remove DockerMySQLManager and fix import formatting
edeno Oct 3, 2025
bdfb0cd
Refactor test imports and formatting in test_install.py
edeno Oct 3, 2025
3478130
Merge branch 'master' into golden-path
edeno Nov 3, 2025
e7da335
Add config_schema.yml for directory structure and TLS
edeno Nov 10, 2025
1d56f0e
Refactor config schema to JSON and centralize directory structure
edeno Nov 11, 2025
df56596
Replace DockerMySQLManager with _TestDatabaseManager in tests
edeno Nov 11, 2025
a48ae80
Improve install script error handling and messaging
edeno Nov 11, 2025
b4aeced
Fix port handling in _TestDatabaseManager to resolve CI test failures
edeno Nov 11, 2025
aad2387
Add missing connected property to _TestDatabaseManager
edeno Nov 11, 2025
03a33c6
Fix connected property to actually verify database connection
edeno Nov 11, 2025
a941b24
Add container property to _TestDatabaseManager
edeno Nov 11, 2025
3125f3a
Make wait() actually poll for database connectivity
edeno Nov 11, 2025
3e1581a
Remove premature dj.config calls from _TestDatabaseManager
edeno Nov 11, 2025
d99d873
Restore dj.config.update() in connected property
edeno Nov 11, 2025
7d97d90
Fix test_mode propagation by checking dj.config directly
edeno Nov 11, 2025
c8dc6ed
Remove @cached_property from _test_mode to prevent stale value caching
edeno Nov 11, 2025
96d841e
Fix pytest collection hang by moving SpyglassConfig imports inside te…
edeno Nov 11, 2025
285c65a
Fix test_install.py to use temporary directories instead of fake paths
edeno Nov 11, 2025
e68c610
Merge branch 'master' into golden-path
edeno Nov 13, 2025
9944aef
Fix QUICKSTART.md to reference install.py instead of quickstart.py
edeno Nov 13, 2025
d135748
Align installer with primary use case: new lab members joining existi…
edeno Nov 13, 2025
d3b8fdc
Refactor password change to use DataJoint's built-in dj.set_password()
edeno Nov 13, 2025
f85360a
Address all critical issues from code and UX review
edeno Nov 13, 2025
f3deccb
Implement password change during installation using subprocess
edeno Nov 13, 2025
e80b822
Fix critical SQL injection vulnerability in password change function
edeno Nov 13, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
96 changes: 96 additions & 0 deletions QUICKSTART.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,96 @@
# Spyglass Quickstart (5 minutes)

Get from zero to analyzing neural data with Spyglass in just a few commands.

## Prerequisites

- **Python**: Version 3.9 or higher
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

A desire to bump to 3.10 has come up in a fer recent PR reviews for easier type hinting

- **Disk Space**: ~10GB for installation + data storage
- **Operating System**: macOS or Linux (Windows experimental)
- **Package Manager**: [conda](https://docs.conda.io/) (23.10.0+ recommended) or [mamba](https://mamba.readthedocs.io/)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ideally, I would have markdownlint break long lines for easier future edits/reviews. This opinionated formatter would also chage spacing around H3s below


If you don't have mamba/conda, install [miniforge](https://github.com/conda-forge/miniforge#install) first.

## Installation (2 commands)

### 1. Download and run quickstart
```bash
# Clone the repository
git clone https://github.com/LorenFrankLab/spyglass.git
cd spyglass

# Run quickstart (minimal installation)
python scripts/quickstart.py
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

scripts is a new root level directory - I would like for it to either...

  1. have a more specific name for folks new to the repository - 'onboarding_scrips'?
  2. User 'scripts' as a prefix across this and 'maintenance_scrips'
  3. have it get merged with the core package or maintenance folders

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think naming the directory onboarding or onboarding_scripts would help

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

quickstart.py isn't a file in this folder. Should it be install.py?

```

### 2. Validate installation
```bash
# Activate the environment
conda activate spyglass

# Run validation
python scripts/validate_spyglass.py -v
```

**That's it!** Total time: ~5-10 minutes

## Next Steps

### Run first tutorial
```bash
cd notebooks
jupyter notebook 01_Concepts.ipynb
```

### Configure for your data
- Place NWB files in `~/spyglass_data/raw/`
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should 'where do you want your base dir?' be an additional question? This seems like it would set up a default few people would use, and then creates the issue of having the low-tech user change the config after the fact

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Agreed, if nothing else this approach means all our lab members would need to change it after running this. Seems a reasonable option to ask for input

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It appears the code is updated to allow this but not this doc

- See [Data Import Guide](https://lorenfranklab.github.io/spyglass/latest/notebooks/01_Insert_Data/) for details

### Join community
- 📖 [Documentation](https://lorenfranklab.github.io/spyglass/)
- 💬 [GitHub Discussions](https://github.com/LorenFrankLab/spyglass/discussions)
- 🐛 [Report Issues](https://github.com/LorenFrankLab/spyglass/issues)

---

## Installation Options

Need something different? The quickstart supports these options:

```bash
python scripts/quickstart.py --full # All optional dependencies
python scripts/quickstart.py --pipeline=dlc # DeepLabCut pipeline
python scripts/quickstart.py --no-database # Skip database setup
python scripts/quickstart.py --help # See all options
```

## What Gets Installed

The quickstart creates:
- **Conda environment** with Spyglass and core dependencies
- **MySQL database** (local Docker container)
- **Data directories** in `~/spyglass_data/`
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

See above comment

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Doc needs updated here too

- **Jupyter environment** for running tutorials

## Troubleshooting

### Installation fails?
```bash
# Remove environment and retry
conda env remove -n spyglass
python scripts/quickstart.py
```

### Validation fails?
1. Check error messages for specific issues
2. Ensure Docker is running (for database)
3. Try: `python scripts/quickstart.py --no-database`

### Need help?
- Check [Advanced Setup Guide](https://lorenfranklab.github.io/spyglass/latest/notebooks/00_Setup/) for manual installation
- Ask questions in [GitHub Discussions](https://github.com/LorenFrankLab/spyglass/discussions)

---

**Next tutorial**: [01_Concepts.ipynb](notebooks/01_Concepts.ipynb)
**Full documentation**: [lorenfranklab.github.io/spyglass](https://lorenfranklab.github.io/spyglass/)
19 changes: 18 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,24 @@ Documentation can be found at -

## Installation

For installation instructions see -
### Quick Start (Recommended)

Get up and running in 5 minutes with our automated installer:

```bash
# Clone the repository
git clone https://github.com/LorenFrankLab/spyglass.git
cd spyglass

# Run quickstart
python scripts/quickstart.py
```

See [QUICKSTART.md](QUICKSTART.md) for detailed options and troubleshooting.

### Full Installation Guide

For manual installation and advanced configuration options see -
[https://lorenfranklab.github.io/spyglass/latest/notebooks/00_Setup/](https://lorenfranklab.github.io/spyglass/latest/notebooks/00_Setup/)

Typical installation time is: 5-10 minutes
Expand Down
30 changes: 30 additions & 0 deletions environment-min.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
# Minimal Spyglass Environment - Core Dependencies Only
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I intentionally spaced the yml files so it wold be easy to diff them at a glance by flipping between and seeing what was commented out in one vs another. I find that helps maintenance

This is now our 5th environment file, so I think they belong in a directory rather than at root

# 1. Install: `mamba env create -f environment-min.yml`
# 2. Activate: `conda activate spyglass`
# 3. For full features, use environment.yml instead
#
# This environment includes only the essential dependencies needed for basic
# spyglass functionality. Optional dependencies for specific pipelines
# (LFP analysis, spike sorting, etc.) are excluded.

name: spyglass
channels:
- conda-forge
- franklab
- edeno
dependencies:
# Core Python scientific stack
- python>=3.9,<3.13
- pip
- numpy
- matplotlib
- bottleneck
- seaborn

# Core Jupyter environment for notebooks
- jupyterlab>=3.*
- ipympl

# Essential spyglass dependencies (installed via pip for development mode)
- pip:
- .
129 changes: 129 additions & 0 deletions scripts/common.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,129 @@
"""
Common utilities shared between Spyglass scripts.

This module provides shared functionality for quickstart.py and validate_spyglass.py
to improve consistency and reduce code duplication.
"""

from collections import namedtuple
from enum import Enum
from typing import NamedTuple


# Shared color definitions using namedtuple (immutable and functional)
Colors = namedtuple(
"Colors",
[
"HEADER",
"OKBLUE",
"OKCYAN",
"OKGREEN",
"WARNING",
"FAIL",
"ENDC",
"BOLD",
"UNDERLINE",
],
)(
HEADER="\033[95m",
OKBLUE="\033[94m",
OKCYAN="\033[96m",
OKGREEN="\033[92m",
WARNING="\033[93m",
FAIL="\033[91m",
ENDC="\033[0m",
BOLD="\033[1m",
UNDERLINE="\033[4m",
)

# Disabled colors for no-color mode
DisabledColors = namedtuple(
"DisabledColors",
[
"HEADER",
"OKBLUE",
"OKCYAN",
"OKGREEN",
"WARNING",
"FAIL",
"ENDC",
"BOLD",
"UNDERLINE",
],
)(*([""] * 9))


# Shared exception hierarchy
class SpyglassSetupError(Exception):
"""Base exception for setup errors."""

pass


class SystemRequirementError(SpyglassSetupError):
"""System doesn't meet requirements."""

pass


class EnvironmentCreationError(SpyglassSetupError):
"""Environment creation failed."""

pass


class DatabaseSetupError(SpyglassSetupError):
"""Database setup failed."""

pass


# Enums for type-safe choices
class MenuChoice(Enum):
"""User menu choices for installation type."""

MINIMAL = 1
FULL = 2
PIPELINE = 3


class DatabaseChoice(Enum):
"""Database setup choices."""

DOCKER = 1
EXISTING = 2
SKIP = 3


class ConfigLocationChoice(Enum):
"""Configuration file location choices."""

REPO_ROOT = 1
CURRENT_DIR = 2
CUSTOM = 3


class PipelineChoice(Enum):
"""Pipeline-specific installation choices."""

DLC = 1
MOSEQ_CPU = 2
MOSEQ_GPU = 3
LFP = 4
DECODING = 5


# Configuration constants
class Config:
"""Centralized configuration constants."""

DEFAULT_TIMEOUT = 1800 # 30 minutes
DEFAULT_DB_PORT = 3306
DEFAULT_ENV_NAME = "spyglass"

# Timeouts for different operations
TIMEOUTS = {
"environment_create": 1800, # 30 minutes for env creation
"package_install": 600, # 10 minutes for packages
"database_check": 60, # 1 minute for DB readiness
}
5 changes: 5 additions & 0 deletions scripts/core/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
"""Core business logic modules for Spyglass setup.

This package contains pure functions and business logic extracted from
the main setup scripts, as recommended in REVIEW.md.
"""
Loading
Loading