feat(iplist): add iplist database support with independent refresh cadence

Author: AppsGanin

cmd/rospanel/service.go

@@ -157,6 +157,9 @@ func runServer(dataDir string) {
 	if err := geo.Ensure(geoDir); err != nil {
 		log.Printf("geo: %v", err)
 	}
+	// The iplist databases are fetched separately, below: Xray never reads them
+	// (the panel compiles "iplist:" rules into plain matchers itself), so they must
+	// not hold up its start.
 
 	sup := xray.NewSupervisor(xrayBin, xrayConfig, geoDir)
 	mgr := core.New(st, sup, xray.Options{PanelDest: panelDest(adminAddr)},
@@ -177,6 +180,25 @@ func runServer(dataDir string) {
 		log.Printf("initial reconcile failed (panel still starting): %v", err)
 	}
 
+	// Fetch the iplist databases if this box has never had them (~2.7 MB), then
+	// reconcile so a config already referencing a group picks it up now rather than
+	// routing without those rules until the next refresh tick. Backgrounded: Xray is
+	// already serving and does not depend on these.
+	go func() {
+		missing := false
+		for _, f := range geo.StatusLists(geoDir) {
+			missing = missing || !f.Present
+		}
+		if !missing {
+			return
+		}
+		if err := geo.EnsureLists(geoDir); err != nil {
+			log.Printf("geo: %v", err)
+			return
+		}
+		mgr.TriggerReconcile()
+	}()
+
 	// Daily TLS check: renews ACME certs near expiry and reloads Xray on change.
 	go tlsLoop(mgr)
 	// Periodic traffic accounting + quota/expiry enforcement.

internal/core/manager.go

@@ -10,6 +10,7 @@ import (
 	"sync/atomic"
 	"time"
 
+	"github.com/AppsGanin/rospanel/internal/geo"
 	"github.com/AppsGanin/rospanel/internal/logbuf"
 	"github.com/AppsGanin/rospanel/internal/model"
 	"github.com/AppsGanin/rospanel/internal/nodeapi"
@@ -98,9 +99,10 @@ type Manager struct {
 	lastVPNT    time.Time
 	vpnViewers  atomic.Int32 // active dashboard-stream subscribers; gates vpnSpeedLoop
 
-	geoMu   sync.Mutex
-	geoSite []string // cached geosite category codes
-	geoIP   []string // cached geoip category codes
+	geoMu     sync.Mutex
+	geoSite   []string     // cached geosite category codes
+	geoIP     []string     // cached geoip category codes
+	geoGroups geo.GroupSet // cached iplist groups ("<source>/<group>" → rules)
 
 	proxyMu sync.Mutex
 	// proxies holds the local server's current egress proxies of each lane, keyed by
@@ -206,7 +208,8 @@ func New(st *store.Store, sup *xray.Supervisor, opts xray.Options, tls TLSPaths,
 	m.sup.SetOnCrash(m.onXrayCrash) // alert admins when Xray exits unexpectedly
 	go m.reconcileLoop()
 	go m.proxyLoop()
-	go m.geoLoop() // auto-refresh geo databases on the operator's cadence
+	go m.geoLoop()    // auto-refresh geo databases on the operator's cadence
+	go m.ipListLoop() // ...and the iplist lists on their own, separate cadence
 	go m.bruteGuardLoop()
 	go m.healthLoop()              // probe Opera/Hola lane liveness for the UI
 	m.startWebhookWorkers()        // drain the outbound-webhook delivery queue
@@ -390,7 +393,7 @@ func (m *Manager) syncUsers() error {
 	}
 	// Keep config.json current (no restart) so the monitor's crash-restart loads
 	// the right user set.
-	cfg, err := xray.Generate(set, users, m.opts, m.getProxies())
+	cfg, err := xray.Generate(set, users, m.genOpts(), m.getProxies())
 	if err != nil {
 		return err
 	}
@@ -459,7 +462,7 @@ func (m *Manager) reconcileLocked() error {
 	if err != nil {
 		return err
 	}
-	cfg, err := xray.Generate(set, users, m.opts, m.getProxies())
+	cfg, err := xray.Generate(set, users, m.genOpts(), m.getProxies())
 	if err != nil {
 		logErr("reconcile: config generation failed", "err", err)
 		_ = m.store.SetConfigError(err.Error())

internal/core/manager_nodes.go

@@ -143,7 +143,7 @@ func (m *Manager) NodeDesiredState(n *model.Node) (*nodeapi.NodeState, error) {
 	ns.KeyPath = nodeapi.KeyPathSentinel
 	// The node's own fallback points at its local decoy/panel loopback, same as the
 	// panel's own layout. Egress lanes resolve against the node's OWN proxy pool.
-	cfg, err := xray.Generate(ns, users, m.opts, m.getNodeProxies(n.ID))
+	cfg, err := xray.Generate(ns, users, m.genOpts(), m.getNodeProxies(n.ID))
 	if err != nil {
 		return nil, err
 	}

internal/core/manager_settings.go

@@ -16,6 +16,7 @@ import (
 	"github.com/AppsGanin/rospanel/internal/model"
 	"github.com/AppsGanin/rospanel/internal/netguard"
 	"github.com/AppsGanin/rospanel/internal/warp"
+	"github.com/AppsGanin/rospanel/internal/xray"
 )
 
 // SetTimezone validates and persists the operator's IANA timezone, then updates
@@ -137,6 +138,17 @@ func (m *Manager) SetXrayDNS(dns string) error {
 // Settings returns the current settings row (read-only handlers).
 func (m *Manager) Settings() (*model.Settings, error) { return m.store.GetSettings() }
 
+// assetDir is the directory holding the geo + iplist databases, or "" when the
+// Manager has no supervisor (unit tests build a bare Manager). The geo readers
+// treat "" as "no databases present" and error rather than panicking, so every
+// caller degrades to "no categories / no groups" instead of taking the panel down.
+func (m *Manager) assetDir() string {
+	if m.sup == nil {
+		return ""
+	}
+	return m.sup.AssetDir()
+}
+
 // GeoCategories returns the geosite + geoip category codes from the on-disk
 // databases, parsed once and cached (the .dat files only change on refresh).
 func (m *Manager) GeoCategories() (geosite, geoip []string, err error) {
@@ -145,30 +157,82 @@ func (m *Manager) GeoCategories() (geosite, geoip []string, err error) {
 	if m.geoSite != nil || m.geoIP != nil {
 		return m.geoSite, m.geoIP, nil
 	}
-	gs, gi, err := geo.Categories(m.sup.AssetDir())
+	gs, gi, err := geo.Categories(m.assetDir())
 	if err != nil {
 		return nil, nil, err
 	}
 	m.geoSite, m.geoIP = gs, gi
 	return gs, gi, nil
 }
 
-// GeoStatus reports the on-disk state of the geoip/geosite databases (presence,
-// size, last-download time) for the settings UI.
-func (m *Manager) GeoStatus() []geo.FileInfo { return geo.Status(m.sup.AssetDir()) }
+// GeoGroups returns the iplist groups parsed from the on-disk databases, cached
+// like GeoCategories (they only change on a refresh). Callers must not mutate
+// the returned set.
+func (m *Manager) GeoGroups() (geo.GroupSet, error) {
+	m.geoMu.Lock()
+	defer m.geoMu.Unlock()
+	if m.geoGroups != nil {
+		return m.geoGroups, nil
+	}
+	g, err := geo.Groups(m.assetDir())
+	if err != nil {
+		return nil, err
+	}
+	m.geoGroups = g
+	return g, nil
+}
 
-// RefreshGeo re-downloads the geo databases to their latest version, drops the
-// parsed-category cache, and reloads Xray so routing rules pick up the new data.
-func (m *Manager) RefreshGeo() ([]geo.FileInfo, error) {
-	dir := m.sup.AssetDir()
-	if err := geo.Refresh(dir); err != nil {
-		return geo.Status(dir), err
+// genOpts returns the generation options with the iplist groups resolved, so
+// "iplist:" routing entries compile to real matchers. A parse failure (databases
+// not downloaded yet) degrades to no groups rather than blocking generation —
+// those rules are skipped and their traffic falls through to the next lane.
+func (m *Manager) genOpts() xray.Options {
+	opts := m.opts
+	if g, err := m.GeoGroups(); err == nil {
+		opts.Groups = g
 	}
+	return opts
+}
+
+// GeoStatus reports the on-disk state of the Xray geo databases (presence, size,
+// last-download time) for the settings UI.
+func (m *Manager) GeoStatus() []geo.FileInfo { return geo.Status(m.assetDir()) }
+
+// IPListStatus reports the on-disk state of the iplist databases. Separate from
+// GeoStatus because they are a separate concern with their own panel tab: Xray
+// reads the geo .dat files, while the iplist lists are the panel's own source for
+// "iplist:" rules.
+func (m *Manager) IPListStatus() []geo.FileInfo { return geo.StatusLists(m.assetDir()) }
+
+// dropGeoCache forces a re-parse of the categories and groups on next use. Called
+// after every refresh — including a partial failure, since each file is written
+// atomically and independently, so whatever did land must be picked up.
+func (m *Manager) dropGeoCache() {
 	m.geoMu.Lock()
-	m.geoSite, m.geoIP = nil, nil // force re-parse on next GeoCategories
+	m.geoSite, m.geoIP, m.geoGroups = nil, nil, nil
 	m.geoMu.Unlock()
-	m.TriggerReconcile() // reload Xray with the refreshed databases
-	return geo.Status(dir), nil
+}
+
+// RefreshGeo re-downloads the Xray geo databases to their latest version, drops
+// the parsed caches, and reloads Xray so routing rules pick up the new data.
+func (m *Manager) RefreshGeo() ([]geo.FileInfo, error) {
+	if err := geo.Refresh(m.assetDir()); err != nil {
+		return m.GeoStatus(), err
+	}
+	m.dropGeoCache()
+	m.TriggerReconcile()
+	return m.GeoStatus(), nil
+}
+
+// RefreshIPLists re-downloads the iplist databases, drops the parsed caches and
+// reloads Xray, so a changed group takes effect at once.
+func (m *Manager) RefreshIPLists() ([]geo.FileInfo, error) {
+	if err := geo.RefreshLists(m.assetDir()); err != nil {
+		return m.IPListStatus(), err
+	}
+	m.dropGeoCache()
+	m.TriggerReconcile()
+	return m.IPListStatus(), nil
 }
 
 // GeoRefreshHours returns the configured geo auto-refresh cadence (hours; 0 ⇒ off).
@@ -189,32 +253,76 @@ func (m *Manager) currentGeoRefresh() time.Duration {
 	return time.Duration(set.GeoRefreshHours) * time.Hour
 }
 
-// geoStale reports whether any geo database is missing or older than maxAge.
-func (m *Manager) geoStale(maxAge time.Duration) bool {
+// IPListRefreshHours returns the configured iplist auto-refresh cadence (hours;
+// 0 ⇒ off).
+func (m *Manager) IPListRefreshHours() int {
+	set, err := m.store.GetSettings()
+	if err != nil {
+		return 0
+	}
+	return set.IPListRefreshHours
+}
+
+// currentIPListRefresh reads the iplist auto-refresh cadence as a duration (0 ⇒ off).
+func (m *Manager) currentIPListRefresh() time.Duration {
+	set, err := m.store.GetSettings()
+	if err != nil || set.IPListRefreshHours <= 0 {
+		return 0
+	}
+	return time.Duration(set.IPListRefreshHours) * time.Hour
+}
+
+// stale reports whether any file in the set is missing or older than maxAge.
+func stale(files []geo.FileInfo, maxAge time.Duration) bool {
 	cutoff := time.Now().Add(-maxAge).Unix()
-	for _, f := range geo.Status(m.sup.AssetDir()) {
+	for _, f := range files {
 		if !f.Present || f.ModifiedAt < cutoff {
 			return true
 		}
 	}
 	return false
 }
 
+// geoStale reports whether any geo database is missing or older than maxAge.
+func (m *Manager) geoStale(maxAge time.Duration) bool { return stale(m.GeoStatus(), maxAge) }
+
+// ipListStale reports whether any iplist database is missing or older than maxAge.
+func (m *Manager) ipListStale(maxAge time.Duration) bool { return stale(m.IPListStatus(), maxAge) }
+
 // geoLoop auto-refreshes the geo databases when they go stale, on the operator's
 // cadence (0 ⇒ off). It re-checks hourly so a cadence change takes effect without a
 // restart and a reboot doesn't reset a long timer. Sleeps first so boot stays quiet;
 // enabling the cadence refreshes promptly via SetGeoRefresh.
 func (m *Manager) geoLoop() {
+	refreshLoop("geo", m.currentGeoRefresh, m.geoStale, func() error {
+		_, err := m.RefreshGeo()
+		return err
+	})
+}
+
+// ipListLoop is geoLoop's twin for the iplist databases, on their OWN cadence —
+// they follow a different upstream clock (~12h) and are panel-only, so tying them
+// to the geo schedule would either poll the lists too rarely or drag ~28 MB of
+// .dat files down far too often.
+func (m *Manager) ipListLoop() {
+	refreshLoop("iplist", m.currentIPListRefresh, m.ipListStale, func() error {
+		_, err := m.RefreshIPLists()
+		return err
+	})
+}
+
+// refreshLoop is the shared hourly staleness poll behind geoLoop/ipListLoop.
+func refreshLoop(what string, cadence func() time.Duration, isStale func(time.Duration) bool, refresh func() error) {
 	for {
 		time.Sleep(time.Hour)
-		d := m.currentGeoRefresh()
-		if d <= 0 || !m.geoStale(d) {
+		d := cadence()
+		if d <= 0 || !isStale(d) {
 			continue
 		}
-		if _, err := m.RefreshGeo(); err != nil {
-			logWarn("geo: auto-refresh failed", "err", err)
+		if err := refresh(); err != nil {
+			logWarn(what+": auto-refresh failed", "err", err)
 		} else {
-			logInfo("geo: auto-refreshed", "cadence_hours", int(d/time.Hour))
+			logInfo(what+": auto-refreshed", "cadence_hours", int(d/time.Hour))
 		}
 	}
 }
@@ -239,6 +347,25 @@ func (m *Manager) SetGeoRefresh(hours int) error {
 	return nil
 }
 
+// SetIPListRefresh persists the iplist auto-refresh cadence (hours; 0 ⇒ never),
+// refreshing at once if enabling with the lists already stale.
+func (m *Manager) SetIPListRefresh(hours int) error {
+	if hours < 0 {
+		hours = 0
+	}
+	if err := m.store.SetIPListRefresh(hours); err != nil {
+		return err
+	}
+	if d := time.Duration(hours) * time.Hour; d > 0 && m.ipListStale(d) {
+		go func() {
+			if _, err := m.RefreshIPLists(); err != nil {
+				logWarn("iplist: refresh on enable failed", "err", err)
+			}
+		}()
+	}
+	return nil
+}
+
 // SetProxyMode persists the forward-proxy inbound (proxy mode) and reloads Xray.
 func (m *Manager) SetProxyMode(enabled bool, typ string, port int, user, pass string) error {
 	if typ != "socks" && typ != "http" {