Skip to content

Commit 035960b

Browse files
Moustafa-MoustafaMoustafa-Moustafa
committed
feat(db): make rollout state runtime-local for distributed Nebraska
The `rollout_in_progress` indicator for a group is the only field on the admin-managed `groups` table that is written by the update-serving engine, not the admin. This is a step towards the distributed topology described in RFC #1375: admin state needs to flow one-way from the admin node to each runtime node, while runtime state must stay local so each runtime node records what it observes from its own clients. This change moves that single field onto its own runtime-local table. Standalone-mode behavior is unchanged. Note on `policy_safe_mode`: in distributed mode there is no path for runtime nodes to send observations back to the admin node, which means the `safe_mode` auto-pause cannot work as it does today. I'm proposing we do not support `safe_mode` while distributed mode is enabled, at least within the scope of this effort. A proper solution (e.g. some form of runtime-to-admin state sharing) can be designed separately. Refs: #1375 Signed-off-by: Moustafa Moustafa <momousta@microsoft.com>
1 parent ff97d17 commit 035960b

5 files changed

Lines changed: 71 additions & 17 deletions

File tree

‎CHANGELOG.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
2121
- Package list UI now updates immediately after blacklist changes
2222
- Channel edit dialog filters out blacklisted packages from selection
2323
- Floor package selection prevents choosing blacklisted packages with clear visual feedback
24+
- **Group rollout state moved to runtime-local table:** The `rollout_in_progress` flag on `groups` is now stored in a separate `group_state` table, in preparation for the distributed Nebraska topology described in [RFC #1375](https://github.com/flatcar/nebraska/issues/1375). The JSON contract is unchanged. ([#1396](https://github.com/flatcar/nebraska/pull/1396))
2425

2526
### Removed
2627
### Bugfixes

‎backend/pkg/api/db/drop_all_tables.sql‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ drop table if exists package cascade;
55
drop table if exists flatcar_action cascade;
66
drop table if exists channel cascade;
77
drop table if exists groups cascade;
8+
drop table if exists group_state cascade;
89
drop table if exists instance cascade;
910
drop table if exists instance_status cascade;
1011
drop table if exists instance_application cascade;
@@ -14,5 +15,6 @@ drop table if exists event cascade;
1415
drop table if exists activity cascade;
1516
drop table if exists package_channel_blacklist cascade;
1617
drop table if exists database_migrations;
18+
drop function if exists create_group_state_for_group();
1719
-- Legacy tables if we're dropping tables in a non-migrated DB
1820
drop table if exists coreos_action cascade;

‎backend/pkg/api/db/migrations/0022_add_group_state_table.sql‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
-- +migrate Up
2+
3+
create table group_state (
4+
group_id uuid primary key references groups (id) on delete cascade,
5+
rollout_in_progress boolean default false not null,
6+
created_ts timestamptz default current_timestamp not null
7+
);
8+
9+
insert into group_state (group_id, rollout_in_progress, created_ts)
10+
select id, rollout_in_progress, created_ts from groups;
11+
12+
-- +migrate StatementBegin
13+
create or replace function create_group_state_for_group() returns trigger as $$
14+
begin
15+
insert into public.group_state (group_id) values (new.id)
16+
on conflict (group_id) do nothing;
17+
return new;
18+
end;
19+
$$ language plpgsql;
20+
-- +migrate StatementEnd
21+
22+
create trigger groups_create_group_state
23+
after insert on groups
24+
for each row
25+
execute function create_group_state_for_group();
26+
27+
alter table groups enable always trigger groups_create_group_state;
28+
29+
alter table groups drop column rollout_in_progress;
30+
31+
-- +migrate Down
32+
33+
alter table groups add column rollout_in_progress boolean default false not null;
34+
35+
update groups set rollout_in_progress = gs.rollout_in_progress
36+
from group_state gs where groups.id = gs.group_id;
37+
38+
drop trigger if exists groups_create_group_state on groups;
39+
drop function if exists create_group_state_for_group();
40+
drop table if exists group_state;

‎backend/pkg/api/db/sample_data.sql‎

Lines changed: 9 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -25,9 +25,10 @@ insert into package values ('284d295b-518f-4d67-999e-94968d0eed90', 1, '829.0.0'
2525
insert into channel values ('e06064ad-4414-4904-9a6e-fd465593d1b2', 'stable', '#14b9d6', '2019-08-19 05:09:34.261241', 'e96281a6-d1af-4bde-9a0a-97b76e56dc57', '337b3f7e-ff29-47e8-a052-f0834d25bdb5', 1);
2626
insert into channel values ('128b8c29-5058-4643-8e67-a1a0e3c641c9', 'beta', '#fc7f33', '2019-08-19 05:09:34.264334', 'e96281a6-d1af-4bde-9a0a-97b76e56dc57', '337b3f7e-ff29-47e8-a052-f0834d25bdb5', 1);
2727
insert into channel values ('a87a03ad-4984-47a1-8dc4-3507bae91ee1', 'alpha', '#1fbb86', '2019-08-19 05:09:34.265754', 'e96281a6-d1af-4bde-9a0a-97b76e56dc57', '284d295b-518f-4d67-999e-94968d0eed90', 1);
28-
insert into groups values ('9a2deb70-37be-4026-853f-bfdd6b347bbe', 'Stable', 'For production clusters', false, true, true, false, 'Europe/Berlin', '15 minutes', 2, '60 minutes', '2019-08-19 05:09:34.269062', 'e96281a6-d1af-4bde-9a0a-97b76e56dc57', 'e06064ad-4414-4904-9a6e-fd465593d1b2', 'stable');
29-
insert into groups values ('3fe10490-dd73-4b49-b72a-28ac19acfcdc', 'Beta', 'Promoted alpha releases, to catch bugs specific to your configuration', true, true, true, false, 'Europe/Berlin', '15 minutes', 2, '60 minutes', '2019-08-19 05:09:34.273244', 'e96281a6-d1af-4bde-9a0a-97b76e56dc57', '128b8c29-5058-4643-8e67-a1a0e3c641c9', 'beta');
30-
insert into groups values ('5b810680-e36a-4879-b98a-4f989e80b899', 'Alpha', 'Tracks current development work and is released frequently', false, true, true, false, 'Europe/Berlin', '15 minutes', 1, '30 minutes', '2019-08-19 05:09:34.274911', 'e96281a6-d1af-4bde-9a0a-97b76e56dc57', 'a87a03ad-4984-47a1-8dc4-3507bae91ee1', 'alpha');
28+
insert into groups values ('9a2deb70-37be-4026-853f-bfdd6b347bbe', 'Stable', 'For production clusters', true, true, false, 'Europe/Berlin', '15 minutes', 2, '60 minutes', '2019-08-19 05:09:34.269062', 'e96281a6-d1af-4bde-9a0a-97b76e56dc57', 'e06064ad-4414-4904-9a6e-fd465593d1b2', 'stable');
29+
insert into groups values ('3fe10490-dd73-4b49-b72a-28ac19acfcdc', 'Beta', 'Promoted alpha releases, to catch bugs specific to your configuration', true, true, false, 'Europe/Berlin', '15 minutes', 2, '60 minutes', '2019-08-19 05:09:34.273244', 'e96281a6-d1af-4bde-9a0a-97b76e56dc57', '128b8c29-5058-4643-8e67-a1a0e3c641c9', 'beta');
30+
update group_state set rollout_in_progress = true where group_id = '3fe10490-dd73-4b49-b72a-28ac19acfcdc';
31+
insert into groups values ('5b810680-e36a-4879-b98a-4f989e80b899', 'Alpha', 'Tracks current development work and is released frequently', true, true, false, 'Europe/Berlin', '15 minutes', 1, '30 minutes', '2019-08-19 05:09:34.274911', 'e96281a6-d1af-4bde-9a0a-97b76e56dc57', 'a87a03ad-4984-47a1-8dc4-3507bae91ee1', 'alpha');
3132
insert into flatcar_action values ('b2b16e2e-57f8-4775-827f-8f0b11ae9bd2', 'postinstall', '', 'k8CB8tMe0M8DyZ5RZwzDLyTdkHjO/YgfKVn2RgUMokc=', false, false, true, '', '', '', '2019-08-20 00:12:37.532281', '2ba4c984-5e9b-411e-b7c3-b3eb14f7a261');
3233
insert into flatcar_action values ('d5a2cbf3-b810-4e8c-88e8-6df91fc264c6', 'postinstall', '', 'QUGnmP51hp7zy+++o5fBIwElInTAms7/njnkxutn/QI=', false, false, true, '', '', '', '2019-08-20 06:15:29.11685', '337b3f7e-ff29-47e8-a052-f0834d25bdb5');
3334
insert into flatcar_action values ('299c54d1-3344-4ae9-8ad2-5c63d56d6c14', 'postinstall', '', 'SCv89GYzx7Ix+TljqbNsd7on65ooWqBzcCrLFL4wChQ=', false, false, true, '', '', '', '2019-08-20 00:09:06.927461', 'c2a36312-b989-403e-ab57-06c055a7eac2');
@@ -43,10 +44,10 @@ insert into package (id, type, url, filename, version, application_id, arch) val
4344
insert into channel (id, name, color, application_id, package_id, arch) values ('bfe32b4a-5f8c-11e5-9d70-feff819cdc9f', 'Master', '#00CC00', 'b6458005-8f40-4627-b33b-be70a718c48e', '8004bad8-5f97-11e5-9d70-feff819cdc9f', 1);
4445
insert into channel (id, name, color, application_id, package_id, arch) values ('cb2deea8-5f83-11e5-9d70-feff819cdc9f', 'Stable', '#0099FF', 'b6458005-8f40-4627-b33b-be70a718c48e', '12697fa4-5f83-11e5-9d70-feff819cdc9f', 1);
4546
insert into channel (id, name, color, application_id, package_id, arch) values ('dddddddd-5f83-11e5-9d70-feff819cdc9f', 'Failing', '#AA99FF', 'b6458005-8f40-4627-b33b-be70a718c48e', 'aaaaaaaa-5f98-11e5-9d70-feff819cdc9f', 1);
46-
insert into groups values ('bcaa68bc-5f82-11e5-9d70-feff819cdc9f', 'Prod EC2 us-west-2', 'Production servers, west coast', false, true, true, false, 'Europe/Berlin', '15 minutes', 2, '60 minutes', '2019-08-19 05:09:34.269062', 'b6458005-8f40-4627-b33b-be70a718c48e', 'cb2deea8-5f83-11e5-9d70-feff819cdc9f', 'bcaa68bc-5f82-11e5-9d70-feff819cdc9f');
47-
insert into groups values ('7074264a-2070-4b84-96ed-8a269dba5021', 'Prod EC2 us-east-1', 'Production servers, east coast', false, true, true, false, 'Europe/Berlin', '15 minutes', 2, '60 minutes', '2019-08-19 05:09:34.269062', 'b6458005-8f40-4627-b33b-be70a718c48e', 'cb2deea8-5f83-11e5-9d70-feff819cdc9f', '7074264a-2070-4b84-96ed-8a269dba5021');
48-
insert into groups values ('b110813a-5f82-11e5-9d70-feff819cdc9f', 'Qa-Dev', 'QA and development servers, Sydney', false, true, true, false, 'Europe/Berlin', '15 minutes', 2, '60 minutes', '2019-08-19 05:09:34.269062', 'b6458005-8f40-4627-b33b-be70a718c48e', 'bfe32b4a-5f8c-11e5-9d70-feff819cdc9f', 'b110813a-5f82-11e5-9d70-feff819cdc9f');
49-
insert into groups values ('cccccccc-5f82-11e5-9d70-feff819cdc9f', 'Failing Qa-Dev', 'Failing QA and development servers, Sydney', false, true, true, false, 'Europe/Berlin', '15 minutes', 2, '60 minutes', '2019-08-19 05:09:34.269062', 'b6458005-8f40-4627-b33b-be70a718c48e', 'dddddddd-5f83-11e5-9d70-feff819cdc9f', 'cccccccc-5f82-11e5-9d70-feff819cdc9f');
47+
insert into groups values ('bcaa68bc-5f82-11e5-9d70-feff819cdc9f', 'Prod EC2 us-west-2', 'Production servers, west coast', true, true, false, 'Europe/Berlin', '15 minutes', 2, '60 minutes', '2019-08-19 05:09:34.269062', 'b6458005-8f40-4627-b33b-be70a718c48e', 'cb2deea8-5f83-11e5-9d70-feff819cdc9f', 'bcaa68bc-5f82-11e5-9d70-feff819cdc9f');
48+
insert into groups values ('7074264a-2070-4b84-96ed-8a269dba5021', 'Prod EC2 us-east-1', 'Production servers, east coast', true, true, false, 'Europe/Berlin', '15 minutes', 2, '60 minutes', '2019-08-19 05:09:34.269062', 'b6458005-8f40-4627-b33b-be70a718c48e', 'cb2deea8-5f83-11e5-9d70-feff819cdc9f', '7074264a-2070-4b84-96ed-8a269dba5021');
49+
insert into groups values ('b110813a-5f82-11e5-9d70-feff819cdc9f', 'Qa-Dev', 'QA and development servers, Sydney', true, true, false, 'Europe/Berlin', '15 minutes', 2, '60 minutes', '2019-08-19 05:09:34.269062', 'b6458005-8f40-4627-b33b-be70a718c48e', 'bfe32b4a-5f8c-11e5-9d70-feff819cdc9f', 'b110813a-5f82-11e5-9d70-feff819cdc9f');
50+
insert into groups values ('cccccccc-5f82-11e5-9d70-feff819cdc9f', 'Failing Qa-Dev', 'Failing QA and development servers, Sydney', true, true, false, 'Europe/Berlin', '15 minutes', 2, '60 minutes', '2019-08-19 05:09:34.269062', 'b6458005-8f40-4627-b33b-be70a718c48e', 'dddddddd-5f83-11e5-9d70-feff819cdc9f', 'cccccccc-5f82-11e5-9d70-feff819cdc9f');
5051
insert into instance (id, ip) values ('instance1', '10.0.0.1');
5152
insert into instance (id, ip) values ('instance2', '10.0.0.2');
5253
insert into instance (id, ip) values ('instance3', '10.0.0.3');
@@ -93,4 +94,4 @@ insert into application (id, name, description, team_id) values ('780d6940-9a48-
9394
insert into package (id, type, url, filename, version, application_id, arch) values ('efb186c9-d5cb-4df2-9382-c4821e4dcc4b', 4, 'http://localhost:8000/', 'demo_v1.0.0', '1.0.0', '780d6940-9a48-4414-88df-95ba63bbe9cb', 1);
9495
insert into package (id, type, url, filename, version, application_id, arch) values ('ba28af48-b5b9-460e-946a-eba906ce7daf', 4, 'http://localhost:8000/', 'demo_v1.0.1', '1.0.1', '780d6940-9a48-4414-88df-95ba63bbe9cb', 1);
9596
insert into channel (id, name, color, application_id, package_id, arch) values ('a7c8c9a4-d2a3-475d-be64-911ff8d6e997', 'Master', '#14b9d6', '780d6940-9a48-4414-88df-95ba63bbe9cb', 'efb186c9-d5cb-4df2-9382-c4821e4dcc4b', 1);
96-
insert into groups values ('51a32aa9-3552-49fc-a28c-6543bccf0069', 'Master - dev', 'The latest stuff will be always here', false, true, true, false, 'Europe/Berlin', '15 minutes', 2, '60 minutes', '2019-08-19 05:09:34.269062', '780d6940-9a48-4414-88df-95ba63bbe9cb', 'a7c8c9a4-d2a3-475d-be64-911ff8d6e997', '51a32aa9-3552-49fc-a28c-6543bccf0069');
97+
insert into groups values ('51a32aa9-3552-49fc-a28c-6543bccf0069', 'Master - dev', 'The latest stuff will be always here', true, true, false, 'Europe/Berlin', '15 minutes', 2, '60 minutes', '2019-08-19 05:09:34.269062', '780d6940-9a48-4414-88df-95ba63bbe9cb', 'a7c8c9a4-d2a3-475d-be64-911ff8d6e997', '51a32aa9-3552-49fc-a28c-6543bccf0069');

‎backend/pkg/api/groups.go‎

Lines changed: 19 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -198,7 +198,7 @@ func (api *API) AddGroup(group *Group) (*Group, error) {
198198
return nil, err
199199
}
200200
api.updateCachedGroups()
201-
return group, nil
201+
return api.GetGroup(group.ID)
202202
}
203203

204204
// UpdateGroup updates an existing group using the context of the group
@@ -284,7 +284,7 @@ func (api *API) DeleteGroup(groupID string) error {
284284
func (api *API) GetGroup(groupID string) (*Group, error) {
285285
var group Group
286286

287-
query, _, err := goqu.From("groups").
287+
query, _, err := api.groupsQuery().
288288
Where(goqu.C("id").Eq(groupID)).
289289
ToSQL()
290290
if err != nil {
@@ -504,11 +504,13 @@ func (api *API) disableUpdates(groupID string) error {
504504
}
505505

506506
// setGroupRolloutInProgress updates the value of the rollout_in_progress flag
507-
// for a given group, indicating if a rollout is taking place now or not.
507+
// for a given group, indicating if a rollout is taking place now or not. The
508+
// flag lives on the runtime-local group_state table. The row is guaranteed to
509+
// exist by the AFTER INSERT trigger on groups.
508510
func (api *API) setGroupRolloutInProgress(groupID string, inProgress bool) error {
509-
query, _, err := goqu.Update("groups").
511+
query, _, err := goqu.Update("group_state").
510512
Set(goqu.Record{"rollout_in_progress": inProgress}).
511-
Where(goqu.C("id").Eq(groupID)).
513+
Where(goqu.C("group_id").Eq(groupID)).
512514
ToSQL()
513515
if err != nil {
514516
return err
@@ -521,11 +523,19 @@ func (api *API) setGroupRolloutInProgress(groupID string, inProgress bool) error
521523
// groupsQuery returns a SelectDataset prepared to return all groups. This
522524
// query is meant to be extended later in the methods using it to filter by a
523525
// specific group id, all groups of a given app, specify how to query the rows
524-
// or their destination.
526+
// or their destination. The join with group_state pulls in rollout_in_progress,
527+
// which lives on the group_state table.
525528
func (api *API) groupsQuery() *goqu.SelectDataset {
526-
query := goqu.From("groups").Order(goqu.I("created_ts").Desc())
527-
528-
return query
529+
return goqu.From("groups").
530+
InnerJoin(
531+
goqu.T("group_state"),
532+
goqu.On(goqu.I("groups.id").Eq(goqu.I("group_state.group_id"))),
533+
).
534+
Select(
535+
goqu.T("groups").All(),
536+
goqu.I("group_state.rollout_in_progress"),
537+
).
538+
Order(goqu.I("groups.created_ts").Desc())
529539
}
530540

531541
// GetGroupVersionBreakdown returns a version breakdown of all instances running on a given group.

0 commit comments

Comments
 (0)