4
4
# module in a Thor command processor, providing a command-line interface to
5
5
# common Jetpants functionality.
6
6
7
- %w[ thor pry highline/import colored ] . each { |g | require g }
7
+ %w[ thor pry pry-rescue highline/import colored ] . each { |g | require g }
8
8
9
9
module Jetpants
10
10
@@ -222,15 +222,15 @@ module Jetpants
222
222
end
223
223
224
224
source . start_mysql if ! source . running?
225
- error "source (#{ source } ) is not a standby slave" unless source . is_standby?
225
+ error "source (#{ source } ) is not a standby or backup slave" unless ( source . is_standby? || source . for_backups? )
226
226
227
227
targets . each do |t |
228
228
error "target #{ t } already has a master; please clear out node (including in asset tracker) before proceeding" if t . master
229
229
error "target #{ t } is running a different version of MySQL than source #{ source } ! Cannot proceed with clone operation." if t . version_cmp ( source ) != 0
230
230
end
231
231
232
232
source . enslave_siblings! ( targets )
233
- targets . concurrent_each { |t | t . resume_replication ; t . catch_up_to_master }
233
+ targets . concurrent_each { |t | t . resume_replication ; t . catch_up_to_master ( 21600 ) }
234
234
source . pool . sync_configuration
235
235
puts "Cloning complete."
236
236
Jetpants . topology . write_config
@@ -264,7 +264,12 @@ module Jetpants
264
264
puts "This task turns an active slave into a standby slave."
265
265
node = ask_node ( 'Please enter node IP: ' , options [ :node ] )
266
266
describe node
267
- raise "Node is not an active slave" unless node . role == :active_slave
267
+ if node . running?
268
+ raise "Node is not an active slave" unless node . role == :active_slave
269
+ else
270
+ inform "Unable to connect to node #{ node } to pull slave"
271
+ error "Unable to pull slave" unless agree "Please confirm that #{ node } is offline [yes/no]: "
272
+ end
268
273
node . pool . mark_slave_standby ( node )
269
274
Jetpants . topology . write_config
270
275
end
@@ -391,6 +396,8 @@ module Jetpants
391
396
# (ie, if app configuration is a static file that needs to be deployed to webs.)
392
397
desc 'shard_split_child_reads' , 'shard split step 2 of 4: move reads to child shards'
393
398
def shard_split_child_reads
399
+ s = ask_shard_being_split
400
+ s . move_reads_to_children
394
401
Jetpants . topology . write_config
395
402
end
396
403
def self . after_shard_split_child_reads
@@ -453,8 +460,9 @@ module Jetpants
453
460
# because the new last shard can't be created yet (chicken-and-egg problem -- master
454
461
# must exist before we create the pool). The assumption is the hardware spec
455
462
# of the new last shard and previous last shard will be the same.
456
- raise "Not enough total spare machines!" unless Jetpants . topology . count_spares ( like : last_shard_master ) >= Jetpants . standby_slaves_per_pool + 1
457
- raise "Not enough standby_slave role spare machines!" unless Jetpants . topology . count_spares ( role : :standby_slave , like : last_shard_master ) >= Jetpants . standby_slaves_per_pool
463
+ raise "Not enough total spare machines!" unless Jetpants . topology . count_spares ( like : last_shard_master ) >= last_shard . slaves_layout [ :standby_slave ] + last_shard . slaves_layout [ :backup_slave ] + 1
464
+ raise "Not enough standby_slave role spare machines!" unless Jetpants . topology . count_spares ( role : :standby_slave , like : last_shard_master ) >= last_shard . slaves_layout [ :standby_slave ]
465
+ raise "Not enough backup_slave role spare machines!" unless Jetpants . topology . count_spares ( role : :backup_slave ) >= last_shard . slaves_layout [ :backup_slave ]
458
466
raise "Cannot find a spare master-role machine!" unless Jetpants . topology . count_spares ( role : :master , like : last_shard_master ) >= 1
459
467
460
468
# In asset tracker, remove the last shard pool and replace it with a new pool. The new pool
@@ -470,15 +478,26 @@ module Jetpants
470
478
# NOT to actually set the pool of the returned object.)
471
479
new_last_shard_master = Jetpants . topology . claim_spare ( role : :master , like : last_shard_master )
472
480
new_last_shard_master . disable_read_only! if new_last_shard_master . running?
473
- if Jetpants . standby_slaves_per_pool > 0
481
+ if last_shard . slaves_layout [ :standby_slave ] > 0
474
482
# Verify spare count again, now that we can actually supply the new master as the :like context
475
- raise "Not enough standby_slave role spare machines!" unless Jetpants . topology . count_spares ( role : :standby_slave , like : new_last_shard_master ) >= Jetpants . standby_slaves_per_pool
476
- new_last_shard_slaves = Jetpants . topology . claim_spares ( Jetpants . standby_slaves_per_pool , role : :standby_slave , like : new_last_shard_master )
483
+ raise "Not enough standby_slave role spare machines!" unless Jetpants . topology . count_spares ( role : :standby_slave , like : new_last_shard_master ) >= last_shard . slaves_layout [ :standby_slave ]
484
+ new_last_shard_slaves = Jetpants . topology . claim_spares ( last_shard . slaves_layout [ :standby_slave ] , role : :standby_slave , like : new_last_shard_master )
477
485
new_last_shard_slaves . each do |x |
478
486
x . change_master_to new_last_shard_master
479
487
x . resume_replication
480
488
end
481
489
end
490
+
491
+ # Set up backup slaves
492
+ if last_shard . slaves_layout [ :backup_slave ] > 0
493
+ raise "Not enough backup_slave role spare machines!" unless Jetpants . topology . count_spares ( role : :backup_slave ) >= last_shard . slaves_layout [ :backup_slave ]
494
+ new_last_shard_backup_slaves = Jetpants . topology . claim_spares ( last_shard . slaves_layout [ :backup_slave ] , role : :backup_slave )
495
+ new_last_shard_backup_slaves . each do |x |
496
+ x . change_master_to new_last_shard_master
497
+ x . resume_replication
498
+ end
499
+ end
500
+
482
501
new_last_shard = Shard . new ( cutover_id , 'INFINITY' , new_last_shard_master )
483
502
new_last_shard . sync_configuration
484
503
Jetpants . topology . pools << new_last_shard
@@ -498,7 +517,53 @@ module Jetpants
498
517
'Deploy the configuration to all machines.' ,
499
518
)
500
519
end
501
-
520
+
521
+ desc 'rebalance_backup_slaves' , 'Add backup slaves to pools which contain too few (does not destroy existing slaves)'
522
+ def rebalance_backup_slaves
523
+ spares_available = Jetpants . topology . count_spares ( role : :backup_slave )
524
+
525
+ raise "No backup slaves available" if spares_available == 0
526
+
527
+ # find shards that need a backup slave
528
+ need_backup_shards = Jetpants . topology . shards . reject { |shard | shard . backup_slaves . count >= shard . slaves_layout [ :backup_slave ] }
529
+
530
+ possible_iterations = spares_available < need_backup_shards . count ? spares_available : need_backup_shards . count
531
+
532
+ spares = Jetpants . topology . claim_spares ( possible_iterations , role : :backup_slave )
533
+
534
+ need_backup_shards . first ( possible_iterations )
535
+
536
+ # loop through and place a backup slave per pool that is without
537
+ need_backup_shards . limited_concurrent_map ( 5 ) do |shard |
538
+
539
+ if spares . count > 0
540
+
541
+ output "Considering shard #{ shard } "
542
+
543
+ source = shard . standby_slaves . last
544
+ source . master . probe if source . master # fail early if there are any replication issues in this pool
545
+
546
+
547
+ targets = [ spares . pop ]
548
+
549
+ source . start_mysql if ! source . running?
550
+ error "Source (#{ source } ) is not a standby slave" unless source . is_standby?
551
+
552
+ targets . each do |t |
553
+ error "Target #{ t } already has a master; please clear out node (including in asset tracker) before proceeding" if t . master
554
+ error "Target #{ t } is running a different version of MySQL than source #{ source } ! Cannot proceed with clone operation." if t . version_cmp ( source ) != 0
555
+ error "Target #{ t } already has a pool!" if t . pool
556
+ end
557
+
558
+ source . enslave_siblings! ( targets )
559
+ targets . concurrent_each { |t | t . resume_replication ; t . catch_up_to_master ( 21600 ) }
560
+ source . pool . sync_configuration
561
+ output "Rebalance complete for #{ shard } "
562
+
563
+ end
564
+
565
+ end
566
+ end
502
567
503
568
no_tasks do
504
569
def is_ip? address
@@ -516,7 +581,7 @@ module Jetpants
516
581
def describe node
517
582
puts "Node #{ node } (#{ node . hostname } :#{ node . port } ) has role #{ node . role } in pool #{ node . pool ( true ) } ." . green
518
583
end
519
-
584
+
520
585
def ask_node ( prompt , supplied_node = false )
521
586
node = supplied_node || ask ( prompt )
522
587
error "Node (#{ node } ) does not appear to be an IP address." unless is_ip? node
@@ -540,7 +605,7 @@ module Jetpants
540
605
s = Jetpants . topology . shard shard_min , shard_max
541
606
raise "Shard not found" unless s
542
607
end
543
- raise "Shard isn't in expected state" unless s . state == :deprecated
608
+ raise "Shard does not have children" if ( s . children . nil? or s . children . count == 0 )
544
609
s
545
610
end
546
611
end
558
623
# We load jetpants last so that plugins can monkeypatch Jetpants::CommandSuite if desired.
559
624
require 'jetpants'
560
625
561
- Jetpants ::CommandSuite . start
626
+ def with_debug
627
+ Pry ::rescue do
628
+ begin
629
+ yield if block_given?
630
+ rescue => e
631
+ Pry . config . prompt = proc { |_ , _ , _ | "# debug > " }
632
+ puts "Entering debugging session (debug_exceptions is enabled). Resume with ctrl+d."
633
+ Pry ::rescued e
634
+ end
635
+ end
636
+ end
637
+
638
+ if Jetpants . debug_exceptions
639
+ with_debug { Jetpants ::CommandSuite . start }
640
+ else
641
+ Jetpants ::CommandSuite . start
642
+ end
0 commit comments