Skip to content
This repository was archived by the owner on May 6, 2021. It is now read-only.

BUGFIX | Bad connection quality generates infinitely loop in synchronization proccess (in worker) #32

Open
wants to merge 2 commits into
base: master
Choose a base branch
from
Open

BUGFIX | Bad connection quality generates infinitely loop in synchronization proccess (in worker) #32

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
8006254
BUGFIX | Connection quality generates infinitly loop in synchronizati…
bieli Jul 16, 2015
5adb299
BUGFIX | Bad connection quality generates infinitely loop in synchron…
bieli Jul 16, 2015
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 22 additions & 7 deletions radosgw_agent/worker.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -256,6 +256,9 @@ def sync_object(self, bucket, obj):
if found:
client.remove_op_state(self.dest_conn, self.daemon_id,
local_op_id, bucket, obj)
log.debug('op state ID: "%s" removed' % local_op_id)
else:
log.debug('op state ID: "%s" not removed' % local_op_id)
except NotFound:
log.debug('op state already gone')
except Exception:
Expand All @@ -264,26 +267,38 @@ def sync_object(self, bucket, obj):

return True

def wait_for_object(self, bucket, obj, until, local_op_id):
def wait_for_object(self, bucket, obj, until, local_op_id, error_max_count=10):
err_counts = 0
while time.time() < until:
try:
state = client.get_op_state(self.dest_conn,
self.daemon_id,
local_op_id,
bucket, obj)
log.debug('op state is %s', state)
state = state[0]['state']
if state == 'complete':
return
elif state != 'in-progress':
raise SyncFailed('state is {0}'.format(state))
try:
state = state[0]['state']
if state == 'complete':
return
elif state != 'in-progress':
raise SyncFailed('state is {0}'.format(state))
except IndexError:
raise SyncFailed('client.get_op_state() returns bad element key')

time.sleep(1)
except SyncFailed:
raise
except NotFound:
raise SyncFailed('object copy state not found')
except Exception as e:
log.debug('error geting op state: %s', e, exc_info=True)
err_counts += 1
if err_counts > error_max_count:
log.error('error counter >= %d - possible infinitely loop !',
error_max_count, exc_info=True)
raise SyncFailed('Infinitely loop ! Check connections quality between clusters !')
else:
log.exception('error (count: %d) when geting op state: %s',
err_counts, e, exc_info=True)
time.sleep(1)
# timeout expired
raise SyncTimedOut()
Expand Down