Skip to content

(Housekeeping, refactor) Normalize whitespace for all *.py files #63

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

(Housekeeping, refactor) Normalize whitespace for all *.py files #63

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
426 changes: 213 additions & 213 deletions api/coalition.py
View file
Open in desktop

Large diffs are not rendered by default.

140 changes: 70 additions & 70 deletions cloud/aws.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,88 +13,88 @@


def startInstance(name, config):
"""
Run the aws command to start a worker instance.
Return the created instanceid in case of dedicated ec2 instance or the spotinstancerequestid
in case of a spot instance.
"""

if config.get("worker", "spot"):
cmd = ["aws", "ec2", "request-spot-instances",
"--spot-price", config.get("spot", "spotprice"),
"--instance-count", config.get("spot", "instancecount"),
"--type", config.get("spot", "type"),
"--launch-specification", _getLaunchSpecification(name, config),]
else:
cmd = ["aws", "ec2", "run-instances",
"--key-name", config.get("authentication", "keyname"),
"--image-id", config.get("worker", "imageid"),
"--instance-type", config.get("worker", "instancetype"),
"--subnet-id", config.get("worker", "subnetid"),
"--security-group-ids",
config.get("worker", "securitygroupid"),
"--iam-instance-profile",
"Arn=%s" % config.get("worker", "iaminstanceprofile"),
"--user-data", _getUserData(name, config),]
common._run_or_none(cmd)
"""
Run the aws command to start a worker instance.
Return the created instanceid in case of dedicated ec2 instance or the spotinstancerequestid
in case of a spot instance.
"""

if config.get("worker", "spot"):
cmd = ["aws", "ec2", "request-spot-instances",
"--spot-price", config.get("spot", "spotprice"),
"--instance-count", config.get("spot", "instancecount"),
"--type", config.get("spot", "type"),
"--launch-specification", _getLaunchSpecification(name, config),]
else:
cmd = ["aws", "ec2", "run-instances",
"--key-name", config.get("authentication", "keyname"),
"--image-id", config.get("worker", "imageid"),
"--instance-type", config.get("worker", "instancetype"),
"--subnet-id", config.get("worker", "subnetid"),
"--security-group-ids",
config.get("worker", "securitygroupid"),
"--iam-instance-profile",
"Arn=%s" % config.get("worker", "iaminstanceprofile"),
"--user-data", _getUserData(name, config),]
common._run_or_none(cmd)


def stopInstance(name, config):
"""Run the aws command to terminate the instance."""
cmd = ["aws", "ec2", "terminate-instances", "--instance-ids",
_getInstanceIdByName(name)]
common._run_or_none(cmd)
"""Run the aws command to terminate the instance."""
cmd = ["aws", "ec2", "terminate-instances", "--instance-ids",
_getInstanceIdByName(name)]
common._run_or_none(cmd)


def _getLaunchSpecification(name, config):
with open("cloud/aws_worker_spot_launchspecification.json.template", 'r') as f:
template = Template(f.read())
values = {
"image_id": config.get("worker", "imageid"),
"keyname": config.get("authentication", "keyname"),
"security_group_id": config.get("worker", "securitygroupid"),
"instance_type": config.get("worker", "instancetype"),
"user_data": encodestring(_getUserData(name, config)), }
return template.substitute(values).replace('\n', '')
with open("cloud/aws_worker_spot_launchspecification.json.template", 'r') as f:
template = Template(f.read())
values = {
"image_id": config.get("worker", "imageid"),
"keyname": config.get("authentication", "keyname"),
"security_group_id": config.get("worker", "securitygroupid"),
"instance_type": config.get("worker", "instancetype"),
"user_data": encodestring(_getUserData(name, config)), }
return template.substitute(values).replace('\n', '')


def _getUserData(name, config):
"""
Prepare the user-data script in cloud-init syntax.
Return the script as a string.
"""

with open("cloud/aws_worker_cloud_init.template", 'r') as f:
template = Template(f.read())
values = {
"hostname": name,
"region": config.get("authentication", "region"),
"access_key": config.get("authentication", "accesskey"),
"secret_access_key":
config.get("authentication", "secretaccesskey"),
"bucket_name": config.get("storage", "name"),
"mount_point": config.get("storage", "mountpoint"),
"guerilla_render_filename":
config.get("storage", "guerillarenderfilename"),
"coalition_filename":
config.get("storage", "coalitionfilename"),
"coalition_server_ip": config.get("coalition", "ip"),
"coalition_server_port": config.get("coalition", "port"),}
return template.substitute(values)
"""
Prepare the user-data script in cloud-init syntax.
Return the script as a string.
"""

with open("cloud/aws_worker_cloud_init.template", 'r') as f:
template = Template(f.read())
values = {
"hostname": name,
"region": config.get("authentication", "region"),
"access_key": config.get("authentication", "accesskey"),
"secret_access_key":
config.get("authentication", "secretaccesskey"),
"bucket_name": config.get("storage", "name"),
"mount_point": config.get("storage", "mountpoint"),
"guerilla_render_filename":
config.get("storage", "guerillarenderfilename"),
"coalition_filename":
config.get("storage", "coalitionfilename"),
"coalition_server_ip": config.get("coalition", "ip"),
"coalition_server_port": config.get("coalition", "port"),}
return template.substitute(values)


def _getInstanceIdByName(name):
"""Return instanceid from name."""
cmd = ["aws", "ec2", "describe-instances"]
output = common._check_output_or_none(cmd)
if output:
for resources in json.loads(output)["Reservations"]:
instance = resources["Instances"][0]
if instance.has_key("Tags"):
for tags in instance["Tags"]:
if tags["Key"] == "Name" and tags["Value"] == name:
return instance["InstanceId"]
return None
"""Return instanceid from name."""
cmd = ["aws", "ec2", "describe-instances"]
output = common._check_output_or_none(cmd)
if output:
for resources in json.loads(output)["Reservations"]:
instance = resources["Instances"][0]
if instance.has_key("Tags"):
for tags in instance["Tags"]:
if tags["Key"] == "Name" and tags["Value"] == name:
return instance["InstanceId"]
return None

# vim: tabstop=4 noexpandtab shiftwidth=4 softtabstop=4 textwidth=79

42 changes: 21 additions & 21 deletions cloud/common.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,27 +9,27 @@


def createWorkerInstanceName(prefix):
"""Return a unique name based on prefix and timestamp."""
return "%s%s" % (prefix, int(time()))


def _run_or_none(cmd):
"""Execute command. Returns None in case of exception."""
try:
return subprocess.Popen(cmd, stderr=subprocess.STDOUT,
universal_newlines=True)
except Exception as e:
print(e)
return None

def _check_output_or_none(cmd):
"""Execute command. Returns None in case of exception."""
try:
return subprocess.check_output(cmd, stderr=subprocess.STDOUT,
universal_newlines=True)
except Exception as e:
print(e)
return None
"""Return a unique name based on prefix and timestamp."""
return "%s%s" % (prefix, int(time()))


def _run_or_none(cmd):
"""Execute command. Returns None in case of exception."""
try:
return subprocess.Popen(cmd, stderr=subprocess.STDOUT,
universal_newlines=True)
except Exception as e:
print(e)
return None

def _check_output_or_none(cmd):
"""Execute command. Returns None in case of exception."""
try:
return subprocess.check_output(cmd, stderr=subprocess.STDOUT,
universal_newlines=True)
except Exception as e:
print(e)
return None

# vim: tabstop=4 noexpandtab shiftwidth=4 softtabstop=4 textwidth=79

110 changes: 55 additions & 55 deletions cloud/gcloud.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,68 +15,68 @@


def startInstance(name, config):
"""
Run the gcloud command to start a worker instance.
Return the created FIXME
"""

# gcloud command line tool is picky with params escaping (eg. key-id in json)
# So we use a real temporary file
startup_script_file = tempfile.NamedTemporaryFile(delete=False)
startup_script_file.write(_getStartupScript(name, config))
startup_script_file.flush()
os.fsync(startup_script_file.fileno())


cmd = ["gcloud", "compute", "--project", config.get("authentication", "project"),
"instances", "create", name,
"--zone", config.get("worker", "zone"),
"--machine-type", config.get("worker", "machinetype"),
"--subnet", config.get("worker", "subnet"),
"--maintenance-policy", config.get("worker", "maintenancepolicy"),
"--service-account", config.get("authentication", "serviceaccount"),
"--scopes", config.get("authentication", "scopes"),
"--image", config.get("worker", "image"),
"--image-project", config.get("worker", "imageproject"),
"--boot-disk-size", config.get("worker", "bootdisksize"),
"--boot-disk-type", config.get("worker", "bootdisktype"),
"--boot-disk-device-name", name,
"--metadata-from-file", "startup-script={}".format(startup_script_file.name),]
if config.getboolean("worker", "preemptible") == True:
cmd.append("--preemptible")
common._run_or_none(cmd)
"""
Run the gcloud command to start a worker instance.
Return the created FIXME
"""

# gcloud command line tool is picky with params escaping (eg. key-id in json)
# So we use a real temporary file
startup_script_file = tempfile.NamedTemporaryFile(delete=False)
startup_script_file.write(_getStartupScript(name, config))
startup_script_file.flush()
os.fsync(startup_script_file.fileno())


cmd = ["gcloud", "compute", "--project", config.get("authentication", "project"),
"instances", "create", name,
"--zone", config.get("worker", "zone"),
"--machine-type", config.get("worker", "machinetype"),
"--subnet", config.get("worker", "subnet"),
"--maintenance-policy", config.get("worker", "maintenancepolicy"),
"--service-account", config.get("authentication", "serviceaccount"),
"--scopes", config.get("authentication", "scopes"),
"--image", config.get("worker", "image"),
"--image-project", config.get("worker", "imageproject"),
"--boot-disk-size", config.get("worker", "bootdisksize"),
"--boot-disk-type", config.get("worker", "bootdisktype"),
"--boot-disk-device-name", name,
"--metadata-from-file", "startup-script={}".format(startup_script_file.name),]
if config.getboolean("worker", "preemptible") == True:
cmd.append("--preemptible")
common._run_or_none(cmd)


def stopInstance(name, config):
"""Run the gcloud command to terminate the instance."""
zone = config.get("worker", "zone")
cmd = ["gcloud", "compute", "instances", "delete", "--quiet", "--zone", zone, name]
common._run_or_none(cmd)
"""Run the gcloud command to terminate the instance."""
zone = config.get("worker", "zone")
cmd = ["gcloud", "compute", "instances", "delete", "--quiet", "--zone", zone, name]
common._run_or_none(cmd)


def _getStartupScript(name, config):
"""
Prepare the startup-script in bash script syntax.
Return the script as a string.
"""

with open(config.get("authentication", "keyfile"), 'r') as f:
key_id_data = f.read()

with open("cloud/gcloud_worker_startup_script.template", 'r') as f:
template = Template(f.read())
values = {
"key_id_json": key_id_data,
"hostname": name,
"mount_point": config.get("storage", "mountpoint"),
"bucket_name": config.get("storage", "name"),
"""
Prepare the startup-script in bash script syntax.
Return the script as a string.
"""

with open(config.get("authentication", "keyfile"), 'r') as f:
key_id_data = f.read()

with open("cloud/gcloud_worker_startup_script.template", 'r') as f:
template = Template(f.read())
values = {
"key_id_json": key_id_data,
"hostname": name,
"mount_point": config.get("storage", "mountpoint"),
"bucket_name": config.get("storage", "name"),
"install_dir": config.get("worker", "installdir"),
"coalition_package": config.get("storage", "coalitionpackage"),
"main_program_package": config.get("main_program", "package"),
"main_program_environment": config.get("main_program", "environment"),
"coalition_server_ip": config.get("coalition", "ip"),
"coalition_server_port": config.get("coalition", "port"),}
return template.substitute(values)
"coalition_package": config.get("storage", "coalitionpackage"),
"main_program_package": config.get("main_program", "package"),
"main_program_environment": config.get("main_program", "environment"),
"coalition_server_ip": config.get("coalition", "ip"),
"coalition_server_port": config.get("coalition", "port"),}
return template.substitute(values)

# vim: tabstop=4 noexpandtab shiftwidth=4 softtabstop=4 textwidth=79

Loading