Add requester pays configuration

Author: freeman-lab

python/thunder/utils/ec2.py

@@ -63,6 +63,10 @@ def install_thunder(master, opts):
     access, secret = get_s3_keys()
     filled = configstring.replace('ACCESS', access).replace('SECRET', secret)
     ssh(master, opts, "sed -i'f' 's,.*</configuration>.*,"+filled+"&,' /root/ephemeral-hdfs/conf/core-site.xml")
+    # configure requester pays
+    ssh(master, opts, "touch /root/spark/conf/jets3t.properties")
+    ssh(master, opts, "echo 'httpclient.requester-pays-buckets-enabled = true' >> /root/spark/conf/jets3t.properties")
+    ssh(master, opts, "~/spark-ec2/copy-dir /root/spark/conf")
 
     print "\n\n"
     print "-------------------------------"