diff --git a/.cfignore b/.cfignore new file mode 120000 index 00000000..3e4e48b0 --- /dev/null +++ b/.cfignore @@ -0,0 +1 @@ +.gitignore \ No newline at end of file diff --git a/.gitignore b/.gitignore index a395c345..ca37456c 100644 --- a/.gitignore +++ b/.gitignore @@ -9,7 +9,6 @@ db/*.db .*.sw* .env .*.env -.cfignore .vagrant .idea/ *profile* diff --git a/create-space.sh b/create-space.sh new file mode 100755 index 00000000..ea3c9a66 --- /dev/null +++ b/create-space.sh @@ -0,0 +1,51 @@ +#!/bin/bash +set -e + +if [ ! -n "$1" ]; then + echo "Usage: From inside open-data-maker directory..." + echo " create-space.sh spacename" + exit 1 +fi +echo "Creating space: $1" +SPACE=$1 + +# Creating the indexing space +cf create-space ${SPACE} -o ed + +echo "# When creating the space, the user which creates it is a SpaceManager" +echo "# Add developers using: cf set-space-role USERNAME ed ${SPACE} SpaceDeveloper" + +# Target the space +cf target -o ed -s ${SPACE} + +# Create a bservice instance used by snapshot tools +cf create-service s3 basic data-files + + +#cf create-user-provided-service bservice -p '{"BSERVICE_ACCESS_KEY":"YOUR_S3_ACCESS_KEY", "BSERVICE_SECRET_KEY": "YOUR_S3_SECRET_KEY", "BSERVICE_BUCKET": "YOUR_S3_BUCKET"}' + +# Create backup service +echo "# To create a backup service in this space run:" +echo "# cf create-service s3 basic backup" + +# Create the ElasticSearch service +cf create-service elasticsearch-swarm-1.7.5 3x eservice + +echo "Creating the API server by pushing the ccapi-${SPACE} app:" +cf push -f manifest-${SPACE}.yml +echo "By default the app will use the data-files bucket, leaving DATA_PATH env blank" + +echo "# For data archive / downloads, these are served via a S3 proxy" +echo "# The /downloads path is redirected via CloudFront to" +echo "# ed-public-download.apps.cloud.gov which is in the production space" +echo "# To create additional S3 proxies: https://github.com/18F/cg-s3-proxy" + +echo "TODO: how to put files in the bucket" + +echo "# now you need to index" +echo "cf-ssh -f manifest-${SPACE}.yml --verbose" +echo "# wait several minutes for this to connect" +echo "echo $DATA_PATH" +echo "# it should be blank, meaning you will get default cities data" +echo "rake import" +echo "# when this is done, go to https://ccapi-${SPACE}.18f.gov and explore" diff --git a/lib/data_magic.rb b/lib/data_magic.rb index 1000d5cf..b48879ca 100644 --- a/lib/data_magic.rb +++ b/lib/data_magic.rb @@ -41,8 +41,11 @@ class InvalidDictionary < StandardError def self.s3 if @s3.nil? - if ENV['VCAP_APPLICATION'] - s3cred = ::CF::App::Credentials.find_by_service_name(ENV['s3_bucket_service'] || 'bservice') + if ENV['VCAP_APPLICATION'] # in Cloud Foundry + cfcred = ::CF::App::Credentials.find_by_service_name(ENV['s3_bucket_service'] || 'data-files') + s3cred = {'access_key'=> cfcred['access_key_id'], 'secret_key' => cfcred['secret_access_key']} + bucket_name = cfcred['bucket'] + # TODO: default DATA_PATH to s3://{bucket_name} else s3cred = {'access_key'=> ENV['s3_access_key'], 'secret_key' => ENV['s3_secret_key']} end diff --git a/manifest-testing.yml b/manifest-testing.yml new file mode 100644 index 00000000..94f74014 --- /dev/null +++ b/manifest-testing.yml @@ -0,0 +1,16 @@ +--- +applications: +- name: ccapi + host: ccapi-testing + domain: 18f.gov + command: bundle exec puma -C ./config/puma.rb + instances: 1 + memory: 1G + services: + - data-files + - eservice + env: + MAX_THREADS: 5 + WEB_CONCURRENCY: 1 + INDEX_APP: enable + NPROCS: 1 diff --git a/script/s3config.rb b/script/s3config.rb index d54fb6c3..c0d993dd 100644 --- a/script/s3config.rb +++ b/script/s3config.rb @@ -4,6 +4,7 @@ # @s3 = ::Aws::S3::Client.new require 'dotenv' +require 'json' branch = `echo $(git symbolic-ref --short HEAD)`.chomp @@ -12,26 +13,34 @@ puts "using APP_ENV from environment #{APP_ENV}" else case branch - when "master" - APP_ENV = "production" - when "staging" - APP_ENV = "staging" - else - puts "not on master or staging branch lets use dev" - APP_ENV = "dev" + when "master" + APP_ENV = "production" + when "staging" + APP_ENV = "staging" + else + puts "not on master or staging branch lets use dev" + APP_ENV = "dev" # FIXME: shouldn't the APP_ENV be testing? end end -Dotenv.load( - File.expand_path("../../.#{APP_ENV}.env", __FILE__), - File.expand_path("../../.env", __FILE__)) +cf_credentials = `cf target -o ed -s #{APP_ENV} && echo "$(cf env ccapi)" | tail -n +5 | sed -n -e :a -e '1,10!{P;N;D;};N;ba'` +cf_json_str = cf_credentials.gsub("\n", '').gsub(/^[^{]+{/, '{').gsub(/{\s+"VCAP_APPLICATION".+$/, '') + +cf_json = JSON.parse(cf_json_str) +cf_data_files = cf_json['VCAP_SERVICES']['s3'].detect {|j| j['name'] == 'data-files'} + +fail "Unable to find data-files configuration" if cf_data_files.nil? + +ENV['CF_CREDENTIALS'] = cf_json_str +ENV['AWS_ACCESS_KEY_ID'] = cf_data_files['credentials']['access_key_id'] +ENV['AWS_SECRET_ACCESS_KEY'] = cf_data_files['credentials']['secret_access_key'] +ENV['BUCKET_NAME'] = cf_data_files['credentials']['bucket'] require 'aws-sdk' puts "app env: #{APP_ENV}" -puts "bucket name: #{ENV['s3_bucket']}" - +puts "bucket name: #{ENV['BUCKET_NAME']}" -s3cred = {'access_key'=> ENV['s3_access_key'], 'secret_key' => ENV['s3_secret_key']} +s3cred = {'access_key'=> ENV['AWS_ACCESS_KEY_ID'], 'secret_key' => ENV['AWS_SECRET_ACCESS_KEY']} ::Aws.config[:credentials] = ::Aws::Credentials.new(s3cred['access_key'], s3cred['secret_key']) ::Aws.config[:region] = 'us-east-1' diff --git a/script/s3pull b/script/s3pull index 7cb48ec3..e9647f9c 100755 --- a/script/s3pull +++ b/script/s3pull @@ -4,7 +4,7 @@ require_relative 's3config.rb' @s3 = ::Aws::S3::Client.new -bucket = ENV['s3_bucket'] +bucket = ENV['BUCKET_NAME'] dirname = 'real-data' unless File.directory?(dirname) diff --git a/script/s3push b/script/s3push index 6d300280..c4ed1bee 100755 --- a/script/s3push +++ b/script/s3push @@ -5,7 +5,7 @@ require_relative 's3config.rb' @s3 = ::Aws::S3::Client.new dirname = 'real-data' -bucket_name = ENV['s3_bucket'] +bucket_name = ENV['BUCKET_NAME'] datayamlpath = File.expand_path("../../#{dirname}/#{bucket_name}.yaml", __FILE__) puts "copying #{datayamlpath}"