From 96f77649010f90ab31b7731f844fef556425e556 Mon Sep 17 00:00:00 2001 From: Russell Seymour Date: Thu, 7 Sep 2017 11:10:38 +0100 Subject: [PATCH 1/9] Added support for creating a marketplace image in Alibaba Signed-off-by: Russell Seymour --- .kitchen.yml | 13 +++++ attributes/alibaba.rb | 53 +++++++++++++++++++ libraries/helpers.rb | 27 ++++++++-- recipes/_packer.rb | 6 +++ recipes/_publish.rb | 19 +++++-- .../default/install_marketplace_oss.sh.erb | 9 ++++ 6 files changed, 118 insertions(+), 9 deletions(-) create mode 100644 attributes/alibaba.rb create mode 100644 templates/default/install_marketplace_oss.sh.erb diff --git a/.kitchen.yml b/.kitchen.yml index d40d819..2634ef0 100644 --- a/.kitchen.yml +++ b/.kitchen.yml @@ -16,7 +16,20 @@ platforms: cpus: 2 memory: 4096 + suites: + - name: alibaba_automate + run_list: "marketplace_image::default" + attributes: + marketplace_image: + alibaba: + automate: + enabled: true + packman: + version: 1.0.4 + checksums: + "1.0.4": 646da085cbcb8c666474d500a44d933df533cf4f1ff286193d67b51372c3c59e + - name: azure_compliance run_list: "marketplace_image::default" attributes: diff --git a/attributes/alibaba.rb b/attributes/alibaba.rb new file mode 100644 index 0000000..a2ffa4e --- /dev/null +++ b/attributes/alibaba.rb @@ -0,0 +1,53 @@ +# frozen_string_literal: true +cred_dir = ::File.expand_path(::File.join('~', '.alibaba')) +credential_file = ::File.join(cred_dir, 'credentials') + +TIMESTAMP = Time.now.strftime('%Y_%m_%d_%H_%M_%S').freeze +INVALID_AMI_NAME_CHARACTERS = /[^\w\(\)\.\-\/]/ + +def normalize_name(name) + # Append timestamp + # Remove invalid characters + # Ensure it's 128 characters or less + "#{name}_#{TIMESTAMP}".gsub(INVALID_AMI_NAME_CHARACTERS, '').strip[0..127] +end + +default['marketplace_image']['alibaba']['cred_dir'] = cred_dir +default['marketplace_image']['alibaba']['credential_file'] = credential_file + +default['marketplace_image']['alibaba']['public']['compliance']['enabled'] = false +default['marketplace_image']['alibaba']['public']['automate']['enabled'] = false + +default['marketplace_image']['alibaba']['marketplace']['url'] = ' +http://chef-software.oss-cn-beijing.aliyuncs.com/chef-marketplace_0.2.5%2B20170906073057-1_amd64.deb' + +default_marketplace_config = { + 'role' => 'automate', + 'platform' => 'alibaba', + 'user' => 'root', + 'support_email' => 'aws@chef.io', + 'sales_email' => 'awesome@chef.io', + 'reporting_cron_enabled' => false, + 'doc_url' => 'https://docs.chef.io/aws_marketplace.html', + 'disable_outbound_traffic' => false, + 'license_type' => 'flexible', + 'free_node_count' => 5, +} + +alibaba_builder_config = { + 'type' => 'alicloud-ecs', + 'region' => 'cn-beijing', + 'image_name' => 'chef_automate', + 'source_image' => 'ubuntu_14_0405_64_40G_alibase_20170711.vhd', + 'ssh_username' => 'root', + 'instance_type' => 'ecs.n2.medium', + 'io_optimized' => 'true', + 'internet_charge_type' => 'PayByTraffic', + 'image_force_delete' => 'true', +} + +default['marketplace_image']['alibaba']['automate'] = { + 'name' => 'alibaba_automate_BYOL', + 'builder_options' => alibaba_builder_config, + 'marketplace_config_options' => default_marketplace_config, +} diff --git a/libraries/helpers.rb b/libraries/helpers.rb index 13bde50..97da207 100644 --- a/libraries/helpers.rb +++ b/libraries/helpers.rb @@ -19,11 +19,11 @@ module MarketplaceImageCookbook module Helpers def marketplace_products - aws_products + azure_products + gce_products + aws_products + azure_products + gce_products + alibaba_products end def enabled_products - enabled_aws_products + enabled_azure_products + enabled_gce_products + enabled_aws_products + enabled_azure_products + enabled_gce_products + enabled_alibaba_products end def enabled_builders @@ -122,13 +122,32 @@ def enabled_gce_builders enabled_gce_products.map { |p| p['name'] } end + def alibaba_products + products = [] + products << node['marketplace_image']['alibaba']['automate'] + products + end + + def enabled_alibaba_products + products = [] + products << node['marketplace_image']['alibaba']['automate'] if + node['marketplace_image']['alibaba']['automate']['enabled'] + products + end + + def alibaba_builders + alibaba_products.map { |p| p['name'] } + end + def automate_products [node['marketplace_image']['aws']['public']['automate']] end def automate_builders - [node['marketplace_image']['aws']['public']['automate']['name'], - node['marketplace_image']['azure']['automate']['name'], + [ + node['marketplace_image']['aws']['public']['automate']['name'], + node['marketplace_image']['azure']['automate']['name'], + node['marketplace_image']['alibaba']['automate']['name'], ] end diff --git a/recipes/_packer.rb b/recipes/_packer.rb index e8b2d68..6161bdd 100644 --- a/recipes/_packer.rb +++ b/recipes/_packer.rb @@ -40,3 +40,9 @@ content JSON.pretty_generate(creds['gce']['account']) end end + +# Set the credentials for Alibaba Cloud as environment variables +if creds['alibaba'] + ENV['ALICLOUD_ACCESS_KEY'] = creds['alibaba']['access_key'] + ENV['ALICLOUD_SECRET_KEY'] = creds['alibaba']['secret_key'] +end diff --git a/recipes/_publish.rb b/recipes/_publish.rb index a1f6770..7246a96 100644 --- a/recipes/_publish.rb +++ b/recipes/_publish.rb @@ -22,14 +22,14 @@ packer_provisioner 'setup_apt_current' do type 'shell' source 'setup_apt_current.sh.erb' - only azure_builders + only azure_builders + alibaba_builders only_if { use_current_repo? } end packer_provisioner 'setup_apt_stable' do type 'shell' source 'setup_apt_stable.sh.erb' - only azure_builders + only azure_builders + alibaba_builders not_if { use_current_repo? } end @@ -39,6 +39,15 @@ only azure_builders end +packer_provisioner 'install_marketplace_oss' do + type 'shell' + source 'install_marketplace_oss.sh.erb' + variables( + 'url' => node['marketplace_image']['alibaba']['marketplace']['url'] + ) + only alibaba_builders +end + packer_provisioner 'enable_ipv6_loopback' do type 'shell' source 'enable_ipv6_loopback.sh.erb' @@ -47,21 +56,21 @@ packer_provisioner 'setup_yum_current' do type 'shell' source 'setup_yum_current.sh.erb' - except azure_builders + except azure_builders + alibaba_builders only_if { use_current_repo? } end packer_provisioner 'setup_yum_stable' do type 'shell' source 'setup_yum_stable.sh.erb' - except azure_builders + except azure_builders + alibaba_builders not_if { use_current_repo? } end packer_provisioner 'install_marketplace_yum' do type 'shell' source 'install_marketplace_yum.sh.erb' - except azure_builders + except azure_builders + alibaba_builders end packer_provisioner 'prepare_automate_for_publishing' do diff --git a/templates/default/install_marketplace_oss.sh.erb b/templates/default/install_marketplace_oss.sh.erb new file mode 100644 index 0000000..d1e7244 --- /dev/null +++ b/templates/default/install_marketplace_oss.sh.erb @@ -0,0 +1,9 @@ +#!/bin/bash + +# Download an install Marketplace from the Alibaba Object Storage service + +set -ex + +wget -q -O chef-marketplace.deb http://chef-software.oss-cn-beijing.aliyuncs.com/chef-marketplace_0.2.5%2B20170906073057-1_amd64.deb + +dpkg -i chef-marketplace.deb From 90a31f77188b02537d221bb196f9d186728dbf0d Mon Sep 17 00:00:00 2001 From: Russell Seymour Date: Fri, 8 Sep 2017 16:03:42 +0100 Subject: [PATCH 2/9] Fixed issue with packer_provisioner template Added URLs to download products to attributes. A new packer file is created so that the Omnibus marketplace recipe reads it in to get the URLs for download Signed-off-by: Russell Seymour --- attributes/alibaba.rb | 11 +++++++---- recipes/_publish.rb | 11 ++++++++++- templates/default/install_marketplace_oss.sh.erb | 5 ++--- templates/default/marketplace.rb.erb | 2 +- templates/default/product_download_urls.json.erb | 3 +++ 5 files changed, 23 insertions(+), 9 deletions(-) create mode 100644 templates/default/product_download_urls.json.erb diff --git a/attributes/alibaba.rb b/attributes/alibaba.rb index a2ffa4e..e757c20 100644 --- a/attributes/alibaba.rb +++ b/attributes/alibaba.rb @@ -18,13 +18,16 @@ def normalize_name(name) default['marketplace_image']['alibaba']['public']['compliance']['enabled'] = false default['marketplace_image']['alibaba']['public']['automate']['enabled'] = false -default['marketplace_image']['alibaba']['marketplace']['url'] = ' -http://chef-software.oss-cn-beijing.aliyuncs.com/chef-marketplace_0.2.5%2B20170906073057-1_amd64.deb' +default['marketplace_image']['alibaba']['product_urls'] = { + 'marketplace' => 'http://chef-software.oss-cn-beijing.aliyuncs.com/chef-marketplace_0.2.5%2B20170908141726.git.1.8d5bc1c-1_amd64.deb', + 'automate' => 'http://chef-software.oss-cn-beijing.aliyuncs.com/automate_1.6.95-1_amd64.deb', + 'chef_server' => 'http://chef-software.oss-cn-beijing.aliyuncs.com/chef-server-core_12.15.8-1_amd64.deb', +} default_marketplace_config = { 'role' => 'automate', 'platform' => 'alibaba', - 'user' => 'root', + 'user' => 'ubuntu', 'support_email' => 'aws@chef.io', 'sales_email' => 'awesome@chef.io', 'reporting_cron_enabled' => false, @@ -38,7 +41,7 @@ def normalize_name(name) 'type' => 'alicloud-ecs', 'region' => 'cn-beijing', 'image_name' => 'chef_automate', - 'source_image' => 'ubuntu_14_0405_64_40G_alibase_20170711.vhd', + 'source_image' => 'ubuntu_14_0405_64_20G_alibase_20170824.vhd', 'ssh_username' => 'root', 'instance_type' => 'ecs.n2.medium', 'io_optimized' => 'true', diff --git a/recipes/_publish.rb b/recipes/_publish.rb index 7246a96..4c2a405 100644 --- a/recipes/_publish.rb +++ b/recipes/_publish.rb @@ -43,7 +43,16 @@ type 'shell' source 'install_marketplace_oss.sh.erb' variables( - 'url' => node['marketplace_image']['alibaba']['marketplace']['url'] + url: node['marketplace_image']['alibaba']['product_urls']['marketplace'] + ) + only alibaba_builders +end + +packer_provisioner '/tmp/product_download_urls.json' do + type 'file' + source 'product_download_urls.json.erb' + variables( + urls: node['marketplace_image']['alibaba']['product_urls'].to_json ) only alibaba_builders end diff --git a/templates/default/install_marketplace_oss.sh.erb b/templates/default/install_marketplace_oss.sh.erb index d1e7244..6996f37 100644 --- a/templates/default/install_marketplace_oss.sh.erb +++ b/templates/default/install_marketplace_oss.sh.erb @@ -4,6 +4,5 @@ set -ex -wget -q -O chef-marketplace.deb http://chef-software.oss-cn-beijing.aliyuncs.com/chef-marketplace_0.2.5%2B20170906073057-1_amd64.deb - -dpkg -i chef-marketplace.deb +wget -q -O chef-marketplace.deb <%= @url %> +sudo dpkg -i chef-marketplace.deb diff --git a/templates/default/marketplace.rb.erb b/templates/default/marketplace.rb.erb index c7ef6e0..8ad19a5 100644 --- a/templates/default/marketplace.rb.erb +++ b/templates/default/marketplace.rb.erb @@ -20,4 +20,4 @@ manage.marketplace.sign_up.enabled = true <% end -%> <% if @role != 'aio' -%> analytics.trimmer.enabled = false -<% end -%> +<% end -%> \ No newline at end of file diff --git a/templates/default/product_download_urls.json.erb b/templates/default/product_download_urls.json.erb new file mode 100644 index 0000000..2f5d6ea --- /dev/null +++ b/templates/default/product_download_urls.json.erb @@ -0,0 +1,3 @@ +{ + "urls": <%= @urls %> +} \ No newline at end of file From 88ef67496e7dc8e8f0208fde85ac67b4c124fef3 Mon Sep 17 00:00:00 2001 From: Russell Seymour Date: Mon, 11 Sep 2017 12:53:11 +0100 Subject: [PATCH 3/9] Updated URL for chef-marketplace Signed-off-by: Russell Seymour --- attributes/alibaba.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/attributes/alibaba.rb b/attributes/alibaba.rb index e757c20..227f2bd 100644 --- a/attributes/alibaba.rb +++ b/attributes/alibaba.rb @@ -19,7 +19,7 @@ def normalize_name(name) default['marketplace_image']['alibaba']['public']['automate']['enabled'] = false default['marketplace_image']['alibaba']['product_urls'] = { - 'marketplace' => 'http://chef-software.oss-cn-beijing.aliyuncs.com/chef-marketplace_0.2.5%2B20170908141726.git.1.8d5bc1c-1_amd64.deb', + 'marketplace' => 'http://chef-software.oss-cn-beijing.aliyuncs.com/chef-marketplace_0.2.5%2B20170911103840.git.2.1c6a67f-1_amd64.deb', 'automate' => 'http://chef-software.oss-cn-beijing.aliyuncs.com/automate_1.6.95-1_amd64.deb', 'chef_server' => 'http://chef-software.oss-cn-beijing.aliyuncs.com/chef-server-core_12.15.8-1_amd64.deb', } From 3bcf07ec896e9599062337fcc5911adfd4ab7b86 Mon Sep 17 00:00:00 2001 From: Russell Seymour Date: Thu, 14 Sep 2017 10:26:28 +0100 Subject: [PATCH 4/9] Updated URL for chef-marketplace Signed-off-by: Russell Seymour --- attributes/alibaba.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/attributes/alibaba.rb b/attributes/alibaba.rb index 227f2bd..eeb4d14 100644 --- a/attributes/alibaba.rb +++ b/attributes/alibaba.rb @@ -19,7 +19,7 @@ def normalize_name(name) default['marketplace_image']['alibaba']['public']['automate']['enabled'] = false default['marketplace_image']['alibaba']['product_urls'] = { - 'marketplace' => 'http://chef-software.oss-cn-beijing.aliyuncs.com/chef-marketplace_0.2.5%2B20170911103840.git.2.1c6a67f-1_amd64.deb', + 'marketplace' => 'http://chef-software.oss-cn-beijing.aliyuncs.com/chef-marketplace_0.2.5%2B20170913175530.git.3.4b24680-1_amd64.deb', 'automate' => 'http://chef-software.oss-cn-beijing.aliyuncs.com/automate_1.6.95-1_amd64.deb', 'chef_server' => 'http://chef-software.oss-cn-beijing.aliyuncs.com/chef-server-core_12.15.8-1_amd64.deb', } @@ -40,7 +40,7 @@ def normalize_name(name) alibaba_builder_config = { 'type' => 'alicloud-ecs', 'region' => 'cn-beijing', - 'image_name' => 'chef_automate', + 'image_name' => 'chef_automate_3', 'source_image' => 'ubuntu_14_0405_64_20G_alibase_20170824.vhd', 'ssh_username' => 'root', 'instance_type' => 'ecs.n2.medium', From 61d812e5d46dae861bea51ba0d13fe29c8facc81 Mon Sep 17 00:00:00 2001 From: Russell Seymour Date: Thu, 14 Sep 2017 10:30:30 +0100 Subject: [PATCH 5/9] Changed support email address. Signed-off-by: Russell Seymour --- attributes/alibaba.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/attributes/alibaba.rb b/attributes/alibaba.rb index eeb4d14..68f72d1 100644 --- a/attributes/alibaba.rb +++ b/attributes/alibaba.rb @@ -28,7 +28,7 @@ def normalize_name(name) 'role' => 'automate', 'platform' => 'alibaba', 'user' => 'ubuntu', - 'support_email' => 'aws@chef.io', + 'support_email' => 'alibaba@chef.io', 'sales_email' => 'awesome@chef.io', 'reporting_cron_enabled' => false, 'doc_url' => 'https://docs.chef.io/aws_marketplace.html', From 109a88b75501f82ce44595e6cf43f8f0bdb137fe Mon Sep 17 00:00:00 2001 From: Russell Seymour Date: Thu, 14 Sep 2017 10:31:30 +0100 Subject: [PATCH 6/9] Modified Doc URL Signed-off-by: Russell Seymour --- attributes/alibaba.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/attributes/alibaba.rb b/attributes/alibaba.rb index 68f72d1..62b5b3a 100644 --- a/attributes/alibaba.rb +++ b/attributes/alibaba.rb @@ -31,7 +31,7 @@ def normalize_name(name) 'support_email' => 'alibaba@chef.io', 'sales_email' => 'awesome@chef.io', 'reporting_cron_enabled' => false, - 'doc_url' => 'https://docs.chef.io/aws_marketplace.html', + 'doc_url' => 'https://docs.chef.io/alibaba_marketplace.html', 'disable_outbound_traffic' => false, 'license_type' => 'flexible', 'free_node_count' => 5, From bf32d9c7ea65daedf4cadb5e021fe25f07969b22 Mon Sep 17 00:00:00 2001 From: Russell Seymour Date: Thu, 14 Sep 2017 10:33:39 +0100 Subject: [PATCH 7/9] Changed to fixed licence type Signed-off-by: Russell Seymour --- attributes/alibaba.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/attributes/alibaba.rb b/attributes/alibaba.rb index 62b5b3a..46e2f19 100644 --- a/attributes/alibaba.rb +++ b/attributes/alibaba.rb @@ -33,7 +33,7 @@ def normalize_name(name) 'reporting_cron_enabled' => false, 'doc_url' => 'https://docs.chef.io/alibaba_marketplace.html', 'disable_outbound_traffic' => false, - 'license_type' => 'flexible', + 'license_type' => 'fixed', 'free_node_count' => 5, } From c5a20870df4c29e5c68479eb78ac6135b2daa1c6 Mon Sep 17 00:00:00 2001 From: Russell Seymour Date: Tue, 3 Oct 2017 14:27:15 +0100 Subject: [PATCH 8/9] Added script to ensure that the ipv6 loopback is enabled Changed URL for the marketplace image Signed-off-by: Russell Seymour --- attributes/alibaba.rb | 8 ++++---- templates/default/enable_ipv6_loopback.sh.erb | 5 +++++ 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/attributes/alibaba.rb b/attributes/alibaba.rb index 46e2f19..f097335 100644 --- a/attributes/alibaba.rb +++ b/attributes/alibaba.rb @@ -19,9 +19,9 @@ def normalize_name(name) default['marketplace_image']['alibaba']['public']['automate']['enabled'] = false default['marketplace_image']['alibaba']['product_urls'] = { - 'marketplace' => 'http://chef-software.oss-cn-beijing.aliyuncs.com/chef-marketplace_0.2.5%2B20170913175530.git.3.4b24680-1_amd64.deb', - 'automate' => 'http://chef-software.oss-cn-beijing.aliyuncs.com/automate_1.6.95-1_amd64.deb', - 'chef_server' => 'http://chef-software.oss-cn-beijing.aliyuncs.com/chef-server-core_12.15.8-1_amd64.deb', + 'marketplace' => 'http://chef-software.oss-cn-beijing.aliyuncs.com/chef-marketplace_0.2.5%2B20171003114422.git.4.cce18c9-1_amd64.deb', + 'automate' => 'http://chef-software.oss-cn-beijing.aliyuncs.com/automate_1.6.179-1_amd64.deb', + 'chef_server' => 'http://chef-software.oss-cn-beijing.aliyuncs.com/chef-server-core_12.16.9-1_amd64.deb', } default_marketplace_config = { @@ -40,7 +40,7 @@ def normalize_name(name) alibaba_builder_config = { 'type' => 'alicloud-ecs', 'region' => 'cn-beijing', - 'image_name' => 'chef_automate_3', + 'image_name' => 'chef_automate_20171003_1', 'source_image' => 'ubuntu_14_0405_64_20G_alibase_20170824.vhd', 'ssh_username' => 'root', 'instance_type' => 'ecs.n2.medium', diff --git a/templates/default/enable_ipv6_loopback.sh.erb b/templates/default/enable_ipv6_loopback.sh.erb index d5b88a0..d511710 100644 --- a/templates/default/enable_ipv6_loopback.sh.erb +++ b/templates/default/enable_ipv6_loopback.sh.erb @@ -6,3 +6,8 @@ if ! grep net.ipv6.conf.lo.disable_ipv6 /etc/sysctl.conf; then echo "net.ipv6.conf.lo.disable_ipv6=0" | sudo tee -a /etc/sysctl.conf fi + +# Alibaba images have the loopback explicitly disabled +if grep "net.ipv6.conf.lo.disable_ipv6 = 1" /etc/sysctl.conf; then + sed -i.bak "s/net\.ipv6\.conf\.lo\.disable_ipv6 = 1/net.ipv6.conf.lo.disable_ipv6 = 0/g" /etc/sysctl.conf +fi From 71efa9c2c41a678e20883a8b0196b2745e555a10 Mon Sep 17 00:00:00 2001 From: Russell Seymour Date: Fri, 20 Oct 2017 16:23:14 +0100 Subject: [PATCH 9/9] Changed instance size as the one we were using is now not available in Alibaba Added script to perform a dist-upgrade on Ubuntu 14.04 Signed-off-by: Russell Seymour --- attributes/alibaba.rb | 4 ++-- recipes/_publish.rb | 6 ++++++ templates/default/apt_dist_upgrade.sh.erb | 4 ++++ 3 files changed, 12 insertions(+), 2 deletions(-) create mode 100644 templates/default/apt_dist_upgrade.sh.erb diff --git a/attributes/alibaba.rb b/attributes/alibaba.rb index f097335..76005c4 100644 --- a/attributes/alibaba.rb +++ b/attributes/alibaba.rb @@ -40,10 +40,10 @@ def normalize_name(name) alibaba_builder_config = { 'type' => 'alicloud-ecs', 'region' => 'cn-beijing', - 'image_name' => 'chef_automate_20171003_1', + 'image_name' => 'chef_automate_20171020_1', 'source_image' => 'ubuntu_14_0405_64_20G_alibase_20170824.vhd', 'ssh_username' => 'root', - 'instance_type' => 'ecs.n2.medium', + 'instance_type' => 'ecs.n4.xlarge', 'io_optimized' => 'true', 'internet_charge_type' => 'PayByTraffic', 'image_force_delete' => 'true', diff --git a/recipes/_publish.rb b/recipes/_publish.rb index 4c2a405..6afef38 100644 --- a/recipes/_publish.rb +++ b/recipes/_publish.rb @@ -33,6 +33,12 @@ not_if { use_current_repo? } end +packer_provisioner 'apt_dist_upgrade' do + type 'shell' + source 'apt_dist_upgrade.sh.erb' + only alibaba_builders +end + packer_provisioner 'install_marketplace_apt' do type 'shell' source 'install_marketplace_apt.sh.erb' diff --git a/templates/default/apt_dist_upgrade.sh.erb b/templates/default/apt_dist_upgrade.sh.erb new file mode 100644 index 0000000..0b9f14b --- /dev/null +++ b/templates/default/apt_dist_upgrade.sh.erb @@ -0,0 +1,4 @@ +#!/bin/bash + +# Perform a distribution upgrade +sudo apt-get -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" dist-upgrade -y \ No newline at end of file