Staging pgtune

chef/cookbooks/postgresql/recipes/config_pgtune.rb

@@ -1,5 +1,5 @@
 #
-# Cookbook Name:: postgresql
+# Cookbook:: postgresql
 # Recipe:: config_pgtune
 # Author:: David Crane (<[email protected]>)
 #
@@ -31,7 +31,7 @@
 # general range for a dedicated Postgresql system.
 #
 # This recipe takes three optional parameters that may be passed in as
-# node['postgresql']['config_pgtune'] attributes:
+# node["postgresql"]["config_pgtune"] attributes:
 #   * db_type -- Specifies database type as one of: dw, oltp,
 #     web, mixed, desktop. If not specified, the default is mixed.
 #   * max_connections -- Specifies number of maximum connections
@@ -40,7 +40,7 @@
 #     it will be detected from the Ohai automatic attributes.
 #
 # Using those inputs, this recipe will compute and set the following
-# node.default['postgresql']['config'] attributes:
+# node.default["postgresql"]["config"] attributes:
 #   * max_connections
 #   * shared_buffers
 #   * effective_cache_size
@@ -58,7 +58,7 @@
 
 #######
 # These are the workload characteristics of the five database types
-# that can be specified as node['postgresql']['config_pgtune']['db_type']:
+# that can be specified as node["postgresql"]["config_pgtune"]["db_type"]:
 #
 # dw -- Data Warehouse
 #   * Typically I/O- or RAM-bound
@@ -86,51 +86,53 @@
 # Parse out db_type option, or use default.
 db_type = "mixed"
 
-if (node["postgresql"].attribute?("config_pgtune") && node["postgresql"]["config_pgtune"].attribute?("db_type"))
+if node["postgresql"].attribute?("config_pgtune") && node["postgresql"]["config_pgtune"].attribute?("db_type")
   db_type = node["postgresql"]["config_pgtune"]["db_type"]
-  if (!(["dw","oltp","web","mixed","desktop"].include?(db_type)))
-    Chef::Application.fatal!([
-        "Bad value (#{db_type})",
-        "for node['postgresql']['config_pgtune']['db_type'] attribute.",
-        "Valid values are one of dw, oltp, web, mixed, desktop."
-      ].join(" "))
+  unless %w(dw oltp web mixed desktop).include?(db_type)
+    Chef::Log.fatal([
+      "Bad value (#{db_type})",
+      "for node['postgresql']['config_pgtune']['db_type'] attribute.",
+      "Valid values are one of dw, oltp, web, mixed, desktop.",
+    ].join(" "))
+    raise
   end
 end
 
 # Parse out max_connections option, or use a value based on db_type.
 con =
-{ "web" => 200,
-  "oltp" => 300,
-  "dw" => 20,
-  "mixed" => 80,
-  "desktop" => 5
-}.fetch(db_type)
-
-if (node["postgresql"].attribute?("config_pgtune") && node["postgresql"]["config_pgtune"].attribute?("max_connections"))
-  max_connections = node["postgresql"]["config_pgtune"]["max_connections"]
-  if (max_connections.match(/\A[1-9]\d*\Z/) == nil)
-    Chef::Application.fatal!([
-        "Bad value (#{max_connections})",
-        "for node['postgresql']['config_pgtune']['max_connections'] attribute.",
-        "Valid values are non-zero integers only."
-      ].join(" "))
+  { "web" => 200,
+    "oltp" => 300,
+    "dw" => 20,
+    "mixed" => 80,
+    "desktop" => 5,
+  }.fetch(db_type)
+
+if node["postgresql"].attribute?("config_pgtune") && node["postgresql"]["config_pgtune"].attribute?("max_connections")
+  max_connections = node["postgresql"]["config_pgtune"]["max_connections"].to_i
+  if max_connections <= 0
+    Chef::Log.fatal([
+      "Bad value (#{max_connections})",
+      "for node['postgresql']['config_pgtune']['max_connections'] attribute.",
+      "Valid values are non-zero integers only.",
+    ].join(" "))
+    raise
   end
-  con = max_connections.to_i
+  con = max_connections
 end
 
 # Parse out total_memory option, or use value detected by Ohai.
 total_memory = node["memory"]["total"]
 
 # Override max_connections with a node attribute if DevOps desires.
 # For example, on a system *not* dedicated to Postgresql.
-if (node["postgresql"].attribute?("config_pgtune") && node["postgresql"]["config_pgtune"].attribute?("total_memory"))
+if node["postgresql"].attribute?("config_pgtune") && node["postgresql"]["config_pgtune"].attribute?("total_memory")
   total_memory = node["postgresql"]["config_pgtune"]["total_memory"]
-  if (total_memory.match(/\A[1-9]\d*kB\Z/) == nil)
+  if total_memory.match(/\A[1-9]\d*kB\Z/).nil?
     Chef::Application.fatal!([
-        "Bad value (#{total_memory})",
-        "for node['postgresql']['config_pgtune']['total_memory'] attribute.",
-        "Valid values are non-zero integers followed by kB (e.g., 49416564kB)."
-      ].join(" "))
+      "Bad value (#{total_memory})",
+      "for node['postgresql']['config_pgtune']['total_memory'] attribute.",
+      "Valid values are non-zero integers followed by kB (e.g., 49416564kB).",
+    ].join(" "))
   end
 end
 
@@ -150,77 +152,73 @@
 # for low memory systems. In that case, the calculation is skipped,
 # leaving the built-in Postgresql settings, which are actually
 # intended for those low memory systems.
-if (mem >= 256)
+if mem >= 256
 
   # (2) shared_buffers
   #     Sets the number of shared memory buffers used by the server.
   shared_buffers =
-  { "web" => mem/4,
-    "oltp" => mem/4,
-    "dw" => mem/4,
-    "mixed" => mem/4,
-    "desktop" => mem/16
-  }.fetch(db_type)
+    { "web" => mem / 4,
+      "oltp" => mem / 4,
+      "dw" => mem / 4,
+      "mixed" => mem / 4,
+      "desktop" => mem / 16,
+    }.fetch(db_type)
 
   # Robert Haas has advised to cap the size of shared_buffers based on
   # the memory architecture: 2GB on 32-bit and 8GB on 64-bit machines.
   # http://rhaas.blogspot.com/2012/03/tuning-sharedbuffers-and-walbuffers.html
   case node["kernel"]["machine"]
   when "i386" # 32-bit machines
-    if shared_buffers > 2*1024
-      shared_buffers = 2*1024
-    end
+    shared_buffers = 2 * 1024 if shared_buffers > 2 * 1024
   when "x86_64" # 64-bit machines
-    if shared_buffers > 8*1024
-      shared_buffers = 8*1024
-    end
+    shared_buffers = 8 * 1024 if shared_buffers > 8 * 1024
   end
 
-  node.default["postgresql"]["config"]["shared_buffers"] = binaryround(shared_buffers*1024*1024)
+  node.default["postgresql"]["config"]["shared_buffers"] = binaryround(shared_buffers * 1024 * 1024)
 
   # (3) effective_cache_size
   #     Sets the planner's assumption about the size of the disk cache.
   #     That is, the portion of the kernel's disk cache that will be
   #     used for PostgreSQL data files.
   effective_cache_size =
-  { "web" => mem * 3 / 4,
-    "oltp" => mem * 3 / 4,
-    "dw" => mem * 3 / 4,
-    "mixed" => mem * 3 / 4,
-    "desktop" => mem / 4
-  }.fetch(db_type)
+    { "web" => mem * 3 / 4,
+      "oltp" => mem * 3 / 4,
+      "dw" => mem * 3 / 4,
+      "mixed" => mem * 3 / 4,
+      "desktop" => mem / 4,
+    }.fetch(db_type)
 
-  node.default["postgresql"]["config"]["effective_cache_size"] = binaryround(effective_cache_size*1024*1024)
+  node.default["postgresql"]["config"]["effective_cache_size"] = binaryround(effective_cache_size * 1024 * 1024)
 
   # (4) work_mem
   #     Sets the maximum memory to be used for query workspaces.
+  mem_con_v = (mem.to_f / con).ceil
+
   work_mem =
-  { "web" => mem / con,
-    "oltp" => mem / con,
-    "dw" => mem / con / 2,
-    "mixed" => mem / con / 2,
-    "desktop" => mem / con / 6
-  }.fetch(db_type)
+    { "web" => mem_con_v,
+      "oltp" => mem_con_v,
+      "dw" => mem_con_v / 2,
+      "mixed" => mem_con_v / 2,
+      "desktop" => mem_con_v / 6,
+    }.fetch(db_type)
 
-  node.default["postgresql"]["config"]["work_mem"] = binaryround(work_mem*1024*1024)
+  node.default["postgresql"]["config"]["work_mem"] = binaryround(work_mem * 1024 * 1024)
 
   # (5) maintenance_work_mem
   #     Sets the maximum memory to be used for maintenance operations.
   #     This includes operations such as VACUUM and CREATE INDEX.
   maintenance_work_mem =
-  { "web" => mem / 16,
-    "oltp" => mem / 16,
-    "dw" => mem / 8,
-    "mixed" => mem / 16,
-    "desktop" => mem / 16
-  }.fetch(db_type)
+    { "web" => mem / 16,
+      "oltp" => mem / 16,
+      "dw" => mem / 8,
+      "mixed" => mem / 16,
+      "desktop" => mem / 16,
+    }.fetch(db_type)
 
   # Cap maintenence RAM at 1GB on servers with lots of memory
-  if (maintenance_work_mem > 1*1024)
-      maintenance_work_mem = 1*1024
-  end
+  maintenance_work_mem = 1 * 1024 if maintenance_work_mem > 1 * 1024
 
-  node.default["postgresql"]["config"]["maintenance_work_mem"] = binaryround(maintenance_work_mem*1024*1024)
+  node.default["postgresql"]["config"]["maintenance_work_mem"] = binaryround(maintenance_work_mem * 1024 * 1024)
 
 end
 
@@ -231,25 +229,29 @@
 # (6) checkpoint_segments
 #     Sets the maximum distance in log segments between automatic WAL checkpoints.
 checkpoint_segments =
-{ "web" => 8,
-  "oltp" => 16,
-  "dw" => 64,
-  "mixed" => 16,
-  "desktop" => 3
-}.fetch(db_type)
+  { "web" => 8,
+    "oltp" => 16,
+    "dw" => 64,
+    "mixed" => 16,
+    "desktop" => 3,
+  }.fetch(db_type)
 
-node.default["postgresql"]["config"]["checkpoint_segments"] = checkpoint_segments
+if node["postgresql"]["version"].to_f >= 9.5
+  node.default["postgresql"]["config"]["max_wal_size"] = ((3 * checkpoint_segments) * 16).to_s + "MB"
+else
+  node.default["postgresql"]["config"]["checkpoint_segments"] = checkpoint_segments
+end
 
 # (7) checkpoint_completion_target
 #     Time spent flushing dirty buffers during checkpoint, as fraction
 #     of checkpoint interval.
 checkpoint_completion_target =
-{ "web" => "0.7",
-  "oltp" => "0.9",
-  "dw" => "0.9",
-  "mixed" => "0.9",
-  "desktop" => "0.5"
-}.fetch(db_type)
+  { "web" => "0.7",
+    "oltp" => "0.9",
+    "dw" => "0.9",
+    "mixed" => "0.9",
+    "desktop" => "0.5",
+  }.fetch(db_type)
 
 node.default["postgresql"]["config"]["checkpoint_completion_target"] = checkpoint_completion_target
 
@@ -260,7 +262,7 @@
 if node["postgresql"]["version"].to_f < 9.1
   wal_buffers = 512 * checkpoint_segments
   # The pgtune seems to use 1kB units for wal_buffers
-  node.default["postgresql"]["config"]["wal_buffers"] = binaryround(wal_buffers*1024)
+  node.default["postgresql"]["config"]["wal_buffers"] = binaryround(wal_buffers * 1024)
 else
   node.default["postgresql"]["config"]["wal_buffers"] = "-1"
 end
@@ -270,11 +272,11 @@
 #     that have not had a column-specific target set via
 #     ALTER TABLE SET STATISTICS.
 default_statistics_target =
-{ "web" => 100,
-  "oltp" => 100,
-  "dw" => 500,
-  "mixed" => 100,
-  "desktop" => 100
-}.fetch(db_type)
+  { "web" => 100,
+    "oltp" => 100,
+    "dw" => 500,
+    "mixed" => 100,
+    "desktop" => 100,
+  }.fetch(db_type)
 
 node.default["postgresql"]["config"]["default_statistics_target"] = default_statistics_target

chef/cookbooks/postgresql/recipes/server.rb

@@ -24,6 +24,7 @@
 ::Chef::Recipe.send(:include, Opscode::OpenSSL::Password)
 
 include_recipe "postgresql::client"
+include_recipe "postgresql::config_pgtune"
 
 # For Crowbar, we need to set the address to bind - default to admin node.
 newaddr = CrowbarDatabaseHelper.get_listen_address(node)

chef/data_bags/crowbar/migrate/database/010_pgtune.rb

@@ -0,0 +1,19 @@
+def upgrade(ta, td, a, d)
+  a["postgresql"]["config_pgtune"] ||= {}
+  unless a["postgresql"]["config_pgtune"].key?("max_connections")
+    a["postgresql"]["config_pgtune"]["max_connections"] = a["postgresql"]["config"]["max_connections"]
+  end
+  a["postgresql"]["config"].delete("max_connections")
+
+  return a, d
+end
+
+def downgrade(ta, td, a, d)
+  if ta["postgresql"]["config"].key?("max_connections")
+    a["postgresql"]["config"]["max_connections"] = a["postgresql"]["config_pgtune"]["max_connections"]
+    a["postgresql"]["config_pgtune"].delete("max_connections")
+    a["postgresql"].delete("config_pgtune") if a["postgresql"]["config_pgtune"].empty?
+  end
+
+  return a, d
+end

chef/data_bags/crowbar/template-database.json

@@ -8,8 +8,10 @@
         "datadir": "/var/lib/mysql"
       },
       "postgresql": {
+        "config_pgtune": {
+          "max_connections": 1000
+        },
         "config": {
-          "max_connections": 1000,
           "log_filename": "postgresql.log-%Y%m%d%H%M",
           "log_truncate_on_rotation": false
         }
@@ -33,7 +35,7 @@
     "database": {
       "crowbar-revision": 0,
       "crowbar-applied": false,
-      "schema-revision": 4,
+      "schema-revision": 10,
       "element_states": {
         "database-server": [ "readying", "ready", "applying" ]
       },

chef/data_bags/crowbar/template-database.schema

@@ -28,11 +28,17 @@
               "type": "map",
               "required": false,
               "mapping": {
+                "config_pgtune": {
+                  "type": "map",
+                  "required": false,
+                  "mapping": {
+                    "max_connections": { "type": "int" }
+                  }
+                },
                 "config": {
                   "type": "map",
                   "required": false,
                   "mapping": {
-                    "max_connections": { "type": "int" },
                     "log_truncate_on_rotation": { "type": "bool" },
                     "log_filename": {"type": "str" }
                   }

crowbar_framework/app/views/barclamp/database/_edit_attributes.html.haml

@@ -17,7 +17,7 @@
         %legend
           = t('.postgresql_attributes')
 
-        = integer_field %w(postgresql config max_connections)
+        = integer_field %w(postgresql config_pgtune max_connections)
 
       -# As HA is only supported for postgresql, we put this section in #postgresql_container
       %fieldset#ha-setup{ "data-show-for-clusters-only" => "true", "data-elements-path" => "database-server" }

crowbar_framework/config/locales/database/en.yml

@@ -27,7 +27,7 @@ en:
           datadir: 'Datadir'
         postgresql_attributes: 'PostgreSQL Options'
         postgresql:
-          config:
+          config_pgtune:
             max_connections: 'Global Connection Limit (max_connections)'
         ha_header: 'High Availability'
         ha: