Skip to content

Add support in migrations for collations in Postgres #662

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 8 commits into from
May 2, 2025
Merged
Show file tree
Hide file tree
Changes from 4 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
61 changes: 61 additions & 0 deletions integration_test/myxql/migrations_test.exs
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,42 @@ defmodule Ecto.Integration.MigrationsTest do
end
end

text_variants = ~w/tinytext text mediumtext longtext/a
@text_variants text_variants

collation = "utf8mb4_bin"
@collation collation

defmodule CollateMigration do
use Ecto.Migration

@text_variants text_variants
@collation collation

def change do
create table(:collate_reference) do
add :name, :string, collation: @collation
end

create unique_index(:collate_reference, :name)

create table(:collate) do
add :string, :string, collation: @collation
add :varchar, :varchar, size: 255, collation: @collation
add :integer, :integer, collation: @collation
add :name_string, references(:collate_reference, type: :string, column: :name), collation: @collation

for type <- @text_variants do
add type, type, collation: @collation
end
end

alter table(:collate) do
modify :string, :string, collation: "utf8mb4_general_ci"
end
end
end

describe "Migrator" do
@get_lock_command ~s[SELECT GET_LOCK('ecto_Ecto.Integration.PoolRepo', -1)]
@release_lock_command ~s[SELECT RELEASE_LOCK('ecto_Ecto.Integration.PoolRepo')]
Expand Down Expand Up @@ -107,5 +143,30 @@ defmodule Ecto.Integration.MigrationsTest do

assert log =~ "ALTER TABLE `alter_table` ADD `column2` varchar(255) COMMENT 'second column' AFTER `column1`"
end

test "collation can be set on a column" do
num = @base_migration + System.unique_integer([:positive])
assert :ok = Ecto.Migrator.up(PoolRepo, num, CollateMigration, log: false)
query = fn column -> """
SELECT collation_name
FROM information_schema.columns
WHERE table_name = 'collate' AND column_name = '#{column}';
"""
end

assert %{
rows: [["utf8mb4_general_ci"]]
} = Ecto.Adapters.SQL.query!(PoolRepo, query.("string"), [])

for type <- ~w/text name_string/ ++ @text_variants do
assert %{
rows: [[@collation]]
} = Ecto.Adapters.SQL.query!(PoolRepo, query.(type), [])
end

assert %{
rows: [[nil]]
} = Ecto.Adapters.SQL.query!(PoolRepo, query.("integer"), [])
end
end
end
62 changes: 62 additions & 0 deletions integration_test/pg/migrations_test.exs
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,41 @@ defmodule Ecto.Integration.MigrationsTest do
end
end

collation = "POSIX"
@collation collation

text_types = ~w/char varchar text/a
@text_types text_types

defmodule CollateMigration do
use Ecto.Migration

@collation collation
@text_types text_types

def change do
create table(:collate_reference) do
add :name, :string, primary_key: true, collation: @collation
end

create unique_index(:collate_reference, :name)

create table(:collate) do
add :string, :string, collation: @collation
for type <- @text_types do
add type, type, collation: @collation
end

add :integer, :integer, collation: @collation
add :name_string, references(:collate_reference, type: :string, column: :name), collation: @collation
end

alter table(:collate) do
modify :string, :string, collation: "C"
end
end
end

test "logs Postgres notice messages" do
log =
capture_log(fn ->
Expand Down Expand Up @@ -145,5 +180,32 @@ defmodule Ecto.Integration.MigrationsTest do
refute down_log =~ @version_delete
refute down_log =~ "commit []"
end

test "collation can be set on a column" do
num = @base_migration + System.unique_integer([:positive])

assert :ok = Ecto.Migrator.up(PoolRepo, num, CollateMigration, log: :info)

query = fn column -> """
SELECT collation_name
FROM information_schema.columns
WHERE table_name = 'collate' AND column_name = '#{column}';
"""
end

assert %{
rows: [["C"]]
} = Ecto.Adapters.SQL.query!(PoolRepo, query.("string"), [])

for type <- @text_types do
assert %{
rows: [[@collation]]
} = Ecto.Adapters.SQL.query!(PoolRepo, query.(type), [])
end

assert %{
rows: [[nil]]
} = Ecto.Adapters.SQL.query!(PoolRepo, query.("integer"), [])
end
end
end
1 change: 1 addition & 0 deletions integration_test/sql/migration.exs
Original file line number Diff line number Diff line change
Expand Up @@ -463,6 +463,7 @@ defmodule Ecto.Integration.MigrationTest do
end
end


import Ecto.Query, only: [from: 2]
import Ecto.Migrator, only: [up: 4, down: 4]

Expand Down
58 changes: 58 additions & 0 deletions integration_test/tds/migrations_test.exs
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,38 @@ defmodule Ecto.Integration.MigrationsTest do
end
end

collation = "Latin1_General_CS_AS"
@collation collation

defmodule CollateMigration do
use Ecto.Migration
@collation collation

def change do
create table(:collate_reference) do
add :name, :string, collation: @collation
end

create unique_index(:collate_reference, :name)

create table(:collate) do
add :string, :string, collation: @collation
add :char, :char, size: 255, collation: @collation
add :nchar, :nchar, size: 255, collation: @collation
add :varchar, :varchar, size: 255, collation: @collation
add :nvarchar, :nvarchar, size: 255, collation: @collation
add :text, :text, collation: @collation
add :ntext, :ntext, collation: @collation
add :integer, :integer, collation: @collation
add :name_string, references(:collate_reference, type: :string, column: :name), collation: @collation
end

alter table(:collate) do
modify :string, :string, collation: "Japanese_Bushu_Kakusu_100_CS_AS_KS_WS"
end
end
end

describe "Migrator" do
@get_lock_command ~s(sp_getapplock @Resource = 'ecto_Ecto.Integration.PoolRepo', @LockMode = 'Exclusive', @LockOwner = 'Transaction', @LockTimeout = -1)
@create_table_sql ~s(CREATE TABLE [log_mode_table])
Expand Down Expand Up @@ -77,5 +109,31 @@ defmodule Ecto.Integration.MigrationsTest do
refute down_log =~ @version_delete
refute down_log =~ "commit []"
end

test "collation can be set on a column" do
num = @base_migration + System.unique_integer([:positive])
assert :ok = Ecto.Migrator.up(PoolRepo, num, CollateMigration, log: :info)

query = fn column -> """
SELECT collation_name
FROM information_schema.columns
WHERE table_name = 'collate' AND column_name = '#{column}';
"""
end

assert %{
rows: [["Japanese_Bushu_Kakusu_100_CS_AS_KS_WS"]]
} = Ecto.Adapters.SQL.query!(PoolRepo, query.("string"), [])

for type <- ~w/char varchar nchar nvarchar text ntext/ do
assert %{
rows: [[@collation]]
} = Ecto.Adapters.SQL.query!(PoolRepo, query.(type), [])
end

assert %{
rows: [[nil]]
} = Ecto.Adapters.SQL.query!(PoolRepo, query.("integer"), [])
end
end
end
40 changes: 30 additions & 10 deletions lib/ecto/adapters/myxql/connection.ex
Original file line number Diff line number Diff line change
Expand Up @@ -1171,13 +1171,13 @@ if Code.ensure_loaded?(MyXQL) do
quote_name(name),
?\s,
reference_column_type(ref.type, opts),
column_options(opts),
column_options(ref.type, opts),
reference_expr(ref, table, name)
]
end

defp column_definition(_table, {:add, name, type, opts}) do
[quote_name(name), ?\s, column_type(type, opts), column_options(opts)]
[quote_name(name), ?\s, column_type(type, opts), column_options(type, opts)]
end

defp column_changes(table, columns) do
Expand All @@ -1194,13 +1194,13 @@ if Code.ensure_loaded?(MyXQL) do
quote_name(name),
?\s,
reference_column_type(ref.type, opts),
column_options(opts),
column_options(ref.type, opts),
constraint_expr(ref, table, name)
]
end

defp column_change(_table, {:add, name, type, opts}) do
["ADD ", quote_name(name), ?\s, column_type(type, opts), column_options(opts)]
["ADD ", quote_name(name), ?\s, column_type(type, opts), column_options(type, opts)]
end

defp column_change(table, {:add_if_not_exists, name, %Reference{} = ref, opts}) do
Expand All @@ -1209,13 +1209,19 @@ if Code.ensure_loaded?(MyXQL) do
quote_name(name),
?\s,
reference_column_type(ref.type, opts),
column_options(opts),
column_options(ref.type, opts),
constraint_if_not_exists_expr(ref, table, name)
]
end

defp column_change(_table, {:add_if_not_exists, name, type, opts}) do
["ADD IF NOT EXISTS ", quote_name(name), ?\s, column_type(type, opts), column_options(opts)]
[
"ADD IF NOT EXISTS ",
quote_name(name),
?\s,
column_type(type, opts),
column_options(type, opts)
]
end

defp column_change(table, {:modify, name, %Reference{} = ref, opts}) do
Expand All @@ -1225,7 +1231,7 @@ if Code.ensure_loaded?(MyXQL) do
quote_name(name),
?\s,
reference_column_type(ref.type, opts),
column_options(opts),
column_options(ref.type, opts),
constraint_expr(ref, table, name)
]
end
Expand All @@ -1237,7 +1243,7 @@ if Code.ensure_loaded?(MyXQL) do
quote_name(name),
?\s,
column_type(type, opts),
column_options(opts)
column_options(type, opts)
]
end

Expand All @@ -1259,13 +1265,20 @@ if Code.ensure_loaded?(MyXQL) do
defp column_change(_table, {:remove_if_exists, name}),
do: ["DROP IF EXISTS ", quote_name(name)]

defp column_options(opts) do
defp column_options(type, opts) do
default = Keyword.fetch(opts, :default)
null = Keyword.get(opts, :null)
after_column = Keyword.get(opts, :after)
comment = Keyword.get(opts, :comment)
collation = Keyword.fetch(opts, :collation)

[default_expr(default), null_expr(null), comment_expr(comment), after_expr(after_column)]
[
default_expr(default),
collation_expr(collation, type),
null_expr(null),
comment_expr(comment),
after_expr(after_column)
]
end

defp comment_expr(comment, create_table? \\ false)
Expand All @@ -1286,6 +1299,13 @@ if Code.ensure_loaded?(MyXQL) do
defp null_expr(true), do: " NULL"
defp null_expr(_), do: []

defp collation_expr({:ok, collation_name}, text_type)
when text_type in ~w/string char varchar text tinytext mediumtext longtext/a do
" COLLATE \"#{collation_name}\""
end

defp collation_expr(_, _), do: []
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think we should either raise or not check the type and let the underlying SQL operation fail. Perhaps the second option is best, because I don’t know if Postgres supports collations for custom types (which would then be impossible to check).

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Postgres does support collations for custom types, so letting the SQL fail is the best option.


defp new_constraint_expr(%Constraint{check: check} = constraint) when is_binary(check) do
[
"CONSTRAINT ",
Expand Down
20 changes: 17 additions & 3 deletions lib/ecto/adapters/postgres/connection.ex
Original file line number Diff line number Diff line change
Expand Up @@ -1555,6 +1555,8 @@ if Code.ensure_loaded?(Postgrex) do
end

defp column_change(table, {:modify, name, %Reference{} = ref, opts}) do
collation = Keyword.fetch(opts, :collation)

[
drop_reference_expr(opts[:from], table, name),
"ALTER COLUMN ",
Expand All @@ -1564,19 +1566,23 @@ if Code.ensure_loaded?(Postgrex) do
", ADD ",
reference_expr(ref, table, name),
modify_null(name, opts),
modify_default(name, ref.type, opts)
modify_default(name, ref.type, opts),
collation_expr(collation, ref.type)
]
end

defp column_change(table, {:modify, name, type, opts}) do
collation = Keyword.fetch(opts, :collation)

[
drop_reference_expr(opts[:from], table, name),
"ALTER COLUMN ",
quote_name(name),
" TYPE ",
column_type(type, opts),
modify_null(name, opts),
modify_default(name, type, opts)
modify_default(name, type, opts),
collation_expr(collation, type)
]
end

Expand Down Expand Up @@ -1624,14 +1630,22 @@ if Code.ensure_loaded?(Postgrex) do
defp column_options(type, opts) do
default = Keyword.fetch(opts, :default)
null = Keyword.get(opts, :null)
collation = Keyword.fetch(opts, :collation)

[default_expr(default, type), null_expr(null)]
[default_expr(default, type), null_expr(null), collation_expr(collation, type)]
end

defp null_expr(false), do: " NOT NULL"
defp null_expr(true), do: " NULL"
defp null_expr(_), do: []

defp collation_expr({:ok, collation_name}, text_type)
when text_type in ~w/string text char varchar/a do
" COLLATE \"#{collation_name}\""
end

defp collation_expr(_, _), do: []

defp new_constraint_expr(%Constraint{check: check} = constraint) when is_binary(check) do
[
"CONSTRAINT ",
Expand Down
Loading
Loading