Add ability to skip validation on postgres FK creation (#244)

Validation on FK constraints in postgres can cause large tables to stay locked for a relatively long time because it has to check all rows for valid data, while also preventing new entries.

Postgres implemented a way to not validate the constraint immediately so the FK can be immediately inserted, improving concurrency, with the intention that a `VALIDATE CONSTRAINT` query will be following.

The validation query uses a less expensive lock, because it only needs to check rows that existed before the constraint. This allows safer migrations, and also allows for users to clean up known violations while still enforcing the constraint for new entries.

This commit implements the ability to skip validation, but it does not implement the validation query.

Author: TylerWitt

lib/ecto/adapters/myxql/connection.ex

@@ -854,6 +854,10 @@ if Code.ensure_loaded?(MyXQL) do
       intersperse_map(columns, ", ", &column_change(table, &1))
     end
 
+    defp column_change(_table, {_command, _name, %Reference{validate: false}, _opts}) do
+      error!(nil, "validate: false on references is not supported in MyXQL")
+    end
+
     defp column_change(table, {:add, name, %Reference{} = ref, opts}) do
       ["ADD ", quote_name(name), ?\s, reference_column_type(ref.type, opts),
        column_options(opts), constraint_expr(ref, table, name)]

lib/ecto/adapters/postgres/connection.ex

@@ -1121,13 +1121,13 @@ if Code.ensure_loaded?(Postgrex) do
     defp reference_expr(%Reference{} = ref, table, name),
       do: [" CONSTRAINT ", reference_name(ref, table, name), " REFERENCES ",
            quote_table(ref.prefix || table.prefix, ref.table), ?(, quote_name(ref.column), ?),
-           reference_on_delete(ref.on_delete), reference_on_update(ref.on_update)]
+           reference_on_delete(ref.on_delete), reference_on_update(ref.on_update), validate(ref.validate)]
 
     defp constraint_expr(%Reference{} = ref, table, name),
       do: [", ADD CONSTRAINT ", reference_name(ref, table, name), ?\s,
            "FOREIGN KEY (", quote_name(name), ") REFERENCES ",
            quote_table(ref.prefix || table.prefix, ref.table), ?(, quote_name(ref.column), ?),
-           reference_on_delete(ref.on_delete), reference_on_update(ref.on_update)]
+           reference_on_delete(ref.on_delete), reference_on_update(ref.on_update), validate(ref.validate)]
 
     defp drop_constraint_expr(%Reference{} = ref, table, name),
       do: ["DROP CONSTRAINT ", reference_name(ref, table, name), ", "]
@@ -1158,6 +1158,9 @@ if Code.ensure_loaded?(Postgrex) do
     defp reference_on_update(:restrict), do: " ON UPDATE RESTRICT"
     defp reference_on_update(_), do: []
 
+    defp validate(false), do: " NOT VALID"
+    defp validate(_), do: []
+
     ## Helpers
 
     defp get_source(query, sources, ix, source) do

lib/ecto/adapters/tds/connection.ex

@@ -1195,6 +1195,10 @@ if Code.ensure_loaded?(Tds) do
       end
     end
 
+    defp column_change(_statement_prefix, _table, {_command, _name, %Reference{validate: false}, _opts}) do
+      error!(nil, "validate: false on references is not supported in Tds")
+    end
+
     defp column_change(statement_prefix, table, {:add, name, %Reference{} = ref, opts}) do
       [
         [

lib/ecto/migration.ex

@@ -370,8 +370,8 @@ defmodule Ecto.Migration do
 
     To define a reference in a migration, see `Ecto.Migration.references/2`.
     """
-    defstruct name: nil, prefix: nil, table: nil, column: :id, type: :bigserial, on_delete: :nothing, on_update: :nothing
-    @type t :: %__MODULE__{table: String.t, prefix: atom | nil, column: atom, type: atom, on_delete: atom, on_update: atom}
+    defstruct name: nil, prefix: nil, table: nil, column: :id, type: :bigserial, on_delete: :nothing, on_update: :nothing, validate: true
+    @type t :: %__MODULE__{table: String.t, prefix: atom | nil, column: atom, type: atom, on_delete: atom, on_update: atom, validate: boolean}
   end
 
   defmodule Constraint do
@@ -1094,6 +1094,9 @@ defmodule Ecto.Migration do
       `:nothing` (default), `:delete_all`, `:nilify_all`, or `:restrict`.
     * `:on_update` - What to do if the referenced entry is updated. May be
       `:nothing` (default), `:update_all`, `:nilify_all`, or `:restrict`.
+    * `:validate` - Whether or not to validate the foreign key constraint on
+       creation or not. Only available in PostgreSQL, and should be followed by
+       a command to validate the foreign key in a following migration if false.
 
   """
   def references(table, opts \\ [])

test/ecto/adapters/myxql_test.exs

@@ -1283,6 +1283,14 @@ defmodule Ecto.Adapters.MyXQLTest do
     """ |> remove_newlines]
   end
 
+  test "alter table with invalid reference opts" do
+    alter = {:alter, table(:posts), [{:add, :author_id, %Reference{table: :author, validate: false}, []}]}
+
+    assert_raise ArgumentError, "validate: false on references is not supported in MyXQL", fn ->
+      execute_ddl(alter)
+    end
+  end
+
   test "create index" do
     create = {:create, index(:posts, [:category_id, :permalink])}
     assert execute_ddl(create) ==

test/ecto/adapters/postgres_test.exs

@@ -1435,6 +1435,7 @@ defmodule Ecto.Adapters.PostgresTest do
     alter = {:alter, table(:posts),
              [{:add, :title, :string, [default: "Untitled", size: 100, null: false]},
              {:add, :author_id, %Reference{table: :author}, []},
+             {:add, :category_id, %Reference{table: :categories, validate: false}, []},
              {:add_if_not_exists, :subtitle, :string, [size: 100, null: false]},
              {:add_if_not_exists, :editor_id, %Reference{table: :editor}, []},
              {:modify, :price, :numeric, [precision: 8, scale: 2, null: true]},
@@ -1453,6 +1454,7 @@ defmodule Ecto.Adapters.PostgresTest do
     ALTER TABLE "posts"
     ADD COLUMN "title" varchar(100) DEFAULT 'Untitled' NOT NULL,
     ADD COLUMN "author_id" bigint CONSTRAINT "posts_author_id_fkey" REFERENCES "author"("id"),
+    ADD COLUMN "category_id" bigint CONSTRAINT "posts_category_id_fkey" REFERENCES "categories"("id") NOT VALID,
     ADD COLUMN IF NOT EXISTS "subtitle" varchar(100) NOT NULL,
     ADD COLUMN IF NOT EXISTS "editor_id" bigint CONSTRAINT "posts_editor_id_fkey" REFERENCES "editor"("id"),
     ALTER COLUMN "price" TYPE numeric(8,2),

test/ecto/adapters/tds_test.exs

@@ -1214,6 +1214,14 @@ defmodule Ecto.Adapters.TdsTest do
              ]
   end
 
+  test "alter table with invalid reference opts" do
+    alter = {:alter, table(:posts), [{:add, :author_id, %Reference{table: :author, validate: false}, []}]}
+
+    assert_raise ArgumentError, "validate: false on references is not supported in Tds", fn ->
+      execute_ddl(alter)
+    end
+  end
+
   test "create check constraint" do
     create = {:create, constraint(:products, "price_must_be_positive", check: "price > 0")}
 

test/ecto/migration_test.exs

@@ -100,6 +100,11 @@ defmodule Ecto.MigrationTest do
            %Reference{table: "posts", column: :id, type: :binary_id}
   end
 
+  test "creates a reference without validating" do
+    assert references(:posts, validate: false) ==
+      %Reference{table: "posts", column: :id, type: :bigserial, validate: false}
+  end
+
   test "creates a constraint" do
     assert constraint(:posts, :price_is_positive, check: "price > 0") ==
            %Constraint{table: "posts", name: :price_is_positive, check: "price > 0"}