diff --git a/lib/phoenix/endpoint/server.ex b/lib/phoenix/endpoint/server.ex
index 7a3a5727a3..d5bf72f757 100644
--- a/lib/phoenix/endpoint/server.ex
+++ b/lib/phoenix/endpoint/server.ex
@@ -20,12 +20,11 @@ defmodule Phoenix.Endpoint.Server do
   end
 
   defp default(config, otp_app, port) do
-    config =
-      config
-      |> Keyword.put_new(:otp_app, otp_app)
-      |> Keyword.put_new(:port, port)
+    port =  Keyword.get(config, :port, port) |> to_port()
 
-    Keyword.put(config, :port, to_port(config[:port]))
+    :proplists.delete(:port, config)
+      |> Keyword.put_new(:port, port)
+      |> Keyword.put_new(:otp_app, otp_app)
   end
 
   defp to_port(nil), do: raise "server can't start because :port in config is nil, please use a valid port number"
diff --git a/test/phoenix/endpoint/endpoint_test.exs b/test/phoenix/endpoint/endpoint_test.exs
index adf3a49313..2bc3d55680 100644
--- a/test/phoenix/endpoint/endpoint_test.exs
+++ b/test/phoenix/endpoint/endpoint_test.exs
@@ -4,7 +4,7 @@ defmodule Phoenix.Endpoint.EndpointTest do
 
   @config [url: [host: "example.com", path: "/api"],
            static_url: [host: "static.example.com"],
-           server: false, http: [port: 80], https: [port: 443],
+           server: false, http: [{:port, 80}, {:raw, 1, 15, <<1,0,0,0>>}], https: [port: 443],
            force_ssl: [subdomains: true],
            cache_static_manifest: "../../../../test/fixtures/manifest.json",
            pubsub: [adapter: Phoenix.PubSub.PG2, name: :endpoint_pub]]