From c52982e9c5381c824595a0410d3a6db0ea9f1cfc Mon Sep 17 00:00:00 2001 From: floatingghost Date: Sun, 14 Aug 2022 23:13:49 +0000 Subject: [PATCH] add finch outbound proxy support (#158) Reviewed-on: https://akkoma.dev/AkkomaGang/akkoma/pulls/158 --- CHANGELOG.md | 5 ++ config/description.exs | 7 +- docs/docs/configuration/cheatsheet.md | 2 +- lib/pleroma/application.ex | 4 + lib/pleroma/http/adapter_helper.ex | 76 ++++++++++++----- lib/pleroma/http/adapter_helper/default.ex | 2 +- lib/pleroma/http/adapter_helper/gun.ex | 82 ------------------- lib/pleroma/http/adapter_helper/hackney.ex | 40 --------- test/pleroma/http/adapter_helper/gun_test.exs | 77 ----------------- .../http/adapter_helper/hackney_test.exs | 35 -------- test/pleroma/http/adapter_helper_test.exs | 30 ++++++- 11 files changed, 98 insertions(+), 262 deletions(-) delete mode 100644 lib/pleroma/http/adapter_helper/gun.ex delete mode 100644 lib/pleroma/http/adapter_helper/hackney.ex delete mode 100644 test/pleroma/http/adapter_helper/gun_test.exs delete mode 100644 test/pleroma/http/adapter_helper/hackney_test.exs diff --git a/CHANGELOG.md b/CHANGELOG.md index c658af460..7c7cd8601 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,11 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ## [Unreleased] +### Removed +- Non-finch HTTP adapters. `:tesla, :adapter` is now highly recommended to be set to the default. + +## 2022.08 + ### Added - extended runtime module support, see config cheatsheet - quote posting; quotes are limited to public posts diff --git a/config/description.exs b/config/description.exs index a44ab2432..9f93265d1 100644 --- a/config/description.exs +++ b/config/description.exs @@ -2598,9 +2598,10 @@ %{ key: :proxy_url, label: "Proxy URL", - type: [:string, :tuple], - description: "Proxy URL", - suggestions: ["localhost:9020", {:socks5, :localhost, 3090}] + type: :string, + description: + "Proxy URL - of the format http://host:port. Advise setting in .exs instead of admin-fe due to this being set at boot-time.", + suggestions: ["http://localhost:3128"] }, %{ key: :user_agent, diff --git a/docs/docs/configuration/cheatsheet.md b/docs/docs/configuration/cheatsheet.md index 71ebf28dc..8fa188de1 100644 --- a/docs/docs/configuration/cheatsheet.md +++ b/docs/docs/configuration/cheatsheet.md @@ -521,7 +521,7 @@ Available caches: ### :http -* `proxy_url`: an upstream proxy to fetch posts and/or media with, (default: `nil`) +* `proxy_url`: an upstream proxy to fetch posts and/or media with, (default: `nil`); for example `http://127.0.0.1:3192`. Does not support SOCKS5 proxy, only http(s). * `send_user_agent`: should we include a user agent with HTTP requests? (default: `true`) * `user_agent`: what user agent should we use? (default: `:default`), must be string or `:default` * `adapter`: array of adapter options diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index e29bf3ca3..cb619232f 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -248,9 +248,13 @@ def limiters_setup do end defp http_children do + proxy_url = Config.get([:http, :proxy_url]) + proxy = Pleroma.HTTP.AdapterHelper.format_proxy(proxy_url) + config = [:http, :adapter] |> Config.get([]) + |> Pleroma.HTTP.AdapterHelper.maybe_add_proxy_pool(proxy) |> Keyword.put(:name, MyFinch) [{Finch, config}] diff --git a/lib/pleroma/http/adapter_helper.ex b/lib/pleroma/http/adapter_helper.ex index f9b489616..4949dd727 100644 --- a/lib/pleroma/http/adapter_helper.ex +++ b/lib/pleroma/http/adapter_helper.ex @@ -6,7 +6,7 @@ defmodule Pleroma.HTTP.AdapterHelper do @moduledoc """ Configure Tesla.Client with default and customized adapter options. """ - @defaults [name: MyFinch, connect_timeout: 5_000, recv_timeout: 5_000] + @defaults [name: MyFinch, pool_timeout: 5_000, receive_timeout: 5_000] @type proxy_type() :: :socks4 | :socks5 @type host() :: charlist() | :inet.ip_address() @@ -25,15 +25,58 @@ def format_proxy(nil), do: nil def format_proxy(proxy_url) do case parse_proxy(proxy_url) do - {:ok, host, port} -> {host, port} - {:ok, type, host, port} -> {type, host, port} + {:ok, host, port} -> {:http, host, port, []} + {:ok, type, host, port} -> {type, host, port, []} _ -> nil end end @spec maybe_add_proxy(keyword(), proxy() | nil) :: keyword() def maybe_add_proxy(opts, nil), do: opts - def maybe_add_proxy(opts, proxy), do: Keyword.put_new(opts, :proxy, proxy) + + def maybe_add_proxy(opts, proxy) do + Keyword.put(opts, :proxy, proxy) + end + + def maybe_add_proxy_pool(opts, nil), do: opts + + def maybe_add_proxy_pool(opts, proxy) do + Logger.info("Using HTTP Proxy: #{inspect(proxy)}") + + opts + |> maybe_add_pools() + |> maybe_add_default_pool() + |> maybe_add_conn_opts() + |> put_in([:pools, :default, :conn_opts, :proxy], proxy) + end + + defp maybe_add_pools(opts) do + if Keyword.has_key?(opts, :pools) do + opts + else + Keyword.put(opts, :pools, %{}) + end + end + + defp maybe_add_default_pool(opts) do + pools = Keyword.get(opts, :pools) + + if Map.has_key?(pools, :default) do + opts + else + put_in(opts, [:pools, :default], []) + end + end + + defp maybe_add_conn_opts(opts) do + conn_opts = get_in(opts, [:pools, :default, :conn_opts]) + + unless is_nil(conn_opts) do + opts + else + put_in(opts, [:pools, :default, :conn_opts], []) + end + end @doc """ Merge default connection & adapter options with received ones. @@ -46,36 +89,31 @@ def options(%URI{} = uri, opts \\ []) do |> AdapterHelper.Default.options(uri) end + defp proxy_type("http"), do: {:ok, :http} + defp proxy_type("https"), do: {:ok, :https} + defp proxy_type(_), do: {:error, :unknown} + @spec parse_proxy(String.t() | tuple() | nil) :: {:ok, host(), pos_integer()} | {:ok, proxy_type(), host(), pos_integer()} | {:error, atom()} | nil - def parse_proxy(nil), do: nil def parse_proxy(proxy) when is_binary(proxy) do - with [host, port] <- String.split(proxy, ":"), - {port, ""} <- Integer.parse(port) do - {:ok, parse_host(host), port} + with %URI{} = uri <- URI.parse(proxy), + {:ok, type} <- proxy_type(uri.scheme) do + {:ok, type, uri.host, uri.port} else - {_, _} -> - Logger.warn("Parsing port failed #{inspect(proxy)}") - {:error, :invalid_proxy_port} - - :error -> - Logger.warn("Parsing port failed #{inspect(proxy)}") - {:error, :invalid_proxy_port} - - _ -> - Logger.warn("Parsing proxy failed #{inspect(proxy)}") + e -> + Logger.warn("Parsing proxy failed #{inspect(proxy)}, #{inspect(e)}") {:error, :invalid_proxy} end end def parse_proxy(proxy) when is_tuple(proxy) do with {type, host, port} <- proxy do - {:ok, type, parse_host(host), port} + {:ok, type, host, port} else _ -> Logger.warn("Parsing proxy failed #{inspect(proxy)}") diff --git a/lib/pleroma/http/adapter_helper/default.ex b/lib/pleroma/http/adapter_helper/default.ex index a1614b9c5..630536871 100644 --- a/lib/pleroma/http/adapter_helper/default.ex +++ b/lib/pleroma/http/adapter_helper/default.ex @@ -9,7 +9,7 @@ defmodule Pleroma.HTTP.AdapterHelper.Default do @spec options(keyword(), URI.t()) :: keyword() def options(opts, _uri) do - proxy = Pleroma.Config.get([:http, :proxy_url], nil) + proxy = Pleroma.Config.get([:http, :proxy_url]) AdapterHelper.maybe_add_proxy(opts, AdapterHelper.format_proxy(proxy)) end diff --git a/lib/pleroma/http/adapter_helper/gun.ex b/lib/pleroma/http/adapter_helper/gun.ex deleted file mode 100644 index 251539f34..000000000 --- a/lib/pleroma/http/adapter_helper/gun.ex +++ /dev/null @@ -1,82 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2021 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.HTTP.AdapterHelper.Gun do - @behaviour Pleroma.HTTP.AdapterHelper - - alias Pleroma.Config - alias Pleroma.HTTP.AdapterHelper - - require Logger - - @defaults [ - retry: 1, - retry_timeout: 1_000 - ] - - @type pool() :: :federation | :upload | :media | :default - - @spec options(keyword(), URI.t()) :: keyword() - def options(incoming_opts \\ [], %URI{} = uri) do - proxy = - [:http, :proxy_url] - |> Config.get() - |> AdapterHelper.format_proxy() - - config_opts = Config.get([:http, :adapter], []) - - @defaults - |> Keyword.merge(config_opts) - |> add_scheme_opts(uri) - |> AdapterHelper.maybe_add_proxy(proxy) - |> Keyword.merge(incoming_opts) - |> put_timeout() - end - - defp add_scheme_opts(opts, %{scheme: "http"}), do: opts - - defp add_scheme_opts(opts, %{scheme: "https"}) do - Keyword.put(opts, :certificates_verification, true) - end - - defp put_timeout(opts) do - {recv_timeout, opts} = Keyword.pop(opts, :recv_timeout, pool_timeout(opts[:pool])) - # this is the timeout to receive a message from Gun - # `:timeout` key is used in Tesla - Keyword.put(opts, :timeout, recv_timeout) - end - - @spec pool_timeout(pool()) :: non_neg_integer() - def pool_timeout(pool) do - default = Config.get([:pools, :default, :recv_timeout], 5_000) - - Config.get([:pools, pool, :recv_timeout], default) - end - - def limiter_setup do - prefix = Pleroma.Gun.ConnectionPool - wait = Config.get([:connections_pool, :connection_acquisition_wait]) - retries = Config.get([:connections_pool, :connection_acquisition_retries]) - - :pools - |> Config.get([]) - |> Enum.each(fn {name, opts} -> - max_running = Keyword.get(opts, :size, 50) - max_waiting = Keyword.get(opts, :max_waiting, 10) - - result = - ConcurrentLimiter.new(:"#{prefix}.#{name}", max_running, max_waiting, - wait: wait, - max_retries: retries - ) - - case result do - :ok -> :ok - {:error, :existing} -> :ok - end - end) - - :ok - end -end diff --git a/lib/pleroma/http/adapter_helper/hackney.ex b/lib/pleroma/http/adapter_helper/hackney.ex deleted file mode 100644 index af0ada1e7..000000000 --- a/lib/pleroma/http/adapter_helper/hackney.ex +++ /dev/null @@ -1,40 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2021 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.HTTP.AdapterHelper.Hackney do - @behaviour Pleroma.HTTP.AdapterHelper - - @defaults [ - follow_redirect: true, - force_redirect: true - ] - - @spec options(keyword(), URI.t()) :: keyword() - def options(connection_opts \\ [], %URI{} = uri) do - proxy = Pleroma.Config.get([:http, :proxy_url]) - - config_opts = Pleroma.Config.get([:http, :adapter], []) - - @defaults - |> Keyword.merge(config_opts) - |> Keyword.merge(connection_opts) - |> add_scheme_opts(uri) - |> maybe_add_with_body() - |> Pleroma.HTTP.AdapterHelper.maybe_add_proxy(proxy) - end - - defp add_scheme_opts(opts, %URI{scheme: "https"}) do - Keyword.put(opts, :ssl_options, versions: [:"tlsv1.3", :"tlsv1.2", :"tlsv1.1", :tlsv1]) - end - - defp add_scheme_opts(opts, _), do: opts - - defp maybe_add_with_body(opts) do - if opts[:max_body] do - Keyword.put(opts, :with_body, true) - else - opts - end - end -end diff --git a/test/pleroma/http/adapter_helper/gun_test.exs b/test/pleroma/http/adapter_helper/gun_test.exs deleted file mode 100644 index cfb68557d..000000000 --- a/test/pleroma/http/adapter_helper/gun_test.exs +++ /dev/null @@ -1,77 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2021 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.HTTP.AdapterHelper.GunTest do - use ExUnit.Case - use Pleroma.Tests.Helpers - - import Mox - - alias Pleroma.HTTP.AdapterHelper.Gun - - setup :verify_on_exit! - - describe "options/1" do - setup do: clear_config([:http, :adapter], a: 1, b: 2) - - test "https url with default port" do - uri = URI.parse("https://example.com") - - opts = Gun.options([receive_conn: false], uri) - assert opts[:certificates_verification] - end - - test "https ipv4 with default port" do - uri = URI.parse("https://127.0.0.1") - - opts = Gun.options([receive_conn: false], uri) - assert opts[:certificates_verification] - end - - test "https ipv6 with default port" do - uri = URI.parse("https://[2a03:2880:f10c:83:face:b00c:0:25de]") - - opts = Gun.options([receive_conn: false], uri) - assert opts[:certificates_verification] - end - - test "https url with non standart port" do - uri = URI.parse("https://example.com:115") - - opts = Gun.options([receive_conn: false], uri) - - assert opts[:certificates_verification] - end - - test "merges with defaul http adapter config" do - defaults = Gun.options([receive_conn: false], URI.parse("https://example.com")) - assert Keyword.has_key?(defaults, :a) - assert Keyword.has_key?(defaults, :b) - end - - test "parses string proxy host & port" do - clear_config([:http, :proxy_url], "localhost:8123") - - uri = URI.parse("https://some-domain.com") - opts = Gun.options([receive_conn: false], uri) - assert opts[:proxy] == {'localhost', 8123} - end - - test "parses tuple proxy scheme host and port" do - clear_config([:http, :proxy_url], {:socks, 'localhost', 1234}) - - uri = URI.parse("https://some-domain.com") - opts = Gun.options([receive_conn: false], uri) - assert opts[:proxy] == {:socks, 'localhost', 1234} - end - - test "passed opts have more weight than defaults" do - clear_config([:http, :proxy_url], {:socks5, 'localhost', 1234}) - uri = URI.parse("https://some-domain.com") - opts = Gun.options([receive_conn: false, proxy: {'example.com', 4321}], uri) - - assert opts[:proxy] == {'example.com', 4321} - end - end -end diff --git a/test/pleroma/http/adapter_helper/hackney_test.exs b/test/pleroma/http/adapter_helper/hackney_test.exs deleted file mode 100644 index 85150a65c..000000000 --- a/test/pleroma/http/adapter_helper/hackney_test.exs +++ /dev/null @@ -1,35 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2021 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.HTTP.AdapterHelper.HackneyTest do - use ExUnit.Case, async: true - use Pleroma.Tests.Helpers - - alias Pleroma.HTTP.AdapterHelper.Hackney - - setup_all do - uri = URI.parse("http://domain.com") - {:ok, uri: uri} - end - - describe "options/2" do - setup do: clear_config([:http, :adapter], a: 1, b: 2) - - test "add proxy and opts from config", %{uri: uri} do - opts = Hackney.options([proxy: "localhost:8123"], uri) - - assert opts[:a] == 1 - assert opts[:b] == 2 - assert opts[:proxy] == "localhost:8123" - end - - test "respect connection opts and no proxy", %{uri: uri} do - opts = Hackney.options([a: 2, b: 1], uri) - - assert opts[:a] == 2 - assert opts[:b] == 1 - refute Keyword.has_key?(opts, :proxy) - end - end -end diff --git a/test/pleroma/http/adapter_helper_test.exs b/test/pleroma/http/adapter_helper_test.exs index 3c8c89164..55ffe4921 100644 --- a/test/pleroma/http/adapter_helper_test.exs +++ b/test/pleroma/http/adapter_helper_test.exs @@ -13,16 +13,38 @@ test "with nil" do end test "with string" do - assert AdapterHelper.format_proxy("127.0.0.1:8123") == {{127, 0, 0, 1}, 8123} + assert AdapterHelper.format_proxy("http://127.0.0.1:8123") == {:http, "127.0.0.1", 8123, []} end test "localhost with port" do - assert AdapterHelper.format_proxy("localhost:8123") == {'localhost', 8123} + assert AdapterHelper.format_proxy("https://localhost:8123") == + {:https, "localhost", 8123, []} end test "tuple" do - assert AdapterHelper.format_proxy({:socks4, :localhost, 9050}) == - {:socks4, 'localhost', 9050} + assert AdapterHelper.format_proxy({:http, "localhost", 9050}) == + {:http, "localhost", 9050, []} + end + end + + describe "maybe_add_proxy_pool/1" do + test "should do nothing with nil" do + assert AdapterHelper.maybe_add_proxy_pool([], nil) == [] + end + + test "should create pools" do + assert AdapterHelper.maybe_add_proxy_pool([], "proxy") == [ + pools: %{default: [conn_opts: [proxy: "proxy"]]} + ] + end + + test "should not override conn_opts if set" do + assert AdapterHelper.maybe_add_proxy_pool( + [pools: %{default: [conn_opts: [already: "set"]]}], + "proxy" + ) == [ + pools: %{default: [conn_opts: [proxy: "proxy", already: "set"]]} + ] end end end