Allow listing languages, setting source language (#192)

Co-authored-by: FloatingGhost <hannah@coffee-and-dreams.uk>
Reviewed-on: https://akkoma.dev/AkkomaGang/akkoma/pulls/192

lib/pleroma/akkoma/translators/deepl.ex

@@ -22,8 +22,27 @@ defp tier do
   end
 
   @impl Pleroma.Akkoma.Translator
-  def translate(string, to_language) do
+  def languages do
-    with {:ok, %{status: 200} = response} <- do_request(api_key(), tier(), string, to_language),
+    with {:ok, %{status: 200} = response} <- do_languages(),
+         {:ok, body} <- Jason.decode(response.body) do
+      resp =
+        Enum.map(body, fn %{"language" => code, "name" => name} -> %{code: code, name: name} end)
+
+      {:ok, resp}
+    else
+      {:ok, %{status: status} = response} ->
+        Logger.warning("DeepL: Request rejected: #{inspect(response)}")
+        {:error, "DeepL request failed (code #{status})"}
+
+      {:error, reason} ->
+        {:error, reason}
+    end
+  end
+
+  @impl Pleroma.Akkoma.Translator
+  def translate(string, from_language, to_language) do
+    with {:ok, %{status: 200} = response} <-
+           do_request(api_key(), tier(), string, from_language, to_language),
          {:ok, body} <- Jason.decode(response.body) do
       %{"translations" => [%{"text" => translated, "detected_source_language" => detected}]} =
         body
@@ -39,14 +58,16 @@ def translate(string, to_language) do
     end
   end
 
-  defp do_request(api_key, tier, string, to_language) do
+  defp do_request(api_key, tier, string, from_language, to_language) do
     HTTP.post(
       base_url(tier) <> "translate",
       URI.encode_query(
         %{
           text: string,
-          target_lang: to_language
+          target_lang: to_language,
-        },
+          tag_handling: "html"
+        }
+        |> maybe_add_source(from_language),
         :rfc3986
       ),
       [
@@ -55,4 +76,16 @@ defp do_request(api_key, tier, string, to_language) do
       ]
     )
   end
+
+  defp maybe_add_source(opts, nil), do: opts
+  defp maybe_add_source(opts, lang), do: Map.put(opts, :source_lang, lang)
+
+  defp do_languages() do
+    HTTP.get(
+      base_url(tier()) <> "languages?type=target",
+      [
+        {"authorization", "DeepL-Auth-Key #{api_key()}"}
+      ]
+    )
+  end
 end

lib/pleroma/akkoma/translators/libre_translate.ex

@@ -14,10 +14,33 @@ defp url do
   end
 
   @impl Pleroma.Akkoma.Translator
-  def translate(string, to_language) do
+  def languages do
-    with {:ok, %{status: 200} = response} <- do_request(string, to_language),
+    with {:ok, %{status: 200} = response} <- do_languages(),
          {:ok, body} <- Jason.decode(response.body) do
-      %{"translatedText" => translated, "detectedLanguage" => %{"language" => detected}} = body
+      resp = Enum.map(body, fn %{"code" => code, "name" => name} -> %{code: code, name: name} end)
+      {:ok, resp}
+    else
+      {:ok, %{status: status} = response} ->
+        Logger.warning("LibreTranslate: Request rejected: #{inspect(response)}")
+        {:error, "LibreTranslate request failed (code #{status})"}
+
+      {:error, reason} ->
+        {:error, reason}
+    end
+  end
+
+  @impl Pleroma.Akkoma.Translator
+  def translate(string, from_language, to_language) do
+    with {:ok, %{status: 200} = response} <- do_request(string, from_language, to_language),
+         {:ok, body} <- Jason.decode(response.body) do
+      %{"translatedText" => translated} = body
+
+      detected =
+        if Map.has_key?(body, "detectedLanguage") do
+          get_in(body, ["detectedLanguage", "language"])
+        else
+          from_language
+        end
 
       {:ok, detected, translated}
     else
@@ -30,7 +53,7 @@ def translate(string, to_language) do
     end
   end
 
-  defp do_request(string, to_language) do
+  defp do_request(string, from_language, to_language) do
     url = URI.parse(url())
     url = %{url | path: "/translate"}
 
@@ -38,7 +61,7 @@ defp do_request(string, to_language) do
       to_string(url),
       Jason.encode!(%{
         q: string,
-        source: "auto",
+        source: if(is_nil(from_language), do: "auto", else: from_language),
         target: to_language,
         format: "html",
         api_key: api_key()
@@ -48,4 +71,11 @@ defp do_request(string, to_language) do
       ]
     )
   end
+
+  defp do_languages() do
+    url = URI.parse(url())
+    url = %{url | path: "/languages"}
+
+    HTTP.get(to_string(url))
+  end
 end

lib/pleroma/akkoma/translators/translator.ex

@@ -1,3 +1,5 @@
 defmodule Pleroma.Akkoma.Translator do
-  @callback translate(String.t(), String.t()) :: {:ok, String.t(), String.t()} | {:error, any()}
+  @callback translate(String.t(), String.t() | nil, String.t()) ::
+              {:ok, String.t(), String.t()} | {:error, any()}
+  @callback languages() :: {:ok, [%{name: String.t(), code: String.t()}]} | {:error, any()}
 end

lib/pleroma/web/akkoma_api/controllers/translation_controller.ex

@@ -0,0 +1,43 @@
+defmodule Pleroma.Web.AkkomaAPI.TranslationController do
+  use Pleroma.Web, :controller
+
+  alias Pleroma.Web.Plugs.OAuthScopesPlug
+
+  @cachex Pleroma.Config.get([:cachex, :provider], Cachex)
+
+  @unauthenticated_access %{fallback: :proceed_unauthenticated, scopes: []}
+  plug(
+    OAuthScopesPlug,
+    %{@unauthenticated_access | scopes: ["read:statuses"]}
+    when action in [
+           :languages
+         ]
+  )
+
+  plug(Pleroma.Web.ApiSpec.CastAndValidate)
+  defdelegate open_api_operation(action), to: Pleroma.Web.ApiSpec.TranslationOperation
+
+  action_fallback(Pleroma.Web.MastodonAPI.FallbackController)
+
+  @doc "GET /api/v1/akkoma/translation/languages"
+  def languages(conn, _params) do
+    with {:ok, languages} <- get_languages() do
+      conn
+      |> json(languages)
+    else
+      e -> IO.inspect(e)
+    end
+  end
+
+  defp get_languages do
+    module = Pleroma.Config.get([:translator, :module])
+
+    @cachex.fetch!(:translations_cache, "languages:#{module}}", fn _ ->
+      with {:ok, languages} <- module.languages() do
+        {:ok, languages}
+      else
+        {:error, err} -> {:ignore, {:error, err}}
+      end
+    end)
+  end
+end

lib/pleroma/web/api_spec/operations/status_operation.ex

@@ -413,7 +413,7 @@ def translate_operation do
       description: "View the translation of a given status",
       operationId: "StatusController.translation",
       security: [%{"oAuth" => ["read:statuses"]}],
-      parameters: [id_param(), language_param()],
+      parameters: [id_param(), language_param(), source_language_param()],
       responses: %{
         200 => Operation.response("Translation", "application/json", translation()),
         400 => Operation.response("Error", "application/json", ApiError),
@@ -572,6 +572,10 @@ defp language_param do
     Operation.parameter(:language, :path, :string, "ISO 639 language code", example: "en")
   end
 
+  defp source_language_param do
+    Operation.parameter(:from, :query, :string, "ISO 639 language code", example: "en")
+  end
+
   defp status_response do
     Operation.response("Status", "application/json", Status)
   end

lib/pleroma/web/api_spec/operations/translate_operation.ex

@@ -0,0 +1,41 @@
+defmodule Pleroma.Web.ApiSpec.TranslationOperation do
+  alias OpenApiSpex.Operation
+  alias OpenApiSpex.Schema
+
+  @spec open_api_operation(atom) :: Operation.t()
+  def open_api_operation(action) do
+    operation = String.to_existing_atom("#{action}_operation")
+    apply(__MODULE__, operation, [])
+  end
+
+  @spec languages_operation() :: Operation.t()
+  def languages_operation() do
+    %Operation{
+      tags: ["Retrieve status translation"],
+      summary: "Translate status",
+      description: "View the translation of a given status",
+      operationId: "AkkomaAPI.TranslationController.languages",
+      security: [%{"oAuth" => ["read:statuses"]}],
+      responses: %{
+        200 => Operation.response("Translation", "application/json", languages_schema())
+      }
+    }
+  end
+
+  defp languages_schema do
+    %Schema{
+      type: "array",
+      items: %Schema{
+        type: "object",
+        properties: %{
+          code: %Schema{
+            type: "string"
+          },
+          name: %Schema{
+            type: "string"
+          }
+        }
+      }
+    }
+  end
+end

lib/pleroma/web/mastodon_api/controllers/status_controller.ex

@@ -422,7 +422,7 @@ def bookmarks(%{assigns: %{user: user}} = conn, params) do
   end
 
   @doc "GET /api/v1/statuses/:id/translations/:language"
-  def translate(%{assigns: %{user: user}} = conn, %{id: id, language: language}) do
+  def translate(%{assigns: %{user: user}} = conn, %{id: id, language: language} = params) do
     with {:enabled, true} <- {:enabled, Config.get([:translator, :enabled])},
          %Activity{} = activity <- Activity.get_by_id_with_object(id),
          {:visible, true} <- {:visible, Visibility.visible_for_user?(activity, user)},
@@ -431,6 +431,7 @@ def translate(%{assigns: %{user: user}} = conn, %{id: id, language: language}) d
            fetch_or_translate(
              activity.id,
              activity.object.data["content"],
+             Map.get(params, :from, nil),
              language,
              translation_module
            ) do
@@ -449,16 +450,20 @@ def translate(%{assigns: %{user: user}} = conn, %{id: id, language: language}) d
     end
   end
 
-  defp fetch_or_translate(status_id, text, language, translation_module) do
+  defp fetch_or_translate(status_id, text, source_language, target_language, translation_module) do
-    @cachex.fetch!(:user_cache, "translations:#{status_id}:#{language}", fn _ ->
+    @cachex.fetch!(
-      value = translation_module.translate(text, language)
+      :translations_cache,
+      "translations:#{status_id}:#{source_language}:#{target_language}",
+      fn _ ->
+        value = translation_module.translate(text, source_language, target_language)
 
-      with {:ok, _, _} <- value do
+        with {:ok, _, _} <- value do
-        value
+          value
-      else
+        else
-        _ -> {:ignore, value}
+          _ -> {:ignore, value}
+        end
       end
-    end)
+    )
   end
 
   defp put_application(params, %{assigns: %{token: %Token{user: %User{} = user} = token}} = _conn) do

lib/pleroma/web/router.ex

@@ -462,6 +462,11 @@ defmodule Pleroma.Web.Router do
     put("/statuses/:id/emoji_reactions/:emoji", EmojiReactionController, :create)
   end
 
+  scope "/api/v1/akkoma", Pleroma.Web.AkkomaAPI do
+    pipe_through(:authenticated_api)
+    get("/translation/languages", TranslationController, :languages)
+  end
+
   scope "/api/v1", Pleroma.Web.MastodonAPI do
     pipe_through(:authenticated_api)
 

test/pleroma/translators/deepl_test.exs

@@ -8,6 +8,36 @@ defmodule Pleroma.Akkoma.Translators.DeepLTest do
       clear_config([:deepl, :api_key], "deepl_api_key")
     end
 
+    test "should list supported languages" do
+      clear_config([:deepl, :tier], :free)
+
+      Tesla.Mock.mock(fn
+        %{method: :get, url: "https://api-free.deepl.com/v2/languages?type=target"} = env ->
+          auth_header = Enum.find(env.headers, fn {k, _v} -> k == "authorization" end)
+          assert {"authorization", "DeepL-Auth-Key deepl_api_key"} = auth_header
+
+          %Tesla.Env{
+            status: 200,
+            body:
+              Jason.encode!([
+                %{
+                  "language" => "BG",
+                  "name" => "Bulgarian",
+                  "supports_formality" => false
+                },
+                %{
+                  "language" => "CS",
+                  "name" => "Czech",
+                  "supports_formality" => false
+                }
+              ])
+          }
+      end)
+
+      assert {:ok, [%{code: "BG", name: "Bulgarian"}, %{code: "CS", name: "Czech"}]} =
+               DeepL.languages()
+    end
+
     test "should work with the free tier" do
       clear_config([:deepl, :tier], :free)
 
@@ -30,7 +60,7 @@ test "should work with the free tier" do
           }
       end)
 
-      assert {:ok, "ja", "I will crush you"} = DeepL.translate("ギュギュ握りつぶしちゃうぞ", "en")
+      assert {:ok, "ja", "I will crush you"} = DeepL.translate("ギュギュ握りつぶしちゃうぞ", nil, "en")
     end
 
     test "should work with the pro tier" do
@@ -55,7 +85,33 @@ test "should work with the pro tier" do
           }
       end)
 
-      assert {:ok, "ja", "I will crush you"} = DeepL.translate("ギュギュ握りつぶしちゃうぞ", "en")
+      assert {:ok, "ja", "I will crush you"} = DeepL.translate("ギュギュ握りつぶしちゃうぞ", nil, "en")
+    end
+
+    test "should assign source language if set" do
+      clear_config([:deepl, :tier], :pro)
+
+      Tesla.Mock.mock(fn
+        %{method: :post, url: "https://api.deepl.com/v2/translate"} = env ->
+          auth_header = Enum.find(env.headers, fn {k, _v} -> k == "authorization" end)
+          assert {"authorization", "DeepL-Auth-Key deepl_api_key"} = auth_header
+          assert String.contains?(env.body, "source_lang=ja")
+
+          %Tesla.Env{
+            status: 200,
+            body:
+              Jason.encode!(%{
+                translations: [
+                  %{
+                    "text" => "I will crush you",
+                    "detected_source_language" => "ja"
+                  }
+                ]
+              })
+          }
+      end)
+
+      assert {:ok, "ja", "I will crush you"} = DeepL.translate("ギュギュ握りつぶしちゃうぞ", "ja", "en")
     end
 
     test "should gracefully fail if the API errors" do
@@ -69,7 +125,8 @@ test "should gracefully fail if the API errors" do
           }
       end)
 
-      assert {:error, "DeepL request failed (code 403)"} = DeepL.translate("ギュギュ握りつぶしちゃうぞ", "en")
+      assert {:error, "DeepL request failed (code 403)"} =
+               DeepL.translate("ギュギュ握りつぶしちゃうぞ", nil, "en")
     end
   end
 end

test/pleroma/translators/libre_translate_test.exs

@@ -8,10 +8,35 @@ defmodule Pleroma.Akkoma.Translators.LibreTranslateTest do
       clear_config([:libre_translate, :url], "http://libre.translate/translate")
     end
 
+    test "should list supported languages" do
+      clear_config([:deepl, :tier], :free)
+
+      Tesla.Mock.mock(fn
+        %{method: :get, url: "http://libre.translate/languages"} = _ ->
+          %Tesla.Env{
+            status: 200,
+            body:
+              Jason.encode!([
+                %{
+                  "code" => "en",
+                  "name" => "English"
+                },
+                %{
+                  "code" => "ar",
+                  "name" => "Arabic"
+                }
+              ])
+          }
+      end)
+
+      assert {:ok, [%{code: "en", name: "English"}, %{code: "ar", name: "Arabic"}]} =
+               LibreTranslate.languages()
+    end
+
     test "should work without an API key" do
       Tesla.Mock.mock(fn
         %{method: :post, url: "http://libre.translate/translate"} = env ->
-          assert {:ok, %{"api_key" => nil}} = Jason.decode(env.body)
+          assert {:ok, %{"api_key" => nil, "source" => "auto"}} = Jason.decode(env.body)
 
           %Tesla.Env{
             status: 200,
@@ -26,7 +51,8 @@ test "should work without an API key" do
           }
       end)
 
-      assert {:ok, "ja", "I will crush you"} = LibreTranslate.translate("ギュギュ握りつぶしちゃうぞ", "en")
+      assert {:ok, "ja", "I will crush you"} =
+               LibreTranslate.translate("ギュギュ握りつぶしちゃうぞ", nil, "en")
     end
 
     test "should work with an API key" do
@@ -49,7 +75,8 @@ test "should work with an API key" do
           }
       end)
 
-      assert {:ok, "ja", "I will crush you"} = LibreTranslate.translate("ギュギュ握りつぶしちゃうぞ", "en")
+      assert {:ok, "ja", "I will crush you"} =
+               LibreTranslate.translate("ギュギュ握りつぶしちゃうぞ", nil, "en")
     end
 
     test "should gracefully handle API key errors" do
@@ -67,7 +94,25 @@ test "should gracefully handle API key errors" do
       end)
 
       assert {:error, "libre_translate: request failed (code 403)"} =
-               LibreTranslate.translate("ギュギュ握りつぶしちゃうぞ", "en")
+               LibreTranslate.translate("ギュギュ握りつぶしちゃうぞ", nil, "en")
+    end
+
+    test "should set a source language if requested" do
+      Tesla.Mock.mock(fn
+        %{method: :post, url: "http://libre.translate/translate"} = env ->
+          assert {:ok, %{"api_key" => nil, "source" => "ja"}} = Jason.decode(env.body)
+
+          %Tesla.Env{
+            status: 200,
+            body:
+              Jason.encode!(%{
+                translatedText: "I will crush you"
+              })
+          }
+      end)
+
+      assert {:ok, "ja", "I will crush you"} =
+               LibreTranslate.translate("ギュギュ握りつぶしちゃうぞ", "ja", "en")
     end
 
     test "should gracefully handle an unsupported language" do
@@ -85,7 +130,7 @@ test "should gracefully handle an unsupported language" do
       end)
 
       assert {:error, "libre_translate: request failed (code 400)"} =
-               LibreTranslate.translate("ギュギュ握りつぶしちゃうぞ", "zoop")
+               LibreTranslate.translate("ギュギュ握りつぶしちゃうぞ", nil, "zoop")
     end
   end
 end