From 6e1ec4c5da6da4bd301080a8c35f8483d89e095f Mon Sep 17 00:00:00 2001 From: lain Date: Mon, 26 Aug 2019 16:29:51 -0500 Subject: [PATCH 001/107] ActivityPub: Basic EmojiReactions. --- lib/pleroma/web/activity_pub/activity_pub.ex | 10 +++++++++ lib/pleroma/web/activity_pub/utils.ex | 10 +++++++++ test/web/activity_pub/activity_pub_test.exs | 22 ++++++++++++++++++++ 3 files changed, 42 insertions(+) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 172c952d4..a6088a012 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -312,6 +312,16 @@ def update(%{to: to, cc: cc, actor: actor, object: object} = params) do end end + def react_with_emoji(user, object, emoji, options \\ []) do + with local <- Keyword.get(options, :local, true), + activity_id <- Keyword.get(options, :activity_id, nil), + is_emoji?(emoji), + reaction_data <- make_emoji_reaction_data(user, object, emoji, activity_id), + {:ok, activity} <- insert(reaction_data, local) do + {:ok, activity, object} + end + end + # TODO: This is weird, maybe we shouldn't check here if we can make the activity. def like( %User{ap_id: ap_id} = user, diff --git a/lib/pleroma/web/activity_pub/utils.ex b/lib/pleroma/web/activity_pub/utils.ex index 1c3058658..d57830da3 100644 --- a/lib/pleroma/web/activity_pub/utils.ex +++ b/lib/pleroma/web/activity_pub/utils.ex @@ -296,6 +296,16 @@ def get_object_likes(%{data: %{"id" => id}}) do Repo.all(query) end + def is_emoji?(emoji) do + String.length(emoji) == 1 + end + + def make_emoji_reaction_data(user, object, emoji, activity_id) do + make_like_data(user, object, activity_id) + |> Map.put("type", "EmojiReaction") + |> Map.put("content", emoji) + end + def make_like_data( %User{ap_id: ap_id} = actor, %{data: %{"actor" => object_actor_id, "id" => id}} = object, diff --git a/test/web/activity_pub/activity_pub_test.exs b/test/web/activity_pub/activity_pub_test.exs index 1515f4eb6..cf93f624a 100644 --- a/test/web/activity_pub/activity_pub_test.exs +++ b/test/web/activity_pub/activity_pub_test.exs @@ -675,6 +675,28 @@ test "returns reblogs for users for whom reblogs have not been muted" do end end + describe "react to an object" do + test "adds an emoji reaction activity to the db" do + user = insert(:user) + reactor = insert(:user) + {:ok, activity} = CommonAPI.post(user, %{"status" => "YASSSS queen slay"}) + assert object = Object.normalize(activity) + + {:ok, reaction_activity, object} = ActivityPub.react_with_emoji(reactor, object, "๐Ÿ”ฅ") + + assert reaction_activity + + assert reaction_activity.data["actor"] == reactor.ap_id + assert reaction_activity.data["type"] == "EmojiReaction" + assert reaction_activity.data["content"] == "๐Ÿ”ฅ" + assert reaction_activity.data["object"] == object.data["id"] + assert reaction_activity.data["to"] == [User.ap_followers(reactor), activity.data["actor"]] + assert reaction_activity.data["context"] == object.data["context"] + # assert object.data["reaction_count"] == 1 + # assert object.data["reactions"] == [user.ap_id] + end + end + describe "like an object" do test "adds a like activity to the db" do note_activity = insert(:note_activity) From a0b21c89284304bea90f2774f17d5b2b7b3c1359 Mon Sep 17 00:00:00 2001 From: lain Date: Mon, 26 Aug 2019 16:47:31 -0500 Subject: [PATCH 002/107] Transmogrifier: Handle incoming emoji reactions. --- .../web/activity_pub/transmogrifier.ex | 21 +++++++++++++ test/fixtures/emoji-reaction.json | 30 +++++++++++++++++++ test/web/activity_pub/transmogrifier_test.exs | 18 +++++++++++ 3 files changed, 69 insertions(+) create mode 100644 test/fixtures/emoji-reaction.json diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index 36340a3a1..9132df8cb 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -572,6 +572,27 @@ def handle_incoming( end end + def handle_incoming( + %{ + "type" => "EmojiReaction", + "object" => object_id, + "actor" => _actor, + "id" => id, + "content" => emoji + } = data, + _options + ) do + with actor <- Containment.get_actor(data), + {:ok, %User{} = actor} <- User.get_or_fetch_by_ap_id(actor), + {:ok, object} <- get_obj_helper(object_id), + {:ok, activity, _object} <- + ActivityPub.react_with_emoji(actor, object, emoji, activity_id: id, local: false) do + {:ok, activity} + else + _e -> :error + end + end + def handle_incoming( %{"type" => "Announce", "object" => object_id, "actor" => _actor, "id" => id} = data, _options diff --git a/test/fixtures/emoji-reaction.json b/test/fixtures/emoji-reaction.json new file mode 100644 index 000000000..3812e43ad --- /dev/null +++ b/test/fixtures/emoji-reaction.json @@ -0,0 +1,30 @@ +{ + "type": "EmojiReaction", + "signature": { + "type": "RsaSignature2017", + "signatureValue": "fdxMfQSMwbC6wP6sh6neS/vM5879K67yQkHTbiT5Npr5wAac0y6+o3Ij+41tN3rL6wfuGTosSBTHOtta6R4GCOOhCaCSLMZKypnp1VltCzLDoyrZELnYQIC8gpUXVmIycZbREk22qWUe/w7DAFaKK4UscBlHDzeDVcA0K3Se5Sluqi9/Zh+ldAnEzj/rSEPDjrtvf5wGNf3fHxbKSRKFt90JvKK6hS+vxKUhlRFDf6/SMETw+EhwJSNW4d10yMUakqUWsFv4Acq5LW7l+HpYMvlYY1FZhNde1+uonnCyuQDyvzkff8zwtEJmAXC4RivO/VVLa17SmqheJZfI8oluVg==", + "creator": "http://mastodon.example.org/users/admin#main-key", + "created": "2018-02-17T18:57:49Z" + }, + "object": "http://localtesting.pleroma.lol/objects/eb92579d-3417-42a8-8652-2492c2d4f454", + "content": "๐Ÿ‘Œ", + "nickname": "lain", + "id": "http://mastodon.example.org/users/admin#reactions/2", + "actor": "http://mastodon.example.org/users/admin", + "@context": [ + "https://www.w3.org/ns/activitystreams", + "https://w3id.org/security/v1", + { + "toot": "http://joinmastodon.org/ns#", + "sensitive": "as:sensitive", + "ostatus": "http://ostatus.org#", + "movedTo": "as:movedTo", + "manuallyApprovesFollowers": "as:manuallyApprovesFollowers", + "inReplyToAtomUri": "ostatus:inReplyToAtomUri", + "conversation": "ostatus:conversation", + "atomUri": "ostatus:atomUri", + "Hashtag": "as:Hashtag", + "Emoji": "toot:Emoji" + } + ] +} diff --git a/test/web/activity_pub/transmogrifier_test.exs b/test/web/activity_pub/transmogrifier_test.exs index 0661d5d7c..6df707370 100644 --- a/test/web/activity_pub/transmogrifier_test.exs +++ b/test/web/activity_pub/transmogrifier_test.exs @@ -311,6 +311,24 @@ test "it works for incoming likes" do assert data["object"] == activity.data["object"] end + test "it works for incoming emoji reactions" do + user = insert(:user) + {:ok, activity} = CommonAPI.post(user, %{"status" => "hello"}) + + data = + File.read!("test/fixtures/emoji-reaction.json") + |> Poison.decode!() + |> Map.put("object", activity.data["object"]) + + {:ok, %Activity{data: data, local: false}} = Transmogrifier.handle_incoming(data) + + assert data["actor"] == "http://mastodon.example.org/users/admin" + assert data["type"] == "EmojiReaction" + assert data["id"] == "http://mastodon.example.org/users/admin#reactions/2" + assert data["object"] == activity.data["object"] + assert data["content"] == "๐Ÿ‘Œ" + end + test "it returns an error for incoming unlikes wihout a like activity" do user = insert(:user) {:ok, activity} = CommonAPI.post(user, %{"status" => "leave a like pls"}) From b770ed1d9940230d4bd97113abdc220ca7d8eb1a Mon Sep 17 00:00:00 2001 From: lain Date: Tue, 27 Aug 2019 17:56:28 -0500 Subject: [PATCH 003/107] CommonAPI: Support emoji reactions. --- lib/pleroma/web/common_api/common_api.ex | 10 ++++++++++ test/web/common_api/common_api_test.exs | 14 ++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/lib/pleroma/web/common_api/common_api.ex b/lib/pleroma/web/common_api/common_api.ex index 5faddc9f4..3e1aa4818 100644 --- a/lib/pleroma/web/common_api/common_api.ex +++ b/lib/pleroma/web/common_api/common_api.ex @@ -123,6 +123,16 @@ def unfavorite(id_or_ap_id, user) do end end + def react_with_emoji(id, user, emoji) do + with %Activity{} = activity <- Activity.get_by_id(id), + object <- Object.normalize(activity) do + ActivityPub.react_with_emoji(user, object, emoji) + else + _ -> + {:error, dgettext("errors", "Could not add reaction emoji")} + end + end + def vote(user, object, choices) do with "Question" <- object.data["type"], {:author, false} <- {:author, object.data["actor"] == user.ap_id}, diff --git a/test/web/common_api/common_api_test.exs b/test/web/common_api/common_api_test.exs index f28a66090..7cb1202fc 100644 --- a/test/web/common_api/common_api_test.exs +++ b/test/web/common_api/common_api_test.exs @@ -222,6 +222,20 @@ test "it can handle activities that expire" do end describe "reactions" do + test "reacting to a status with an emoji" do + user = insert(:user) + other_user = insert(:user) + + {:ok, activity} = CommonAPI.post(other_user, %{"status" => "cofe"}) + + {:ok, reaction, _} = CommonAPI.react_with_emoji(activity.id, user, "๐Ÿ‘") + + assert reaction.data["actor"] == user.ap_id + assert reaction.data["content"] == "๐Ÿ‘" + + # TODO: test error case. + end + test "repeating a status" do user = insert(:user) other_user = insert(:user) From 9bc12b88b3b4b90ee1b55ebf49a13665a751ef1a Mon Sep 17 00:00:00 2001 From: lain Date: Tue, 3 Sep 2019 16:50:04 -0500 Subject: [PATCH 004/107] ActivityPub: Save emoji reactions in object. --- lib/pleroma/web/activity_pub/activity_pub.ex | 3 ++- lib/pleroma/web/activity_pub/utils.ex | 27 +++++++++++++++++++- test/web/activity_pub/activity_pub_test.exs | 4 +-- 3 files changed, 30 insertions(+), 4 deletions(-) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 532db17c4..6cd168427 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -317,7 +317,8 @@ def react_with_emoji(user, object, emoji, options \\ []) do activity_id <- Keyword.get(options, :activity_id, nil), is_emoji?(emoji), reaction_data <- make_emoji_reaction_data(user, object, emoji, activity_id), - {:ok, activity} <- insert(reaction_data, local) do + {:ok, activity} <- insert(reaction_data, local), + {:ok, object} <- add_emoji_reaction_to_object(activity, object) do {:ok, activity, object} end end diff --git a/lib/pleroma/web/activity_pub/utils.ex b/lib/pleroma/web/activity_pub/utils.ex index 32e22d8d7..1e6a67deb 100644 --- a/lib/pleroma/web/activity_pub/utils.ex +++ b/lib/pleroma/web/activity_pub/utils.ex @@ -321,10 +321,21 @@ def make_like_data( @spec update_element_in_object(String.t(), list(any), Object.t()) :: {:ok, Object.t()} | {:error, Ecto.Changeset.t()} def update_element_in_object(property, element, object) do + length = + if is_map(element) do + element + |> Map.values() + |> List.flatten() + |> length() + else + element + |> length() + end + data = Map.merge( object.data, - %{"#{property}_count" => length(element), "#{property}s" => element} + %{"#{property}_count" => length, "#{property}s" => element} ) object @@ -332,6 +343,20 @@ def update_element_in_object(property, element, object) do |> Object.update_and_set_cache() end + @spec add_emoji_reaction_to_object(Activity.t(), Object.t()) :: + {:ok, Object.t()} | {:error, Ecto.Changeset.t()} + + def add_emoji_reaction_to_object( + %Activity{data: %{"content" => emoji, "actor" => actor}}, + object + ) do + reactions = object.data["reactions"] || %{} + emoji_actors = reactions[emoji] || [] + new_emoji_actors = [actor | emoji_actors] |> Enum.uniq() + new_reactions = Map.put(reactions, emoji, new_emoji_actors) + update_element_in_object("reaction", new_reactions, object) + end + @spec add_like_to_object(Activity.t(), Object.t()) :: {:ok, Object.t()} | {:error, Ecto.Changeset.t()} def add_like_to_object(%Activity{data: %{"actor" => actor}}, object) do diff --git a/test/web/activity_pub/activity_pub_test.exs b/test/web/activity_pub/activity_pub_test.exs index b168e85a0..67dfb9394 100644 --- a/test/web/activity_pub/activity_pub_test.exs +++ b/test/web/activity_pub/activity_pub_test.exs @@ -694,8 +694,8 @@ test "adds an emoji reaction activity to the db" do assert reaction_activity.data["object"] == object.data["id"] assert reaction_activity.data["to"] == [User.ap_followers(reactor), activity.data["actor"]] assert reaction_activity.data["context"] == object.data["context"] - # assert object.data["reaction_count"] == 1 - # assert object.data["reactions"] == [user.ap_id] + assert object.data["reaction_count"] == 1 + assert object.data["reactions"]["๐Ÿ”ฅ"] == [reactor.ap_id] end end From 99ea990a16a417cd316b3464ef380746171ecb55 Mon Sep 17 00:00:00 2001 From: lain Date: Wed, 4 Sep 2019 12:20:35 -0500 Subject: [PATCH 005/107] PleromaAPIController: Add emoji reactions. --- .../web/pleroma_api/pleroma_api_controller.ex | 11 +++++++++++ lib/pleroma/web/router.ex | 1 + .../pleroma_api/pleroma_api_controller_test.exs | 15 +++++++++++++++ 3 files changed, 27 insertions(+) diff --git a/lib/pleroma/web/pleroma_api/pleroma_api_controller.ex b/lib/pleroma/web/pleroma_api/pleroma_api_controller.ex index b6d2bf86b..740ea4747 100644 --- a/lib/pleroma/web/pleroma_api/pleroma_api_controller.ex +++ b/lib/pleroma/web/pleroma_api/pleroma_api_controller.ex @@ -7,11 +7,22 @@ defmodule Pleroma.Web.PleromaAPI.PleromaAPIController do import Pleroma.Web.ControllerHelper, only: [add_link_headers: 7] + alias Pleroma.Activity alias Pleroma.Conversation.Participation alias Pleroma.Web.ActivityPub.ActivityPub + alias Pleroma.Web.CommonAPI alias Pleroma.Web.MastodonAPI.ConversationView alias Pleroma.Web.MastodonAPI.StatusView + def react_with_emoji(%{assigns: %{user: user}} = conn, %{"id" => activity_id, "emoji" => emoji}) do + with {:ok, _activity, _object} <- CommonAPI.react_with_emoji(activity_id, user, emoji), + activity = Activity.get_by_id(activity_id) do + conn + |> put_view(StatusView) + |> render("status.json", %{activity: activity, for: user, as: :activity}) + end + end + def conversation(%{assigns: %{user: user}} = conn, %{"id" => participation_id}) do with %Participation{} = participation <- Participation.get(participation_id), true <- user.id == participation.user_id do diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 969dc66fd..6cca54211 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -277,6 +277,7 @@ defmodule Pleroma.Web.Router do scope [] do pipe_through(:oauth_write) patch("/conversations/:id", PleromaAPIController, :update_conversation) + post("/statuses/:id/react_with_emoji", PleromaAPIController, :react_with_emoji) end end diff --git a/test/web/pleroma_api/pleroma_api_controller_test.exs b/test/web/pleroma_api/pleroma_api_controller_test.exs index ed6b79727..aab0c774e 100644 --- a/test/web/pleroma_api/pleroma_api_controller_test.exs +++ b/test/web/pleroma_api/pleroma_api_controller_test.exs @@ -11,6 +11,21 @@ defmodule Pleroma.Web.PleromaAPI.PleromaAPIControllerTest do import Pleroma.Factory + test "POST /api/v1/pleroma/statuses/:id/react_with_emoji", %{conn: conn} do + user = insert(:user) + other_user = insert(:user) + + {:ok, activity} = CommonAPI.post(user, %{"status" => "#cofe"}) + + result = + conn + |> assign(:user, other_user) + |> post("/api/v1/pleroma/statuses/#{activity.id}/react_with_emoji", %{"emoji" => "โ˜•"}) + + assert %{"id" => id} = json_response(result, 200) + assert to_string(activity.id) == id + end + test "/api/v1/pleroma/conversations/:id", %{conn: conn} do user = insert(:user) other_user = insert(:user) From 05e9776517498370ab8f7b7afa0408f6ee979844 Mon Sep 17 00:00:00 2001 From: lain Date: Thu, 12 Sep 2019 18:48:25 +0200 Subject: [PATCH 006/107] PleromaAPIController: Add endpoint to fetch emoji reactions. --- .../web/pleroma_api/pleroma_api_controller.ex | 28 +++++++++++++++++++ lib/pleroma/web/router.ex | 6 ++++ .../pleroma_api_controller_test.exs | 25 +++++++++++++++++ 3 files changed, 59 insertions(+) diff --git a/lib/pleroma/web/pleroma_api/pleroma_api_controller.ex b/lib/pleroma/web/pleroma_api/pleroma_api_controller.ex index 740ea4747..bb090d37f 100644 --- a/lib/pleroma/web/pleroma_api/pleroma_api_controller.ex +++ b/lib/pleroma/web/pleroma_api/pleroma_api_controller.ex @@ -8,12 +8,40 @@ defmodule Pleroma.Web.PleromaAPI.PleromaAPIController do import Pleroma.Web.ControllerHelper, only: [add_link_headers: 7] alias Pleroma.Activity + alias Pleroma.Object + alias Pleroma.User alias Pleroma.Conversation.Participation alias Pleroma.Web.ActivityPub.ActivityPub alias Pleroma.Web.CommonAPI + alias Pleroma.Web.MastodonAPI.AccountView alias Pleroma.Web.MastodonAPI.ConversationView alias Pleroma.Web.MastodonAPI.StatusView + def emoji_reactions_by(%{assigns: %{user: user}} = conn, %{"id" => activity_id}) do + with %Activity{} = activity <- Activity.get_by_id_with_object(activity_id), + %Object{data: %{"reactions" => emoji_reactions}} <- Object.normalize(activity) do + reactions = + Enum.reduce(emoji_reactions, %{}, fn {emoji, users}, res -> + users = + users + |> Enum.map(&User.get_cached_by_ap_id/1) + + res + |> Map.put( + emoji, + AccountView.render("accounts.json", %{users: users, for: user, as: :user}) + ) + end) + + conn + |> json(reactions) + else + _e -> + conn + |> json(%{}) + end + end + def react_with_emoji(%{assigns: %{user: user}} = conn, %{"id" => activity_id, "emoji" => emoji}) do with {:ok, _activity, _object} <- CommonAPI.react_with_emoji(activity_id, user, emoji), activity = Activity.get_by_id(activity_id) do diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 6cca54211..ec6179420 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -265,6 +265,12 @@ defmodule Pleroma.Web.Router do end end + scope "/api/v1/pleroma", Pleroma.Web.PleromaAPI do + pipe_through(:api) + + get("/statuses/:id/emoji_reactions_by", PleromaAPIController, :emoji_reactions_by) + end + scope "/api/v1/pleroma", Pleroma.Web.PleromaAPI do pipe_through(:authenticated_api) diff --git a/test/web/pleroma_api/pleroma_api_controller_test.exs b/test/web/pleroma_api/pleroma_api_controller_test.exs index aab0c774e..71e4d3e1c 100644 --- a/test/web/pleroma_api/pleroma_api_controller_test.exs +++ b/test/web/pleroma_api/pleroma_api_controller_test.exs @@ -8,6 +8,7 @@ defmodule Pleroma.Web.PleromaAPI.PleromaAPIControllerTest do alias Pleroma.Conversation.Participation alias Pleroma.Repo alias Pleroma.Web.CommonAPI + alias Pleroma.Web.MastodonAPI.AccountView import Pleroma.Factory @@ -26,6 +27,30 @@ test "POST /api/v1/pleroma/statuses/:id/react_with_emoji", %{conn: conn} do assert to_string(activity.id) == id end + test "GET /api/v1/pleroma/statuses/:id/emoji_reactions_by", %{conn: conn} do + user = insert(:user) + other_user = insert(:user) + + {:ok, activity} = CommonAPI.post(user, %{"status" => "#cofe"}) + + result = + conn + |> get("/api/v1/pleroma/statuses/#{activity.id}/emoji_reactions_by") + |> json_response(200) + + assert result == %{} + + {:ok, _, _} = CommonAPI.react_with_emoji(activity.id, other_user, "๐ŸŽ…") + + result = + conn + |> get("/api/v1/pleroma/statuses/#{activity.id}/emoji_reactions_by") + |> json_response(200) + + [represented_user] = result["๐ŸŽ…"] + assert represented_user["id"] == other_user.id + end + test "/api/v1/pleroma/conversations/:id", %{conn: conn} do user = insert(:user) other_user = insert(:user) From 8d4b661ecb549ba554815ca55a29ec5872c68380 Mon Sep 17 00:00:00 2001 From: lain Date: Thu, 12 Sep 2019 18:59:13 +0200 Subject: [PATCH 007/107] Transmogrifier: Strip internal emoji reaction fields. --- lib/pleroma/web/activity_pub/transmogrifier.ex | 4 +++- test/web/activity_pub/transmogrifier_test.exs | 14 ++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index 9132df8cb..0b9cc4499 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -995,9 +995,11 @@ def prepare_attachments(object) do |> Map.put("attachment", attachments) end - defp strip_internal_fields(object) do + def strip_internal_fields(object) do object |> Map.drop([ + "reactions", + "reaction_count", "likes", "like_count", "announcements", diff --git a/test/web/activity_pub/transmogrifier_test.exs b/test/web/activity_pub/transmogrifier_test.exs index 6df707370..20d274a02 100644 --- a/test/web/activity_pub/transmogrifier_test.exs +++ b/test/web/activity_pub/transmogrifier_test.exs @@ -491,6 +491,20 @@ test "it strips internal likes" do refute Map.has_key?(object.data, "likes") end + test "it strips internal reactions" do + user = insert(:user) + {:ok, activity} = CommonAPI.post(user, %{"status" => "#cofe"}) + {:ok, _, _} = CommonAPI.react_with_emoji(activity.id, user, "๐Ÿ“ข") + + %{object: object} = Activity.get_by_id_with_object(activity.id) + assert Map.has_key?(object.data, "reactions") + assert Map.has_key?(object.data, "reaction_count") + + object_data = Transmogrifier.strip_internal_fields(object.data) + refute Map.has_key?(object_data, "reactions") + refute Map.has_key?(object_data, "reaction_count") + end + test "it works for incoming update activities" do data = File.read!("test/fixtures/mastodon-post-activity.json") |> Poison.decode!() From a697f0d79148358da828d24ebfe12bbb9bb33b34 Mon Sep 17 00:00:00 2001 From: lain Date: Fri, 13 Sep 2019 02:11:02 +0200 Subject: [PATCH 008/107] Emoji: Add function to detect if a character is an emoji --- lib/pleroma/emoji-data.txt | 769 +++++++++++++++++++++++++++++++++++++ lib/pleroma/emoji.ex | 25 ++ test/emoji_test.exs | 8 + 3 files changed, 802 insertions(+) create mode 100644 lib/pleroma/emoji-data.txt diff --git a/lib/pleroma/emoji-data.txt b/lib/pleroma/emoji-data.txt new file mode 100644 index 000000000..2fb5c3ff6 --- /dev/null +++ b/lib/pleroma/emoji-data.txt @@ -0,0 +1,769 @@ +# emoji-data.txt +# Date: 2019-01-15, 12:10:05 GMT +# ยฉ 2019 Unicodeยฎ, Inc. +# Unicode and the Unicode Logo are registered trademarks of Unicode, Inc. in the U.S. and other countries. +# For terms of use, see http://www.unicode.org/terms_of_use.html +# +# Emoji Data for UTS #51 +# Version: 12.0 +# +# For documentation and usage, see http://www.unicode.org/reports/tr51 +# +# Format: +# ; # +# Note: there is no guarantee as to the structure of whitespace or comments +# +# Characters and sequences are listed in code point order. Users should be shown a more natural order. +# See the CLDR collation order for Emoji. + + +# ================================================ + +# All omitted code points have Emoji=No +# @missing: 0000..10FFFF ; Emoji ; No + +0023 ; Emoji # 1.1 [1] (#๏ธ) number sign +002A ; Emoji # 1.1 [1] (*๏ธ) asterisk +0030..0039 ; Emoji # 1.1 [10] (0๏ธ..9๏ธ) digit zero..digit nine +00A9 ; Emoji # 1.1 [1] (ยฉ๏ธ) copyright +00AE ; Emoji # 1.1 [1] (ยฎ๏ธ) registered +203C ; Emoji # 1.1 [1] (โ€ผ๏ธ) double exclamation mark +2049 ; Emoji # 3.0 [1] (โ‰๏ธ) exclamation question mark +2122 ; Emoji # 1.1 [1] (โ„ข๏ธ) trade mark +2139 ; Emoji # 3.0 [1] (โ„น๏ธ) information +2194..2199 ; Emoji # 1.1 [6] (โ†”๏ธ..โ†™๏ธ) left-right arrow..down-left arrow +21A9..21AA ; Emoji # 1.1 [2] (โ†ฉ๏ธ..โ†ช๏ธ) right arrow curving left..left arrow curving right +231A..231B ; Emoji # 1.1 [2] (โŒš..โŒ›) watch..hourglass done +2328 ; Emoji # 1.1 [1] (โŒจ๏ธ) keyboard +23CF ; Emoji # 4.0 [1] (โ๏ธ) eject button +23E9..23F3 ; Emoji # 6.0 [11] (โฉ..โณ) fast-forward button..hourglass not done +23F8..23FA ; Emoji # 7.0 [3] (โธ๏ธ..โบ๏ธ) pause button..record button +24C2 ; Emoji # 1.1 [1] (โ“‚๏ธ) circled M +25AA..25AB ; Emoji # 1.1 [2] (โ–ช๏ธ..โ–ซ๏ธ) black small square..white small square +25B6 ; Emoji # 1.1 [1] (โ–ถ๏ธ) play button +25C0 ; Emoji # 1.1 [1] (โ—€๏ธ) reverse button +25FB..25FE ; Emoji # 3.2 [4] (โ—ป๏ธ..โ—พ) white medium square..black medium-small square +2600..2604 ; Emoji # 1.1 [5] (โ˜€๏ธ..โ˜„๏ธ) sun..comet +260E ; Emoji # 1.1 [1] (โ˜Ž๏ธ) telephone +2611 ; Emoji # 1.1 [1] (โ˜‘๏ธ) check box with check +2614..2615 ; Emoji # 4.0 [2] (โ˜”..โ˜•) umbrella with rain drops..hot beverage +2618 ; Emoji # 4.1 [1] (โ˜˜๏ธ) shamrock +261D ; Emoji # 1.1 [1] (โ˜๏ธ) index pointing up +2620 ; Emoji # 1.1 [1] (โ˜ ๏ธ) skull and crossbones +2622..2623 ; Emoji # 1.1 [2] (โ˜ข๏ธ..โ˜ฃ๏ธ) radioactive..biohazard +2626 ; Emoji # 1.1 [1] (โ˜ฆ๏ธ) orthodox cross +262A ; Emoji # 1.1 [1] (โ˜ช๏ธ) star and crescent +262E..262F ; Emoji # 1.1 [2] (โ˜ฎ๏ธ..โ˜ฏ๏ธ) peace symbol..yin yang +2638..263A ; Emoji # 1.1 [3] (โ˜ธ๏ธ..โ˜บ๏ธ) wheel of dharma..smiling face +2640 ; Emoji # 1.1 [1] (โ™€๏ธ) female sign +2642 ; Emoji # 1.1 [1] (โ™‚๏ธ) male sign +2648..2653 ; Emoji # 1.1 [12] (โ™ˆ..โ™“) Aries..Pisces +265F..2660 ; Emoji # 1.1 [2] (โ™Ÿ๏ธ..โ™ ๏ธ) chess pawn..spade suit +2663 ; Emoji # 1.1 [1] (โ™ฃ๏ธ) club suit +2665..2666 ; Emoji # 1.1 [2] (โ™ฅ๏ธ..โ™ฆ๏ธ) heart suit..diamond suit +2668 ; Emoji # 1.1 [1] (โ™จ๏ธ) hot springs +267B ; Emoji # 3.2 [1] (โ™ป๏ธ) recycling symbol +267E..267F ; Emoji # 4.1 [2] (โ™พ๏ธ..โ™ฟ) infinity..wheelchair symbol +2692..2697 ; Emoji # 4.1 [6] (โš’๏ธ..โš—๏ธ) hammer and pick..alembic +2699 ; Emoji # 4.1 [1] (โš™๏ธ) gear +269B..269C ; Emoji # 4.1 [2] (โš›๏ธ..โšœ๏ธ) atom symbol..fleur-de-lis +26A0..26A1 ; Emoji # 4.0 [2] (โš ๏ธ..โšก) warning..high voltage +26AA..26AB ; Emoji # 4.1 [2] (โšช..โšซ) white circle..black circle +26B0..26B1 ; Emoji # 4.1 [2] (โšฐ๏ธ..โšฑ๏ธ) coffin..funeral urn +26BD..26BE ; Emoji # 5.2 [2] (โšฝ..โšพ) soccer ball..baseball +26C4..26C5 ; Emoji # 5.2 [2] (โ›„..โ›…) snowman without snow..sun behind cloud +26C8 ; Emoji # 5.2 [1] (โ›ˆ๏ธ) cloud with lightning and rain +26CE ; Emoji # 6.0 [1] (โ›Ž) Ophiuchus +26CF ; Emoji # 5.2 [1] (โ›๏ธ) pick +26D1 ; Emoji # 5.2 [1] (โ›‘๏ธ) rescue workerโ€™s helmet +26D3..26D4 ; Emoji # 5.2 [2] (โ›“๏ธ..โ›”) chains..no entry +26E9..26EA ; Emoji # 5.2 [2] (โ›ฉ๏ธ..โ›ช) shinto shrine..church +26F0..26F5 ; Emoji # 5.2 [6] (โ›ฐ๏ธ..โ›ต) mountain..sailboat +26F7..26FA ; Emoji # 5.2 [4] (โ›ท๏ธ..โ›บ) skier..tent +26FD ; Emoji # 5.2 [1] (โ›ฝ) fuel pump +2702 ; Emoji # 1.1 [1] (โœ‚๏ธ) scissors +2705 ; Emoji # 6.0 [1] (โœ…) check mark button +2708..2709 ; Emoji # 1.1 [2] (โœˆ๏ธ..โœ‰๏ธ) airplane..envelope +270A..270B ; Emoji # 6.0 [2] (โœŠ..โœ‹) raised fist..raised hand +270C..270D ; Emoji # 1.1 [2] (โœŒ๏ธ..โœ๏ธ) victory hand..writing hand +270F ; Emoji # 1.1 [1] (โœ๏ธ) pencil +2712 ; Emoji # 1.1 [1] (โœ’๏ธ) black nib +2714 ; Emoji # 1.1 [1] (โœ”๏ธ) check mark +2716 ; Emoji # 1.1 [1] (โœ–๏ธ) multiplication sign +271D ; Emoji # 1.1 [1] (โœ๏ธ) latin cross +2721 ; Emoji # 1.1 [1] (โœก๏ธ) star of David +2728 ; Emoji # 6.0 [1] (โœจ) sparkles +2733..2734 ; Emoji # 1.1 [2] (โœณ๏ธ..โœด๏ธ) eight-spoked asterisk..eight-pointed star +2744 ; Emoji # 1.1 [1] (โ„๏ธ) snowflake +2747 ; Emoji # 1.1 [1] (โ‡๏ธ) sparkle +274C ; Emoji # 6.0 [1] (โŒ) cross mark +274E ; Emoji # 6.0 [1] (โŽ) cross mark button +2753..2755 ; Emoji # 6.0 [3] (โ“..โ•) question mark..white exclamation mark +2757 ; Emoji # 5.2 [1] (โ—) exclamation mark +2763..2764 ; Emoji # 1.1 [2] (โฃ๏ธ..โค๏ธ) heart exclamation..red heart +2795..2797 ; Emoji # 6.0 [3] (โž•..โž—) plus sign..division sign +27A1 ; Emoji # 1.1 [1] (โžก๏ธ) right arrow +27B0 ; Emoji # 6.0 [1] (โžฐ) curly loop +27BF ; Emoji # 6.0 [1] (โžฟ) double curly loop +2934..2935 ; Emoji # 3.2 [2] (โคด๏ธ..โคต๏ธ) right arrow curving up..right arrow curving down +2B05..2B07 ; Emoji # 4.0 [3] (โฌ…๏ธ..โฌ‡๏ธ) left arrow..down arrow +2B1B..2B1C ; Emoji # 5.1 [2] (โฌ›..โฌœ) black large square..white large square +2B50 ; Emoji # 5.1 [1] (โญ) star +2B55 ; Emoji # 5.2 [1] (โญ•) hollow red circle +3030 ; Emoji # 1.1 [1] (ใ€ฐ๏ธ) wavy dash +303D ; Emoji # 3.2 [1] (ใ€ฝ๏ธ) part alternation mark +3297 ; Emoji # 1.1 [1] (ใŠ—๏ธ) Japanese โ€œcongratulationsโ€ button +3299 ; Emoji # 1.1 [1] (ใŠ™๏ธ) Japanese โ€œsecretโ€ button +1F004 ; Emoji # 5.1 [1] (๐Ÿ€„) mahjong red dragon +1F0CF ; Emoji # 6.0 [1] (๐Ÿƒ) joker +1F170..1F171 ; Emoji # 6.0 [2] (๐Ÿ…ฐ๏ธ..๐Ÿ…ฑ๏ธ) A button (blood type)..B button (blood type) +1F17E ; Emoji # 6.0 [1] (๐Ÿ…พ๏ธ) O button (blood type) +1F17F ; Emoji # 5.2 [1] (๐Ÿ…ฟ๏ธ) P button +1F18E ; Emoji # 6.0 [1] (๐Ÿ†Ž) AB button (blood type) +1F191..1F19A ; Emoji # 6.0 [10] (๐Ÿ†‘..๐Ÿ†š) CL button..VS button +1F1E6..1F1FF ; Emoji # 6.0 [26] (๐Ÿ‡ฆ..๐Ÿ‡ฟ) regional indicator symbol letter a..regional indicator symbol letter z +1F201..1F202 ; Emoji # 6.0 [2] (๐Ÿˆ..๐Ÿˆ‚๏ธ) Japanese โ€œhereโ€ button..Japanese โ€œservice chargeโ€ button +1F21A ; Emoji # 5.2 [1] (๐Ÿˆš) Japanese โ€œfree of chargeโ€ button +1F22F ; Emoji # 5.2 [1] (๐Ÿˆฏ) Japanese โ€œreservedโ€ button +1F232..1F23A ; Emoji # 6.0 [9] (๐Ÿˆฒ..๐Ÿˆบ) Japanese โ€œprohibitedโ€ button..Japanese โ€œopen for businessโ€ button +1F250..1F251 ; Emoji # 6.0 [2] (๐Ÿ‰..๐Ÿ‰‘) Japanese โ€œbargainโ€ button..Japanese โ€œacceptableโ€ button +1F300..1F320 ; Emoji # 6.0 [33] (๐ŸŒ€..๐ŸŒ ) cyclone..shooting star +1F321 ; Emoji # 7.0 [1] (๐ŸŒก๏ธ) thermometer +1F324..1F32C ; Emoji # 7.0 [9] (๐ŸŒค๏ธ..๐ŸŒฌ๏ธ) sun behind small cloud..wind face +1F32D..1F32F ; Emoji # 8.0 [3] (๐ŸŒญ..๐ŸŒฏ) hot dog..burrito +1F330..1F335 ; Emoji # 6.0 [6] (๐ŸŒฐ..๐ŸŒต) chestnut..cactus +1F336 ; Emoji # 7.0 [1] (๐ŸŒถ๏ธ) hot pepper +1F337..1F37C ; Emoji # 6.0 [70] (๐ŸŒท..๐Ÿผ) tulip..baby bottle +1F37D ; Emoji # 7.0 [1] (๐Ÿฝ๏ธ) fork and knife with plate +1F37E..1F37F ; Emoji # 8.0 [2] (๐Ÿพ..๐Ÿฟ) bottle with popping cork..popcorn +1F380..1F393 ; Emoji # 6.0 [20] (๐ŸŽ€..๐ŸŽ“) ribbon..graduation cap +1F396..1F397 ; Emoji # 7.0 [2] (๐ŸŽ–๏ธ..๐ŸŽ—๏ธ) military medal..reminder ribbon +1F399..1F39B ; Emoji # 7.0 [3] (๐ŸŽ™๏ธ..๐ŸŽ›๏ธ) studio microphone..control knobs +1F39E..1F39F ; Emoji # 7.0 [2] (๐ŸŽž๏ธ..๐ŸŽŸ๏ธ) film frames..admission tickets +1F3A0..1F3C4 ; Emoji # 6.0 [37] (๐ŸŽ ..๐Ÿ„) carousel horse..person surfing +1F3C5 ; Emoji # 7.0 [1] (๐Ÿ…) sports medal +1F3C6..1F3CA ; Emoji # 6.0 [5] (๐Ÿ†..๐ŸŠ) trophy..person swimming +1F3CB..1F3CE ; Emoji # 7.0 [4] (๐Ÿ‹๏ธ..๐ŸŽ๏ธ) person lifting weights..racing car +1F3CF..1F3D3 ; Emoji # 8.0 [5] (๐Ÿ..๐Ÿ“) cricket game..ping pong +1F3D4..1F3DF ; Emoji # 7.0 [12] (๐Ÿ”๏ธ..๐ŸŸ๏ธ) snow-capped mountain..stadium +1F3E0..1F3F0 ; Emoji # 6.0 [17] (๐Ÿ ..๐Ÿฐ) house..castle +1F3F3..1F3F5 ; Emoji # 7.0 [3] (๐Ÿณ๏ธ..๐Ÿต๏ธ) white flag..rosette +1F3F7 ; Emoji # 7.0 [1] (๐Ÿท๏ธ) label +1F3F8..1F3FF ; Emoji # 8.0 [8] (๐Ÿธ..๐Ÿฟ) badminton..dark skin tone +1F400..1F43E ; Emoji # 6.0 [63] (๐Ÿ€..๐Ÿพ) rat..paw prints +1F43F ; Emoji # 7.0 [1] (๐Ÿฟ๏ธ) chipmunk +1F440 ; Emoji # 6.0 [1] (๐Ÿ‘€) eyes +1F441 ; Emoji # 7.0 [1] (๐Ÿ‘๏ธ) eye +1F442..1F4F7 ; Emoji # 6.0[182] (๐Ÿ‘‚..๐Ÿ“ท) ear..camera +1F4F8 ; Emoji # 7.0 [1] (๐Ÿ“ธ) camera with flash +1F4F9..1F4FC ; Emoji # 6.0 [4] (๐Ÿ“น..๐Ÿ“ผ) video camera..videocassette +1F4FD ; Emoji # 7.0 [1] (๐Ÿ“ฝ๏ธ) film projector +1F4FF ; Emoji # 8.0 [1] (๐Ÿ“ฟ) prayer beads +1F500..1F53D ; Emoji # 6.0 [62] (๐Ÿ”€..๐Ÿ”ฝ) shuffle tracks button..downwards button +1F549..1F54A ; Emoji # 7.0 [2] (๐Ÿ•‰๏ธ..๐Ÿ•Š๏ธ) om..dove +1F54B..1F54E ; Emoji # 8.0 [4] (๐Ÿ•‹..๐Ÿ•Ž) kaaba..menorah +1F550..1F567 ; Emoji # 6.0 [24] (๐Ÿ•..๐Ÿ•ง) one oโ€™clock..twelve-thirty +1F56F..1F570 ; Emoji # 7.0 [2] (๐Ÿ•ฏ๏ธ..๐Ÿ•ฐ๏ธ) candle..mantelpiece clock +1F573..1F579 ; Emoji # 7.0 [7] (๐Ÿ•ณ๏ธ..๐Ÿ•น๏ธ) hole..joystick +1F57A ; Emoji # 9.0 [1] (๐Ÿ•บ) man dancing +1F587 ; Emoji # 7.0 [1] (๐Ÿ–‡๏ธ) linked paperclips +1F58A..1F58D ; Emoji # 7.0 [4] (๐Ÿ–Š๏ธ..๐Ÿ–๏ธ) pen..crayon +1F590 ; Emoji # 7.0 [1] (๐Ÿ–๏ธ) hand with fingers splayed +1F595..1F596 ; Emoji # 7.0 [2] (๐Ÿ–•..๐Ÿ––) middle finger..vulcan salute +1F5A4 ; Emoji # 9.0 [1] (๐Ÿ–ค) black heart +1F5A5 ; Emoji # 7.0 [1] (๐Ÿ–ฅ๏ธ) desktop computer +1F5A8 ; Emoji # 7.0 [1] (๐Ÿ–จ๏ธ) printer +1F5B1..1F5B2 ; Emoji # 7.0 [2] (๐Ÿ–ฑ๏ธ..๐Ÿ–ฒ๏ธ) computer mouse..trackball +1F5BC ; Emoji # 7.0 [1] (๐Ÿ–ผ๏ธ) framed picture +1F5C2..1F5C4 ; Emoji # 7.0 [3] (๐Ÿ—‚๏ธ..๐Ÿ—„๏ธ) card index dividers..file cabinet +1F5D1..1F5D3 ; Emoji # 7.0 [3] (๐Ÿ—‘๏ธ..๐Ÿ—“๏ธ) wastebasket..spiral calendar +1F5DC..1F5DE ; Emoji # 7.0 [3] (๐Ÿ—œ๏ธ..๐Ÿ—ž๏ธ) clamp..rolled-up newspaper +1F5E1 ; Emoji # 7.0 [1] (๐Ÿ—ก๏ธ) dagger +1F5E3 ; Emoji # 7.0 [1] (๐Ÿ—ฃ๏ธ) speaking head +1F5E8 ; Emoji # 7.0 [1] (๐Ÿ—จ๏ธ) left speech bubble +1F5EF ; Emoji # 7.0 [1] (๐Ÿ—ฏ๏ธ) right anger bubble +1F5F3 ; Emoji # 7.0 [1] (๐Ÿ—ณ๏ธ) ballot box with ballot +1F5FA ; Emoji # 7.0 [1] (๐Ÿ—บ๏ธ) world map +1F5FB..1F5FF ; Emoji # 6.0 [5] (๐Ÿ—ป..๐Ÿ—ฟ) mount fuji..moai +1F600 ; Emoji # 6.1 [1] (๐Ÿ˜€) grinning face +1F601..1F610 ; Emoji # 6.0 [16] (๐Ÿ˜..๐Ÿ˜) beaming face with smiling eyes..neutral face +1F611 ; Emoji # 6.1 [1] (๐Ÿ˜‘) expressionless face +1F612..1F614 ; Emoji # 6.0 [3] (๐Ÿ˜’..๐Ÿ˜”) unamused face..pensive face +1F615 ; Emoji # 6.1 [1] (๐Ÿ˜•) confused face +1F616 ; Emoji # 6.0 [1] (๐Ÿ˜–) confounded face +1F617 ; Emoji # 6.1 [1] (๐Ÿ˜—) kissing face +1F618 ; Emoji # 6.0 [1] (๐Ÿ˜˜) face blowing a kiss +1F619 ; Emoji # 6.1 [1] (๐Ÿ˜™) kissing face with smiling eyes +1F61A ; Emoji # 6.0 [1] (๐Ÿ˜š) kissing face with closed eyes +1F61B ; Emoji # 6.1 [1] (๐Ÿ˜›) face with tongue +1F61C..1F61E ; Emoji # 6.0 [3] (๐Ÿ˜œ..๐Ÿ˜ž) winking face with tongue..disappointed face +1F61F ; Emoji # 6.1 [1] (๐Ÿ˜Ÿ) worried face +1F620..1F625 ; Emoji # 6.0 [6] (๐Ÿ˜ ..๐Ÿ˜ฅ) angry face..sad but relieved face +1F626..1F627 ; Emoji # 6.1 [2] (๐Ÿ˜ฆ..๐Ÿ˜ง) frowning face with open mouth..anguished face +1F628..1F62B ; Emoji # 6.0 [4] (๐Ÿ˜จ..๐Ÿ˜ซ) fearful face..tired face +1F62C ; Emoji # 6.1 [1] (๐Ÿ˜ฌ) grimacing face +1F62D ; Emoji # 6.0 [1] (๐Ÿ˜ญ) loudly crying face +1F62E..1F62F ; Emoji # 6.1 [2] (๐Ÿ˜ฎ..๐Ÿ˜ฏ) face with open mouth..hushed face +1F630..1F633 ; Emoji # 6.0 [4] (๐Ÿ˜ฐ..๐Ÿ˜ณ) anxious face with sweat..flushed face +1F634 ; Emoji # 6.1 [1] (๐Ÿ˜ด) sleeping face +1F635..1F640 ; Emoji # 6.0 [12] (๐Ÿ˜ต..๐Ÿ™€) dizzy face..weary cat +1F641..1F642 ; Emoji # 7.0 [2] (๐Ÿ™..๐Ÿ™‚) slightly frowning face..slightly smiling face +1F643..1F644 ; Emoji # 8.0 [2] (๐Ÿ™ƒ..๐Ÿ™„) upside-down face..face with rolling eyes +1F645..1F64F ; Emoji # 6.0 [11] (๐Ÿ™…..๐Ÿ™) person gesturing NO..folded hands +1F680..1F6C5 ; Emoji # 6.0 [70] (๐Ÿš€..๐Ÿ›…) rocket..left luggage +1F6CB..1F6CF ; Emoji # 7.0 [5] (๐Ÿ›‹๏ธ..๐Ÿ›๏ธ) couch and lamp..bed +1F6D0 ; Emoji # 8.0 [1] (๐Ÿ›) place of worship +1F6D1..1F6D2 ; Emoji # 9.0 [2] (๐Ÿ›‘..๐Ÿ›’) stop sign..shopping cart +1F6D5 ; Emoji # 12.0 [1] (๐Ÿ›•) hindu temple +1F6E0..1F6E5 ; Emoji # 7.0 [6] (๐Ÿ› ๏ธ..๐Ÿ›ฅ๏ธ) hammer and wrench..motor boat +1F6E9 ; Emoji # 7.0 [1] (๐Ÿ›ฉ๏ธ) small airplane +1F6EB..1F6EC ; Emoji # 7.0 [2] (๐Ÿ›ซ..๐Ÿ›ฌ) airplane departure..airplane arrival +1F6F0 ; Emoji # 7.0 [1] (๐Ÿ›ฐ๏ธ) satellite +1F6F3 ; Emoji # 7.0 [1] (๐Ÿ›ณ๏ธ) passenger ship +1F6F4..1F6F6 ; Emoji # 9.0 [3] (๐Ÿ›ด..๐Ÿ›ถ) kick scooter..canoe +1F6F7..1F6F8 ; Emoji # 10.0 [2] (๐Ÿ›ท..๐Ÿ›ธ) sled..flying saucer +1F6F9 ; Emoji # 11.0 [1] (๐Ÿ›น) skateboard +1F6FA ; Emoji # 12.0 [1] (๐Ÿ›บ) auto rickshaw +1F7E0..1F7EB ; Emoji # 12.0 [12] (๐ŸŸ ..๐ŸŸซ) orange circle..brown square +1F90D..1F90F ; Emoji # 12.0 [3] (๐Ÿค..๐Ÿค) white heart..pinching hand +1F910..1F918 ; Emoji # 8.0 [9] (๐Ÿค..๐Ÿค˜) zipper-mouth face..sign of the horns +1F919..1F91E ; Emoji # 9.0 [6] (๐Ÿค™..๐Ÿคž) call me hand..crossed fingers +1F91F ; Emoji # 10.0 [1] (๐ŸคŸ) love-you gesture +1F920..1F927 ; Emoji # 9.0 [8] (๐Ÿค ..๐Ÿคง) cowboy hat face..sneezing face +1F928..1F92F ; Emoji # 10.0 [8] (๐Ÿคจ..๐Ÿคฏ) face with raised eyebrow..exploding head +1F930 ; Emoji # 9.0 [1] (๐Ÿคฐ) pregnant woman +1F931..1F932 ; Emoji # 10.0 [2] (๐Ÿคฑ..๐Ÿคฒ) breast-feeding..palms up together +1F933..1F93A ; Emoji # 9.0 [8] (๐Ÿคณ..๐Ÿคบ) selfie..person fencing +1F93C..1F93E ; Emoji # 9.0 [3] (๐Ÿคผ..๐Ÿคพ) people wrestling..person playing handball +1F93F ; Emoji # 12.0 [1] (๐Ÿคฟ) diving mask +1F940..1F945 ; Emoji # 9.0 [6] (๐Ÿฅ€..๐Ÿฅ…) wilted flower..goal net +1F947..1F94B ; Emoji # 9.0 [5] (๐Ÿฅ‡..๐Ÿฅ‹) 1st place medal..martial arts uniform +1F94C ; Emoji # 10.0 [1] (๐ŸฅŒ) curling stone +1F94D..1F94F ; Emoji # 11.0 [3] (๐Ÿฅ..๐Ÿฅ) lacrosse..flying disc +1F950..1F95E ; Emoji # 9.0 [15] (๐Ÿฅ..๐Ÿฅž) croissant..pancakes +1F95F..1F96B ; Emoji # 10.0 [13] (๐ŸฅŸ..๐Ÿฅซ) dumpling..canned food +1F96C..1F970 ; Emoji # 11.0 [5] (๐Ÿฅฌ..๐Ÿฅฐ) leafy green..smiling face with hearts +1F971 ; Emoji # 12.0 [1] (๐Ÿฅฑ) yawning face +1F973..1F976 ; Emoji # 11.0 [4] (๐Ÿฅณ..๐Ÿฅถ) partying face..cold face +1F97A ; Emoji # 11.0 [1] (๐Ÿฅบ) pleading face +1F97B ; Emoji # 12.0 [1] (๐Ÿฅป) sari +1F97C..1F97F ; Emoji # 11.0 [4] (๐Ÿฅผ..๐Ÿฅฟ) lab coat..flat shoe +1F980..1F984 ; Emoji # 8.0 [5] (๐Ÿฆ€..๐Ÿฆ„) crab..unicorn +1F985..1F991 ; Emoji # 9.0 [13] (๐Ÿฆ…..๐Ÿฆ‘) eagle..squid +1F992..1F997 ; Emoji # 10.0 [6] (๐Ÿฆ’..๐Ÿฆ—) giraffe..cricket +1F998..1F9A2 ; Emoji # 11.0 [11] (๐Ÿฆ˜..๐Ÿฆข) kangaroo..swan +1F9A5..1F9AA ; Emoji # 12.0 [6] (๐Ÿฆฅ..๐Ÿฆช) sloth..oyster +1F9AE..1F9AF ; Emoji # 12.0 [2] (๐Ÿฆฎ..๐Ÿฆฏ) guide dog..probing cane +1F9B0..1F9B9 ; Emoji # 11.0 [10] (๐Ÿฆฐ..๐Ÿฆน) red hair..supervillain +1F9BA..1F9BF ; Emoji # 12.0 [6] (๐Ÿฆบ..๐Ÿฆฟ) safety vest..mechanical leg +1F9C0 ; Emoji # 8.0 [1] (๐Ÿง€) cheese wedge +1F9C1..1F9C2 ; Emoji # 11.0 [2] (๐Ÿง..๐Ÿง‚) cupcake..salt +1F9C3..1F9CA ; Emoji # 12.0 [8] (๐Ÿงƒ..๐ŸงŠ) beverage box..ice cube +1F9CD..1F9CF ; Emoji # 12.0 [3] (๐Ÿง..๐Ÿง) person standing..deaf person +1F9D0..1F9E6 ; Emoji # 10.0 [23] (๐Ÿง..๐Ÿงฆ) face with monocle..socks +1F9E7..1F9FF ; Emoji # 11.0 [25] (๐Ÿงง..๐Ÿงฟ) red envelope..nazar amulet +1FA70..1FA73 ; Emoji # 12.0 [4] (๐Ÿฉฐ..๐Ÿฉณ) ballet shoes..shorts +1FA78..1FA7A ; Emoji # 12.0 [3] (๐Ÿฉธ..๐Ÿฉบ) drop of blood..stethoscope +1FA80..1FA82 ; Emoji # 12.0 [3] (๐Ÿช€..๐Ÿช‚) yo-yo..parachute +1FA90..1FA95 ; Emoji # 12.0 [6] (๐Ÿช..๐Ÿช•) ringed planet..banjo + +# Total elements: 1311 + +# ================================================ + +# All omitted code points have Emoji_Presentation=No +# @missing: 0000..10FFFF ; Emoji_Presentation ; No + +231A..231B ; Emoji_Presentation # 1.1 [2] (โŒš..โŒ›) watch..hourglass done +23E9..23EC ; Emoji_Presentation # 6.0 [4] (โฉ..โฌ) fast-forward button..fast down button +23F0 ; Emoji_Presentation # 6.0 [1] (โฐ) alarm clock +23F3 ; Emoji_Presentation # 6.0 [1] (โณ) hourglass not done +25FD..25FE ; Emoji_Presentation # 3.2 [2] (โ—ฝ..โ—พ) white medium-small square..black medium-small square +2614..2615 ; Emoji_Presentation # 4.0 [2] (โ˜”..โ˜•) umbrella with rain drops..hot beverage +2648..2653 ; Emoji_Presentation # 1.1 [12] (โ™ˆ..โ™“) Aries..Pisces +267F ; Emoji_Presentation # 4.1 [1] (โ™ฟ) wheelchair symbol +2693 ; Emoji_Presentation # 4.1 [1] (โš“) anchor +26A1 ; Emoji_Presentation # 4.0 [1] (โšก) high voltage +26AA..26AB ; Emoji_Presentation # 4.1 [2] (โšช..โšซ) white circle..black circle +26BD..26BE ; Emoji_Presentation # 5.2 [2] (โšฝ..โšพ) soccer ball..baseball +26C4..26C5 ; Emoji_Presentation # 5.2 [2] (โ›„..โ›…) snowman without snow..sun behind cloud +26CE ; Emoji_Presentation # 6.0 [1] (โ›Ž) Ophiuchus +26D4 ; Emoji_Presentation # 5.2 [1] (โ›”) no entry +26EA ; Emoji_Presentation # 5.2 [1] (โ›ช) church +26F2..26F3 ; Emoji_Presentation # 5.2 [2] (โ›ฒ..โ›ณ) fountain..flag in hole +26F5 ; Emoji_Presentation # 5.2 [1] (โ›ต) sailboat +26FA ; Emoji_Presentation # 5.2 [1] (โ›บ) tent +26FD ; Emoji_Presentation # 5.2 [1] (โ›ฝ) fuel pump +2705 ; Emoji_Presentation # 6.0 [1] (โœ…) check mark button +270A..270B ; Emoji_Presentation # 6.0 [2] (โœŠ..โœ‹) raised fist..raised hand +2728 ; Emoji_Presentation # 6.0 [1] (โœจ) sparkles +274C ; Emoji_Presentation # 6.0 [1] (โŒ) cross mark +274E ; Emoji_Presentation # 6.0 [1] (โŽ) cross mark button +2753..2755 ; Emoji_Presentation # 6.0 [3] (โ“..โ•) question mark..white exclamation mark +2757 ; Emoji_Presentation # 5.2 [1] (โ—) exclamation mark +2795..2797 ; Emoji_Presentation # 6.0 [3] (โž•..โž—) plus sign..division sign +27B0 ; Emoji_Presentation # 6.0 [1] (โžฐ) curly loop +27BF ; Emoji_Presentation # 6.0 [1] (โžฟ) double curly loop +2B1B..2B1C ; Emoji_Presentation # 5.1 [2] (โฌ›..โฌœ) black large square..white large square +2B50 ; Emoji_Presentation # 5.1 [1] (โญ) star +2B55 ; Emoji_Presentation # 5.2 [1] (โญ•) hollow red circle +1F004 ; Emoji_Presentation # 5.1 [1] (๐Ÿ€„) mahjong red dragon +1F0CF ; Emoji_Presentation # 6.0 [1] (๐Ÿƒ) joker +1F18E ; Emoji_Presentation # 6.0 [1] (๐Ÿ†Ž) AB button (blood type) +1F191..1F19A ; Emoji_Presentation # 6.0 [10] (๐Ÿ†‘..๐Ÿ†š) CL button..VS button +1F1E6..1F1FF ; Emoji_Presentation # 6.0 [26] (๐Ÿ‡ฆ..๐Ÿ‡ฟ) regional indicator symbol letter a..regional indicator symbol letter z +1F201 ; Emoji_Presentation # 6.0 [1] (๐Ÿˆ) Japanese โ€œhereโ€ button +1F21A ; Emoji_Presentation # 5.2 [1] (๐Ÿˆš) Japanese โ€œfree of chargeโ€ button +1F22F ; Emoji_Presentation # 5.2 [1] (๐Ÿˆฏ) Japanese โ€œreservedโ€ button +1F232..1F236 ; Emoji_Presentation # 6.0 [5] (๐Ÿˆฒ..๐Ÿˆถ) Japanese โ€œprohibitedโ€ button..Japanese โ€œnot free of chargeโ€ button +1F238..1F23A ; Emoji_Presentation # 6.0 [3] (๐Ÿˆธ..๐Ÿˆบ) Japanese โ€œapplicationโ€ button..Japanese โ€œopen for businessโ€ button +1F250..1F251 ; Emoji_Presentation # 6.0 [2] (๐Ÿ‰..๐Ÿ‰‘) Japanese โ€œbargainโ€ button..Japanese โ€œacceptableโ€ button +1F300..1F320 ; Emoji_Presentation # 6.0 [33] (๐ŸŒ€..๐ŸŒ ) cyclone..shooting star +1F32D..1F32F ; Emoji_Presentation # 8.0 [3] (๐ŸŒญ..๐ŸŒฏ) hot dog..burrito +1F330..1F335 ; Emoji_Presentation # 6.0 [6] (๐ŸŒฐ..๐ŸŒต) chestnut..cactus +1F337..1F37C ; Emoji_Presentation # 6.0 [70] (๐ŸŒท..๐Ÿผ) tulip..baby bottle +1F37E..1F37F ; Emoji_Presentation # 8.0 [2] (๐Ÿพ..๐Ÿฟ) bottle with popping cork..popcorn +1F380..1F393 ; Emoji_Presentation # 6.0 [20] (๐ŸŽ€..๐ŸŽ“) ribbon..graduation cap +1F3A0..1F3C4 ; Emoji_Presentation # 6.0 [37] (๐ŸŽ ..๐Ÿ„) carousel horse..person surfing +1F3C5 ; Emoji_Presentation # 7.0 [1] (๐Ÿ…) sports medal +1F3C6..1F3CA ; Emoji_Presentation # 6.0 [5] (๐Ÿ†..๐ŸŠ) trophy..person swimming +1F3CF..1F3D3 ; Emoji_Presentation # 8.0 [5] (๐Ÿ..๐Ÿ“) cricket game..ping pong +1F3E0..1F3F0 ; Emoji_Presentation # 6.0 [17] (๐Ÿ ..๐Ÿฐ) house..castle +1F3F4 ; Emoji_Presentation # 7.0 [1] (๐Ÿด) black flag +1F3F8..1F3FF ; Emoji_Presentation # 8.0 [8] (๐Ÿธ..๐Ÿฟ) badminton..dark skin tone +1F400..1F43E ; Emoji_Presentation # 6.0 [63] (๐Ÿ€..๐Ÿพ) rat..paw prints +1F440 ; Emoji_Presentation # 6.0 [1] (๐Ÿ‘€) eyes +1F442..1F4F7 ; Emoji_Presentation # 6.0[182] (๐Ÿ‘‚..๐Ÿ“ท) ear..camera +1F4F8 ; Emoji_Presentation # 7.0 [1] (๐Ÿ“ธ) camera with flash +1F4F9..1F4FC ; Emoji_Presentation # 6.0 [4] (๐Ÿ“น..๐Ÿ“ผ) video camera..videocassette +1F4FF ; Emoji_Presentation # 8.0 [1] (๐Ÿ“ฟ) prayer beads +1F500..1F53D ; Emoji_Presentation # 6.0 [62] (๐Ÿ”€..๐Ÿ”ฝ) shuffle tracks button..downwards button +1F54B..1F54E ; Emoji_Presentation # 8.0 [4] (๐Ÿ•‹..๐Ÿ•Ž) kaaba..menorah +1F550..1F567 ; Emoji_Presentation # 6.0 [24] (๐Ÿ•..๐Ÿ•ง) one oโ€™clock..twelve-thirty +1F57A ; Emoji_Presentation # 9.0 [1] (๐Ÿ•บ) man dancing +1F595..1F596 ; Emoji_Presentation # 7.0 [2] (๐Ÿ–•..๐Ÿ––) middle finger..vulcan salute +1F5A4 ; Emoji_Presentation # 9.0 [1] (๐Ÿ–ค) black heart +1F5FB..1F5FF ; Emoji_Presentation # 6.0 [5] (๐Ÿ—ป..๐Ÿ—ฟ) mount fuji..moai +1F600 ; Emoji_Presentation # 6.1 [1] (๐Ÿ˜€) grinning face +1F601..1F610 ; Emoji_Presentation # 6.0 [16] (๐Ÿ˜..๐Ÿ˜) beaming face with smiling eyes..neutral face +1F611 ; Emoji_Presentation # 6.1 [1] (๐Ÿ˜‘) expressionless face +1F612..1F614 ; Emoji_Presentation # 6.0 [3] (๐Ÿ˜’..๐Ÿ˜”) unamused face..pensive face +1F615 ; Emoji_Presentation # 6.1 [1] (๐Ÿ˜•) confused face +1F616 ; Emoji_Presentation # 6.0 [1] (๐Ÿ˜–) confounded face +1F617 ; Emoji_Presentation # 6.1 [1] (๐Ÿ˜—) kissing face +1F618 ; Emoji_Presentation # 6.0 [1] (๐Ÿ˜˜) face blowing a kiss +1F619 ; Emoji_Presentation # 6.1 [1] (๐Ÿ˜™) kissing face with smiling eyes +1F61A ; Emoji_Presentation # 6.0 [1] (๐Ÿ˜š) kissing face with closed eyes +1F61B ; Emoji_Presentation # 6.1 [1] (๐Ÿ˜›) face with tongue +1F61C..1F61E ; Emoji_Presentation # 6.0 [3] (๐Ÿ˜œ..๐Ÿ˜ž) winking face with tongue..disappointed face +1F61F ; Emoji_Presentation # 6.1 [1] (๐Ÿ˜Ÿ) worried face +1F620..1F625 ; Emoji_Presentation # 6.0 [6] (๐Ÿ˜ ..๐Ÿ˜ฅ) angry face..sad but relieved face +1F626..1F627 ; Emoji_Presentation # 6.1 [2] (๐Ÿ˜ฆ..๐Ÿ˜ง) frowning face with open mouth..anguished face +1F628..1F62B ; Emoji_Presentation # 6.0 [4] (๐Ÿ˜จ..๐Ÿ˜ซ) fearful face..tired face +1F62C ; Emoji_Presentation # 6.1 [1] (๐Ÿ˜ฌ) grimacing face +1F62D ; Emoji_Presentation # 6.0 [1] (๐Ÿ˜ญ) loudly crying face +1F62E..1F62F ; Emoji_Presentation # 6.1 [2] (๐Ÿ˜ฎ..๐Ÿ˜ฏ) face with open mouth..hushed face +1F630..1F633 ; Emoji_Presentation # 6.0 [4] (๐Ÿ˜ฐ..๐Ÿ˜ณ) anxious face with sweat..flushed face +1F634 ; Emoji_Presentation # 6.1 [1] (๐Ÿ˜ด) sleeping face +1F635..1F640 ; Emoji_Presentation # 6.0 [12] (๐Ÿ˜ต..๐Ÿ™€) dizzy face..weary cat +1F641..1F642 ; Emoji_Presentation # 7.0 [2] (๐Ÿ™..๐Ÿ™‚) slightly frowning face..slightly smiling face +1F643..1F644 ; Emoji_Presentation # 8.0 [2] (๐Ÿ™ƒ..๐Ÿ™„) upside-down face..face with rolling eyes +1F645..1F64F ; Emoji_Presentation # 6.0 [11] (๐Ÿ™…..๐Ÿ™) person gesturing NO..folded hands +1F680..1F6C5 ; Emoji_Presentation # 6.0 [70] (๐Ÿš€..๐Ÿ›…) rocket..left luggage +1F6CC ; Emoji_Presentation # 7.0 [1] (๐Ÿ›Œ) person in bed +1F6D0 ; Emoji_Presentation # 8.0 [1] (๐Ÿ›) place of worship +1F6D1..1F6D2 ; Emoji_Presentation # 9.0 [2] (๐Ÿ›‘..๐Ÿ›’) stop sign..shopping cart +1F6D5 ; Emoji_Presentation # 12.0 [1] (๐Ÿ›•) hindu temple +1F6EB..1F6EC ; Emoji_Presentation # 7.0 [2] (๐Ÿ›ซ..๐Ÿ›ฌ) airplane departure..airplane arrival +1F6F4..1F6F6 ; Emoji_Presentation # 9.0 [3] (๐Ÿ›ด..๐Ÿ›ถ) kick scooter..canoe +1F6F7..1F6F8 ; Emoji_Presentation # 10.0 [2] (๐Ÿ›ท..๐Ÿ›ธ) sled..flying saucer +1F6F9 ; Emoji_Presentation # 11.0 [1] (๐Ÿ›น) skateboard +1F6FA ; Emoji_Presentation # 12.0 [1] (๐Ÿ›บ) auto rickshaw +1F7E0..1F7EB ; Emoji_Presentation # 12.0 [12] (๐ŸŸ ..๐ŸŸซ) orange circle..brown square +1F90D..1F90F ; Emoji_Presentation # 12.0 [3] (๐Ÿค..๐Ÿค) white heart..pinching hand +1F910..1F918 ; Emoji_Presentation # 8.0 [9] (๐Ÿค..๐Ÿค˜) zipper-mouth face..sign of the horns +1F919..1F91E ; Emoji_Presentation # 9.0 [6] (๐Ÿค™..๐Ÿคž) call me hand..crossed fingers +1F91F ; Emoji_Presentation # 10.0 [1] (๐ŸคŸ) love-you gesture +1F920..1F927 ; Emoji_Presentation # 9.0 [8] (๐Ÿค ..๐Ÿคง) cowboy hat face..sneezing face +1F928..1F92F ; Emoji_Presentation # 10.0 [8] (๐Ÿคจ..๐Ÿคฏ) face with raised eyebrow..exploding head +1F930 ; Emoji_Presentation # 9.0 [1] (๐Ÿคฐ) pregnant woman +1F931..1F932 ; Emoji_Presentation # 10.0 [2] (๐Ÿคฑ..๐Ÿคฒ) breast-feeding..palms up together +1F933..1F93A ; Emoji_Presentation # 9.0 [8] (๐Ÿคณ..๐Ÿคบ) selfie..person fencing +1F93C..1F93E ; Emoji_Presentation # 9.0 [3] (๐Ÿคผ..๐Ÿคพ) people wrestling..person playing handball +1F93F ; Emoji_Presentation # 12.0 [1] (๐Ÿคฟ) diving mask +1F940..1F945 ; Emoji_Presentation # 9.0 [6] (๐Ÿฅ€..๐Ÿฅ…) wilted flower..goal net +1F947..1F94B ; Emoji_Presentation # 9.0 [5] (๐Ÿฅ‡..๐Ÿฅ‹) 1st place medal..martial arts uniform +1F94C ; Emoji_Presentation # 10.0 [1] (๐ŸฅŒ) curling stone +1F94D..1F94F ; Emoji_Presentation # 11.0 [3] (๐Ÿฅ..๐Ÿฅ) lacrosse..flying disc +1F950..1F95E ; Emoji_Presentation # 9.0 [15] (๐Ÿฅ..๐Ÿฅž) croissant..pancakes +1F95F..1F96B ; Emoji_Presentation # 10.0 [13] (๐ŸฅŸ..๐Ÿฅซ) dumpling..canned food +1F96C..1F970 ; Emoji_Presentation # 11.0 [5] (๐Ÿฅฌ..๐Ÿฅฐ) leafy green..smiling face with hearts +1F971 ; Emoji_Presentation # 12.0 [1] (๐Ÿฅฑ) yawning face +1F973..1F976 ; Emoji_Presentation # 11.0 [4] (๐Ÿฅณ..๐Ÿฅถ) partying face..cold face +1F97A ; Emoji_Presentation # 11.0 [1] (๐Ÿฅบ) pleading face +1F97B ; Emoji_Presentation # 12.0 [1] (๐Ÿฅป) sari +1F97C..1F97F ; Emoji_Presentation # 11.0 [4] (๐Ÿฅผ..๐Ÿฅฟ) lab coat..flat shoe +1F980..1F984 ; Emoji_Presentation # 8.0 [5] (๐Ÿฆ€..๐Ÿฆ„) crab..unicorn +1F985..1F991 ; Emoji_Presentation # 9.0 [13] (๐Ÿฆ…..๐Ÿฆ‘) eagle..squid +1F992..1F997 ; Emoji_Presentation # 10.0 [6] (๐Ÿฆ’..๐Ÿฆ—) giraffe..cricket +1F998..1F9A2 ; Emoji_Presentation # 11.0 [11] (๐Ÿฆ˜..๐Ÿฆข) kangaroo..swan +1F9A5..1F9AA ; Emoji_Presentation # 12.0 [6] (๐Ÿฆฅ..๐Ÿฆช) sloth..oyster +1F9AE..1F9AF ; Emoji_Presentation # 12.0 [2] (๐Ÿฆฎ..๐Ÿฆฏ) guide dog..probing cane +1F9B0..1F9B9 ; Emoji_Presentation # 11.0 [10] (๐Ÿฆฐ..๐Ÿฆน) red hair..supervillain +1F9BA..1F9BF ; Emoji_Presentation # 12.0 [6] (๐Ÿฆบ..๐Ÿฆฟ) safety vest..mechanical leg +1F9C0 ; Emoji_Presentation # 8.0 [1] (๐Ÿง€) cheese wedge +1F9C1..1F9C2 ; Emoji_Presentation # 11.0 [2] (๐Ÿง..๐Ÿง‚) cupcake..salt +1F9C3..1F9CA ; Emoji_Presentation # 12.0 [8] (๐Ÿงƒ..๐ŸงŠ) beverage box..ice cube +1F9CD..1F9CF ; Emoji_Presentation # 12.0 [3] (๐Ÿง..๐Ÿง) person standing..deaf person +1F9D0..1F9E6 ; Emoji_Presentation # 10.0 [23] (๐Ÿง..๐Ÿงฆ) face with monocle..socks +1F9E7..1F9FF ; Emoji_Presentation # 11.0 [25] (๐Ÿงง..๐Ÿงฟ) red envelope..nazar amulet +1FA70..1FA73 ; Emoji_Presentation # 12.0 [4] (๐Ÿฉฐ..๐Ÿฉณ) ballet shoes..shorts +1FA78..1FA7A ; Emoji_Presentation # 12.0 [3] (๐Ÿฉธ..๐Ÿฉบ) drop of blood..stethoscope +1FA80..1FA82 ; Emoji_Presentation # 12.0 [3] (๐Ÿช€..๐Ÿช‚) yo-yo..parachute +1FA90..1FA95 ; Emoji_Presentation # 12.0 [6] (๐Ÿช..๐Ÿช•) ringed planet..banjo + +# Total elements: 1093 + +# ================================================ + +# All omitted code points have Emoji_Modifier=No +# @missing: 0000..10FFFF ; Emoji_Modifier ; No + +1F3FB..1F3FF ; Emoji_Modifier # 8.0 [5] (๐Ÿป..๐Ÿฟ) light skin tone..dark skin tone + +# Total elements: 5 + +# ================================================ + +# All omitted code points have Emoji_Modifier_Base=No +# @missing: 0000..10FFFF ; Emoji_Modifier_Base ; No + +261D ; Emoji_Modifier_Base # 1.1 [1] (โ˜๏ธ) index pointing up +26F9 ; Emoji_Modifier_Base # 5.2 [1] (โ›น๏ธ) person bouncing ball +270A..270B ; Emoji_Modifier_Base # 6.0 [2] (โœŠ..โœ‹) raised fist..raised hand +270C..270D ; Emoji_Modifier_Base # 1.1 [2] (โœŒ๏ธ..โœ๏ธ) victory hand..writing hand +1F385 ; Emoji_Modifier_Base # 6.0 [1] (๐ŸŽ…) Santa Claus +1F3C2..1F3C4 ; Emoji_Modifier_Base # 6.0 [3] (๐Ÿ‚..๐Ÿ„) snowboarder..person surfing +1F3C7 ; Emoji_Modifier_Base # 6.0 [1] (๐Ÿ‡) horse racing +1F3CA ; Emoji_Modifier_Base # 6.0 [1] (๐ŸŠ) person swimming +1F3CB..1F3CC ; Emoji_Modifier_Base # 7.0 [2] (๐Ÿ‹๏ธ..๐ŸŒ๏ธ) person lifting weights..person golfing +1F442..1F443 ; Emoji_Modifier_Base # 6.0 [2] (๐Ÿ‘‚..๐Ÿ‘ƒ) ear..nose +1F446..1F450 ; Emoji_Modifier_Base # 6.0 [11] (๐Ÿ‘†..๐Ÿ‘) backhand index pointing up..open hands +1F466..1F478 ; Emoji_Modifier_Base # 6.0 [19] (๐Ÿ‘ฆ..๐Ÿ‘ธ) boy..princess +1F47C ; Emoji_Modifier_Base # 6.0 [1] (๐Ÿ‘ผ) baby angel +1F481..1F483 ; Emoji_Modifier_Base # 6.0 [3] (๐Ÿ’..๐Ÿ’ƒ) person tipping hand..woman dancing +1F485..1F487 ; Emoji_Modifier_Base # 6.0 [3] (๐Ÿ’…..๐Ÿ’‡) nail polish..person getting haircut +1F48F ; Emoji_Modifier_Base # 6.0 [1] (๐Ÿ’) kiss +1F491 ; Emoji_Modifier_Base # 6.0 [1] (๐Ÿ’‘) couple with heart +1F4AA ; Emoji_Modifier_Base # 6.0 [1] (๐Ÿ’ช) flexed biceps +1F574..1F575 ; Emoji_Modifier_Base # 7.0 [2] (๐Ÿ•ด๏ธ..๐Ÿ•ต๏ธ) man in suit levitating..detective +1F57A ; Emoji_Modifier_Base # 9.0 [1] (๐Ÿ•บ) man dancing +1F590 ; Emoji_Modifier_Base # 7.0 [1] (๐Ÿ–๏ธ) hand with fingers splayed +1F595..1F596 ; Emoji_Modifier_Base # 7.0 [2] (๐Ÿ–•..๐Ÿ––) middle finger..vulcan salute +1F645..1F647 ; Emoji_Modifier_Base # 6.0 [3] (๐Ÿ™…..๐Ÿ™‡) person gesturing NO..person bowing +1F64B..1F64F ; Emoji_Modifier_Base # 6.0 [5] (๐Ÿ™‹..๐Ÿ™) person raising hand..folded hands +1F6A3 ; Emoji_Modifier_Base # 6.0 [1] (๐Ÿšฃ) person rowing boat +1F6B4..1F6B6 ; Emoji_Modifier_Base # 6.0 [3] (๐Ÿšด..๐Ÿšถ) person biking..person walking +1F6C0 ; Emoji_Modifier_Base # 6.0 [1] (๐Ÿ›€) person taking bath +1F6CC ; Emoji_Modifier_Base # 7.0 [1] (๐Ÿ›Œ) person in bed +1F90F ; Emoji_Modifier_Base # 12.0 [1] (๐Ÿค) pinching hand +1F918 ; Emoji_Modifier_Base # 8.0 [1] (๐Ÿค˜) sign of the horns +1F919..1F91E ; Emoji_Modifier_Base # 9.0 [6] (๐Ÿค™..๐Ÿคž) call me hand..crossed fingers +1F91F ; Emoji_Modifier_Base # 10.0 [1] (๐ŸคŸ) love-you gesture +1F926 ; Emoji_Modifier_Base # 9.0 [1] (๐Ÿคฆ) person facepalming +1F930 ; Emoji_Modifier_Base # 9.0 [1] (๐Ÿคฐ) pregnant woman +1F931..1F932 ; Emoji_Modifier_Base # 10.0 [2] (๐Ÿคฑ..๐Ÿคฒ) breast-feeding..palms up together +1F933..1F939 ; Emoji_Modifier_Base # 9.0 [7] (๐Ÿคณ..๐Ÿคน) selfie..person juggling +1F93C..1F93E ; Emoji_Modifier_Base # 9.0 [3] (๐Ÿคผ..๐Ÿคพ) people wrestling..person playing handball +1F9B5..1F9B6 ; Emoji_Modifier_Base # 11.0 [2] (๐Ÿฆต..๐Ÿฆถ) leg..foot +1F9B8..1F9B9 ; Emoji_Modifier_Base # 11.0 [2] (๐Ÿฆธ..๐Ÿฆน) superhero..supervillain +1F9BB ; Emoji_Modifier_Base # 12.0 [1] (๐Ÿฆป) ear with hearing aid +1F9CD..1F9CF ; Emoji_Modifier_Base # 12.0 [3] (๐Ÿง..๐Ÿง) person standing..deaf person +1F9D1..1F9DD ; Emoji_Modifier_Base # 10.0 [13] (๐Ÿง‘..๐Ÿง) person..elf + +# Total elements: 120 + +# ================================================ + +# All omitted code points have Emoji_Component=No +# @missing: 0000..10FFFF ; Emoji_Component ; No + +0023 ; Emoji_Component # 1.1 [1] (#๏ธ) number sign +002A ; Emoji_Component # 1.1 [1] (*๏ธ) asterisk +0030..0039 ; Emoji_Component # 1.1 [10] (0๏ธ..9๏ธ) digit zero..digit nine +200D ; Emoji_Component # 1.1 [1] (โ€) zero width joiner +20E3 ; Emoji_Component # 3.0 [1] (โƒฃ) combining enclosing keycap +FE0F ; Emoji_Component # 3.2 [1] () VARIATION SELECTOR-16 +1F1E6..1F1FF ; Emoji_Component # 6.0 [26] (๐Ÿ‡ฆ..๐Ÿ‡ฟ) regional indicator symbol letter a..regional indicator symbol letter z +1F3FB..1F3FF ; Emoji_Component # 8.0 [5] (๐Ÿป..๐Ÿฟ) light skin tone..dark skin tone +1F9B0..1F9B3 ; Emoji_Component # 11.0 [4] (๐Ÿฆฐ..๐Ÿฆณ) red hair..white hair +E0020..E007F ; Emoji_Component # 3.1 [96] (๓ € ..๓ ฟ) tag space..cancel tag + +# Total elements: 146 + +# ================================================ + +# All omitted code points have Extended_Pictographic=No +# @missing: 0000..10FFFF ; Extended_Pictographic ; No + +00A9 ; Extended_Pictographic# 1.1 [1] (ยฉ๏ธ) copyright +00AE ; Extended_Pictographic# 1.1 [1] (ยฎ๏ธ) registered +203C ; Extended_Pictographic# 1.1 [1] (โ€ผ๏ธ) double exclamation mark +2049 ; Extended_Pictographic# 3.0 [1] (โ‰๏ธ) exclamation question mark +2122 ; Extended_Pictographic# 1.1 [1] (โ„ข๏ธ) trade mark +2139 ; Extended_Pictographic# 3.0 [1] (โ„น๏ธ) information +2194..2199 ; Extended_Pictographic# 1.1 [6] (โ†”๏ธ..โ†™๏ธ) left-right arrow..down-left arrow +21A9..21AA ; Extended_Pictographic# 1.1 [2] (โ†ฉ๏ธ..โ†ช๏ธ) right arrow curving left..left arrow curving right +231A..231B ; Extended_Pictographic# 1.1 [2] (โŒš..โŒ›) watch..hourglass done +2328 ; Extended_Pictographic# 1.1 [1] (โŒจ๏ธ) keyboard +2388 ; Extended_Pictographic# 3.0 [1] (โŽˆ) HELM SYMBOL +23CF ; Extended_Pictographic# 4.0 [1] (โ๏ธ) eject button +23E9..23F3 ; Extended_Pictographic# 6.0 [11] (โฉ..โณ) fast-forward button..hourglass not done +23F8..23FA ; Extended_Pictographic# 7.0 [3] (โธ๏ธ..โบ๏ธ) pause button..record button +24C2 ; Extended_Pictographic# 1.1 [1] (โ“‚๏ธ) circled M +25AA..25AB ; Extended_Pictographic# 1.1 [2] (โ–ช๏ธ..โ–ซ๏ธ) black small square..white small square +25B6 ; Extended_Pictographic# 1.1 [1] (โ–ถ๏ธ) play button +25C0 ; Extended_Pictographic# 1.1 [1] (โ—€๏ธ) reverse button +25FB..25FE ; Extended_Pictographic# 3.2 [4] (โ—ป๏ธ..โ—พ) white medium square..black medium-small square +2600..2605 ; Extended_Pictographic# 1.1 [6] (โ˜€๏ธ..โ˜…) sun..BLACK STAR +2607..2612 ; Extended_Pictographic# 1.1 [12] (โ˜‡..โ˜’) LIGHTNING..BALLOT BOX WITH X +2614..2615 ; Extended_Pictographic# 4.0 [2] (โ˜”..โ˜•) umbrella with rain drops..hot beverage +2616..2617 ; Extended_Pictographic# 3.2 [2] (โ˜–..โ˜—) WHITE SHOGI PIECE..BLACK SHOGI PIECE +2618 ; Extended_Pictographic# 4.1 [1] (โ˜˜๏ธ) shamrock +2619 ; Extended_Pictographic# 3.0 [1] (โ˜™) REVERSED ROTATED FLORAL HEART BULLET +261A..266F ; Extended_Pictographic# 1.1 [86] (โ˜š..โ™ฏ) BLACK LEFT POINTING INDEX..MUSIC SHARP SIGN +2670..2671 ; Extended_Pictographic# 3.0 [2] (โ™ฐ..โ™ฑ) WEST SYRIAC CROSS..EAST SYRIAC CROSS +2672..267D ; Extended_Pictographic# 3.2 [12] (โ™ฒ..โ™ฝ) UNIVERSAL RECYCLING SYMBOL..PARTIALLY-RECYCLED PAPER SYMBOL +267E..267F ; Extended_Pictographic# 4.1 [2] (โ™พ๏ธ..โ™ฟ) infinity..wheelchair symbol +2680..2685 ; Extended_Pictographic# 3.2 [6] (โš€..โš…) DIE FACE-1..DIE FACE-6 +2690..2691 ; Extended_Pictographic# 4.0 [2] (โš..โš‘) WHITE FLAG..BLACK FLAG +2692..269C ; Extended_Pictographic# 4.1 [11] (โš’๏ธ..โšœ๏ธ) hammer and pick..fleur-de-lis +269D ; Extended_Pictographic# 5.1 [1] (โš) OUTLINED WHITE STAR +269E..269F ; Extended_Pictographic# 5.2 [2] (โšž..โšŸ) THREE LINES CONVERGING RIGHT..THREE LINES CONVERGING LEFT +26A0..26A1 ; Extended_Pictographic# 4.0 [2] (โš ๏ธ..โšก) warning..high voltage +26A2..26B1 ; Extended_Pictographic# 4.1 [16] (โšข..โšฑ๏ธ) DOUBLED FEMALE SIGN..funeral urn +26B2 ; Extended_Pictographic# 5.0 [1] (โšฒ) NEUTER +26B3..26BC ; Extended_Pictographic# 5.1 [10] (โšณ..โšผ) CERES..SESQUIQUADRATE +26BD..26BF ; Extended_Pictographic# 5.2 [3] (โšฝ..โšฟ) soccer ball..SQUARED KEY +26C0..26C3 ; Extended_Pictographic# 5.1 [4] (โ›€..โ›ƒ) WHITE DRAUGHTS MAN..BLACK DRAUGHTS KING +26C4..26CD ; Extended_Pictographic# 5.2 [10] (โ›„..โ›) snowman without snow..DISABLED CAR +26CE ; Extended_Pictographic# 6.0 [1] (โ›Ž) Ophiuchus +26CF..26E1 ; Extended_Pictographic# 5.2 [19] (โ›๏ธ..โ›ก) pick..RESTRICTED LEFT ENTRY-2 +26E2 ; Extended_Pictographic# 6.0 [1] (โ›ข) ASTRONOMICAL SYMBOL FOR URANUS +26E3 ; Extended_Pictographic# 5.2 [1] (โ›ฃ) HEAVY CIRCLE WITH STROKE AND TWO DOTS ABOVE +26E4..26E7 ; Extended_Pictographic# 6.0 [4] (โ›ค..โ›ง) PENTAGRAM..INVERTED PENTAGRAM +26E8..26FF ; Extended_Pictographic# 5.2 [24] (โ›จ..โ›ฟ) BLACK CROSS ON SHIELD..WHITE FLAG WITH HORIZONTAL MIDDLE BLACK STRIPE +2700 ; Extended_Pictographic# 7.0 [1] (โœ€) BLACK SAFETY SCISSORS +2701..2704 ; Extended_Pictographic# 1.1 [4] (โœ..โœ„) UPPER BLADE SCISSORS..WHITE SCISSORS +2705 ; Extended_Pictographic# 6.0 [1] (โœ…) check mark button +2708..2709 ; Extended_Pictographic# 1.1 [2] (โœˆ๏ธ..โœ‰๏ธ) airplane..envelope +270A..270B ; Extended_Pictographic# 6.0 [2] (โœŠ..โœ‹) raised fist..raised hand +270C..2712 ; Extended_Pictographic# 1.1 [7] (โœŒ๏ธ..โœ’๏ธ) victory hand..black nib +2714 ; Extended_Pictographic# 1.1 [1] (โœ”๏ธ) check mark +2716 ; Extended_Pictographic# 1.1 [1] (โœ–๏ธ) multiplication sign +271D ; Extended_Pictographic# 1.1 [1] (โœ๏ธ) latin cross +2721 ; Extended_Pictographic# 1.1 [1] (โœก๏ธ) star of David +2728 ; Extended_Pictographic# 6.0 [1] (โœจ) sparkles +2733..2734 ; Extended_Pictographic# 1.1 [2] (โœณ๏ธ..โœด๏ธ) eight-spoked asterisk..eight-pointed star +2744 ; Extended_Pictographic# 1.1 [1] (โ„๏ธ) snowflake +2747 ; Extended_Pictographic# 1.1 [1] (โ‡๏ธ) sparkle +274C ; Extended_Pictographic# 6.0 [1] (โŒ) cross mark +274E ; Extended_Pictographic# 6.0 [1] (โŽ) cross mark button +2753..2755 ; Extended_Pictographic# 6.0 [3] (โ“..โ•) question mark..white exclamation mark +2757 ; Extended_Pictographic# 5.2 [1] (โ—) exclamation mark +2763..2767 ; Extended_Pictographic# 1.1 [5] (โฃ๏ธ..โง) heart exclamation..ROTATED FLORAL HEART BULLET +2795..2797 ; Extended_Pictographic# 6.0 [3] (โž•..โž—) plus sign..division sign +27A1 ; Extended_Pictographic# 1.1 [1] (โžก๏ธ) right arrow +27B0 ; Extended_Pictographic# 6.0 [1] (โžฐ) curly loop +27BF ; Extended_Pictographic# 6.0 [1] (โžฟ) double curly loop +2934..2935 ; Extended_Pictographic# 3.2 [2] (โคด๏ธ..โคต๏ธ) right arrow curving up..right arrow curving down +2B05..2B07 ; Extended_Pictographic# 4.0 [3] (โฌ…๏ธ..โฌ‡๏ธ) left arrow..down arrow +2B1B..2B1C ; Extended_Pictographic# 5.1 [2] (โฌ›..โฌœ) black large square..white large square +2B50 ; Extended_Pictographic# 5.1 [1] (โญ) star +2B55 ; Extended_Pictographic# 5.2 [1] (โญ•) hollow red circle +3030 ; Extended_Pictographic# 1.1 [1] (ใ€ฐ๏ธ) wavy dash +303D ; Extended_Pictographic# 3.2 [1] (ใ€ฝ๏ธ) part alternation mark +3297 ; Extended_Pictographic# 1.1 [1] (ใŠ—๏ธ) Japanese โ€œcongratulationsโ€ button +3299 ; Extended_Pictographic# 1.1 [1] (ใŠ™๏ธ) Japanese โ€œsecretโ€ button +1F000..1F02B ; Extended_Pictographic# 5.1 [44] (๐Ÿ€€..๐Ÿ€ซ) MAHJONG TILE EAST WIND..MAHJONG TILE BACK +1F02C..1F02F ; Extended_Pictographic# NA [4] (๐Ÿ€ฌ..๐Ÿ€ฏ) .. +1F030..1F093 ; Extended_Pictographic# 5.1[100] (๐Ÿ€ฐ..๐Ÿ‚“) DOMINO TILE HORIZONTAL BACK..DOMINO TILE VERTICAL-06-06 +1F094..1F09F ; Extended_Pictographic# NA [12] (๐Ÿ‚”..๐Ÿ‚Ÿ) .. +1F0A0..1F0AE ; Extended_Pictographic# 6.0 [15] (๐Ÿ‚ ..๐Ÿ‚ฎ) PLAYING CARD BACK..PLAYING CARD KING OF SPADES +1F0AF..1F0B0 ; Extended_Pictographic# NA [2] (๐Ÿ‚ฏ..๐Ÿ‚ฐ) .. +1F0B1..1F0BE ; Extended_Pictographic# 6.0 [14] (๐Ÿ‚ฑ..๐Ÿ‚พ) PLAYING CARD ACE OF HEARTS..PLAYING CARD KING OF HEARTS +1F0BF ; Extended_Pictographic# 7.0 [1] (๐Ÿ‚ฟ) PLAYING CARD RED JOKER +1F0C0 ; Extended_Pictographic# NA [1] (๐Ÿƒ€) +1F0C1..1F0CF ; Extended_Pictographic# 6.0 [15] (๐Ÿƒ..๐Ÿƒ) PLAYING CARD ACE OF DIAMONDS..joker +1F0D0 ; Extended_Pictographic# NA [1] (๐Ÿƒ) +1F0D1..1F0DF ; Extended_Pictographic# 6.0 [15] (๐Ÿƒ‘..๐ŸƒŸ) PLAYING CARD ACE OF CLUBS..PLAYING CARD WHITE JOKER +1F0E0..1F0F5 ; Extended_Pictographic# 7.0 [22] (๐Ÿƒ ..๐Ÿƒต) PLAYING CARD FOOL..PLAYING CARD TRUMP-21 +1F0F6..1F0FF ; Extended_Pictographic# NA [10] (๐Ÿƒถ..๐Ÿƒฟ) .. +1F10D..1F10F ; Extended_Pictographic# NA [3] (๐Ÿ„..๐Ÿ„) .. +1F12F ; Extended_Pictographic# 11.0 [1] (๐Ÿ„ฏ) COPYLEFT SYMBOL +1F16C ; Extended_Pictographic# 12.0 [1] (๐Ÿ…ฌ) RAISED MR SIGN +1F16D..1F16F ; Extended_Pictographic# NA [3] (๐Ÿ…ญ..๐Ÿ…ฏ) .. +1F170..1F171 ; Extended_Pictographic# 6.0 [2] (๐Ÿ…ฐ๏ธ..๐Ÿ…ฑ๏ธ) A button (blood type)..B button (blood type) +1F17E ; Extended_Pictographic# 6.0 [1] (๐Ÿ…พ๏ธ) O button (blood type) +1F17F ; Extended_Pictographic# 5.2 [1] (๐Ÿ…ฟ๏ธ) P button +1F18E ; Extended_Pictographic# 6.0 [1] (๐Ÿ†Ž) AB button (blood type) +1F191..1F19A ; Extended_Pictographic# 6.0 [10] (๐Ÿ†‘..๐Ÿ†š) CL button..VS button +1F1AD..1F1E5 ; Extended_Pictographic# NA [57] (๐Ÿ†ญ..๐Ÿ‡ฅ) .. +1F201..1F202 ; Extended_Pictographic# 6.0 [2] (๐Ÿˆ..๐Ÿˆ‚๏ธ) Japanese โ€œhereโ€ button..Japanese โ€œservice chargeโ€ button +1F203..1F20F ; Extended_Pictographic# NA [13] (๐Ÿˆƒ..๐Ÿˆ) .. +1F21A ; Extended_Pictographic# 5.2 [1] (๐Ÿˆš) Japanese โ€œfree of chargeโ€ button +1F22F ; Extended_Pictographic# 5.2 [1] (๐Ÿˆฏ) Japanese โ€œreservedโ€ button +1F232..1F23A ; Extended_Pictographic# 6.0 [9] (๐Ÿˆฒ..๐Ÿˆบ) Japanese โ€œprohibitedโ€ button..Japanese โ€œopen for businessโ€ button +1F23C..1F23F ; Extended_Pictographic# NA [4] (๐Ÿˆผ..๐Ÿˆฟ) .. +1F249..1F24F ; Extended_Pictographic# NA [7] (๐Ÿ‰‰..๐Ÿ‰) .. +1F250..1F251 ; Extended_Pictographic# 6.0 [2] (๐Ÿ‰..๐Ÿ‰‘) Japanese โ€œbargainโ€ button..Japanese โ€œacceptableโ€ button +1F252..1F25F ; Extended_Pictographic# NA [14] (๐Ÿ‰’..๐Ÿ‰Ÿ) .. +1F260..1F265 ; Extended_Pictographic# 10.0 [6] (๐Ÿ‰ ..๐Ÿ‰ฅ) ROUNDED SYMBOL FOR FU..ROUNDED SYMBOL FOR CAI +1F266..1F2FF ; Extended_Pictographic# NA[154] (๐Ÿ‰ฆ..๐Ÿ‹ฟ) .. +1F300..1F320 ; Extended_Pictographic# 6.0 [33] (๐ŸŒ€..๐ŸŒ ) cyclone..shooting star +1F321..1F32C ; Extended_Pictographic# 7.0 [12] (๐ŸŒก๏ธ..๐ŸŒฌ๏ธ) thermometer..wind face +1F32D..1F32F ; Extended_Pictographic# 8.0 [3] (๐ŸŒญ..๐ŸŒฏ) hot dog..burrito +1F330..1F335 ; Extended_Pictographic# 6.0 [6] (๐ŸŒฐ..๐ŸŒต) chestnut..cactus +1F336 ; Extended_Pictographic# 7.0 [1] (๐ŸŒถ๏ธ) hot pepper +1F337..1F37C ; Extended_Pictographic# 6.0 [70] (๐ŸŒท..๐Ÿผ) tulip..baby bottle +1F37D ; Extended_Pictographic# 7.0 [1] (๐Ÿฝ๏ธ) fork and knife with plate +1F37E..1F37F ; Extended_Pictographic# 8.0 [2] (๐Ÿพ..๐Ÿฟ) bottle with popping cork..popcorn +1F380..1F393 ; Extended_Pictographic# 6.0 [20] (๐ŸŽ€..๐ŸŽ“) ribbon..graduation cap +1F394..1F39F ; Extended_Pictographic# 7.0 [12] (๐ŸŽ”..๐ŸŽŸ๏ธ) HEART WITH TIP ON THE LEFT..admission tickets +1F3A0..1F3C4 ; Extended_Pictographic# 6.0 [37] (๐ŸŽ ..๐Ÿ„) carousel horse..person surfing +1F3C5 ; Extended_Pictographic# 7.0 [1] (๐Ÿ…) sports medal +1F3C6..1F3CA ; Extended_Pictographic# 6.0 [5] (๐Ÿ†..๐ŸŠ) trophy..person swimming +1F3CB..1F3CE ; Extended_Pictographic# 7.0 [4] (๐Ÿ‹๏ธ..๐ŸŽ๏ธ) person lifting weights..racing car +1F3CF..1F3D3 ; Extended_Pictographic# 8.0 [5] (๐Ÿ..๐Ÿ“) cricket game..ping pong +1F3D4..1F3DF ; Extended_Pictographic# 7.0 [12] (๐Ÿ”๏ธ..๐ŸŸ๏ธ) snow-capped mountain..stadium +1F3E0..1F3F0 ; Extended_Pictographic# 6.0 [17] (๐Ÿ ..๐Ÿฐ) house..castle +1F3F1..1F3F7 ; Extended_Pictographic# 7.0 [7] (๐Ÿฑ..๐Ÿท๏ธ) WHITE PENNANT..label +1F3F8..1F3FA ; Extended_Pictographic# 8.0 [3] (๐Ÿธ..๐Ÿบ) badminton..amphora +1F400..1F43E ; Extended_Pictographic# 6.0 [63] (๐Ÿ€..๐Ÿพ) rat..paw prints +1F43F ; Extended_Pictographic# 7.0 [1] (๐Ÿฟ๏ธ) chipmunk +1F440 ; Extended_Pictographic# 6.0 [1] (๐Ÿ‘€) eyes +1F441 ; Extended_Pictographic# 7.0 [1] (๐Ÿ‘๏ธ) eye +1F442..1F4F7 ; Extended_Pictographic# 6.0[182] (๐Ÿ‘‚..๐Ÿ“ท) ear..camera +1F4F8 ; Extended_Pictographic# 7.0 [1] (๐Ÿ“ธ) camera with flash +1F4F9..1F4FC ; Extended_Pictographic# 6.0 [4] (๐Ÿ“น..๐Ÿ“ผ) video camera..videocassette +1F4FD..1F4FE ; Extended_Pictographic# 7.0 [2] (๐Ÿ“ฝ๏ธ..๐Ÿ“พ) film projector..PORTABLE STEREO +1F4FF ; Extended_Pictographic# 8.0 [1] (๐Ÿ“ฟ) prayer beads +1F500..1F53D ; Extended_Pictographic# 6.0 [62] (๐Ÿ”€..๐Ÿ”ฝ) shuffle tracks button..downwards button +1F546..1F54A ; Extended_Pictographic# 7.0 [5] (๐Ÿ•†..๐Ÿ•Š๏ธ) WHITE LATIN CROSS..dove +1F54B..1F54F ; Extended_Pictographic# 8.0 [5] (๐Ÿ•‹..๐Ÿ•) kaaba..BOWL OF HYGIEIA +1F550..1F567 ; Extended_Pictographic# 6.0 [24] (๐Ÿ•..๐Ÿ•ง) one oโ€™clock..twelve-thirty +1F568..1F579 ; Extended_Pictographic# 7.0 [18] (๐Ÿ•จ..๐Ÿ•น๏ธ) RIGHT SPEAKER..joystick +1F57A ; Extended_Pictographic# 9.0 [1] (๐Ÿ•บ) man dancing +1F57B..1F5A3 ; Extended_Pictographic# 7.0 [41] (๐Ÿ•ป..๐Ÿ–ฃ) LEFT HAND TELEPHONE RECEIVER..BLACK DOWN POINTING BACKHAND INDEX +1F5A4 ; Extended_Pictographic# 9.0 [1] (๐Ÿ–ค) black heart +1F5A5..1F5FA ; Extended_Pictographic# 7.0 [86] (๐Ÿ–ฅ๏ธ..๐Ÿ—บ๏ธ) desktop computer..world map +1F5FB..1F5FF ; Extended_Pictographic# 6.0 [5] (๐Ÿ—ป..๐Ÿ—ฟ) mount fuji..moai +1F600 ; Extended_Pictographic# 6.1 [1] (๐Ÿ˜€) grinning face +1F601..1F610 ; Extended_Pictographic# 6.0 [16] (๐Ÿ˜..๐Ÿ˜) beaming face with smiling eyes..neutral face +1F611 ; Extended_Pictographic# 6.1 [1] (๐Ÿ˜‘) expressionless face +1F612..1F614 ; Extended_Pictographic# 6.0 [3] (๐Ÿ˜’..๐Ÿ˜”) unamused face..pensive face +1F615 ; Extended_Pictographic# 6.1 [1] (๐Ÿ˜•) confused face +1F616 ; Extended_Pictographic# 6.0 [1] (๐Ÿ˜–) confounded face +1F617 ; Extended_Pictographic# 6.1 [1] (๐Ÿ˜—) kissing face +1F618 ; Extended_Pictographic# 6.0 [1] (๐Ÿ˜˜) face blowing a kiss +1F619 ; Extended_Pictographic# 6.1 [1] (๐Ÿ˜™) kissing face with smiling eyes +1F61A ; Extended_Pictographic# 6.0 [1] (๐Ÿ˜š) kissing face with closed eyes +1F61B ; Extended_Pictographic# 6.1 [1] (๐Ÿ˜›) face with tongue +1F61C..1F61E ; Extended_Pictographic# 6.0 [3] (๐Ÿ˜œ..๐Ÿ˜ž) winking face with tongue..disappointed face +1F61F ; Extended_Pictographic# 6.1 [1] (๐Ÿ˜Ÿ) worried face +1F620..1F625 ; Extended_Pictographic# 6.0 [6] (๐Ÿ˜ ..๐Ÿ˜ฅ) angry face..sad but relieved face +1F626..1F627 ; Extended_Pictographic# 6.1 [2] (๐Ÿ˜ฆ..๐Ÿ˜ง) frowning face with open mouth..anguished face +1F628..1F62B ; Extended_Pictographic# 6.0 [4] (๐Ÿ˜จ..๐Ÿ˜ซ) fearful face..tired face +1F62C ; Extended_Pictographic# 6.1 [1] (๐Ÿ˜ฌ) grimacing face +1F62D ; Extended_Pictographic# 6.0 [1] (๐Ÿ˜ญ) loudly crying face +1F62E..1F62F ; Extended_Pictographic# 6.1 [2] (๐Ÿ˜ฎ..๐Ÿ˜ฏ) face with open mouth..hushed face +1F630..1F633 ; Extended_Pictographic# 6.0 [4] (๐Ÿ˜ฐ..๐Ÿ˜ณ) anxious face with sweat..flushed face +1F634 ; Extended_Pictographic# 6.1 [1] (๐Ÿ˜ด) sleeping face +1F635..1F640 ; Extended_Pictographic# 6.0 [12] (๐Ÿ˜ต..๐Ÿ™€) dizzy face..weary cat +1F641..1F642 ; Extended_Pictographic# 7.0 [2] (๐Ÿ™..๐Ÿ™‚) slightly frowning face..slightly smiling face +1F643..1F644 ; Extended_Pictographic# 8.0 [2] (๐Ÿ™ƒ..๐Ÿ™„) upside-down face..face with rolling eyes +1F645..1F64F ; Extended_Pictographic# 6.0 [11] (๐Ÿ™…..๐Ÿ™) person gesturing NO..folded hands +1F680..1F6C5 ; Extended_Pictographic# 6.0 [70] (๐Ÿš€..๐Ÿ›…) rocket..left luggage +1F6C6..1F6CF ; Extended_Pictographic# 7.0 [10] (๐Ÿ›†..๐Ÿ›๏ธ) TRIANGLE WITH ROUNDED CORNERS..bed +1F6D0 ; Extended_Pictographic# 8.0 [1] (๐Ÿ›) place of worship +1F6D1..1F6D2 ; Extended_Pictographic# 9.0 [2] (๐Ÿ›‘..๐Ÿ›’) stop sign..shopping cart +1F6D3..1F6D4 ; Extended_Pictographic# 10.0 [2] (๐Ÿ›“..๐Ÿ›”) STUPA..PAGODA +1F6D5 ; Extended_Pictographic# 12.0 [1] (๐Ÿ›•) hindu temple +1F6D6..1F6DF ; Extended_Pictographic# NA [10] (๐Ÿ›–..๐Ÿ›Ÿ) .. +1F6E0..1F6EC ; Extended_Pictographic# 7.0 [13] (๐Ÿ› ๏ธ..๐Ÿ›ฌ) hammer and wrench..airplane arrival +1F6ED..1F6EF ; Extended_Pictographic# NA [3] (๐Ÿ›ญ..๐Ÿ›ฏ) .. +1F6F0..1F6F3 ; Extended_Pictographic# 7.0 [4] (๐Ÿ›ฐ๏ธ..๐Ÿ›ณ๏ธ) satellite..passenger ship +1F6F4..1F6F6 ; Extended_Pictographic# 9.0 [3] (๐Ÿ›ด..๐Ÿ›ถ) kick scooter..canoe +1F6F7..1F6F8 ; Extended_Pictographic# 10.0 [2] (๐Ÿ›ท..๐Ÿ›ธ) sled..flying saucer +1F6F9 ; Extended_Pictographic# 11.0 [1] (๐Ÿ›น) skateboard +1F6FA ; Extended_Pictographic# 12.0 [1] (๐Ÿ›บ) auto rickshaw +1F6FB..1F6FF ; Extended_Pictographic# NA [5] (๐Ÿ›ป..๐Ÿ›ฟ) .. +1F774..1F77F ; Extended_Pictographic# NA [12] (๐Ÿด..๐Ÿฟ) .. +1F7D5..1F7D8 ; Extended_Pictographic# 11.0 [4] (๐ŸŸ•..๐ŸŸ˜) CIRCLED TRIANGLE..NEGATIVE CIRCLED SQUARE +1F7D9..1F7DF ; Extended_Pictographic# NA [7] (๐ŸŸ™..๐ŸŸŸ) .. +1F7E0..1F7EB ; Extended_Pictographic# 12.0 [12] (๐ŸŸ ..๐ŸŸซ) orange circle..brown square +1F7EC..1F7FF ; Extended_Pictographic# NA [20] (๐ŸŸฌ..๐ŸŸฟ) .. +1F80C..1F80F ; Extended_Pictographic# NA [4] (๐Ÿ Œ..๐Ÿ ) .. +1F848..1F84F ; Extended_Pictographic# NA [8] (๐Ÿกˆ..๐Ÿก) .. +1F85A..1F85F ; Extended_Pictographic# NA [6] (๐Ÿกš..๐ŸกŸ) .. +1F888..1F88F ; Extended_Pictographic# NA [8] (๐Ÿขˆ..๐Ÿข) .. +1F8AE..1F8FF ; Extended_Pictographic# NA [82] (๐Ÿขฎ..๐Ÿฃฟ) .. +1F90C ; Extended_Pictographic# NA [1] (๐ŸคŒ) +1F90D..1F90F ; Extended_Pictographic# 12.0 [3] (๐Ÿค..๐Ÿค) white heart..pinching hand +1F910..1F918 ; Extended_Pictographic# 8.0 [9] (๐Ÿค..๐Ÿค˜) zipper-mouth face..sign of the horns +1F919..1F91E ; Extended_Pictographic# 9.0 [6] (๐Ÿค™..๐Ÿคž) call me hand..crossed fingers +1F91F ; Extended_Pictographic# 10.0 [1] (๐ŸคŸ) love-you gesture +1F920..1F927 ; Extended_Pictographic# 9.0 [8] (๐Ÿค ..๐Ÿคง) cowboy hat face..sneezing face +1F928..1F92F ; Extended_Pictographic# 10.0 [8] (๐Ÿคจ..๐Ÿคฏ) face with raised eyebrow..exploding head +1F930 ; Extended_Pictographic# 9.0 [1] (๐Ÿคฐ) pregnant woman +1F931..1F932 ; Extended_Pictographic# 10.0 [2] (๐Ÿคฑ..๐Ÿคฒ) breast-feeding..palms up together +1F933..1F93A ; Extended_Pictographic# 9.0 [8] (๐Ÿคณ..๐Ÿคบ) selfie..person fencing +1F93C..1F93E ; Extended_Pictographic# 9.0 [3] (๐Ÿคผ..๐Ÿคพ) people wrestling..person playing handball +1F93F ; Extended_Pictographic# 12.0 [1] (๐Ÿคฟ) diving mask +1F940..1F945 ; Extended_Pictographic# 9.0 [6] (๐Ÿฅ€..๐Ÿฅ…) wilted flower..goal net +1F947..1F94B ; Extended_Pictographic# 9.0 [5] (๐Ÿฅ‡..๐Ÿฅ‹) 1st place medal..martial arts uniform +1F94C ; Extended_Pictographic# 10.0 [1] (๐ŸฅŒ) curling stone +1F94D..1F94F ; Extended_Pictographic# 11.0 [3] (๐Ÿฅ..๐Ÿฅ) lacrosse..flying disc +1F950..1F95E ; Extended_Pictographic# 9.0 [15] (๐Ÿฅ..๐Ÿฅž) croissant..pancakes +1F95F..1F96B ; Extended_Pictographic# 10.0 [13] (๐ŸฅŸ..๐Ÿฅซ) dumpling..canned food +1F96C..1F970 ; Extended_Pictographic# 11.0 [5] (๐Ÿฅฌ..๐Ÿฅฐ) leafy green..smiling face with hearts +1F971 ; Extended_Pictographic# 12.0 [1] (๐Ÿฅฑ) yawning face +1F972 ; Extended_Pictographic# NA [1] (๐Ÿฅฒ) +1F973..1F976 ; Extended_Pictographic# 11.0 [4] (๐Ÿฅณ..๐Ÿฅถ) partying face..cold face +1F977..1F979 ; Extended_Pictographic# NA [3] (๐Ÿฅท..๐Ÿฅน) .. +1F97A ; Extended_Pictographic# 11.0 [1] (๐Ÿฅบ) pleading face +1F97B ; Extended_Pictographic# 12.0 [1] (๐Ÿฅป) sari +1F97C..1F97F ; Extended_Pictographic# 11.0 [4] (๐Ÿฅผ..๐Ÿฅฟ) lab coat..flat shoe +1F980..1F984 ; Extended_Pictographic# 8.0 [5] (๐Ÿฆ€..๐Ÿฆ„) crab..unicorn +1F985..1F991 ; Extended_Pictographic# 9.0 [13] (๐Ÿฆ…..๐Ÿฆ‘) eagle..squid +1F992..1F997 ; Extended_Pictographic# 10.0 [6] (๐Ÿฆ’..๐Ÿฆ—) giraffe..cricket +1F998..1F9A2 ; Extended_Pictographic# 11.0 [11] (๐Ÿฆ˜..๐Ÿฆข) kangaroo..swan +1F9A3..1F9A4 ; Extended_Pictographic# NA [2] (๐Ÿฆฃ..๐Ÿฆค) .. +1F9A5..1F9AA ; Extended_Pictographic# 12.0 [6] (๐Ÿฆฅ..๐Ÿฆช) sloth..oyster +1F9AB..1F9AD ; Extended_Pictographic# NA [3] (๐Ÿฆซ..๐Ÿฆญ) .. +1F9AE..1F9AF ; Extended_Pictographic# 12.0 [2] (๐Ÿฆฎ..๐Ÿฆฏ) guide dog..probing cane +1F9B0..1F9B9 ; Extended_Pictographic# 11.0 [10] (๐Ÿฆฐ..๐Ÿฆน) red hair..supervillain +1F9BA..1F9BF ; Extended_Pictographic# 12.0 [6] (๐Ÿฆบ..๐Ÿฆฟ) safety vest..mechanical leg +1F9C0 ; Extended_Pictographic# 8.0 [1] (๐Ÿง€) cheese wedge +1F9C1..1F9C2 ; Extended_Pictographic# 11.0 [2] (๐Ÿง..๐Ÿง‚) cupcake..salt +1F9C3..1F9CA ; Extended_Pictographic# 12.0 [8] (๐Ÿงƒ..๐ŸงŠ) beverage box..ice cube +1F9CB..1F9CC ; Extended_Pictographic# NA [2] (๐Ÿง‹..๐ŸงŒ) .. +1F9CD..1F9CF ; Extended_Pictographic# 12.0 [3] (๐Ÿง..๐Ÿง) person standing..deaf person +1F9D0..1F9E6 ; Extended_Pictographic# 10.0 [23] (๐Ÿง..๐Ÿงฆ) face with monocle..socks +1F9E7..1F9FF ; Extended_Pictographic# 11.0 [25] (๐Ÿงง..๐Ÿงฟ) red envelope..nazar amulet +1FA00..1FA53 ; Extended_Pictographic# 12.0 [84] (๐Ÿจ€..๐Ÿฉ“) NEUTRAL CHESS KING..BLACK CHESS KNIGHT-BISHOP +1FA54..1FA5F ; Extended_Pictographic# NA [12] (๐Ÿฉ”..๐ŸฉŸ) .. +1FA60..1FA6D ; Extended_Pictographic# 11.0 [14] (๐Ÿฉ ..๐Ÿฉญ) XIANGQI RED GENERAL..XIANGQI BLACK SOLDIER +1FA6E..1FA6F ; Extended_Pictographic# NA [2] (๐Ÿฉฎ..๐Ÿฉฏ) .. +1FA70..1FA73 ; Extended_Pictographic# 12.0 [4] (๐Ÿฉฐ..๐Ÿฉณ) ballet shoes..shorts +1FA74..1FA77 ; Extended_Pictographic# NA [4] (๐Ÿฉด..๐Ÿฉท) .. +1FA78..1FA7A ; Extended_Pictographic# 12.0 [3] (๐Ÿฉธ..๐Ÿฉบ) drop of blood..stethoscope +1FA7B..1FA7F ; Extended_Pictographic# NA [5] (๐Ÿฉป..๐Ÿฉฟ) .. +1FA80..1FA82 ; Extended_Pictographic# 12.0 [3] (๐Ÿช€..๐Ÿช‚) yo-yo..parachute +1FA83..1FA8F ; Extended_Pictographic# NA [13] (๐Ÿชƒ..๐Ÿช) .. +1FA90..1FA95 ; Extended_Pictographic# 12.0 [6] (๐Ÿช..๐Ÿช•) ringed planet..banjo +1FA96..1FFFD ; Extended_Pictographic# NA[1384] (๐Ÿช–..๐Ÿฟฝ) .. + +# Total elements: 3793 + +#EOF diff --git a/lib/pleroma/emoji.ex b/lib/pleroma/emoji.ex index 66e20f0e4..3f5169007 100644 --- a/lib/pleroma/emoji.ex +++ b/lib/pleroma/emoji.ex @@ -260,4 +260,29 @@ defp match_group_patterns(group_patterns, matcher) do Enum.any?(patterns, matcher) && group end) end + + {:ok, file} = File.read("lib/pleroma/emoji-data.txt") + + @unicode_emoji file + |> String.split("\n") + |> Enum.filter(fn line -> String.starts_with?(line, ["1", "2", "3", "4"]) end) + |> Enum.map(fn line -> String.split(line) |> List.first() end) + |> Enum.map(fn line -> + case String.split(line, "..") do + [number] -> + String.to_integer(number, 16) + + [first, last] -> + Range.new(String.to_integer(first, 16), String.to_integer(last, 16)) + |> Enum.to_list() + end + end) + |> List.flatten() + |> Enum.filter(&is_integer/1) + |> Enum.uniq() + |> Enum.map(fn n -> :unicode.characters_to_binary([n], :utf32) end) + + def is_unicode_emoji?(str) do + str in @unicode_emoji + end end diff --git a/test/emoji_test.exs b/test/emoji_test.exs index 07ac6ff1d..467357291 100644 --- a/test/emoji_test.exs +++ b/test/emoji_test.exs @@ -6,6 +6,14 @@ defmodule Pleroma.EmojiTest do use ExUnit.Case, async: true alias Pleroma.Emoji + describe "is_unicode_emoji?/1" do + test "tells if a string is an unicode emoji" do + refute Emoji.is_unicode_emoji?("X") + assert Emoji.is_unicode_emoji?("โ˜‚") + assert Emoji.is_unicode_emoji?("๐Ÿฅบ") + end + end + describe "get_all/0" do setup do emoji_list = Emoji.get_all() From e5b3ad3d049a7c665285f724c53f6cafb0e10118 Mon Sep 17 00:00:00 2001 From: lain Date: Fri, 13 Sep 2019 16:06:34 +0200 Subject: [PATCH 009/107] ActivityPub: Use is_unicode_emoji? function. --- lib/pleroma/web/activity_pub/activity_pub.ex | 2 +- lib/pleroma/web/activity_pub/utils.ex | 4 ---- test/web/pleroma_api/pleroma_api_controller_test.exs | 1 - 3 files changed, 1 insertion(+), 6 deletions(-) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 6cd168427..4ee9b1885 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -315,7 +315,7 @@ def update(%{to: to, cc: cc, actor: actor, object: object} = params) do def react_with_emoji(user, object, emoji, options \\ []) do with local <- Keyword.get(options, :local, true), activity_id <- Keyword.get(options, :activity_id, nil), - is_emoji?(emoji), + Pleroma.Emoji.is_unicode_emoji?(emoji), reaction_data <- make_emoji_reaction_data(user, object, emoji, activity_id), {:ok, activity} <- insert(reaction_data, local), {:ok, object} <- add_emoji_reaction_to_object(activity, object) do diff --git a/lib/pleroma/web/activity_pub/utils.ex b/lib/pleroma/web/activity_pub/utils.ex index 1e6a67deb..95e040c6c 100644 --- a/lib/pleroma/web/activity_pub/utils.ex +++ b/lib/pleroma/web/activity_pub/utils.ex @@ -277,10 +277,6 @@ def get_object_likes(%{data: %{"id" => id}}) do |> Repo.all() end - def is_emoji?(emoji) do - String.length(emoji) == 1 - end - def make_emoji_reaction_data(user, object, emoji, activity_id) do make_like_data(user, object, activity_id) |> Map.put("type", "EmojiReaction") diff --git a/test/web/pleroma_api/pleroma_api_controller_test.exs b/test/web/pleroma_api/pleroma_api_controller_test.exs index 71e4d3e1c..fa85d9174 100644 --- a/test/web/pleroma_api/pleroma_api_controller_test.exs +++ b/test/web/pleroma_api/pleroma_api_controller_test.exs @@ -8,7 +8,6 @@ defmodule Pleroma.Web.PleromaAPI.PleromaAPIControllerTest do alias Pleroma.Conversation.Participation alias Pleroma.Repo alias Pleroma.Web.CommonAPI - alias Pleroma.Web.MastodonAPI.AccountView import Pleroma.Factory From f649a2e972b70dfefb7bfc110b27a0194cda51c5 Mon Sep 17 00:00:00 2001 From: lain Date: Fri, 13 Sep 2019 16:19:50 +0200 Subject: [PATCH 010/107] Pleroma API Docs: Documented Emoji reaction endpoints. --- docs/api/pleroma_api.md | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/docs/api/pleroma_api.md b/docs/api/pleroma_api.md index b134b31a8..c846df1da 100644 --- a/docs/api/pleroma_api.md +++ b/docs/api/pleroma_api.md @@ -354,3 +354,28 @@ The status posting endpoint takes an additional parameter, `in_reply_to_conversa * Params: * `recipients`: A list of ids of users that should receive posts to this conversation. This will replace the current list of recipients, so submit the full list. The owner of owner of the conversation will always be part of the set of recipients, though. * Response: JSON, statuses (200 - healthy, 503 unhealthy) + +# Emoji Reactions + +Emoji reactions work a lot like favourites do. They make it possible to react to a post with a single emoji character. + +## `POST /api/v1/pleroma/statuses/:id/react_with_emoji` +### React to a post with a unicode emoji +* Method: `POST` +* Authentication: required +* Params: `emoji`: A single character unicode emoji +* Response: JSON, the status. + +## `GET /api/v1/pleroma/statuses/:id/emoji_reactions_by` +### Get an object of emoji to account mappings with accounts that reacted to the post +* Method: `GET` +* Authentication: optional +* Params: None +* Response: JSON, a map of emoji to account list mappings. +* Example Response: +```json +{ + "๐Ÿ˜€" => [{"id" => "xyz.."...}, {"id" => "zyx..."}], + "๐Ÿ—ก" => [{"id" => "abc..."}] +} +``` From 3ff55322201fa7f6b22368988095963fcb82d6e4 Mon Sep 17 00:00:00 2001 From: lain Date: Fri, 13 Sep 2019 16:41:30 +0200 Subject: [PATCH 011/107] Linting. --- lib/pleroma/web/pleroma_api/pleroma_api_controller.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/web/pleroma_api/pleroma_api_controller.ex b/lib/pleroma/web/pleroma_api/pleroma_api_controller.ex index 110240115..d41091d93 100644 --- a/lib/pleroma/web/pleroma_api/pleroma_api_controller.ex +++ b/lib/pleroma/web/pleroma_api/pleroma_api_controller.ex @@ -8,10 +8,10 @@ defmodule Pleroma.Web.PleromaAPI.PleromaAPIController do import Pleroma.Web.ControllerHelper, only: [add_link_headers: 7] alias Pleroma.Activity - alias Pleroma.Object - alias Pleroma.User alias Pleroma.Conversation.Participation alias Pleroma.Notification + alias Pleroma.Object + alias Pleroma.User alias Pleroma.Web.ActivityPub.ActivityPub alias Pleroma.Web.CommonAPI alias Pleroma.Web.MastodonAPI.AccountView From 6fe2f554c36be1ef03ac1d1104a78d0686f48a26 Mon Sep 17 00:00:00 2001 From: lain Date: Fri, 13 Sep 2019 18:27:32 +0200 Subject: [PATCH 012/107] Emoji: Generate emoji detecting functions at compile time. Suggested by jvalim --- lib/pleroma/emoji.ex | 46 +++++++++++++++++++++++++------------------- 1 file changed, 26 insertions(+), 20 deletions(-) diff --git a/lib/pleroma/emoji.ex b/lib/pleroma/emoji.ex index 3f5169007..65f2d5f39 100644 --- a/lib/pleroma/emoji.ex +++ b/lib/pleroma/emoji.ex @@ -261,28 +261,34 @@ defp match_group_patterns(group_patterns, matcher) do end) end - {:ok, file} = File.read("lib/pleroma/emoji-data.txt") + @external_resource "lib/pleroma/emoji-data.txt" - @unicode_emoji file - |> String.split("\n") - |> Enum.filter(fn line -> String.starts_with?(line, ["1", "2", "3", "4"]) end) - |> Enum.map(fn line -> String.split(line) |> List.first() end) - |> Enum.map(fn line -> - case String.split(line, "..") do - [number] -> - String.to_integer(number, 16) + emojis = + @external_resource + |> File.read!() + |> String.split("\n") + |> Enum.filter(fn line -> line != "" and not String.starts_with?(line, "#") end) + |> Enum.map(fn line -> + line + |> String.split(";", parts: 2) + |> hd() + |> String.trim() + |> String.split("..") + |> case do + [number] -> + <> - [first, last] -> - Range.new(String.to_integer(first, 16), String.to_integer(last, 16)) - |> Enum.to_list() - end - end) - |> List.flatten() - |> Enum.filter(&is_integer/1) - |> Enum.uniq() - |> Enum.map(fn n -> :unicode.characters_to_binary([n], :utf32) end) + [first, last] -> + String.to_integer(first, 16)..String.to_integer(last, 16) + |> Enum.map(&<<&1::utf8>>) + end + end) + |> List.flatten() + |> Enum.uniq() - def is_unicode_emoji?(str) do - str in @unicode_emoji + for emoji <- emojis do + def is_unicode_emoji?(unquote(emoji)), do: true end + + def is_unicode_emoji?(_), do: false end From 0aef18bcf4896adbd6bb7ea59e9399cb09dbfa3c Mon Sep 17 00:00:00 2001 From: lain Date: Mon, 30 Sep 2019 14:26:59 +0200 Subject: [PATCH 013/107] Litepub Context: Add EmojiReaction. --- priv/static/schemas/litepub-0.1.jsonld | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/priv/static/schemas/litepub-0.1.jsonld b/priv/static/schemas/litepub-0.1.jsonld index f01c2c33a..5e44ad819 100644 --- a/priv/static/schemas/litepub-0.1.jsonld +++ b/priv/static/schemas/litepub-0.1.jsonld @@ -32,7 +32,8 @@ "uploadMedia": { "@id": "litepub:uploadMedia", "@type": "@id" - } + }, + "EmojiReaction": "litepub:EmojiReaction" } ] } From 04a2910f33405db368687f8749b405eeac06df63 Mon Sep 17 00:00:00 2001 From: lain Date: Mon, 30 Sep 2019 15:13:05 +0200 Subject: [PATCH 014/107] Pleroma.Constants: Fix typo. --- lib/pleroma/constants.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/constants.ex b/lib/pleroma/constants.ex index 7d775c3c2..1a432e681 100644 --- a/lib/pleroma/constants.ex +++ b/lib/pleroma/constants.ex @@ -10,7 +10,7 @@ defmodule Pleroma.Constants do const(object_internal_fields, do: [ "reactions", - "reactions_count", + "reaction_count", "likes", "like_count", "announcements", From 6068d2254e2ed00260dc840f18824dc0e0835afa Mon Sep 17 00:00:00 2001 From: lain Date: Mon, 30 Sep 2019 15:13:25 +0200 Subject: [PATCH 015/107] PleromaAPIController: Fixes and refactoring. --- .../controllers/pleroma_api_controller.ex | 17 ++++++----------- test/web/activity_pub/transmogrifier_test.exs | 2 +- 2 files changed, 7 insertions(+), 12 deletions(-) diff --git a/lib/pleroma/web/pleroma_api/controllers/pleroma_api_controller.ex b/lib/pleroma/web/pleroma_api/controllers/pleroma_api_controller.ex index 474b8d079..39d371ff7 100644 --- a/lib/pleroma/web/pleroma_api/controllers/pleroma_api_controller.ex +++ b/lib/pleroma/web/pleroma_api/controllers/pleroma_api_controller.ex @@ -23,17 +23,12 @@ def emoji_reactions_by(%{assigns: %{user: user}} = conn, %{"id" => activity_id}) with %Activity{} = activity <- Activity.get_by_id_with_object(activity_id), %Object{data: %{"reactions" => emoji_reactions}} <- Object.normalize(activity) do reactions = - Enum.reduce(emoji_reactions, %{}, fn {emoji, users}, res -> - users = - users - |> Enum.map(&User.get_cached_by_ap_id/1) - - res - |> Map.put( - emoji, - AccountView.render("accounts.json", %{users: users, for: user, as: :user}) - ) + emoji_reactions + |> Enum.map(fn {emoji, users} -> + users = Enum.map(users, &User.get_cached_by_ap_id/1) + {emoji, AccountView.render("accounts.json", %{users: users, for: user, as: :user})} end) + |> Enum.into(%{}) conn |> json(reactions) @@ -49,7 +44,7 @@ def react_with_emoji(%{assigns: %{user: user}} = conn, %{"id" => activity_id, "e activity = Activity.get_by_id(activity_id) do conn |> put_view(StatusView) - |> render("status.json", %{activity: activity, for: user, as: :activity}) + |> render("show.json", %{activity: activity, for: user, as: :activity}) end end diff --git a/test/web/activity_pub/transmogrifier_test.exs b/test/web/activity_pub/transmogrifier_test.exs index ba2a43296..f1ceb20d2 100644 --- a/test/web/activity_pub/transmogrifier_test.exs +++ b/test/web/activity_pub/transmogrifier_test.exs @@ -1263,7 +1263,7 @@ test "it can handle Listen activities" do {:ok, activity} = CommonAPI.listen(user, %{"title" => "lain radio episode 1"}) - {:ok, modified} = Transmogrifier.prepare_outgoing(activity.data) + {:ok, _modified} = Transmogrifier.prepare_outgoing(activity.data) end end From 08256e9299494c0bcd1a295c6079263277b21ba7 Mon Sep 17 00:00:00 2001 From: lain Date: Mon, 30 Sep 2019 15:51:09 +0200 Subject: [PATCH 016/107] ActivityPub: Federate reactions. --- lib/pleroma/web/activity_pub/activity_pub.ex | 3 ++- test/web/activity_pub/activity_pub_test.exs | 12 ++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 77fa23db4..a6fb67a28 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -313,7 +313,8 @@ def react_with_emoji(user, object, emoji, options \\ []) do Pleroma.Emoji.is_unicode_emoji?(emoji), reaction_data <- make_emoji_reaction_data(user, object, emoji, activity_id), {:ok, activity} <- insert(reaction_data, local), - {:ok, object} <- add_emoji_reaction_to_object(activity, object) do + {:ok, object} <- add_emoji_reaction_to_object(activity, object), + :ok <- maybe_federate(activity) do {:ok, activity, object} end end diff --git a/test/web/activity_pub/activity_pub_test.exs b/test/web/activity_pub/activity_pub_test.exs index 3e6f389bc..4dc7d96d2 100644 --- a/test/web/activity_pub/activity_pub_test.exs +++ b/test/web/activity_pub/activity_pub_test.exs @@ -727,6 +727,18 @@ test "returns reblogs for users for whom reblogs have not been muted" do end describe "react to an object" do + test_with_mock "sends an activity to federation", Pleroma.Web.Federator, [:passthrough], [] do + Pleroma.Config.put([:instance, :federating], true) + user = insert(:user) + reactor = insert(:user) + {:ok, activity} = CommonAPI.post(user, %{"status" => "YASSSS queen slay"}) + assert object = Object.normalize(activity) + + {:ok, reaction_activity, _object} = ActivityPub.react_with_emoji(reactor, object, "๐Ÿ”ฅ") + + assert called(Pleroma.Web.Federator.publish(reaction_activity)) + end + test "adds an emoji reaction activity to the db" do user = insert(:user) reactor = insert(:user) From 19bc0b8c79765dc485e081651a4e4c589d18b970 Mon Sep 17 00:00:00 2001 From: lain Date: Mon, 30 Sep 2019 16:38:19 +0200 Subject: [PATCH 017/107] . --- lib/pleroma/web/activity_pub/activity_pub.ex | 7 +++++++ test/web/activity_pub/activity_pub_test.exs | 14 ++++++++++++++ test/web/common_api/common_api_test.exs | 10 ++++++++++ .../pleroma_api/pleroma_api_controller_test.exs | 16 ++++++++++++++++ 4 files changed, 47 insertions(+) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index a6fb67a28..1f201d587 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -319,6 +319,13 @@ def react_with_emoji(user, object, emoji, options \\ []) do end end + def unreact_with_emoji(user, reaction_id, option \\ []) do + with local <- Keyword.get(options, :local, true), + activity_id <- Keyword.get(options, :activity_id, nil), + %Activity{actor: ^user.ap_id} = reaction_activity <- Activity.get_by_ap_id(reaction_id), + unreact_data + end + # TODO: This is weird, maybe we shouldn't check here if we can make the activity. def like( %User{ap_id: ap_id} = user, diff --git a/test/web/activity_pub/activity_pub_test.exs b/test/web/activity_pub/activity_pub_test.exs index 4dc7d96d2..5a6464350 100644 --- a/test/web/activity_pub/activity_pub_test.exs +++ b/test/web/activity_pub/activity_pub_test.exs @@ -760,6 +760,20 @@ test "adds an emoji reaction activity to the db" do end end + describe "unreacting to an object" do + test "adds an emoji reaction activity to the db" do + user = insert(:user) + reactor = insert(:user) + {:ok, activity} = CommonAPI.post(user, %{"status" => "YASSSS queen slay"}) + assert object = Object.normalize(activity) + + {:ok, reaction_activity, object} = ActivityPub.react_with_emoji(reactor, object, "๐Ÿ”ฅ") + {:ok, unreaction_activity} = ActivityPub.unreact_with_emoji(reactor, reaction_activity.id) + + IO.inspect(object) + end + end + describe "like an object" do test_with_mock "sends an activity to federation", Pleroma.Web.Federator, [:passthrough], [] do Pleroma.Config.put([:instance, :federating], true) diff --git a/test/web/common_api/common_api_test.exs b/test/web/common_api/common_api_test.exs index 011fcfca9..9e3bf6cc8 100644 --- a/test/web/common_api/common_api_test.exs +++ b/test/web/common_api/common_api_test.exs @@ -236,6 +236,16 @@ test "reacting to a status with an emoji" do # TODO: test error case. end + test "unreacting to a status with an emoji" do + user = insert(:user) + other_user = insert(:user) + + {:ok, activity} = CommonAPI.post(other_user, %{"status" => "cofe"}) + {:ok, reaction, _} = CommonAPI.react_with_emoji(activity.id, user, "๐Ÿ‘") + + assert false + end + test "repeating a status" do user = insert(:user) other_user = insert(:user) diff --git a/test/web/pleroma_api/pleroma_api_controller_test.exs b/test/web/pleroma_api/pleroma_api_controller_test.exs index 3c2a087ca..82d23ea5b 100644 --- a/test/web/pleroma_api/pleroma_api_controller_test.exs +++ b/test/web/pleroma_api/pleroma_api_controller_test.exs @@ -27,6 +27,22 @@ test "POST /api/v1/pleroma/statuses/:id/react_with_emoji", %{conn: conn} do assert to_string(activity.id) == id end + test "POST /api/v1/pleroma/statuses/:id/unreact_with_emoji", %{conn: conn} do + user = insert(:user) + other_user = insert(:user) + + {:ok, activity} = CommonAPI.post(user, %{"status" => "#cofe"}) + {:ok, activity, _object} = CommonAPI.react_with_emoji(activity.id, other_user, "โ˜•") + + result = + conn + |> assign(:user, other_user) + |> post("/api/v1/pleroma/statuses/#{activity.id}/unreact_with_emoji", %{"emoji" => "โ˜•"}) + + assert %{"id" => id} = json_response(result, 200) + assert to_string(activity.id) == id + end + test "GET /api/v1/pleroma/statuses/:id/emoji_reactions_by", %{conn: conn} do user = insert(:user) other_user = insert(:user) From dfe5c958eb94326f5a7743c9de0de073260db926 Mon Sep 17 00:00:00 2001 From: lain Date: Wed, 2 Oct 2019 15:08:20 +0200 Subject: [PATCH 018/107] ActivityPub: Add undo for emoji reactions. --- lib/pleroma/web/activity_pub/activity_pub.ex | 13 +++++-- lib/pleroma/web/activity_pub/utils.ex | 37 ++++++++++++++++++++ lib/pleroma/web/router.ex | 2 -- test/web/activity_pub/activity_pub_test.exs | 32 ++++++++++++++--- 4 files changed, 75 insertions(+), 9 deletions(-) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index a9e53141d..458d3590d 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -319,11 +319,18 @@ def react_with_emoji(user, object, emoji, options \\ []) do end end - def unreact_with_emoji(user, reaction_id, option \\ []) do + def unreact_with_emoji(user, reaction_id, options \\ []) do with local <- Keyword.get(options, :local, true), activity_id <- Keyword.get(options, :activity_id, nil), - %Activity{actor: ^user.ap_id} = reaction_activity <- Activity.get_by_ap_id(reaction_id), - unreact_data + user_ap_id <- user.ap_id, + %Activity{actor: ^user_ap_id} = reaction_activity <- Activity.get_by_ap_id(reaction_id), + object <- Object.normalize(reaction_activity), + unreact_data <- make_undo_data(user, reaction_activity, activity_id), + {:ok, activity} <- insert(unreact_data, local), + {:ok, object} <- remove_emoji_reaction_from_object(reaction_activity, object), + :ok <- maybe_federate(activity) do + {:ok, activity, object} + end end # TODO: This is weird, maybe we shouldn't check here if we can make the activity. diff --git a/lib/pleroma/web/activity_pub/utils.ex b/lib/pleroma/web/activity_pub/utils.ex index 4c146fd86..7807fa9cd 100644 --- a/lib/pleroma/web/activity_pub/utils.ex +++ b/lib/pleroma/web/activity_pub/utils.ex @@ -337,6 +337,24 @@ def add_emoji_reaction_to_object( update_element_in_object("reaction", new_reactions, object) end + def remove_emoji_reaction_from_object( + %Activity{data: %{"content" => emoji, "actor" => actor}}, + object + ) do + reactions = object.data["reactions"] || %{} + emoji_actors = reactions[emoji] || [] + new_emoji_actors = List.delete(emoji_actors, actor) + + new_reactions = + if new_emoji_actors == [] do + Map.delete(reactions, emoji) + else + Map.put(reactions, emoji, new_emoji_actors) + end + + update_element_in_object("reaction", new_reactions, object) + end + @spec add_like_to_object(Activity.t(), Object.t()) :: {:ok, Object.t()} | {:error, Ecto.Changeset.t()} def add_like_to_object(%Activity{data: %{"actor" => actor}}, object) do @@ -522,6 +540,25 @@ def make_unlike_data( |> maybe_put("id", activity_id) end + def make_undo_data( + %User{ap_id: actor, follower_address: follower_address}, + %Activity{ + data: %{"id" => undone_activity_id, "context" => context}, + actor: undone_activity_actor + }, + activity_id \\ nil + ) do + %{ + "type" => "Undo", + "actor" => actor, + "object" => undone_activity_id, + "to" => [follower_address, undone_activity_actor], + "cc" => [Pleroma.Constants.as_public()], + "context" => context + } + |> maybe_put("id", activity_id) + end + @spec add_announce_to_object(Activity.t(), Object.t()) :: {:ok, Object.t()} | {:error, Ecto.Changeset.t()} def add_announce_to_object( diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index eae1f676b..ff3dc273e 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -293,8 +293,6 @@ defmodule Pleroma.Web.Router do end scope "/api/v1/pleroma", Pleroma.Web.PleromaAPI do - pipe_through(:authenticated_api) - scope [] do pipe_through(:authenticated_api) pipe_through(:oauth_read) diff --git a/test/web/activity_pub/activity_pub_test.exs b/test/web/activity_pub/activity_pub_test.exs index 8c9c2c89e..36a82d6a1 100644 --- a/test/web/activity_pub/activity_pub_test.exs +++ b/test/web/activity_pub/activity_pub_test.exs @@ -761,16 +761,40 @@ test "adds an emoji reaction activity to the db" do end describe "unreacting to an object" do - test "adds an emoji reaction activity to the db" do + test_with_mock "sends an activity to federation", Pleroma.Web.Federator, [:passthrough], [] do + Pleroma.Config.put([:instance, :federating], true) user = insert(:user) reactor = insert(:user) {:ok, activity} = CommonAPI.post(user, %{"status" => "YASSSS queen slay"}) assert object = Object.normalize(activity) - {:ok, reaction_activity, object} = ActivityPub.react_with_emoji(reactor, object, "๐Ÿ”ฅ") - {:ok, unreaction_activity} = ActivityPub.unreact_with_emoji(reactor, reaction_activity.id) + {:ok, reaction_activity, _object} = ActivityPub.react_with_emoji(reactor, object, "๐Ÿ”ฅ") - IO.inspect(object) + assert called(Pleroma.Web.Federator.publish(reaction_activity)) + + {:ok, unreaction_activity, _object} = + ActivityPub.unreact_with_emoji(reactor, reaction_activity.data["id"]) + + assert called(Pleroma.Web.Federator.publish(unreaction_activity)) + end + + test "adds an undo activity to the db" do + user = insert(:user) + reactor = insert(:user) + {:ok, activity} = CommonAPI.post(user, %{"status" => "YASSSS queen slay"}) + assert object = Object.normalize(activity) + + {:ok, reaction_activity, _object} = ActivityPub.react_with_emoji(reactor, object, "๐Ÿ”ฅ") + + {:ok, unreaction_activity, _object} = + ActivityPub.unreact_with_emoji(reactor, reaction_activity.data["id"]) + + assert unreaction_activity.actor == reactor.ap_id + assert unreaction_activity.data["object"] == reaction_activity.data["id"] + + object = Object.get_by_ap_id(object.data["id"]) + assert object.data["reaction_count"] == 0 + assert object.data["reactions"] == %{} end end From 9cfe9a57c5a31318abf02fa510d4fdf78505bd97 Mon Sep 17 00:00:00 2001 From: lain Date: Wed, 2 Oct 2019 15:38:57 +0200 Subject: [PATCH 019/107] CommonAPI: Add unreactions. --- lib/pleroma/web/activity_pub/utils.ex | 13 +++++++++++++ lib/pleroma/web/common_api/common_api.ex | 9 +++++++++ test/web/common_api/common_api_test.exs | 5 ++++- 3 files changed, 26 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/web/activity_pub/utils.ex b/lib/pleroma/web/activity_pub/utils.ex index 7807fa9cd..82e2d8f76 100644 --- a/lib/pleroma/web/activity_pub/utils.ex +++ b/lib/pleroma/web/activity_pub/utils.ex @@ -452,6 +452,19 @@ def fetch_latest_follow(%User{ap_id: follower_id}, %User{ap_id: followed_id}) do |> Repo.one() end + def get_latest_reaction(internal_activity_id, %{ap_id: ap_id}, emoji) do + %{data: %{"object" => object_ap_id}} = Activity.get_by_id(internal_activity_id) + + "EmojiReaction" + |> Activity.Queries.by_type() + |> where(actor: ^ap_id) + |> where([activity], fragment("?->>'content' = ?", activity.data, ^emoji)) + |> Activity.Queries.by_object_id(object_ap_id) + |> order_by([activity], fragment("? desc nulls last", activity.id)) + |> limit(1) + |> Repo.one() + end + #### Announce-related helpers @doc """ diff --git a/lib/pleroma/web/common_api/common_api.ex b/lib/pleroma/web/common_api/common_api.ex index 53ada8fab..995d4b1af 100644 --- a/lib/pleroma/web/common_api/common_api.ex +++ b/lib/pleroma/web/common_api/common_api.ex @@ -125,6 +125,15 @@ def react_with_emoji(id, user, emoji) do end end + def unreact_with_emoji(id, user, emoji) do + with %Activity{} = reaction_activity <- Utils.get_latest_reaction(id, user, emoji) do + ActivityPub.unreact_with_emoji(user, reaction_activity.data["id"]) + else + _ -> + {:error, dgettext("errors", "Could not remove reaction emoji")} + end + end + def vote(user, %{data: %{"type" => "Question"}} = object, choices) do with :ok <- validate_not_author(object, user), :ok <- validate_existing_votes(user, object), diff --git a/test/web/common_api/common_api_test.exs b/test/web/common_api/common_api_test.exs index e048ed217..b9e785885 100644 --- a/test/web/common_api/common_api_test.exs +++ b/test/web/common_api/common_api_test.exs @@ -243,7 +243,10 @@ test "unreacting to a status with an emoji" do {:ok, activity} = CommonAPI.post(other_user, %{"status" => "cofe"}) {:ok, reaction, _} = CommonAPI.react_with_emoji(activity.id, user, "๐Ÿ‘") - assert false + {:ok, unreaction, _} = CommonAPI.unreact_with_emoji(activity.id, user, "๐Ÿ‘") + + assert unreaction.data["type"] == "Undo" + assert unreaction.data["object"] == reaction.data["id"] end test "repeating a status" do From 391c7362921ba94a0084b7a25c4126ce7026dc55 Mon Sep 17 00:00:00 2001 From: lain Date: Wed, 2 Oct 2019 18:13:10 +0200 Subject: [PATCH 020/107] PleromaAPI: Fix emoji_reactions_by --- .../web/pleroma_api/controllers/pleroma_api_controller.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/pleroma_api/controllers/pleroma_api_controller.ex b/lib/pleroma/web/pleroma_api/controllers/pleroma_api_controller.ex index 39d371ff7..884b3d877 100644 --- a/lib/pleroma/web/pleroma_api/controllers/pleroma_api_controller.ex +++ b/lib/pleroma/web/pleroma_api/controllers/pleroma_api_controller.ex @@ -26,7 +26,7 @@ def emoji_reactions_by(%{assigns: %{user: user}} = conn, %{"id" => activity_id}) emoji_reactions |> Enum.map(fn {emoji, users} -> users = Enum.map(users, &User.get_cached_by_ap_id/1) - {emoji, AccountView.render("accounts.json", %{users: users, for: user, as: :user})} + {emoji, AccountView.render("index.json", %{users: users, for: user, as: :user})} end) |> Enum.into(%{}) From 4cb603e1df7c9d13db1aa4285e2a18b9be71cd78 Mon Sep 17 00:00:00 2001 From: lain Date: Wed, 2 Oct 2019 18:19:16 +0200 Subject: [PATCH 021/107] PleromaAPI: Add unreacting. --- .../controllers/pleroma_api_controller.ex | 14 +++++++++++++- lib/pleroma/web/router.ex | 1 + .../controllers/pleroma_api_controller_test.exs | 5 +++++ 3 files changed, 19 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/web/pleroma_api/controllers/pleroma_api_controller.ex b/lib/pleroma/web/pleroma_api/controllers/pleroma_api_controller.ex index 884b3d877..8aee7d7c5 100644 --- a/lib/pleroma/web/pleroma_api/controllers/pleroma_api_controller.ex +++ b/lib/pleroma/web/pleroma_api/controllers/pleroma_api_controller.ex @@ -41,7 +41,19 @@ def emoji_reactions_by(%{assigns: %{user: user}} = conn, %{"id" => activity_id}) def react_with_emoji(%{assigns: %{user: user}} = conn, %{"id" => activity_id, "emoji" => emoji}) do with {:ok, _activity, _object} <- CommonAPI.react_with_emoji(activity_id, user, emoji), - activity = Activity.get_by_id(activity_id) do + activity <- Activity.get_by_id(activity_id) do + conn + |> put_view(StatusView) + |> render("show.json", %{activity: activity, for: user, as: :activity}) + end + end + + def unreact_with_emoji(%{assigns: %{user: user}} = conn, %{ + "id" => activity_id, + "emoji" => emoji + }) do + with {:ok, _activity, _object} <- CommonAPI.unreact_with_emoji(activity_id, user, emoji), + activity <- Activity.get_by_id(activity_id) do conn |> put_view(StatusView) |> render("show.json", %{activity: activity, for: user, as: :activity}) diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index ff3dc273e..87d373f55 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -305,6 +305,7 @@ defmodule Pleroma.Web.Router do pipe_through(:oauth_write) patch("/conversations/:id", PleromaAPIController, :update_conversation) post("/statuses/:id/react_with_emoji", PleromaAPIController, :react_with_emoji) + post("/statuses/:id/unreact_with_emoji", PleromaAPIController, :unreact_with_emoji) post("/notifications/read", PleromaAPIController, :read_notification) patch("/accounts/update_avatar", AccountController, :update_avatar) diff --git a/test/web/pleroma_api/controllers/pleroma_api_controller_test.exs b/test/web/pleroma_api/controllers/pleroma_api_controller_test.exs index 82d23ea5b..3a5dbdeea 100644 --- a/test/web/pleroma_api/controllers/pleroma_api_controller_test.exs +++ b/test/web/pleroma_api/controllers/pleroma_api_controller_test.exs @@ -7,6 +7,7 @@ defmodule Pleroma.Web.PleromaAPI.PleromaAPIControllerTest do alias Pleroma.Conversation.Participation alias Pleroma.Notification + alias Pleroma.Object alias Pleroma.Repo alias Pleroma.Web.CommonAPI @@ -41,6 +42,10 @@ test "POST /api/v1/pleroma/statuses/:id/unreact_with_emoji", %{conn: conn} do assert %{"id" => id} = json_response(result, 200) assert to_string(activity.id) == id + + object = Object.normalize(activity) + + assert object.data["reaction_count"] == 0 end test "GET /api/v1/pleroma/statuses/:id/emoji_reactions_by", %{conn: conn} do From 0e41951eab405ea5a3016bbb0897eba4aa7c6a0b Mon Sep 17 00:00:00 2001 From: lain Date: Wed, 2 Oct 2019 18:20:40 +0200 Subject: [PATCH 022/107] Pleroma API Readme: Document unreaction endpoint. --- docs/api/pleroma_api.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/api/pleroma_api.md b/docs/api/pleroma_api.md index 7862e6301..e672aec96 100644 --- a/docs/api/pleroma_api.md +++ b/docs/api/pleroma_api.md @@ -483,6 +483,13 @@ Emoji reactions work a lot like favourites do. They make it possible to react to * Params: `emoji`: A single character unicode emoji * Response: JSON, the status. +## `POST /api/v1/pleroma/statuses/:id/unreact_with_emoji` +### Remove a reaction to a post with a unicode emoji +* Method: `POST` +* Authentication: required +* Params: `emoji`: A single character unicode emoji +* Response: JSON, the status. + ## `GET /api/v1/pleroma/statuses/:id/emoji_reactions_by` ### Get an object of emoji to account mappings with accounts that reacted to the post * Method: `GET` From c9043c6c808129f4f6236d03bf05e5a46f0c6e14 Mon Sep 17 00:00:00 2001 From: lain Date: Thu, 3 Oct 2019 18:37:23 +0200 Subject: [PATCH 023/107] Transmogrifier: Handle incoming Undos for EmojiReactions. --- .../web/activity_pub/transmogrifier.ex | 22 +++++++++++++++++++ test/web/activity_pub/transmogrifier_test.exs | 20 ++++++++++++++++- 2 files changed, 41 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index cb868c336..ea209b5ea 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -721,6 +721,28 @@ def handle_incoming( end end + def handle_incoming( + %{ + "type" => "Undo", + "object" => %{"type" => "EmojiReaction", "id" => reaction_activity_id}, + "actor" => _actor, + "id" => id + } = data, + _options + ) do + with actor <- Containment.get_actor(data), + {:ok, %User{} = actor} <- User.get_or_fetch_by_ap_id(actor), + {:ok, activity, _} <- + ActivityPub.unreact_with_emoji(actor, reaction_activity_id, + activity_id: id, + local: false + ) do + {:ok, activity} + else + _e -> :error + end + end + def handle_incoming( %{ "type" => "Undo", diff --git a/test/web/activity_pub/transmogrifier_test.exs b/test/web/activity_pub/transmogrifier_test.exs index 5bb435457..a9544caf2 100644 --- a/test/web/activity_pub/transmogrifier_test.exs +++ b/test/web/activity_pub/transmogrifier_test.exs @@ -359,6 +359,25 @@ test "it works for incoming emoji reactions" do assert data["content"] == "๐Ÿ‘Œ" end + test "it works for incoming emoji reaction undos" do + user = insert(:user) + + {:ok, activity} = CommonAPI.post(user, %{"status" => "hello"}) + {:ok, reaction_activity, _object} = CommonAPI.react_with_emoji(activity.id, user, "๐Ÿ‘Œ") + + data = + File.read!("test/fixtures/mastodon-undo-like.json") + |> Poison.decode!() + |> Map.put("object", reaction_activity.data["id"]) + |> Map.put("actor", user.ap_id) + + {:ok, activity} = Transmogrifier.handle_incoming(data) + + assert activity.actor == user.ap_id + assert activity.data["id"] == data["id"] + assert activity.data["type"] == "Undo" + end + test "it returns an error for incoming unlikes wihout a like activity" do user = insert(:user) {:ok, activity} = CommonAPI.post(user, %{"status" => "leave a like pls"}) @@ -1116,7 +1135,6 @@ test "it inlines private announced objects" do {:ok, announce_activity, _} = CommonAPI.repeat(activity.id, user) {:ok, modified} = Transmogrifier.prepare_outgoing(announce_activity.data) - object = modified["object"] assert modified["object"]["content"] == "hey" assert modified["object"]["actor"] == modified["object"]["attributedTo"] From 8e83c8423fa846e82103b20798f0cbde676354ff Mon Sep 17 00:00:00 2001 From: lain Date: Fri, 4 Oct 2019 14:49:59 +0200 Subject: [PATCH 024/107] Update Changelog. --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a71a9dae6..35c062dcf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Pleroma API: `GET /api/v1/pleroma/accounts/:id/scrobbles` to get a list of recently scrobbled items - Pleroma API: `POST /api/v1/pleroma/scrobble` to scrobble a media item - Mastodon API: Add `upload_limit`, `avatar_upload_limit`, `background_upload_limit`, and `banner_upload_limit` to `/api/v1/instance` +- Pleroma API: Add Emoji reactions ### Changed - **Breaking:** Elixir >=1.8 is now required (was >= 1.7) From 43a211bcb111720622a89da3016372f5a3a12184 Mon Sep 17 00:00:00 2001 From: lain Date: Fri, 4 Oct 2019 17:01:04 +0200 Subject: [PATCH 025/107] Transmogrifier: Handle misskey likes with reactions like EmojiReactions. --- .../web/activity_pub/transmogrifier.ex | 27 +++++++++++++++++++ test/fixtures/misskey-like.json | 14 ++++++++++ test/web/activity_pub/transmogrifier_test.exs | 18 +++++++++++++ 3 files changed, 59 insertions(+) create mode 100644 test/fixtures/misskey-like.json diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index ea209b5ea..54c18bc0e 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -560,6 +560,33 @@ def handle_incoming( end end + @misskey_reactions %{ + "like" => "๐Ÿ‘", + "love" => "โค๏ธ", + "laugh" => "๐Ÿ˜†", + "hmm" => "๐Ÿค”", + "surprise" => "๐Ÿ˜ฎ", + "congrats" => "๐ŸŽ‰", + "angry" => "๐Ÿ’ข", + "confused" => "๐Ÿ˜ฅ", + "rip" => "๐Ÿ˜‡", + "pudding" => "๐Ÿฎ" + } + + @doc "Rewrite misskey likes into EmojiReactions" + def handle_incoming( + %{ + "type" => "Like", + "_misskey_reaction" => reaction + } = data, + options + ) do + data + |> Map.put("type", "EmojiReaction") + |> Map.put("content", @misskey_reactions[reaction]) + |> handle_incoming(options) + end + def handle_incoming( %{"type" => "Like", "object" => object_id, "actor" => _actor, "id" => id} = data, _options diff --git a/test/fixtures/misskey-like.json b/test/fixtures/misskey-like.json new file mode 100644 index 000000000..84d56f473 --- /dev/null +++ b/test/fixtures/misskey-like.json @@ -0,0 +1,14 @@ +{ + "@context" : [ + "https://www.w3.org/ns/activitystreams", + "https://w3id.org/security/v1", + {"Hashtag" : "as:Hashtag"} + ], + "_misskey_reaction" : "pudding", + "actor": "http://mastodon.example.org/users/admin", + "cc" : ["https://testing.pleroma.lol/users/lain"], + "id" : "https://misskey.xyz/75149198-2f45-46e4-930a-8b0538297075", + "nickname" : "lain", + "object" : "https://testing.pleroma.lol/objects/c331bbf7-2eb9-4801-a709-2a6103492a5a", + "type" : "Like" +} diff --git a/test/web/activity_pub/transmogrifier_test.exs b/test/web/activity_pub/transmogrifier_test.exs index a9544caf2..530a762fa 100644 --- a/test/web/activity_pub/transmogrifier_test.exs +++ b/test/web/activity_pub/transmogrifier_test.exs @@ -341,6 +341,24 @@ test "it works for incoming likes" do assert data["object"] == activity.data["object"] end + test "it works for incoming misskey likes, turning them into EmojiReactions" do + user = insert(:user) + {:ok, activity} = CommonAPI.post(user, %{"status" => "hello"}) + + data = + File.read!("test/fixtures/misskey-like.json") + |> Poison.decode!() + |> Map.put("object", activity.data["object"]) + + {:ok, %Activity{data: data, local: false}} = Transmogrifier.handle_incoming(data) + + assert data["actor"] == data["actor"] + assert data["type"] == "EmojiReaction" + assert data["id"] == data["id"] + assert data["object"] == activity.data["object"] + assert data["content"] == "๐Ÿฎ" + end + test "it works for incoming emoji reactions" do user = insert(:user) {:ok, activity} = CommonAPI.post(user, %{"status" => "hello"}) From 795ea5dfc2549b50265cea2f7b7a774356a735b4 Mon Sep 17 00:00:00 2001 From: Maxim Filippov Date: Fri, 4 Oct 2019 18:58:44 +0300 Subject: [PATCH 026/107] Move HTTP verb to the header (admin_api.md) --- docs/api/admin_api.md | 190 ++++++++++++++++++++++++------------------ 1 file changed, 109 insertions(+), 81 deletions(-) diff --git a/docs/api/admin_api.md b/docs/api/admin_api.md index ee9e68cb1..045686bf4 100644 --- a/docs/api/admin_api.md +++ b/docs/api/admin_api.md @@ -2,11 +2,10 @@ Authentication is required and the user must be an admin. -## `/api/pleroma/admin/users` +## `GET /api/pleroma/admin/users` ### List users -- Method `GET` - Query Params: - *optional* `query`: **string** search term (e.g. nickname, domain, nickname@domain) - *optional* `filters`: **string** comma-separated string of filters: @@ -47,11 +46,10 @@ Authentication is required and the user must be an admin. } ``` -## `/api/pleroma/admin/users` +## `DELETE /api/pleroma/admin/users` ### Remove a user -- Method `DELETE` - Params: - `nickname` - Response: Userโ€™s nickname @@ -69,31 +67,30 @@ Authentication is required and the user must be an admin. ] - Response: Userโ€™s nickname -## `/api/pleroma/admin/users/follow` +## `POST /api/pleroma/admin/users/follow` + ### Make a user follow another user -- Methods: `POST` - Params: - - `follower`: The nickname of the follower - - `followed`: The nickname of the followed + - `follower`: The nickname of the follower + - `followed`: The nickname of the followed - Response: - - "ok" + - "ok" + +## `POST /api/pleroma/admin/users/unfollow` -## `/api/pleroma/admin/users/unfollow` ### Make a user unfollow another user -- Methods: `POST` - Params: - - `follower`: The nickname of the follower - - `followed`: The nickname of the followed + - `follower`: The nickname of the follower + - `followed`: The nickname of the followed - Response: - - "ok" + - "ok" -## `/api/pleroma/admin/users/:nickname/toggle_activation` +## `PATCH /api/pleroma/admin/users/:nickname/toggle_activation` ### Toggle user activation -- Method: `PATCH` - Params: - `nickname` - Response: Userโ€™s object @@ -106,27 +103,26 @@ Authentication is required and the user must be an admin. } ``` -## `/api/pleroma/admin/users/tag` +## `PUT /api/pleroma/admin/users/tag` ### Tag a list of users -- Method: `PUT` - Params: - `nicknames` (array) - `tags` (array) +## `DELETE /api/pleroma/admin/users/tag` + ### Untag a list of users -- Method: `DELETE` - Params: - `nicknames` (array) - `tags` (array) -## `/api/pleroma/admin/users/:nickname/permission_group` +## `GET /api/pleroma/admin/users/:nickname/permission_group` ### Get user user permission groups membership -- Method: `GET` - Params: none - Response: @@ -137,13 +133,12 @@ Authentication is required and the user must be an admin. } ``` -## `/api/pleroma/admin/users/:nickname/permission_group/:permission_group` +## `GET /api/pleroma/admin/users/:nickname/permission_group/:permission_group` Note: Available `:permission_group` is currently moderator and admin. 404 is returned when the permission group doesnโ€™t exist. ### Get user user permission groups membership per permission group -- Method: `GET` - Params: none - Response: @@ -154,48 +149,47 @@ Note: Available `:permission_group` is currently moderator and admin. 404 is ret } ``` +## `POST /api/pleroma/admin/users/:nickname/permission_group/:permission_group` + ### Add user in permission group -- Method: `POST` - Params: none - Response: - On failure: `{"error": "โ€ฆ"}` - On success: JSON of the `user.info` +## `DELETE /api/pleroma/admin/users/:nickname/permission_group/:permission_group` + ### Remove user from permission group -- Method: `DELETE` - Params: none - Response: - On failure: `{"error": "โ€ฆ"}` - On success: JSON of the `user.info` - Note: An admin cannot revoke their own admin status. -## `/api/pleroma/admin/users/:nickname/activation_status` +## `PUT /api/pleroma/admin/users/:nickname/activation_status` ### Active or deactivate a user -- Method: `PUT` - Params: - `nickname` - `status` BOOLEAN field, false value means deactivation. -## `/api/pleroma/admin/users/:nickname_or_id` +## `GET /api/pleroma/admin/users/:nickname_or_id` ### Retrive the details of a user -- Method: `GET` - Params: - `nickname` or `id` - Response: - On failure: `Not found` - On success: JSON of the user -## `/api/pleroma/admin/users/:nickname_or_id/statuses` +## `GET /api/pleroma/admin/users/:nickname_or_id/statuses` ### Retrive user's latest statuses -- Method: `GET` - Params: - `nickname` or `id` - *optional* `page_size`: number of statuses to return (default is `20`) @@ -204,29 +198,28 @@ Note: Available `:permission_group` is currently moderator and admin. 404 is ret - On failure: `Not found` - On success: JSON array of user's latest statuses -## `/api/pleroma/admin/relay` +## `POST /api/pleroma/admin/relay` ### Follow a Relay -- Methods: `POST` - Params: - `relay_url` - Response: - On success: URL of the followed relay +## `DELETE /api/pleroma/admin/relay` + ### Unfollow a Relay -- Methods: `DELETE` - Params: - `relay_url` - Response: - On success: URL of the unfollowed relay -## `/api/pleroma/admin/users/invite_token` +## `POST /api/pleroma/admin/users/invite_token` ### Create an account registration invite token -- Methods: `POST` - Params: - *optional* `max_use` (integer) - *optional* `expires_at` (date string e.g. "2019-04-07") @@ -244,11 +237,10 @@ Note: Available `:permission_group` is currently moderator and admin. 404 is ret } ``` -## `/api/pleroma/admin/users/invites` +## `GET /api/pleroma/admin/users/invites` ### Get a list of generated invites -- Methods: `GET` - Params: none - Response: @@ -270,11 +262,10 @@ Note: Available `:permission_group` is currently moderator and admin. 404 is ret } ``` -## `/api/pleroma/admin/users/revoke_invite` +## `POST /api/pleroma/admin/users/revoke_invite` ### Revoke invite by token -- Methods: `POST` - Params: - `token` - Response: @@ -292,21 +283,18 @@ Note: Available `:permission_group` is currently moderator and admin. 404 is ret } ``` - -## `/api/pleroma/admin/users/email_invite` +## `POST /api/pleroma/admin/users/email_invite` ### Sends registration invite via email -- Methods: `POST` - Params: - `email` - `name`, optional -## `/api/pleroma/admin/users/:nickname/password_reset` +## `GET /api/pleroma/admin/users/:nickname/password_reset` ### Get a password reset token for a given nickname -- Methods: `GET` - Params: none - Response: @@ -317,18 +305,17 @@ Note: Available `:permission_group` is currently moderator and admin. 404 is ret } ``` - -## `/api/pleroma/admin/users/:nickname/force_password_reset` +## `PATCH /api/pleroma/admin/users/:nickname/force_password_reset` ### Force passord reset for a user with a given nickname -- Methods: `PATCH` - Params: none - Response: none (code `204`) -## `/api/pleroma/admin/reports` +## `GET /api/pleroma/admin/reports` + ### Get a list of reports -- Method `GET` + - Params: - *optional* `state`: **string** the state of reports. Valid values are `open`, `closed` and `resolved` - *optional* `limit`: **integer** the number of records to retrieve @@ -343,7 +330,7 @@ Note: Available `:permission_group` is currently moderator and admin. 404 is ret ```json { - "total" : 1, + "totalReports" : 1, "reports": [ { "account": { @@ -481,13 +468,24 @@ Note: Available `:permission_group` is currently moderator and admin. 404 is ret } ] } + ], + "totalGroupedReports": 1, + "groupedReports": [ + { + "date": "2019-01-01", // date of the latest report + "account": { ... }, // author of the reported status + "status": { ... }, // reported status + "actors": [{ ... }, { ... }], // accounts that sent reports on the status + "reports": [{ ... }] + } ] } ``` -## `/api/pleroma/admin/reports/:id` +## `GET /api/pleroma/admin/reports/:id` + ### Get an individual report -- Method `GET` + - Params: - `id` - Response: @@ -496,22 +494,41 @@ Note: Available `:permission_group` is currently moderator and admin. 404 is ret - 404 Not Found `"Not found"` - On success: JSON, Report object (see above) -## `/api/pleroma/admin/reports/:id` -### Change the state of the report -- Method `PUT` +## `PATCH /api/pleroma/admin/reports` + +### Change the state of one or multiple reports + - Params: - - `id` - - `state`: required, the new state. Valid values are `open`, `closed` and `resolved` + +```json + `reports`: [ + { + `id`, // required, report id + `state` // required, the new state. Valid values are `open`, `closed` and `resolved` + }, + ... + ] +``` + - Response: - On failure: - - 400 Bad Request `"Unsupported state"` - - 403 Forbidden `{"error": "error_msg"}` - - 404 Not Found `"Not found"` - - On success: JSON, Report object (see above) + - 400 Bad Request, JSON: + + ```json + [ + { + `id`, // report id + `error` // error message + } + ] + ``` + + - On success: `204`, empty response + +## `POST /api/pleroma/admin/reports/:id/respond` -## `/api/pleroma/admin/reports/:id/respond` ### Respond to a report -- Method `POST` + - Params: - `id` - `status`: required, the message @@ -581,9 +598,10 @@ Note: Available `:permission_group` is currently moderator and admin. 404 is ret } ``` -## `/api/pleroma/admin/statuses/:id` +## `PUT /api/pleroma/admin/statuses/:id` + ### Change the scope of an individual reported status -- Method `PUT` + - Params: - `id` - `sensitive`: optional, valid values are `true` or `false` @@ -595,9 +613,10 @@ Note: Available `:permission_group` is currently moderator and admin. 404 is ret - 404 Not Found `"Not found"` - On success: JSON, Mastodon Status entity -## `/api/pleroma/admin/statuses/:id` +## `DELETE /api/pleroma/admin/statuses/:id` + ### Delete an individual reported status -- Method `DELETE` + - Params: - `id` - Response: @@ -606,11 +625,12 @@ Note: Available `:permission_group` is currently moderator and admin. 404 is ret - 404 Not Found `"Not found"` - On success: 200 OK `{}` +## `GET /api/pleroma/admin/config/migrate_to_db` -## `/api/pleroma/admin/config/migrate_to_db` ### Run mix task pleroma.config migrate_to_db + Copy settings on key `:pleroma` to DB. -- Method `GET` + - Params: none - Response: @@ -618,9 +638,12 @@ Copy settings on key `:pleroma` to DB. {} ``` -## `/api/pleroma/admin/config/migrate_from_db` +## `GET /api/pleroma/admin/config/migrate_from_db` + ### Run mix task pleroma.config migrate_from_db + Copy all settings from DB to `config/prod.exported_from_db.secret.exs` with deletion from DB. + - Method `GET` - Params: none - Response: @@ -629,10 +652,12 @@ Copy all settings from DB to `config/prod.exported_from_db.secret.exs` with dele {} ``` -## `/api/pleroma/admin/config` +## `GET /api/pleroma/admin/config` + ### List config settings + List config settings only works with `:pleroma => :instance => :dynamic_configuration` setting to `true`. -- Method `GET` + - Params: none - Response: @@ -648,8 +673,10 @@ List config settings only works with `:pleroma => :instance => :dynamic_configur } ``` -## `/api/pleroma/admin/config` +## `POST /api/pleroma/admin/config` + ### Update config settings + Updating config settings only works with `:pleroma => :instance => :dynamic_configuration` setting to `true`. Module name can be passed as string, which starts with `Pleroma`, e.g. `"Pleroma.Upload"`. Atom keys and values can be passed with `:` in the beginning, e.g. `":upload"`. @@ -672,7 +699,6 @@ Compile time settings (need instance reboot): - `Pleroma.Upload` -> `:proxy_remote` - `:instance` -> `:upload_limit` -- Method `POST` - Params: - `configs` => [ - `group` (string) @@ -727,9 +753,10 @@ Compile time settings (need instance reboot): } ``` -## `/api/pleroma/admin/moderation_log` +## `GET /api/pleroma/admin/moderation_log` + ### Get moderation log -- Method `GET` + - Params: - *optional* `page`: **integer** page number - *optional* `page_size`: **integer** number of log entries per page (default is `50`) @@ -756,8 +783,9 @@ Compile time settings (need instance reboot): ``` ## `POST /api/pleroma/admin/reload_emoji` + ### Reload the instance's custom emoji -* Method `POST` -* Authentication: required -* Params: None -* Response: JSON, "ok" and 200 status + +- Authentication: required +- Params: None +- Response: JSON, "ok" and 200 status From 8dcc2f9f5ecbbc81bc026c85582695de4fbc1a0f Mon Sep 17 00:00:00 2001 From: Maxim Filippov Date: Fri, 4 Oct 2019 19:00:58 +0300 Subject: [PATCH 027/107] Admin API: Allow changing the state of multiple reports at once --- CHANGELOG.md | 1 + lib/pleroma/web/activity_pub/utils.ex | 12 +++ .../web/admin_api/admin_api_controller.ex | 29 +++--- lib/pleroma/web/common_api/common_api.ex | 7 ++ lib/pleroma/web/router.ex | 2 +- .../admin_api/admin_api_controller_test.exs | 89 +++++++++++++++---- test/web/common_api/common_api_test.exs | 29 ++++++ 7 files changed, 142 insertions(+), 27 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a71a9dae6..d7afed783 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ### Changed - **Breaking:** Elixir >=1.8 is now required (was >= 1.7) - **Breaking:** Admin API: Return link alongside with token on password reset +- **Breaking:** Admin API: Changing report state now uses `PATCH` (it was `PUT` before) and allows updating multiple reports at once (API changed) - Replaced [pleroma_job_queue](https://git.pleroma.social/pleroma/pleroma_job_queue) and `Pleroma.Web.Federator.RetryQueue` with [Oban](https://github.com/sorentwo/oban) (see [`docs/config.md`](docs/config.md) on migrating customized worker / retry settings) - Introduced [quantum](https://github.com/quantum-elixir/quantum-core) job scheduler - Admin API: Return `total` when querying for reports diff --git a/lib/pleroma/web/activity_pub/utils.ex b/lib/pleroma/web/activity_pub/utils.ex index 0828591ee..824957314 100644 --- a/lib/pleroma/web/activity_pub/utils.ex +++ b/lib/pleroma/web/activity_pub/utils.ex @@ -672,6 +672,18 @@ def update_report_state(%Activity{} = activity, state) when state in @supported_ |> Repo.update() end + def update_report_state(activity_ids, state) when state in @supported_report_states do + activities_num = length(activity_ids) + + from(a in Activity, where: a.id in ^activity_ids) + |> update(set: [data: fragment("jsonb_set(data, '{state}', ?)", ^state)]) + |> Repo.update_all([]) + |> case do + {^activities_num, _} -> :ok + _ -> {:error, activity_ids} + end + end + def update_report_state(_, _), do: {:error, "Unsupported state"} def update_activity_visibility(activity, visibility) when visibility in @valid_visibilities do diff --git a/lib/pleroma/web/admin_api/admin_api_controller.ex b/lib/pleroma/web/admin_api/admin_api_controller.ex index 21da8a7ff..0e8c9dac8 100644 --- a/lib/pleroma/web/admin_api/admin_api_controller.ex +++ b/lib/pleroma/web/admin_api/admin_api_controller.ex @@ -480,17 +480,26 @@ def report_show(conn, %{"id" => id}) do end end - def report_update_state(%{assigns: %{user: admin}} = conn, %{"id" => id, "state" => state}) do - with {:ok, report} <- CommonAPI.update_report_state(id, state) do - ModerationLog.insert_log(%{ - action: "report_update", - actor: admin, - subject: report - }) + def reports_update(%{assigns: %{user: admin}} = conn, %{"reports" => reports}) do + result = + reports + |> Enum.map(fn report -> + with {:ok, activity} <- CommonAPI.update_report_state(report["id"], report["state"]) do + ModerationLog.insert_log(%{ + action: "report_update", + actor: admin, + subject: activity + }) - conn - |> put_view(ReportView) - |> render("show.json", Report.extract_report_info(report)) + activity + else + {:error, message} -> %{id: report["id"], error: message} + end + end) + + case Enum.any?(result, &Map.has_key?(&1, :error)) do + true -> json_response(conn, :bad_request, result) + false -> json_response(conn, :no_content, "") end end diff --git a/lib/pleroma/web/common_api/common_api.ex b/lib/pleroma/web/common_api/common_api.ex index ce73b3270..2b80598ea 100644 --- a/lib/pleroma/web/common_api/common_api.ex +++ b/lib/pleroma/web/common_api/common_api.ex @@ -346,6 +346,13 @@ defp get_reported_account(account_id) do end end + def update_report_state(activity_ids, state) when is_list(activity_ids) do + case Utils.update_report_state(activity_ids, state) do + :ok -> {:ok, activity_ids} + _ -> {:error, dgettext("errors", "Could not update state")} + end + end + def update_report_state(activity_id, state) do with %Activity{} = activity <- Activity.get_by_id(activity_id) do Utils.update_report_state(activity, state) diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index f91af8137..563b01dc5 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -194,7 +194,7 @@ defmodule Pleroma.Web.Router do get("/reports", AdminAPIController, :list_reports) get("/reports/:id", AdminAPIController, :report_show) - put("/reports/:id", AdminAPIController, :report_update_state) + patch("/reports", AdminAPIController, :reports_update) post("/reports/:id/respond", AdminAPIController, :report_respond) put("/statuses/:id", AdminAPIController, :status_update) diff --git a/test/web/admin_api/admin_api_controller_test.exs b/test/web/admin_api/admin_api_controller_test.exs index b5c355e66..cec3570eb 100644 --- a/test/web/admin_api/admin_api_controller_test.exs +++ b/test/web/admin_api/admin_api_controller_test.exs @@ -1224,7 +1224,7 @@ test "returns 404 when report id is invalid", %{conn: conn} do end end - describe "PUT /api/pleroma/admin/reports/:id" do + describe "PATCH /api/pleroma/admin/reports" do setup %{conn: conn} do admin = insert(:user, info: %{is_admin: true}) [reporter, target_user] = insert_pair(:user) @@ -1237,16 +1237,32 @@ test "returns 404 when report id is invalid", %{conn: conn} do "status_ids" => [activity.id] }) - %{conn: assign(conn, :user, admin), id: report_id, admin: admin} + {:ok, %{id: second_report_id}} = + CommonAPI.report(reporter, %{ + "account_id" => target_user.id, + "comment" => "I feel very offended", + "status_ids" => [activity.id] + }) + + %{ + conn: assign(conn, :user, admin), + id: report_id, + admin: admin, + second_report_id: second_report_id + } end test "mark report as resolved", %{conn: conn, id: id, admin: admin} do - response = - conn - |> put("/api/pleroma/admin/reports/#{id}", %{"state" => "resolved"}) - |> json_response(:ok) + conn + |> patch("/api/pleroma/admin/reports", %{ + "reports" => [ + %{"state" => "resolved", "id" => id} + ] + }) + |> json_response(:no_content) - assert response["state"] == "resolved" + activity = Activity.get_by_id(id) + assert activity.data["state"] == "resolved" log_entry = Repo.one(ModerationLog) @@ -1255,12 +1271,16 @@ test "mark report as resolved", %{conn: conn, id: id, admin: admin} do end test "closes report", %{conn: conn, id: id, admin: admin} do - response = - conn - |> put("/api/pleroma/admin/reports/#{id}", %{"state" => "closed"}) - |> json_response(:ok) + conn + |> patch("/api/pleroma/admin/reports", %{ + "reports" => [ + %{"state" => "closed", "id" => id} + ] + }) + |> json_response(:no_content) - assert response["state"] == "closed" + activity = Activity.get_by_id(id) + assert activity.data["state"] == "closed" log_entry = Repo.one(ModerationLog) @@ -1271,17 +1291,54 @@ test "closes report", %{conn: conn, id: id, admin: admin} do test "returns 400 when state is unknown", %{conn: conn, id: id} do conn = conn - |> put("/api/pleroma/admin/reports/#{id}", %{"state" => "test"}) + |> patch("/api/pleroma/admin/reports", %{ + "reports" => [ + %{"state" => "test", "id" => id} + ] + }) - assert json_response(conn, :bad_request) == "Unsupported state" + assert hd(json_response(conn, :bad_request))["error"] == "Unsupported state" end test "returns 404 when report is not exist", %{conn: conn} do conn = conn - |> put("/api/pleroma/admin/reports/test", %{"state" => "closed"}) + |> patch("/api/pleroma/admin/reports", %{ + "reports" => [ + %{"state" => "closed", "id" => "test"} + ] + }) - assert json_response(conn, :not_found) == "Not found" + assert hd(json_response(conn, :bad_request))["error"] == "not_found" + end + + test "updates state of multiple reports", %{ + conn: conn, + id: id, + admin: admin, + second_report_id: second_report_id + } do + conn + |> patch("/api/pleroma/admin/reports", %{ + "reports" => [ + %{"state" => "resolved", "id" => id}, + %{"state" => "closed", "id" => second_report_id} + ] + }) + |> json_response(:no_content) + + activity = Activity.get_by_id(id) + second_activity = Activity.get_by_id(second_report_id) + assert activity.data["state"] == "resolved" + assert second_activity.data["state"] == "closed" + + [first_log_entry, second_log_entry] = Repo.all(ModerationLog) + + assert ModerationLog.get_log_entry_message(first_log_entry) == + "@#{admin.nickname} updated report ##{id} with 'resolved' state" + + assert ModerationLog.get_log_entry_message(second_log_entry) == + "@#{admin.nickname} updated report ##{second_report_id} with 'closed' state" end end diff --git a/test/web/common_api/common_api_test.exs b/test/web/common_api/common_api_test.exs index 2d3c41e82..c57fdb6af 100644 --- a/test/web/common_api/common_api_test.exs +++ b/test/web/common_api/common_api_test.exs @@ -423,6 +423,35 @@ test "does not update report state when state is unsupported" do assert CommonAPI.update_report_state(report_id, "test") == {:error, "Unsupported state"} end + + test "updates state of multiple reports" do + [reporter, target_user] = insert_pair(:user) + activity = insert(:note_activity, user: target_user) + + {:ok, %Activity{id: first_report_id}} = + CommonAPI.report(reporter, %{ + "account_id" => target_user.id, + "comment" => "I feel offended", + "status_ids" => [activity.id] + }) + + {:ok, %Activity{id: second_report_id}} = + CommonAPI.report(reporter, %{ + "account_id" => target_user.id, + "comment" => "I feel very offended!", + "status_ids" => [activity.id] + }) + + {:ok, report_ids} = + CommonAPI.update_report_state([first_report_id, second_report_id], "resolved") + + first_report = Activity.get_by_id(first_report_id) + second_report = Activity.get_by_id(second_report_id) + + assert report_ids -- [first_report_id, second_report_id] == [] + assert first_report.data["state"] == "resolved" + assert second_report.data["state"] == "resolved" + end end describe "reblog muting" do From 6a85f7d1eabd957588a2f9b8dfea5b7f982573be Mon Sep 17 00:00:00 2001 From: lain Date: Sat, 5 Oct 2019 10:45:42 +0200 Subject: [PATCH 028/107] Transmogrifier: Extend misskey like compatibility. --- .../web/activity_pub/transmogrifier.ex | 5 +++-- test/web/activity_pub/transmogrifier_test.exs | 19 +++++++++++++++++++ 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index 54c18bc0e..2c6edb0b1 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -570,7 +570,8 @@ def handle_incoming( "angry" => "๐Ÿ’ข", "confused" => "๐Ÿ˜ฅ", "rip" => "๐Ÿ˜‡", - "pudding" => "๐Ÿฎ" + "pudding" => "๐Ÿฎ", + "star" => "โญ" } @doc "Rewrite misskey likes into EmojiReactions" @@ -583,7 +584,7 @@ def handle_incoming( ) do data |> Map.put("type", "EmojiReaction") - |> Map.put("content", @misskey_reactions[reaction]) + |> Map.put("content", @misskey_reactions[reaction] || reaction) |> handle_incoming(options) end diff --git a/test/web/activity_pub/transmogrifier_test.exs b/test/web/activity_pub/transmogrifier_test.exs index 530a762fa..9156b23e3 100644 --- a/test/web/activity_pub/transmogrifier_test.exs +++ b/test/web/activity_pub/transmogrifier_test.exs @@ -359,6 +359,25 @@ test "it works for incoming misskey likes, turning them into EmojiReactions" do assert data["content"] == "๐Ÿฎ" end + test "it works for incoming misskey likes that contain unicode emojis, turning them into EmojiReactions" do + user = insert(:user) + {:ok, activity} = CommonAPI.post(user, %{"status" => "hello"}) + + data = + File.read!("test/fixtures/misskey-like.json") + |> Poison.decode!() + |> Map.put("object", activity.data["object"]) + |> Map.put("_misskey_reaction", "โญ") + + {:ok, %Activity{data: data, local: false}} = Transmogrifier.handle_incoming(data) + + assert data["actor"] == data["actor"] + assert data["type"] == "EmojiReaction" + assert data["id"] == data["id"] + assert data["object"] == activity.data["object"] + assert data["content"] == "โญ" + end + test "it works for incoming emoji reactions" do user = insert(:user) {:ok, activity} = CommonAPI.post(user, %{"status" => "hello"}) From d580eedfe93ef08fea7869945c0fd4fe908cb82d Mon Sep 17 00:00:00 2001 From: lain Date: Mon, 7 Oct 2019 12:40:33 +0200 Subject: [PATCH 029/107] Linting. --- .../web/pleroma_api/controllers/pleroma_api_controller.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/pleroma_api/controllers/pleroma_api_controller.ex b/lib/pleroma/web/pleroma_api/controllers/pleroma_api_controller.ex index 16c581a95..6db213475 100644 --- a/lib/pleroma/web/pleroma_api/controllers/pleroma_api_controller.ex +++ b/lib/pleroma/web/pleroma_api/controllers/pleroma_api_controller.ex @@ -11,8 +11,8 @@ defmodule Pleroma.Web.PleromaAPI.PleromaAPIController do alias Pleroma.Conversation.Participation alias Pleroma.Notification alias Pleroma.Object - alias Pleroma.User alias Pleroma.Plugs.OAuthScopesPlug + alias Pleroma.User alias Pleroma.Web.ActivityPub.ActivityPub alias Pleroma.Web.CommonAPI alias Pleroma.Web.MastodonAPI.AccountView From 7aceaa517be7b109a9acc15fb4914535b536b66c Mon Sep 17 00:00:00 2001 From: Maxim Filippov Date: Mon, 7 Oct 2019 15:01:18 +0300 Subject: [PATCH 030/107] Admin API: Reports, grouped by status --- CHANGELOG.md | 1 + docs/api/admin_api.md | 30 +++++-- lib/pleroma/activity.ex | 21 +++++ lib/pleroma/web/activity_pub/utils.ex | 89 +++++++++++++++++++ .../web/admin_api/admin_api_controller.ex | 19 ++-- .../web/admin_api/views/report_view.ex | 20 +++++ lib/pleroma/web/router.ex | 1 + .../admin_api/admin_api_controller_test.exs | 69 +++++++++++++- 8 files changed, 230 insertions(+), 20 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d7afed783..7956a6527 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Pleroma API: `GET /api/v1/pleroma/accounts/:id/scrobbles` to get a list of recently scrobbled items - Pleroma API: `POST /api/v1/pleroma/scrobble` to scrobble a media item - Mastodon API: Add `upload_limit`, `avatar_upload_limit`, `background_upload_limit`, and `banner_upload_limit` to `/api/v1/instance` +- Admin API: Add ability to fetch reports, grouped by status `GET /api/pleroma/admin/grouped_reports` ### Changed - **Breaking:** Elixir >=1.8 is now required (was >= 1.7) diff --git a/docs/api/admin_api.md b/docs/api/admin_api.md index 045686bf4..e8232225c 100644 --- a/docs/api/admin_api.md +++ b/docs/api/admin_api.md @@ -468,18 +468,32 @@ Note: Available `:permission_group` is currently moderator and admin. 404 is ret } ] } - ], - "totalGroupedReports": 1, - "groupedReports": [ + ] +} +``` + +## `GET /api/pleroma/admin/grouped_reports` + +### Get a list of reports, grouped by status + +- Params: none +- On success: JSON, returns a list of reports, where: + - `date`: date of the latest report + - `account`: the user who has been reported (see `/api/pleroma/admin/reports` for reference) + - `status`: reported status (see `/api/pleroma/admin/reports` for reference) + - `actors`: users who had reported this status (see `/api/pleroma/admin/reports` for reference) + - `reports`: reports (see `/api/pleroma/admin/reports` for reference) + +```json + "reports": [ { - "date": "2019-01-01", // date of the latest report - "account": { ... }, // author of the reported status - "status": { ... }, // reported status - "actors": [{ ... }, { ... }], // accounts that sent reports on the status + "date": "2019-10-07T12:31:39.615149Z", + "account": { ... }, + "status": { ... }, + "actors": [{ ... }, { ... }], "reports": [{ ... }] } ] -} ``` ## `GET /api/pleroma/admin/reports/:id` diff --git a/lib/pleroma/activity.ex b/lib/pleroma/activity.ex index c1065611b..daf0ed89f 100644 --- a/lib/pleroma/activity.ex +++ b/lib/pleroma/activity.ex @@ -41,6 +41,9 @@ defmodule Pleroma.Activity do field(:actor, :string) field(:recipients, {:array, :string}, default: []) field(:thread_muted?, :boolean, virtual: true) + + # This is a fake relation, do not use outside of with_preloaded_user_actor/with_joined_user_actor + has_one(:user_actor, User, on_delete: :nothing, foreign_key: :id) # This is a fake relation, do not use outside of with_preloaded_bookmark/get_bookmark has_one(:bookmark, Bookmark) has_many(:notifications, Notification, on_delete: :delete_all) @@ -86,6 +89,24 @@ def with_preloaded_object(query, join_type \\ :inner) do |> preload([activity, object: object], object: object) end + def with_joined_user_actor(query, join_type \\ :inner) do + join(query, join_type, [activity], u in User, + on: + fragment( + "? = ?->>'actor'", + u.ap_id, + activity.data + ), + as: :user_actor + ) + end + + def with_preloaded_user_actor(query, join_type \\ :inner) do + query + |> with_joined_user_actor(join_type) + |> preload([activity, user_actor: user_actor], user_actor: user_actor) + end + def with_preloaded_bookmark(query, %User{} = user) do from([a] in query, left_join: b in Bookmark, diff --git a/lib/pleroma/web/activity_pub/utils.ex b/lib/pleroma/web/activity_pub/utils.ex index 824957314..74eb994ab 100644 --- a/lib/pleroma/web/activity_pub/utils.ex +++ b/lib/pleroma/web/activity_pub/utils.ex @@ -6,11 +6,13 @@ defmodule Pleroma.Web.ActivityPub.Utils do alias Ecto.Changeset alias Ecto.UUID alias Pleroma.Activity + alias Pleroma.Activity.Queries alias Pleroma.Notification alias Pleroma.Object alias Pleroma.Repo alias Pleroma.User alias Pleroma.Web + alias Pleroma.Web.ActivityPub.ActivityPub alias Pleroma.Web.ActivityPub.Visibility alias Pleroma.Web.Endpoint alias Pleroma.Web.Router.Helpers @@ -664,6 +666,93 @@ def fetch_ordered_collection(from, pages_left, acc \\ []) do #### Report-related helpers + def get_reports(params, page, page_size) do + params = + params + |> Map.put("type", "Flag") + |> Map.put("skip_preload", true) + |> Map.put("total", true) + |> Map.put("limit", page_size) + |> Map.put("offset", (page - 1) * page_size) + + ActivityPub.fetch_activities([], params, :offset) + end + + @spec get_reports_grouped_by_status() :: %{ + required(:groups) => [ + %{ + required(:date) => String.t(), + required(:account) => %User{}, + required(:status) => %Activity{}, + required(:actors) => [%User{}], + required(:reports) => [%Activity{}] + } + ], + required(:total) => integer + } + def get_reports_grouped_by_status do + paginated_activities = get_reported_status_ids() + + groups = + paginated_activities + |> Enum.map(fn entry -> + status = + Activity + |> Queries.by_ap_id(entry[:activity_id]) + |> Activity.with_preloaded_object(:left) + |> Activity.with_preloaded_user_actor() + |> Repo.one() + + reports = get_reports_by_status_id(status.data["id"]) + + max_date = + Enum.max_by(reports, &Pleroma.Web.CommonAPI.Utils.to_masto_date(&1.data["published"])).data[ + "published" + ] + + actors = Enum.map(reports, & &1.user_actor) + + %{ + date: max_date, + account: status.user_actor, + status: status, + actors: actors, + reports: reports + } + end) + + %{ + groups: groups + } + end + + def get_reports_by_status_id(status_id) do + from(a in Activity, + where: fragment("(?)->>'type' = 'Flag'", a.data), + where: fragment("(?)->'object' \\? (?)", a.data, ^status_id) + ) + |> Activity.with_preloaded_user_actor() + |> Repo.all() + end + + @spec get_reported_status_ids() :: %{ + required(:items) => [%Activity{}], + required(:total) => integer + } + def get_reported_status_ids do + from(a in Activity, + where: fragment("(?)->>'type' = 'Flag'", a.data), + select: %{ + date: fragment("max(?->>'published') date", a.data), + activity_id: + fragment("jsonb_array_elements_text((? #- '{object,0}')->'object') activity_id", a.data) + }, + group_by: fragment("activity_id"), + order_by: fragment("date DESC") + ) + |> Repo.all() + end + def update_report_state(%Activity{} = activity, state) when state in @supported_report_states do new_data = Map.put(activity.data, "state", state) diff --git a/lib/pleroma/web/admin_api/admin_api_controller.ex b/lib/pleroma/web/admin_api/admin_api_controller.ex index 0e8c9dac8..463dd327a 100644 --- a/lib/pleroma/web/admin_api/admin_api_controller.ex +++ b/lib/pleroma/web/admin_api/admin_api_controller.ex @@ -10,6 +10,7 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do alias Pleroma.UserInviteToken alias Pleroma.Web.ActivityPub.ActivityPub alias Pleroma.Web.ActivityPub.Relay + alias Pleroma.Web.ActivityPub.Utils alias Pleroma.Web.AdminAPI.AccountView alias Pleroma.Web.AdminAPI.Config alias Pleroma.Web.AdminAPI.ConfigView @@ -455,19 +456,15 @@ def force_password_reset(conn, %{"nickname" => nickname}) do def list_reports(conn, params) do {page, page_size} = page_params(params) - params = - params - |> Map.put("type", "Flag") - |> Map.put("skip_preload", true) - |> Map.put("total", true) - |> Map.put("limit", page_size) - |> Map.put("offset", (page - 1) * page_size) - - reports = ActivityPub.fetch_activities([], params, :offset) - conn |> put_view(ReportView) - |> render("index.json", %{reports: reports}) + |> render("index.json", %{reports: Utils.get_reports(params, page, page_size)}) + end + + def list_grouped_reports(conn, _params) do + conn + |> put_view(ReportView) + |> render("index_grouped.json", Utils.get_reports_grouped_by_status()) end def report_show(conn, %{"id" => id}) do diff --git a/lib/pleroma/web/admin_api/views/report_view.ex b/lib/pleroma/web/admin_api/views/report_view.ex index 101a74c63..ac25925da 100644 --- a/lib/pleroma/web/admin_api/views/report_view.ex +++ b/lib/pleroma/web/admin_api/views/report_view.ex @@ -42,6 +42,26 @@ def render("show.json", %{report: report, user: user, account: account, statuses } end + def render("index_grouped.json", %{groups: groups}) do + reports = + Enum.map(groups, fn group -> + %{ + date: group[:date], + account: merge_account_views(group[:account]), + status: StatusView.render("show.json", %{activity: group[:status]}), + actors: Enum.map(group[:actors], &merge_account_views/1), + reports: + group[:reports] + |> Enum.map(&Report.extract_report_info(&1)) + |> Enum.map(&render(__MODULE__, "show.json", &1)) + } + end) + + %{ + reports: reports + } + end + defp merge_account_views(%User{} = user) do Pleroma.Web.MastodonAPI.AccountView.render("show.json", %{user: user}) |> Map.merge(Pleroma.Web.AdminAPI.AccountView.render("show.json", %{user: user})) diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 563b01dc5..b895a7b7e 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -193,6 +193,7 @@ defmodule Pleroma.Web.Router do get("/users/:nickname/statuses", AdminAPIController, :list_user_statuses) get("/reports", AdminAPIController, :list_reports) + get("/grouped_reports", AdminAPIController, :list_grouped_reports) get("/reports/:id", AdminAPIController, :report_show) patch("/reports", AdminAPIController, :reports_update) post("/reports/:id/respond", AdminAPIController, :report_respond) diff --git a/test/web/admin_api/admin_api_controller_test.exs b/test/web/admin_api/admin_api_controller_test.exs index cec3570eb..daa0631db 100644 --- a/test/web/admin_api/admin_api_controller_test.exs +++ b/test/web/admin_api/admin_api_controller_test.exs @@ -1461,7 +1461,74 @@ test "returns 403 when requested by anonymous" do end end - # + describe "GET /api/pleroma/admin/grouped_reports" do + setup %{conn: conn} do + admin = insert(:user, info: %{is_admin: true}) + [reporter, target_user] = insert_pair(:user) + + date1 = (DateTime.to_unix(DateTime.utc_now()) + 1000) |> DateTime.from_unix!() + date2 = (DateTime.to_unix(DateTime.utc_now()) + 2000) |> DateTime.from_unix!() + date3 = (DateTime.to_unix(DateTime.utc_now()) + 3000) |> DateTime.from_unix!() + + first_status = + insert(:note_activity, user: target_user, data_attrs: %{"published" => date1}) + + second_status = + insert(:note_activity, user: target_user, data_attrs: %{"published" => date2}) + + third_status = + insert(:note_activity, user: target_user, data_attrs: %{"published" => date3}) + + %{ + conn: assign(conn, :user, admin), + reporter: reporter, + target_user: target_user, + first_status: first_status, + second_status: second_status, + third_status: third_status + } + end + + test "returns reports grouped by status", %{ + conn: conn, + reporter: reporter, + target_user: target_user, + first_status: first_status, + second_status: second_status, + third_status: third_status + } do + {:ok, %{id: _}} = + CommonAPI.report(reporter, %{ + "account_id" => target_user.id, + "status_ids" => [first_status.id, second_status.id, third_status.id] + }) + + {:ok, %{id: _}} = + CommonAPI.report(reporter, %{ + "account_id" => target_user.id, + "status_ids" => [first_status.id, second_status.id] + }) + + {:ok, %{id: _}} = + CommonAPI.report(reporter, %{ + "account_id" => target_user.id, + "status_ids" => [first_status.id] + }) + + response = + conn + |> get("/api/pleroma/admin/grouped_reports") + |> json_response(:ok) + + assert length(response["reports"]) == 3 + [third_group, second_group, first_group] = response["reports"] + + assert length(third_group["reports"]) == 3 + assert length(second_group["reports"]) == 2 + assert length(first_group["reports"]) == 1 + end + end + describe "POST /api/pleroma/admin/reports/:id/respond" do setup %{conn: conn} do admin = insert(:user, info: %{is_admin: true}) From aa7fd616c7cfeb84551af2170886856a815dc498 Mon Sep 17 00:00:00 2001 From: Maxim Filippov Date: Mon, 7 Oct 2019 16:03:23 +0300 Subject: [PATCH 031/107] Line is too long! --- lib/pleroma/activity.ex | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/activity.ex b/lib/pleroma/activity.ex index daf0ed89f..7b77f72c2 100644 --- a/lib/pleroma/activity.ex +++ b/lib/pleroma/activity.ex @@ -42,7 +42,8 @@ defmodule Pleroma.Activity do field(:recipients, {:array, :string}, default: []) field(:thread_muted?, :boolean, virtual: true) - # This is a fake relation, do not use outside of with_preloaded_user_actor/with_joined_user_actor + # This is a fake relation, + # do not use outside of with_preloaded_user_actor/with_joined_user_actor has_one(:user_actor, User, on_delete: :nothing, foreign_key: :id) # This is a fake relation, do not use outside of with_preloaded_bookmark/get_bookmark has_one(:bookmark, Bookmark) From e1fc6cb78f07653300965d212d9c5ece9f5c3de0 Mon Sep 17 00:00:00 2001 From: AkiraFukushima Date: Tue, 5 Nov 2019 23:52:47 +0900 Subject: [PATCH 032/107] Check client and token in GET /oauth/authorize --- lib/pleroma/web/oauth/oauth_controller.ex | 18 +++++++++++++++++- test/web/oauth/oauth_controller_test.exs | 23 +++++++++++++++++++++++ 2 files changed, 40 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/web/oauth/oauth_controller.ex b/lib/pleroma/web/oauth/oauth_controller.ex index fe71aca8c..6fba9f968 100644 --- a/lib/pleroma/web/oauth/oauth_controller.ex +++ b/lib/pleroma/web/oauth/oauth_controller.ex @@ -36,7 +36,7 @@ def authorize(%Plug.Conn{} = conn, %{"authorization" => _} = params) do authorize(conn, Map.merge(params, auth_attrs)) end - def authorize(%Plug.Conn{assigns: %{token: %Token{}}} = conn, params) do + def authorize(%Plug.Conn{assigns: %{token: %Token{}}} = conn, %{"force_login" => _} = params) do if ControllerHelper.truthy_param?(params["force_login"]) do do_authorize(conn, params) else @@ -44,6 +44,22 @@ def authorize(%Plug.Conn{assigns: %{token: %Token{}}} = conn, params) do end end + # Note: the token is set in oauth_plug, but the token and client do not always go together. + # For example, MastodonFE's token is set if user requests with another client, + # after user already authorized to MastodonFE. + # So we have to check client and token. + def authorize( + %Plug.Conn{assigns: %{token: %Token{} = token}} = conn, + %{"client_id" => client_id} = params + ) do + with %Token{} = t <- Repo.get_by(Token, token: token.token) |> Repo.preload(:app), + ^client_id <- t.app.client_id do + handle_existing_authorization(conn, params) + else + _ -> do_authorize(conn, params) + end + end + def authorize(%Plug.Conn{} = conn, params), do: do_authorize(conn, params) defp do_authorize(%Plug.Conn{} = conn, params) do diff --git a/test/web/oauth/oauth_controller_test.exs b/test/web/oauth/oauth_controller_test.exs index ad8d79083..beb995cd8 100644 --- a/test/web/oauth/oauth_controller_test.exs +++ b/test/web/oauth/oauth_controller_test.exs @@ -469,6 +469,29 @@ test "renders authentication page if user is already authenticated but `force_lo assert html_response(conn, 200) =~ ~s(type="submit") end + test "renders authentication page if user is already authenticated but user request with another client", + %{ + app: app, + conn: conn + } do + token = insert(:oauth_token, app_id: app.id) + + conn = + conn + |> put_session(:oauth_token, token.token) + |> get( + "/oauth/authorize", + %{ + "response_type" => "code", + "client_id" => "another_client_id", + "redirect_uri" => OAuthController.default_redirect_uri(app), + "scope" => "read" + } + ) + + assert html_response(conn, 200) =~ ~s(type="submit") + end + test "with existing authentication and non-OOB `redirect_uri`, redirects to app with `token` and `state` params", %{ app: app, From f171095960d172d54015b28e8da302b5745dca86 Mon Sep 17 00:00:00 2001 From: Maxim Filippov Date: Wed, 6 Nov 2019 21:25:46 +1000 Subject: [PATCH 033/107] Grouped reports with status data baked in --- lib/pleroma/web/activity_pub/utils.ex | 58 ++++----- .../web/admin_api/views/report_view.ex | 4 +- .../admin_api/admin_api_controller_test.exs | 121 ++++++++++++++---- 3 files changed, 123 insertions(+), 60 deletions(-) diff --git a/lib/pleroma/web/activity_pub/utils.ex b/lib/pleroma/web/activity_pub/utils.ex index 57349e304..5a51b7884 100644 --- a/lib/pleroma/web/activity_pub/utils.ex +++ b/lib/pleroma/web/activity_pub/utils.ex @@ -6,7 +6,6 @@ defmodule Pleroma.Web.ActivityPub.Utils do alias Ecto.Changeset alias Ecto.UUID alias Pleroma.Activity - alias Pleroma.Activity.Queries alias Pleroma.Notification alias Pleroma.Object alias Pleroma.Repo @@ -697,8 +696,8 @@ def get_reports(params, page, page_size) do required(:groups) => [ %{ required(:date) => String.t(), - required(:account) => %User{}, - required(:status) => %Activity{}, + required(:account) => %{}, + required(:status) => %{}, required(:actors) => [%User{}], required(:reports) => [%Activity{}] } @@ -706,32 +705,23 @@ def get_reports(params, page, page_size) do required(:total) => integer } def get_reports_grouped_by_status do - paginated_activities = get_reported_status_ids() - groups = - paginated_activities + get_reported_status_ids() |> Enum.map(fn entry -> - status = - Activity - |> Queries.by_ap_id(entry[:activity_id]) - |> Activity.with_preloaded_object(:left) - |> Activity.with_preloaded_user_actor() - |> Repo.one() - - reports = get_reports_by_status_id(status.data["id"]) - - max_date = - Enum.max_by(reports, &Pleroma.Web.CommonAPI.Utils.to_masto_date(&1.data["published"])).data[ - "published" - ] - + activity = Jason.decode!(entry.activity) + reports = get_reports_by_status_id(activity["id"]) + max_date = Enum.max_by(reports, &NaiveDateTime.from_iso8601!(&1.data["published"])) actors = Enum.map(reports, & &1.user_actor) %{ - date: max_date, - account: status.user_actor, - status: status, - actors: actors, + date: max_date.data["published"], + account: activity["actor"], + status: %{ + id: activity["id"], + content: activity["content"], + published: activity["published"] + }, + actors: Enum.uniq(actors), reports: reports } end) @@ -741,28 +731,30 @@ def get_reports_grouped_by_status do } end - def get_reports_by_status_id(status_id) do + def get_reports_by_status_id(ap_id) do from(a in Activity, where: fragment("(?)->>'type' = 'Flag'", a.data), - where: fragment("(?)->'object' \\? (?)", a.data, ^status_id) + where: fragment("(?)->'object' @> ?", a.data, ^[%{id: ap_id}]) ) |> Activity.with_preloaded_user_actor() |> Repo.all() end - @spec get_reported_status_ids() :: %{ - required(:items) => [%Activity{}], - required(:total) => integer - } + @spec get_reported_status_ids() :: [ + %{ + required(:activity) => String.t(), + required(:date) => String.t() + } + ] def get_reported_status_ids do from(a in Activity, where: fragment("(?)->>'type' = 'Flag'", a.data), select: %{ date: fragment("max(?->>'published') date", a.data), - activity_id: - fragment("jsonb_array_elements_text((? #- '{object,0}')->'object') activity_id", a.data) + activity: + fragment("jsonb_array_elements_text((? #- '{object,0}')->'object') activity", a.data) }, - group_by: fragment("activity_id"), + group_by: fragment("activity"), order_by: fragment("date DESC") ) |> Repo.all() diff --git a/lib/pleroma/web/admin_api/views/report_view.ex b/lib/pleroma/web/admin_api/views/report_view.ex index ac25925da..ca88595c7 100644 --- a/lib/pleroma/web/admin_api/views/report_view.ex +++ b/lib/pleroma/web/admin_api/views/report_view.ex @@ -47,8 +47,8 @@ def render("index_grouped.json", %{groups: groups}) do Enum.map(groups, fn group -> %{ date: group[:date], - account: merge_account_views(group[:account]), - status: StatusView.render("show.json", %{activity: group[:status]}), + account: group[:account], + status: group[:status], actors: Enum.map(group[:actors], &merge_account_views/1), reports: group[:reports] diff --git a/test/web/admin_api/admin_api_controller_test.exs b/test/web/admin_api/admin_api_controller_test.exs index 35367bed3..4e28c7774 100644 --- a/test/web/admin_api/admin_api_controller_test.exs +++ b/test/web/admin_api/admin_api_controller_test.exs @@ -1551,7 +1551,7 @@ test "returns 403 when requested by anonymous" do describe "GET /api/pleroma/admin/grouped_reports" do setup %{conn: conn} do - admin = insert(:user, info: %{is_admin: true}) + admin = insert(:user, is_admin: true) [reporter, target_user] = insert_pair(:user) date1 = (DateTime.to_unix(DateTime.utc_now()) + 1000) |> DateTime.from_unix!() @@ -1567,53 +1567,124 @@ test "returns 403 when requested by anonymous" do third_status = insert(:note_activity, user: target_user, data_attrs: %{"published" => date3}) - %{ - conn: assign(conn, :user, admin), - reporter: reporter, - target_user: target_user, - first_status: first_status, - second_status: second_status, - third_status: third_status - } - end - - test "returns reports grouped by status", %{ - conn: conn, - reporter: reporter, - target_user: target_user, - first_status: first_status, - second_status: second_status, - third_status: third_status - } do - {:ok, %{id: _}} = + {:ok, first_report} = CommonAPI.report(reporter, %{ "account_id" => target_user.id, "status_ids" => [first_status.id, second_status.id, third_status.id] }) - {:ok, %{id: _}} = + {:ok, second_report} = CommonAPI.report(reporter, %{ "account_id" => target_user.id, "status_ids" => [first_status.id, second_status.id] }) - {:ok, %{id: _}} = + {:ok, third_report} = CommonAPI.report(reporter, %{ "account_id" => target_user.id, "status_ids" => [first_status.id] }) + %{ + conn: assign(conn, :user, admin), + first_status: Activity.get_by_ap_id_with_object(first_status.data["id"]), + second_status: Activity.get_by_ap_id_with_object(second_status.data["id"]), + third_status: Activity.get_by_ap_id_with_object(third_status.data["id"]), + first_status_reports: [first_report, second_report, third_report], + second_status_reports: [first_report, second_report], + third_status_reports: [first_report], + target_user: target_user, + reporter: reporter + } + end + + test "returns reports grouped by status", %{ + conn: conn, + first_status: first_status, + second_status: second_status, + third_status: third_status, + first_status_reports: first_status_reports, + second_status_reports: second_status_reports, + third_status_reports: third_status_reports, + target_user: target_user, + reporter: reporter + } do response = conn |> get("/api/pleroma/admin/grouped_reports") |> json_response(:ok) assert length(response["reports"]) == 3 - [third_group, second_group, first_group] = response["reports"] - assert length(third_group["reports"]) == 3 + first_group = + Enum.find(response["reports"], &(&1["status"]["id"] == first_status.data["id"])) + + second_group = + Enum.find(response["reports"], &(&1["status"]["id"] == second_status.data["id"])) + + third_group = + Enum.find(response["reports"], &(&1["status"]["id"] == third_status.data["id"])) + + assert length(first_group["reports"]) == 3 assert length(second_group["reports"]) == 2 - assert length(first_group["reports"]) == 1 + assert length(third_group["reports"]) == 1 + + assert first_group["date"] == + Enum.max_by(first_status_reports, fn act -> + NaiveDateTime.from_iso8601!(act.data["published"]) + end).data["published"] + + assert first_group["status"] == %{ + "id" => first_status.data["id"], + "content" => first_status.object.data["content"], + "published" => first_status.object.data["published"] + } + + assert first_group["account"]["id"] == target_user.id + + assert length(first_group["actors"]) == 1 + assert hd(first_group["actors"])["id"] == reporter.id + + assert Enum.map(first_group["reports"], & &1["id"]) -- + Enum.map(first_status_reports, & &1.id) == [] + + assert second_group["date"] == + Enum.max_by(second_status_reports, fn act -> + NaiveDateTime.from_iso8601!(act.data["published"]) + end).data["published"] + + assert second_group["status"] == %{ + "id" => second_status.data["id"], + "content" => second_status.object.data["content"], + "published" => second_status.object.data["published"] + } + + assert second_group["account"]["id"] == target_user.id + + assert length(second_group["actors"]) == 1 + assert hd(second_group["actors"])["id"] == reporter.id + + assert Enum.map(second_group["reports"], & &1["id"]) -- + Enum.map(second_status_reports, & &1.id) == [] + + assert third_group["date"] == + Enum.max_by(third_status_reports, fn act -> + NaiveDateTime.from_iso8601!(act.data["published"]) + end).data["published"] + + assert third_group["status"] == %{ + "id" => third_status.data["id"], + "content" => third_status.object.data["content"], + "published" => third_status.object.data["published"] + } + + assert third_group["account"]["id"] == target_user.id + + assert length(third_group["actors"]) == 1 + assert hd(third_group["actors"])["id"] == reporter.id + + assert Enum.map(third_group["reports"], & &1["id"]) -- + Enum.map(third_status_reports, & &1.id) == [] end end From 7258db023e88d5aee5eac06525c42dcb073abd46 Mon Sep 17 00:00:00 2001 From: Maxim Filippov Date: Thu, 7 Nov 2019 22:45:36 +1000 Subject: [PATCH 034/107] Support old flag format --- lib/pleroma/web/activity_pub/utils.ex | 94 +++++++++++-------- .../web/admin_api/admin_api_controller.ex | 4 +- test/web/activity_pub/utils_test.exs | 43 +++++++++ 3 files changed, 100 insertions(+), 41 deletions(-) diff --git a/lib/pleroma/web/activity_pub/utils.ex b/lib/pleroma/web/activity_pub/utils.ex index 5a51b7884..5e7f76bb6 100644 --- a/lib/pleroma/web/activity_pub/utils.ex +++ b/lib/pleroma/web/activity_pub/utils.ex @@ -616,26 +616,31 @@ def make_flag_data(%{actor: actor, context: context, content: content} = params, def make_flag_data(_, _), do: %{} defp build_flag_object(%{account: account, statuses: statuses} = _) do - [account.ap_id] ++ - Enum.map(statuses || [], fn act -> - id = - case act do - %Activity{} = act -> act.data["id"] - act when is_map(act) -> act["id"] - act when is_binary(act) -> act - end + [account.ap_id] ++ build_flag_object(%{statuses: statuses}) + end - activity = Activity.get_by_ap_id_with_object(id) - actor = User.get_by_ap_id(activity.object.data["actor"]) + defp build_flag_object(%{statuses: statuses}) do + Enum.map(statuses || [], &build_flag_object/1) + end - %{ - "type" => "Note", - "id" => activity.data["id"], - "content" => activity.object.data["content"], - "published" => activity.object.data["published"], - "actor" => AccountView.render("show.json", %{user: actor}) - } - end) + defp build_flag_object(act) when is_map(act) or is_binary(act) do + id = + case act do + %Activity{} = act -> act.data["id"] + act when is_map(act) -> act["id"] + act when is_binary(act) -> act + end + + activity = Activity.get_by_ap_id_with_object(id) + actor = User.get_by_ap_id(activity.object.data["actor"]) + + %{ + "type" => "Note", + "id" => activity.data["id"], + "content" => activity.object.data["content"], + "published" => activity.object.data["published"], + "actor" => AccountView.render("show.json", %{user: actor}) + } end defp build_flag_object(_), do: [] @@ -692,7 +697,7 @@ def get_reports(params, page, page_size) do ActivityPub.fetch_activities([], params, :offset) end - @spec get_reports_grouped_by_status() :: %{ + @spec get_reports_grouped_by_status(%{required(:activity) => String.t()}) :: %{ required(:groups) => [ %{ required(:date) => String.t(), @@ -704,30 +709,39 @@ def get_reports(params, page, page_size) do ], required(:total) => integer } - def get_reports_grouped_by_status do - groups = - get_reported_status_ids() + def get_reports_grouped_by_status(groups) do + parsed_groups = + groups |> Enum.map(fn entry -> - activity = Jason.decode!(entry.activity) - reports = get_reports_by_status_id(activity["id"]) - max_date = Enum.max_by(reports, &NaiveDateTime.from_iso8601!(&1.data["published"])) - actors = Enum.map(reports, & &1.user_actor) + activity = + case Jason.decode(entry.activity) do + {:ok, activity} -> activity + _ -> build_flag_object(entry.activity) + end - %{ - date: max_date.data["published"], - account: activity["actor"], - status: %{ - id: activity["id"], - content: activity["content"], - published: activity["published"] - }, - actors: Enum.uniq(actors), - reports: reports - } + parse_report_group(activity) end) %{ - groups: groups + groups: parsed_groups + } + end + + def parse_report_group(activity) do + reports = get_reports_by_status_id(activity["id"]) + max_date = Enum.max_by(reports, &NaiveDateTime.from_iso8601!(&1.data["published"])) + actors = Enum.map(reports, & &1.user_actor) + + %{ + date: max_date.data["published"], + account: activity["actor"], + status: %{ + id: activity["id"], + content: activity["content"], + published: activity["published"] + }, + actors: Enum.uniq(actors), + reports: reports } end @@ -740,13 +754,13 @@ def get_reports_by_status_id(ap_id) do |> Repo.all() end - @spec get_reported_status_ids() :: [ + @spec get_reported_activities() :: [ %{ required(:activity) => String.t(), required(:date) => String.t() } ] - def get_reported_status_ids do + def get_reported_activities do from(a in Activity, where: fragment("(?)->>'type' = 'Flag'", a.data), select: %{ diff --git a/lib/pleroma/web/admin_api/admin_api_controller.ex b/lib/pleroma/web/admin_api/admin_api_controller.ex index 1f48ce8c1..7d5ff7629 100644 --- a/lib/pleroma/web/admin_api/admin_api_controller.ex +++ b/lib/pleroma/web/admin_api/admin_api_controller.ex @@ -625,9 +625,11 @@ def list_reports(conn, params) do end def list_grouped_reports(conn, _params) do + reports = Utils.get_reported_activities() + conn |> put_view(ReportView) - |> render("index_grouped.json", Utils.get_reports_grouped_by_status()) + |> render("index_grouped.json", Utils.get_reports_grouped_by_status(reports)) end def report_show(conn, %{"id" => id}) do diff --git a/test/web/activity_pub/utils_test.exs b/test/web/activity_pub/utils_test.exs index 586eb1d2f..1feb076ba 100644 --- a/test/web/activity_pub/utils_test.exs +++ b/test/web/activity_pub/utils_test.exs @@ -636,4 +636,47 @@ test "removes actor from announcements" do assert updated_object.data["announcement_count"] == 1 end end + + describe "get_reports_grouped_by_status/1" do + setup do + [reporter, target_user] = insert_pair(:user) + first_status = insert(:note_activity, user: target_user) + second_status = insert(:note_activity, user: target_user) + + CommonAPI.report(reporter, %{ + "account_id" => target_user.id, + "comment" => "I feel offended", + "status_ids" => [first_status.id] + }) + + CommonAPI.report(reporter, %{ + "account_id" => target_user.id, + "comment" => "I feel offended2", + "status_ids" => [second_status.id] + }) + + data = [%{activity: first_status.data["id"]}, %{activity: second_status.data["id"]}] + + {:ok, + %{ + first_status: first_status, + second_status: second_status, + data: data + }} + end + + test "works for deprecated reports format", %{ + first_status: first_status, + second_status: second_status, + data: data + } do + groups = Utils.get_reports_grouped_by_status(data).groups + + first_group = Enum.find(groups, &(&1.status.id == first_status.data["id"])) + second_group = Enum.find(groups, &(&1.status.id == second_status.data["id"])) + + assert first_group.status.id == first_status.data["id"] + assert second_group.status.id == second_status.data["id"] + end + end end From a4d3a8ec03ec10bb7a9ba3c3e69d9ddc559ed2a0 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Sat, 2 Mar 2019 14:17:55 +0000 Subject: [PATCH 035/107] static fe: proof of concept --- .../web/static_fe/static_fe_controller.ex | 42 +++++ lib/pleroma/web/static_fe/static_fe_view.ex | 19 +++ .../web/templates/layout/static_fe.html.eex | 150 ++++++++++++++++++ .../static_fe/static_fe/notice.html.eex | 6 + .../static_fe/static_fe/user_card.html.eex | 11 ++ 5 files changed, 228 insertions(+) create mode 100644 lib/pleroma/web/static_fe/static_fe_controller.ex create mode 100644 lib/pleroma/web/static_fe/static_fe_view.ex create mode 100644 lib/pleroma/web/templates/layout/static_fe.html.eex create mode 100644 lib/pleroma/web/templates/static_fe/static_fe/notice.html.eex create mode 100644 lib/pleroma/web/templates/static_fe/static_fe/user_card.html.eex diff --git a/lib/pleroma/web/static_fe/static_fe_controller.ex b/lib/pleroma/web/static_fe/static_fe_controller.ex new file mode 100644 index 000000000..067af9816 --- /dev/null +++ b/lib/pleroma/web/static_fe/static_fe_controller.ex @@ -0,0 +1,42 @@ +# Pleroma: A lightweight social networking server +# Copyright ยฉ 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.StaticFE.StaticFEController do + use Pleroma.Web, :controller + + alias Pleroma.Repo + alias Pleroma.Activity + alias Pleroma.Object + alias Pleroma.User + alias Pleroma.Web.ActivityPub.Visibility + + require Logger + + def show_notice(conn, %{"notice_id" => notice_id}) do + with %Activity{} = activity <- Repo.get(Activity, notice_id), + true <- Visibility.is_public?(activity), + %User{} = user <- User.get_or_fetch(activity.data["actor"]), + %Object{} = object <- Object.normalize(activity.data["object"]) do + conn + |> put_layout(:static_fe) + |> put_status(200) + |> put_view(Pleroma.Web.StaticFE.StaticFEView) + |> render("notice.html", %{notice: activity, object: object, user: user}) + else + _ -> + conn + |> put_status(404) + |> text("Not found") + end + end + + def show(%{path_info: ["notice", notice_id]} = conn, _params), do: show_notice(conn, %{"notice_id" => notice_id}) + + # Fallback for unhandled types + def show(conn, _params) do + conn + |> put_status(404) + |> text("Not found") + end +end diff --git a/lib/pleroma/web/static_fe/static_fe_view.ex b/lib/pleroma/web/static_fe/static_fe_view.ex new file mode 100644 index 000000000..7f58e1b2d --- /dev/null +++ b/lib/pleroma/web/static_fe/static_fe_view.ex @@ -0,0 +1,19 @@ +# Pleroma: A lightweight social networking server +# Copyright ยฉ 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.StaticFE.StaticFEView do + use Pleroma.Web, :view + + alias Pleroma.User + alias Pleroma.Web.MediaProxy + alias Pleroma.Formatter + + def emoji_for_user(%User{} = user) do + (user.info.source_data["tag"] || []) + |> Enum.filter(fn %{"type" => t} -> t == "Emoji" end) + |> Enum.map(fn %{"icon" => %{"url" => url}, "name" => name} -> + {String.trim(name, ":"), url} + end) + end +end diff --git a/lib/pleroma/web/templates/layout/static_fe.html.eex b/lib/pleroma/web/templates/layout/static_fe.html.eex new file mode 100644 index 000000000..c20958162 --- /dev/null +++ b/lib/pleroma/web/templates/layout/static_fe.html.eex @@ -0,0 +1,150 @@ + + + + + + + <%= Application.get_env(:pleroma, :instance)[:name] %> + + + + +
+ <%= render @view_module, @view_template, assigns %> +
+ + diff --git a/lib/pleroma/web/templates/static_fe/static_fe/notice.html.eex b/lib/pleroma/web/templates/static_fe/static_fe/notice.html.eex new file mode 100644 index 000000000..9957697ad --- /dev/null +++ b/lib/pleroma/web/templates/static_fe/static_fe/notice.html.eex @@ -0,0 +1,6 @@ +
+ <%= render("user_card.html", %{user: @user}) %> +
+
<%= @object.data["content"] %>
+
+
diff --git a/lib/pleroma/web/templates/static_fe/static_fe/user_card.html.eex b/lib/pleroma/web/templates/static_fe/static_fe/user_card.html.eex new file mode 100644 index 000000000..8b397c639 --- /dev/null +++ b/lib/pleroma/web/templates/static_fe/static_fe/user_card.html.eex @@ -0,0 +1,11 @@ +
+ +
+ +
+ + <%= @user.name |> Formatter.emojify(emoji_for_user(@user)) %> + <%= @user.nickname %> + + +
From ff8d0902f351f871ab34ae7b127dd3e02e8af4cd Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Sat, 2 Mar 2019 14:23:26 +0000 Subject: [PATCH 036/107] static fe: formatting --- lib/pleroma/web/static_fe/static_fe_controller.ex | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/web/static_fe/static_fe_controller.ex b/lib/pleroma/web/static_fe/static_fe_controller.ex index 067af9816..2ac857759 100644 --- a/lib/pleroma/web/static_fe/static_fe_controller.ex +++ b/lib/pleroma/web/static_fe/static_fe_controller.ex @@ -31,7 +31,8 @@ def show_notice(conn, %{"notice_id" => notice_id}) do end end - def show(%{path_info: ["notice", notice_id]} = conn, _params), do: show_notice(conn, %{"notice_id" => notice_id}) + def show(%{path_info: ["notice", notice_id]} = conn, _params), + do: show_notice(conn, %{"notice_id" => notice_id}) # Fallback for unhandled types def show(conn, _params) do From 8f08da750aaa8e04c4a940a566ec831a559a4852 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Sat, 2 Mar 2019 20:10:30 +0000 Subject: [PATCH 037/107] static fe: use a generic activity representer to render activities --- .../web/static_fe/activity_representer.ex | 52 +++++++++++++++++++ .../web/static_fe/static_fe_controller.ex | 13 ++--- 2 files changed, 55 insertions(+), 10 deletions(-) create mode 100644 lib/pleroma/web/static_fe/activity_representer.ex diff --git a/lib/pleroma/web/static_fe/activity_representer.ex b/lib/pleroma/web/static_fe/activity_representer.ex new file mode 100644 index 000000000..93f8a8813 --- /dev/null +++ b/lib/pleroma/web/static_fe/activity_representer.ex @@ -0,0 +1,52 @@ +# Pleroma: A lightweight social networking server +# Copyright ยฉ 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.StaticFE.ActivityRepresenter do + alias Pleroma.Activity + alias Pleroma.Object + alias Pleroma.User + alias Pleroma.Web.ActivityPub.Visibility + + def prepare_activity(%User{} = user, %Object{} = object) do + %{} + |> set_user(user) + |> set_object(object) + |> set_title(object) + |> set_content(object) + |> set_attachments(object) + end + + defp set_user(data, %User{} = user), do: Map.put(data, :user, user) + + defp set_object(data, %Object{} = object), do: Map.put(data, :object, object) + + defp set_title(data, %Object{data: %{"name" => name}}) when is_binary(name), + do: Map.put(data, :title, name) + + defp set_title(data, %Object{data: %{"summary" => summary}}) when is_binary(summary), + do: Map.put(data, :title, summary) + + defp set_title(data, _), do: Map.put(data, :title, nil) + + defp set_content(data, %Object{data: %{"content" => content}}) when is_binary(content), + do: Map.put(data, :content, content) + + defp set_content(data, _), do: Map.put(data, :content, nil) + + # TODO: attachments + defp set_attachments(data, _), do: Map.put(data, :attachments, []) + + def represent(activity_id) do + with %Activity{data: %{"type" => "Create"}} = activity <- Activity.get_by_id(activity_id), + true <- Visibility.is_public?(activity), + %Object{} = object <- Object.normalize(activity.data["object"]), + %User{} = user <- User.get_or_fetch(activity.data["actor"]), + data <- prepare_activity(user, object) do + {:ok, data} + else + e -> + {:error, e} + end + end +end diff --git a/lib/pleroma/web/static_fe/static_fe_controller.ex b/lib/pleroma/web/static_fe/static_fe_controller.ex index 2ac857759..7d7cb6ddd 100644 --- a/lib/pleroma/web/static_fe/static_fe_controller.ex +++ b/lib/pleroma/web/static_fe/static_fe_controller.ex @@ -5,24 +5,17 @@ defmodule Pleroma.Web.StaticFE.StaticFEController do use Pleroma.Web, :controller - alias Pleroma.Repo - alias Pleroma.Activity - alias Pleroma.Object - alias Pleroma.User - alias Pleroma.Web.ActivityPub.Visibility + alias Pleroma.Web.StaticFE.ActivityRepresenter require Logger def show_notice(conn, %{"notice_id" => notice_id}) do - with %Activity{} = activity <- Repo.get(Activity, notice_id), - true <- Visibility.is_public?(activity), - %User{} = user <- User.get_or_fetch(activity.data["actor"]), - %Object{} = object <- Object.normalize(activity.data["object"]) do + with {:ok, data} <- ActivityRepresenter.represent(notice_id) do conn |> put_layout(:static_fe) |> put_status(200) |> put_view(Pleroma.Web.StaticFE.StaticFEView) - |> render("notice.html", %{notice: activity, object: object, user: user}) + |> render("notice.html", data) else _ -> conn From 2b5bd5236d793edba54366ec9fed447207a09976 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Sat, 2 Mar 2019 21:12:38 +0000 Subject: [PATCH 038/107] static fe: add user profile rendering --- .../web/static_fe/activity_representer.ex | 3 ++ .../web/static_fe/static_fe_controller.ex | 24 ++++++++++++- lib/pleroma/web/static_fe/static_fe_view.ex | 2 ++ lib/pleroma/web/static_fe/user_representer.ex | 35 +++++++++++++++++++ .../web/templates/layout/static_fe.html.eex | 4 +++ .../static_fe/static_fe/notice.html.eex | 4 +-- .../static_fe/static_fe/profile.html.eex | 7 ++++ .../static_fe/static_fe/user_card.html.eex | 2 +- 8 files changed, 77 insertions(+), 4 deletions(-) create mode 100644 lib/pleroma/web/static_fe/user_representer.ex create mode 100644 lib/pleroma/web/templates/static_fe/static_fe/profile.html.eex diff --git a/lib/pleroma/web/static_fe/activity_representer.ex b/lib/pleroma/web/static_fe/activity_representer.ex index 93f8a8813..9c4315610 100644 --- a/lib/pleroma/web/static_fe/activity_representer.ex +++ b/lib/pleroma/web/static_fe/activity_representer.ex @@ -17,6 +17,9 @@ def prepare_activity(%User{} = user, %Object{} = object) do |> set_attachments(object) end + def prepare_activity(%User{} = user, %Activity{} = activity), do: + prepare_activity(user, Object.normalize(activity.data["object"])) + defp set_user(data, %User{} = user), do: Map.put(data, :user, user) defp set_object(data, %Object{} = object), do: Map.put(data, :object, object) diff --git a/lib/pleroma/web/static_fe/static_fe_controller.ex b/lib/pleroma/web/static_fe/static_fe_controller.ex index 7d7cb6ddd..78cd325c8 100644 --- a/lib/pleroma/web/static_fe/static_fe_controller.ex +++ b/lib/pleroma/web/static_fe/static_fe_controller.ex @@ -6,6 +6,7 @@ defmodule Pleroma.Web.StaticFE.StaticFEController do use Pleroma.Web, :controller alias Pleroma.Web.StaticFE.ActivityRepresenter + alias Pleroma.Web.StaticFE.UserRepresenter require Logger @@ -15,7 +16,22 @@ def show_notice(conn, %{"notice_id" => notice_id}) do |> put_layout(:static_fe) |> put_status(200) |> put_view(Pleroma.Web.StaticFE.StaticFEView) - |> render("notice.html", data) + |> render("notice.html", %{data: data}) + else + _ -> + conn + |> put_status(404) + |> text("Not found") + end + end + + def show_user(conn, %{"username_or_id" => username_or_id}) do + with {:ok, data} <- UserRepresenter.represent(username_or_id) do + conn + |> put_layout(:static_fe) + |> put_status(200) + |> put_view(Pleroma.Web.StaticFE.StaticFEView) + |> render("profile.html", %{data: data}) else _ -> conn @@ -27,6 +43,12 @@ def show_notice(conn, %{"notice_id" => notice_id}) do def show(%{path_info: ["notice", notice_id]} = conn, _params), do: show_notice(conn, %{"notice_id" => notice_id}) + def show(%{path_info: ["users", user_id]} = conn, _params), + do: show_user(conn, %{"username_or_id" => user_id}) + + def show(%{path_info: [user_id]} = conn, _params), + do: show_user(conn, %{"username_or_id" => user_id}) + # Fallback for unhandled types def show(conn, _params) do conn diff --git a/lib/pleroma/web/static_fe/static_fe_view.ex b/lib/pleroma/web/static_fe/static_fe_view.ex index 7f58e1b2d..71e47d2c1 100644 --- a/lib/pleroma/web/static_fe/static_fe_view.ex +++ b/lib/pleroma/web/static_fe/static_fe_view.ex @@ -9,6 +9,8 @@ defmodule Pleroma.Web.StaticFE.StaticFEView do alias Pleroma.Web.MediaProxy alias Pleroma.Formatter + import Phoenix.HTML + def emoji_for_user(%User{} = user) do (user.info.source_data["tag"] || []) |> Enum.filter(fn %{"type" => t} -> t == "Emoji" end) diff --git a/lib/pleroma/web/static_fe/user_representer.ex b/lib/pleroma/web/static_fe/user_representer.ex new file mode 100644 index 000000000..9d2f1eb85 --- /dev/null +++ b/lib/pleroma/web/static_fe/user_representer.ex @@ -0,0 +1,35 @@ +# Pleroma: A lightweight social networking server +# Copyright ยฉ 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.StaticFE.UserRepresenter do + alias Pleroma.User + alias Pleroma.Web.ActivityPub.ActivityPub + alias Pleroma.Web.StaticFE.ActivityRepresenter + + def prepare_user(%User{} = user) do + %{} + |> set_user(user) + |> set_timeline(user) + end + + defp set_user(data, %User{} = user), do: Map.put(data, :user, user) + + defp set_timeline(data, %User{} = user) do + activities = + ActivityPub.fetch_user_activities(user, nil, %{}) + |> Enum.map(fn activity -> ActivityRepresenter.prepare_activity(user, activity) end) + + Map.put(data, :timeline, activities) + end + + def represent(username_or_id) do + with %User{} = user <- User.get_cached_by_nickname_or_id(username_or_id), + data <- prepare_user(user) do + {:ok, data} + else + e -> + {:error, e} + end + end +end diff --git a/lib/pleroma/web/templates/layout/static_fe.html.eex b/lib/pleroma/web/templates/layout/static_fe.html.eex index c20958162..40a560460 100644 --- a/lib/pleroma/web/templates/layout/static_fe.html.eex +++ b/lib/pleroma/web/templates/layout/static_fe.html.eex @@ -22,6 +22,10 @@ border-radius: 4px; } + .activity { + margin-bottom: 1em; + } + .avatar { cursor: pointer; } diff --git a/lib/pleroma/web/templates/static_fe/static_fe/notice.html.eex b/lib/pleroma/web/templates/static_fe/static_fe/notice.html.eex index 9957697ad..e8d905d79 100644 --- a/lib/pleroma/web/templates/static_fe/static_fe/notice.html.eex +++ b/lib/pleroma/web/templates/static_fe/static_fe/notice.html.eex @@ -1,6 +1,6 @@
- <%= render("user_card.html", %{user: @user}) %> + <%= render("user_card.html", %{user: @data.user}) %>
-
<%= @object.data["content"] %>
+
<%= raw @data.content %>
diff --git a/lib/pleroma/web/templates/static_fe/static_fe/profile.html.eex b/lib/pleroma/web/templates/static_fe/static_fe/profile.html.eex new file mode 100644 index 000000000..9ae4139ed --- /dev/null +++ b/lib/pleroma/web/templates/static_fe/static_fe/profile.html.eex @@ -0,0 +1,7 @@ +

<%= raw (@data.user.name |> Formatter.emojify(emoji_for_user(@data.user))) %>

+

<%= raw @data.user.bio %>

+
+<%= for activity <- @data.timeline do %> + <%= render("notice.html", %{data: activity}) %> +<% end %> +
diff --git a/lib/pleroma/web/templates/static_fe/static_fe/user_card.html.eex b/lib/pleroma/web/templates/static_fe/static_fe/user_card.html.eex index 8b397c639..c7789f9ac 100644 --- a/lib/pleroma/web/templates/static_fe/static_fe/user_card.html.eex +++ b/lib/pleroma/web/templates/static_fe/static_fe/user_card.html.eex @@ -4,7 +4,7 @@ - <%= @user.name |> Formatter.emojify(emoji_for_user(@user)) %> + <%= raw (@user.name |> Formatter.emojify(emoji_for_user(@user))) %> <%= @user.nickname %> From e2904b5777ecf6c8ffe7fe46f09284bb38b03fc2 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Sat, 2 Mar 2019 21:40:02 +0000 Subject: [PATCH 039/107] static fe: reformat activity representer --- lib/pleroma/web/static_fe/activity_representer.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/web/static_fe/activity_representer.ex b/lib/pleroma/web/static_fe/activity_representer.ex index 9c4315610..446c6023e 100644 --- a/lib/pleroma/web/static_fe/activity_representer.ex +++ b/lib/pleroma/web/static_fe/activity_representer.ex @@ -17,8 +17,8 @@ def prepare_activity(%User{} = user, %Object{} = object) do |> set_attachments(object) end - def prepare_activity(%User{} = user, %Activity{} = activity), do: - prepare_activity(user, Object.normalize(activity.data["object"])) + def prepare_activity(%User{} = user, %Activity{} = activity), + do: prepare_activity(user, Object.normalize(activity.data["object"])) defp set_user(data, %User{} = user), do: Map.put(data, :user, user) From b33fbd58e3852fc9de58917fafbb2c575a21dde1 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Sat, 2 Mar 2019 22:35:54 +0000 Subject: [PATCH 040/107] static fe: add support for message subjects --- .../web/templates/static_fe/static_fe/notice.html.eex | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/web/templates/static_fe/static_fe/notice.html.eex b/lib/pleroma/web/templates/static_fe/static_fe/notice.html.eex index e8d905d79..791bd2562 100644 --- a/lib/pleroma/web/templates/static_fe/static_fe/notice.html.eex +++ b/lib/pleroma/web/templates/static_fe/static_fe/notice.html.eex @@ -1,6 +1,13 @@
<%= render("user_card.html", %{user: @data.user}) %>
-
<%= raw @data.content %>
+ <%= if @data.title do %> +
+ <%= raw @data.title %> + <% end %> +
<%= raw @data.content %>
+ <%= if @data.title do %> +
+ <% end %>
From ca5ef201ef8397981acf0647fe5cffea66299bfa Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Sun, 3 Mar 2019 13:36:59 +0000 Subject: [PATCH 041/107] static fe: add remote follow button --- lib/pleroma/web/static_fe/static_fe_view.ex | 1 + lib/pleroma/web/templates/layout/static_fe.html.eex | 9 +++++++++ .../web/templates/static_fe/static_fe/profile.html.eex | 9 ++++++++- 3 files changed, 18 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/web/static_fe/static_fe_view.ex b/lib/pleroma/web/static_fe/static_fe_view.ex index 71e47d2c1..c01e8d40b 100644 --- a/lib/pleroma/web/static_fe/static_fe_view.ex +++ b/lib/pleroma/web/static_fe/static_fe_view.ex @@ -8,6 +8,7 @@ defmodule Pleroma.Web.StaticFE.StaticFEView do alias Pleroma.User alias Pleroma.Web.MediaProxy alias Pleroma.Formatter + alias Pleroma.Web.Router.Helpers import Phoenix.HTML diff --git a/lib/pleroma/web/templates/layout/static_fe.html.eex b/lib/pleroma/web/templates/layout/static_fe.html.eex index 40a560460..7ce9ead90 100644 --- a/lib/pleroma/web/templates/layout/static_fe.html.eex +++ b/lib/pleroma/web/templates/layout/static_fe.html.eex @@ -70,6 +70,15 @@ text-decoration: none; } + .pull-right { + float: right; + } + + .collapse { + margin: 0; + width: auto; + } + h1 { margin: 0; } diff --git a/lib/pleroma/web/templates/static_fe/static_fe/profile.html.eex b/lib/pleroma/web/templates/static_fe/static_fe/profile.html.eex index 9ae4139ed..47b7d5286 100644 --- a/lib/pleroma/web/templates/static_fe/static_fe/profile.html.eex +++ b/lib/pleroma/web/templates/static_fe/static_fe/profile.html.eex @@ -1,4 +1,11 @@ -

<%= raw (@data.user.name |> Formatter.emojify(emoji_for_user(@data.user))) %>

+

+
+ + + +
+ <%= raw (@data.user.name |> Formatter.emojify(emoji_for_user(@data.user))) %> +

<%= raw @data.user.bio %>

<%= for activity <- @data.timeline do %> From d1320160f436c719ecca8b31463dd16a1ab2bdb9 Mon Sep 17 00:00:00 2001 From: Phil Hagelberg Date: Sun, 27 Oct 2019 15:20:43 -0700 Subject: [PATCH 042/107] Looks like source_data is on user directly now. --- lib/pleroma/web/static_fe/static_fe_view.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/static_fe/static_fe_view.ex b/lib/pleroma/web/static_fe/static_fe_view.ex index c01e8d40b..ba67de835 100644 --- a/lib/pleroma/web/static_fe/static_fe_view.ex +++ b/lib/pleroma/web/static_fe/static_fe_view.ex @@ -13,7 +13,7 @@ defmodule Pleroma.Web.StaticFE.StaticFEView do import Phoenix.HTML def emoji_for_user(%User{} = user) do - (user.info.source_data["tag"] || []) + (user.source_data["tag"] || []) |> Enum.filter(fn %{"type" => t} -> t == "Emoji" end) |> Enum.map(fn %{"icon" => %{"url" => url}, "name" => name} -> {String.trim(name, ":"), url} From c1fc1399860c57460cee173ce8ddb980aabf10de Mon Sep 17 00:00:00 2001 From: Phil Hagelberg Date: Sun, 27 Oct 2019 16:37:57 -0700 Subject: [PATCH 043/107] Add permalinks to the static-fe notice rendering. --- lib/pleroma/web/static_fe/activity_representer.ex | 15 ++++++++++++--- .../templates/static_fe/static_fe/notice.html.eex | 6 ++++-- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/lib/pleroma/web/static_fe/activity_representer.ex b/lib/pleroma/web/static_fe/activity_representer.ex index 446c6023e..e383b8415 100644 --- a/lib/pleroma/web/static_fe/activity_representer.ex +++ b/lib/pleroma/web/static_fe/activity_representer.ex @@ -7,18 +7,21 @@ defmodule Pleroma.Web.StaticFE.ActivityRepresenter do alias Pleroma.Object alias Pleroma.User alias Pleroma.Web.ActivityPub.Visibility + alias Pleroma.Web.Router.Helpers - def prepare_activity(%User{} = user, %Object{} = object) do + def prepare_activity(%User{} = user, %Object{} = object, activity_id) do %{} |> set_user(user) |> set_object(object) |> set_title(object) |> set_content(object) + |> set_link(activity_id) + |> set_published(object) |> set_attachments(object) end def prepare_activity(%User{} = user, %Activity{} = activity), - do: prepare_activity(user, Object.normalize(activity.data["object"])) + do: prepare_activity(user, Object.normalize(activity.data["object"]), activity.id) defp set_user(data, %User{} = user), do: Map.put(data, :user, user) @@ -37,6 +40,12 @@ defp set_content(data, %Object{data: %{"content" => content}}) when is_binary(co defp set_content(data, _), do: Map.put(data, :content, nil) + defp set_link(data, activity_id), + do: Map.put(data, :link, Helpers.o_status_url(Pleroma.Web.Endpoint, :notice, activity_id)) + + defp set_published(data, %Object{data: %{"published" => published}}), + do: Map.put(data, :published, published) + # TODO: attachments defp set_attachments(data, _), do: Map.put(data, :attachments, []) @@ -45,7 +54,7 @@ def represent(activity_id) do true <- Visibility.is_public?(activity), %Object{} = object <- Object.normalize(activity.data["object"]), %User{} = user <- User.get_or_fetch(activity.data["actor"]), - data <- prepare_activity(user, object) do + data <- prepare_activity(user, object, activity_id) do {:ok, data} else e -> diff --git a/lib/pleroma/web/templates/static_fe/static_fe/notice.html.eex b/lib/pleroma/web/templates/static_fe/static_fe/notice.html.eex index 791bd2562..d305d9057 100644 --- a/lib/pleroma/web/templates/static_fe/static_fe/notice.html.eex +++ b/lib/pleroma/web/templates/static_fe/static_fe/notice.html.eex @@ -1,13 +1,15 @@
<%= render("user_card.html", %{user: @data.user}) %>
- <%= if @data.title do %> + <%= if @data.title != "" do %>
<%= raw @data.title %> <% end %>
<%= raw @data.content %>
- <%= if @data.title do %> + <%= if @data.title != "" do %>
<% end %> +

+ <%= @data.published %>

From e79d8985ab90bcad33d9ff13c6a16f006c6abac9 Mon Sep 17 00:00:00 2001 From: Phil Hagelberg Date: Sun, 27 Oct 2019 17:53:20 -0700 Subject: [PATCH 044/107] Don't show 404 in static-fe controller unless it's actually not found. --- lib/pleroma/web/static_fe/static_fe_controller.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/web/static_fe/static_fe_controller.ex b/lib/pleroma/web/static_fe/static_fe_controller.ex index 78cd325c8..8bbd06aa9 100644 --- a/lib/pleroma/web/static_fe/static_fe_controller.ex +++ b/lib/pleroma/web/static_fe/static_fe_controller.ex @@ -18,7 +18,7 @@ def show_notice(conn, %{"notice_id" => notice_id}) do |> put_view(Pleroma.Web.StaticFE.StaticFEView) |> render("notice.html", %{data: data}) else - _ -> + {:error, nil} -> conn |> put_status(404) |> text("Not found") @@ -33,7 +33,7 @@ def show_user(conn, %{"username_or_id" => username_or_id}) do |> put_view(Pleroma.Web.StaticFE.StaticFEView) |> render("profile.html", %{data: data}) else - _ -> + {:error, nil} -> conn |> put_status(404) |> text("Not found") From 0cf04e10880fb0779f34102cacb40950a119d4f8 Mon Sep 17 00:00:00 2001 From: Phil Hagelberg Date: Sun, 27 Oct 2019 19:01:18 -0700 Subject: [PATCH 045/107] Fix OStatus controller to know about StaticFEController. But only when it's configured to be on. --- lib/pleroma/web/ostatus/ostatus_controller.ex | 62 ++++++++++--------- 1 file changed, 33 insertions(+), 29 deletions(-) diff --git a/lib/pleroma/web/ostatus/ostatus_controller.ex b/lib/pleroma/web/ostatus/ostatus_controller.ex index 6958519de..76a244d0f 100644 --- a/lib/pleroma/web/ostatus/ostatus_controller.ex +++ b/lib/pleroma/web/ostatus/ostatus_controller.ex @@ -76,37 +76,41 @@ def activity(%{assigns: %{format: format}} = conn, %{"uuid" => uuid}) do end def notice(%{assigns: %{format: format}} = conn, %{"id" => id}) do - with {_, %Activity{} = activity} <- {:activity, Activity.get_by_id_with_object(id)}, - {_, true} <- {:public?, Visibility.is_public?(activity)}, - %User{} = user <- User.get_cached_by_ap_id(activity.data["actor"]) do - cond do - format == "html" && activity.data["type"] == "Create" -> - %Object{} = object = Object.normalize(activity) - - RedirectController.redirector_with_meta( - conn, - %{ - activity_id: activity.id, - object: object, - url: Router.Helpers.o_status_url(Endpoint, :notice, activity.id), - user: user - } - ) - - format == "html" -> - RedirectController.redirector(conn, nil) - - true -> - represent_activity(conn, format, activity, user) - end + if Pleroma.Config.get([:instance, :static_fe], false) do + Pleroma.Web.StaticFE.StaticFEController.show(conn, %{"notice_id" => id}) else - reason when reason in [{:public?, false}, {:activity, nil}] -> - conn - |> put_status(404) - |> RedirectController.redirector(nil, 404) + with {_, %Activity{} = activity} <- {:activity, Activity.get_by_id_with_object(id)}, + {_, true} <- {:public?, Visibility.is_public?(activity)}, + %User{} = user <- User.get_cached_by_ap_id(activity.data["actor"]) do + cond do + format == "html" && activity.data["type"] == "Create" -> + %Object{} = object = Object.normalize(activity) - e -> - e + RedirectController.redirector_with_meta( + conn, + %{ + activity_id: activity.id, + object: object, + url: Router.Helpers.o_status_url(Endpoint, :notice, activity.id), + user: user + } + ) + + format == "html" -> + RedirectController.redirector(conn, nil) + + true -> + represent_activity(conn, format, activity, user) + end + else + reason when reason in [{:public?, false}, {:activity, nil}] -> + conn + |> put_status(404) + |> RedirectController.redirector(nil, 404) + + e -> + e + end end end From 1d8950798c8aef2cee4458c68d34a72da630ec41 Mon Sep 17 00:00:00 2001 From: Phil Hagelberg Date: Sun, 27 Oct 2019 19:02:19 -0700 Subject: [PATCH 046/107] Fix activity_representer to work with User.get_or_fetch returning tuple. --- lib/pleroma/web/ostatus/ostatus_controller.ex | 2 +- lib/pleroma/web/static_fe/activity_representer.ex | 15 ++++++--------- 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/lib/pleroma/web/ostatus/ostatus_controller.ex b/lib/pleroma/web/ostatus/ostatus_controller.ex index 76a244d0f..ab5fdbc78 100644 --- a/lib/pleroma/web/ostatus/ostatus_controller.ex +++ b/lib/pleroma/web/ostatus/ostatus_controller.ex @@ -81,7 +81,7 @@ def notice(%{assigns: %{format: format}} = conn, %{"id" => id}) do else with {_, %Activity{} = activity} <- {:activity, Activity.get_by_id_with_object(id)}, {_, true} <- {:public?, Visibility.is_public?(activity)}, - %User{} = user <- User.get_cached_by_ap_id(activity.data["actor"]) do + %User{} = user <- User.get_cached_by_ap_id(activity.data["actor"]) do cond do format == "html" && activity.data["type"] == "Create" -> %Object{} = object = Object.normalize(activity) diff --git a/lib/pleroma/web/static_fe/activity_representer.ex b/lib/pleroma/web/static_fe/activity_representer.ex index e383b8415..9bee732d5 100644 --- a/lib/pleroma/web/static_fe/activity_representer.ex +++ b/lib/pleroma/web/static_fe/activity_representer.ex @@ -9,20 +9,19 @@ defmodule Pleroma.Web.StaticFE.ActivityRepresenter do alias Pleroma.Web.ActivityPub.Visibility alias Pleroma.Web.Router.Helpers - def prepare_activity(%User{} = user, %Object{} = object, activity_id) do + def prepare_activity(%User{} = user, %Activity{} = activity) do + object = Object.normalize(activity.data["object"]) + %{} |> set_user(user) |> set_object(object) |> set_title(object) |> set_content(object) - |> set_link(activity_id) + |> set_link(activity.id) |> set_published(object) |> set_attachments(object) end - def prepare_activity(%User{} = user, %Activity{} = activity), - do: prepare_activity(user, Object.normalize(activity.data["object"]), activity.id) - defp set_user(data, %User{} = user), do: Map.put(data, :user, user) defp set_object(data, %Object{} = object), do: Map.put(data, :object, object) @@ -52,10 +51,8 @@ defp set_attachments(data, _), do: Map.put(data, :attachments, []) def represent(activity_id) do with %Activity{data: %{"type" => "Create"}} = activity <- Activity.get_by_id(activity_id), true <- Visibility.is_public?(activity), - %Object{} = object <- Object.normalize(activity.data["object"]), - %User{} = user <- User.get_or_fetch(activity.data["actor"]), - data <- prepare_activity(user, object, activity_id) do - {:ok, data} + {:ok, %User{} = user} <- User.get_or_fetch(activity.data["actor"]) do + {:ok, prepare_activity(user, activity)} else e -> {:error, e} From 748d800acb56cd1e66adf78e5938623b8da7cde1 Mon Sep 17 00:00:00 2001 From: Phil Hagelberg Date: Mon, 28 Oct 2019 19:05:45 -0700 Subject: [PATCH 047/107] Show images, video, and audio attachments to notices. --- .../web/static_fe/activity_representer.ex | 10 ++++++- lib/pleroma/web/static_fe/static_fe_view.ex | 7 +++++ .../web/templates/layout/static_fe.html.eex | 5 ++++ .../static_fe/static_fe/_attachment.html.eex | 8 ++++++ .../static_fe/static_fe/notice.html.eex | 28 ++++++++++++++----- 5 files changed, 50 insertions(+), 8 deletions(-) create mode 100644 lib/pleroma/web/templates/static_fe/static_fe/_attachment.html.eex diff --git a/lib/pleroma/web/static_fe/activity_representer.ex b/lib/pleroma/web/static_fe/activity_representer.ex index 9bee732d5..7b7e1730c 100644 --- a/lib/pleroma/web/static_fe/activity_representer.ex +++ b/lib/pleroma/web/static_fe/activity_representer.ex @@ -19,6 +19,8 @@ def prepare_activity(%User{} = user, %Activity{} = activity) do |> set_content(object) |> set_link(activity.id) |> set_published(object) + |> set_sensitive(object) + |> set_attachment(object.data["attachment"]) |> set_attachments(object) end @@ -39,17 +41,23 @@ defp set_content(data, %Object{data: %{"content" => content}}) when is_binary(co defp set_content(data, _), do: Map.put(data, :content, nil) + defp set_attachment(data, attachment), do: Map.put(data, :attachment, attachment) + defp set_link(data, activity_id), do: Map.put(data, :link, Helpers.o_status_url(Pleroma.Web.Endpoint, :notice, activity_id)) defp set_published(data, %Object{data: %{"published" => published}}), do: Map.put(data, :published, published) + defp set_sensitive(data, %Object{data: %{"sensitive" => sensitive}}), + do: Map.put(data, :sensitive, sensitive) + # TODO: attachments defp set_attachments(data, _), do: Map.put(data, :attachments, []) def represent(activity_id) do - with %Activity{data: %{"type" => "Create"}} = activity <- Activity.get_by_id(activity_id), + with %Activity{data: %{"type" => "Create"}} = activity <- + Activity.get_by_id_with_object(activity_id), true <- Visibility.is_public?(activity), {:ok, %User{} = user} <- User.get_or_fetch(activity.data["actor"]) do {:ok, prepare_activity(user, activity)} diff --git a/lib/pleroma/web/static_fe/static_fe_view.ex b/lib/pleroma/web/static_fe/static_fe_view.ex index ba67de835..2b3b968d3 100644 --- a/lib/pleroma/web/static_fe/static_fe_view.ex +++ b/lib/pleroma/web/static_fe/static_fe_view.ex @@ -8,10 +8,13 @@ defmodule Pleroma.Web.StaticFE.StaticFEView do alias Pleroma.User alias Pleroma.Web.MediaProxy alias Pleroma.Formatter + alias Pleroma.Web.Metadata.Utils alias Pleroma.Web.Router.Helpers import Phoenix.HTML + @media_types ["image", "audio", "video"] + def emoji_for_user(%User{} = user) do (user.source_data["tag"] || []) |> Enum.filter(fn %{"type" => t} -> t == "Emoji" end) @@ -19,4 +22,8 @@ def emoji_for_user(%User{} = user) do {String.trim(name, ":"), url} end) end + + def fetch_media_type(url) do + Utils.fetch_media_type(@media_types, url["mediaType"]) + end end diff --git a/lib/pleroma/web/templates/layout/static_fe.html.eex b/lib/pleroma/web/templates/layout/static_fe.html.eex index 7ce9ead90..c1fbd89cd 100644 --- a/lib/pleroma/web/templates/layout/static_fe.html.eex +++ b/lib/pleroma/web/templates/layout/static_fe.html.eex @@ -36,6 +36,11 @@ margin-right: 4px; } + .activity-content img, video { + max-width: 800px; + max-height: 800px; + } + a { color: white; } diff --git a/lib/pleroma/web/templates/static_fe/static_fe/_attachment.html.eex b/lib/pleroma/web/templates/static_fe/static_fe/_attachment.html.eex new file mode 100644 index 000000000..7e04e9550 --- /dev/null +++ b/lib/pleroma/web/templates/static_fe/static_fe/_attachment.html.eex @@ -0,0 +1,8 @@ +<%= case @mediaType do %> +<% "audio" -> %> + +<% "video" -> %> + +<% _ -> %> +<%= @name %> +<% end %> diff --git a/lib/pleroma/web/templates/static_fe/static_fe/notice.html.eex b/lib/pleroma/web/templates/static_fe/static_fe/notice.html.eex index d305d9057..9a7824a32 100644 --- a/lib/pleroma/web/templates/static_fe/static_fe/notice.html.eex +++ b/lib/pleroma/web/templates/static_fe/static_fe/notice.html.eex @@ -1,15 +1,29 @@
<%= render("user_card.html", %{user: @data.user}) %> +

+ <%= @data.published %>

<%= if @data.title != "" do %> -
- <%= raw @data.title %> - <% end %> +
+ <%= raw @data.title %> +
<%= raw @data.content %>
+
+ <% else %>
<%= raw @data.content %>
- <%= if @data.title != "" do %> -
<% end %> -

- <%= @data.published %>

+ <%= for %{"name" => name, "url" => [url | _]} <- @data.attachment do %> + <%= if @data.sensitive do %> +
+ sensitive media +
+ <%= render("_attachment.html", %{name: name, url: url["href"], + mediaType: fetch_media_type(url)}) %> +
+
+ <% else %> + <%= render("_attachment.html", %{name: name, url: url["href"], + mediaType: fetch_media_type(url)}) %> + <% end %> + <% end %>
From cc1b07132f1c532c623530ed2375ff7fbdc6d559 Mon Sep 17 00:00:00 2001 From: Phil Hagelberg Date: Mon, 28 Oct 2019 19:47:20 -0700 Subject: [PATCH 048/107] Notices should show entire thread from context. --- lib/pleroma/web/static_fe/static_fe_controller.ex | 12 +++++++++++- lib/pleroma/web/templates/layout/static_fe.html.eex | 5 +++++ .../static_fe/{notice.html.eex => _notice.html.eex} | 4 ++-- .../static_fe/static_fe/conversation.html.eex | 5 +++++ .../templates/static_fe/static_fe/profile.html.eex | 6 +++--- 5 files changed, 26 insertions(+), 6 deletions(-) rename lib/pleroma/web/templates/static_fe/static_fe/{notice.html.eex => _notice.html.eex} (93%) create mode 100644 lib/pleroma/web/templates/static_fe/static_fe/conversation.html.eex diff --git a/lib/pleroma/web/static_fe/static_fe_controller.ex b/lib/pleroma/web/static_fe/static_fe_controller.ex index 8bbd06aa9..d2b55767d 100644 --- a/lib/pleroma/web/static_fe/static_fe_controller.ex +++ b/lib/pleroma/web/static_fe/static_fe_controller.ex @@ -5,6 +5,7 @@ defmodule Pleroma.Web.StaticFE.StaticFEController do use Pleroma.Web, :controller + alias Pleroma.Web.ActivityPub.ActivityPub alias Pleroma.Web.StaticFE.ActivityRepresenter alias Pleroma.Web.StaticFE.UserRepresenter @@ -12,11 +13,20 @@ defmodule Pleroma.Web.StaticFE.StaticFEController do def show_notice(conn, %{"notice_id" => notice_id}) do with {:ok, data} <- ActivityRepresenter.represent(notice_id) do + context = data.object.data["context"] + activities = ActivityPub.fetch_activities_for_context(context, %{}) + + data = + for a <- Enum.reverse(activities) do + ActivityRepresenter.prepare_activity(data.user, a) + |> Map.put(:selected, a.object.id == data.object.id) + end + conn |> put_layout(:static_fe) |> put_status(200) |> put_view(Pleroma.Web.StaticFE.StaticFEView) - |> render("notice.html", %{data: data}) + |> render("conversation.html", %{data: data}) else {:error, nil} -> conn diff --git a/lib/pleroma/web/templates/layout/static_fe.html.eex b/lib/pleroma/web/templates/layout/static_fe.html.eex index c1fbd89cd..9d7ee366a 100644 --- a/lib/pleroma/web/templates/layout/static_fe.html.eex +++ b/lib/pleroma/web/templates/layout/static_fe.html.eex @@ -23,6 +23,7 @@ } .activity { + padding: 1em; margin-bottom: 1em; } @@ -41,6 +42,10 @@ max-height: 800px; } + #selected { + background-color: #1b2735; + } + a { color: white; } diff --git a/lib/pleroma/web/templates/static_fe/static_fe/notice.html.eex b/lib/pleroma/web/templates/static_fe/static_fe/_notice.html.eex similarity index 93% rename from lib/pleroma/web/templates/static_fe/static_fe/notice.html.eex rename to lib/pleroma/web/templates/static_fe/static_fe/_notice.html.eex index 9a7824a32..90b5ef67c 100644 --- a/lib/pleroma/web/templates/static_fe/static_fe/notice.html.eex +++ b/lib/pleroma/web/templates/static_fe/static_fe/_notice.html.eex @@ -1,7 +1,7 @@ -
- <%= render("user_card.html", %{user: @data.user}) %> +
id="selected" <% end %>>

<%= @data.published %>

+ <%= render("user_card.html", %{user: @data.user}) %>
<%= if @data.title != "" do %>
diff --git a/lib/pleroma/web/templates/static_fe/static_fe/conversation.html.eex b/lib/pleroma/web/templates/static_fe/static_fe/conversation.html.eex new file mode 100644 index 000000000..35c3c17cd --- /dev/null +++ b/lib/pleroma/web/templates/static_fe/static_fe/conversation.html.eex @@ -0,0 +1,5 @@ +
+ <%= for notice <- @data do %> + <%= render("_notice.html", %{data: notice}) %> + <% end %> +
diff --git a/lib/pleroma/web/templates/static_fe/static_fe/profile.html.eex b/lib/pleroma/web/templates/static_fe/static_fe/profile.html.eex index 47b7d5286..79bf5a729 100644 --- a/lib/pleroma/web/templates/static_fe/static_fe/profile.html.eex +++ b/lib/pleroma/web/templates/static_fe/static_fe/profile.html.eex @@ -8,7 +8,7 @@

<%= raw @data.user.bio %>

-<%= for activity <- @data.timeline do %> - <%= render("notice.html", %{data: activity}) %> -<% end %> + <%= for activity <- @data.timeline do %> + <%= render("_notice.html", %{data: activity}) %> + <% end %>
From 2d1897e8a739a54a07ab0eae5cf11c260428e532 Mon Sep 17 00:00:00 2001 From: Phil Hagelberg Date: Tue, 29 Oct 2019 17:45:26 -0700 Subject: [PATCH 049/107] Apply all suggested changes from reviewers. --- lib/pleroma/web/ostatus/ostatus_controller.ex | 2 +- .../web/static_fe/activity_representer.ex | 4 +- .../web/static_fe/static_fe_controller.ex | 47 +++++++------------ lib/pleroma/web/static_fe/static_fe_view.ex | 9 ++-- lib/pleroma/web/static_fe/user_representer.ex | 9 ++-- .../static_fe/static_fe/_notice.html.eex | 3 +- 6 files changed, 30 insertions(+), 44 deletions(-) diff --git a/lib/pleroma/web/ostatus/ostatus_controller.ex b/lib/pleroma/web/ostatus/ostatus_controller.ex index ab5fdbc78..be275977e 100644 --- a/lib/pleroma/web/ostatus/ostatus_controller.ex +++ b/lib/pleroma/web/ostatus/ostatus_controller.ex @@ -77,7 +77,7 @@ def activity(%{assigns: %{format: format}} = conn, %{"uuid" => uuid}) do def notice(%{assigns: %{format: format}} = conn, %{"id" => id}) do if Pleroma.Config.get([:instance, :static_fe], false) do - Pleroma.Web.StaticFE.StaticFEController.show(conn, %{"notice_id" => id}) + Pleroma.Web.StaticFE.StaticFEController.call(conn, :show_notice) else with {_, %Activity{} = activity} <- {:activity, Activity.get_by_id_with_object(id)}, {_, true} <- {:public?, Visibility.is_public?(activity)}, diff --git a/lib/pleroma/web/static_fe/activity_representer.ex b/lib/pleroma/web/static_fe/activity_representer.ex index 7b7e1730c..8a499195c 100644 --- a/lib/pleroma/web/static_fe/activity_representer.ex +++ b/lib/pleroma/web/static_fe/activity_representer.ex @@ -62,8 +62,8 @@ def represent(activity_id) do {:ok, %User{} = user} <- User.get_or_fetch(activity.data["actor"]) do {:ok, prepare_activity(user, activity)} else - e -> - {:error, e} + {:error, reason} -> {:error, reason} + _error -> {:error, "Not found"} end end end diff --git a/lib/pleroma/web/static_fe/static_fe_controller.ex b/lib/pleroma/web/static_fe/static_fe_controller.ex index d2b55767d..6e8d0d622 100644 --- a/lib/pleroma/web/static_fe/static_fe_controller.ex +++ b/lib/pleroma/web/static_fe/static_fe_controller.ex @@ -9,9 +9,12 @@ defmodule Pleroma.Web.StaticFE.StaticFEController do alias Pleroma.Web.StaticFE.ActivityRepresenter alias Pleroma.Web.StaticFE.UserRepresenter - require Logger + plug(:put_layout, :static_fe) + plug(:put_view, Pleroma.Web.StaticFE.StaticFEView) + plug(:assign_id) + action_fallback(:not_found) - def show_notice(conn, %{"notice_id" => notice_id}) do + def show_notice(%{assigns: %{notice_id: notice_id}} = conn, _params) do with {:ok, data} <- ActivityRepresenter.represent(notice_id) do context = data.object.data["context"] activities = ActivityPub.fetch_activities_for_context(context, %{}) @@ -22,45 +25,29 @@ def show_notice(conn, %{"notice_id" => notice_id}) do |> Map.put(:selected, a.object.id == data.object.id) end - conn - |> put_layout(:static_fe) - |> put_status(200) - |> put_view(Pleroma.Web.StaticFE.StaticFEView) - |> render("conversation.html", %{data: data}) - else - {:error, nil} -> - conn - |> put_status(404) - |> text("Not found") + render(conn, "conversation.html", data: data) end end - def show_user(conn, %{"username_or_id" => username_or_id}) do + def show_user(%{assigns: %{username_or_id: username_or_id}} = conn, _params) do with {:ok, data} <- UserRepresenter.represent(username_or_id) do - conn - |> put_layout(:static_fe) - |> put_status(200) - |> put_view(Pleroma.Web.StaticFE.StaticFEView) - |> render("profile.html", %{data: data}) - else - {:error, nil} -> - conn - |> put_status(404) - |> text("Not found") + render(conn, "profile.html", data: data) end end - def show(%{path_info: ["notice", notice_id]} = conn, _params), - do: show_notice(conn, %{"notice_id" => notice_id}) + def assign_id(%{path_info: ["notice", notice_id]} = conn, _opts), + do: assign(conn, :notice_id, notice_id) - def show(%{path_info: ["users", user_id]} = conn, _params), - do: show_user(conn, %{"username_or_id" => user_id}) + def assign_id(%{path_info: ["users", user_id]} = conn, _opts), + do: assign(conn, :username_or_id, user_id) - def show(%{path_info: [user_id]} = conn, _params), - do: show_user(conn, %{"username_or_id" => user_id}) + def assign_id(%{path_info: [user_id]} = conn, _opts), + do: assign(conn, :username_or_id, user_id) + + def assign_id(conn, _opts), do: conn # Fallback for unhandled types - def show(conn, _params) do + def not_found(conn, _opts) do conn |> put_status(404) |> text("Not found") diff --git a/lib/pleroma/web/static_fe/static_fe_view.ex b/lib/pleroma/web/static_fe/static_fe_view.ex index 2b3b968d3..d633751dd 100644 --- a/lib/pleroma/web/static_fe/static_fe_view.ex +++ b/lib/pleroma/web/static_fe/static_fe_view.ex @@ -11,19 +11,20 @@ defmodule Pleroma.Web.StaticFE.StaticFEView do alias Pleroma.Web.Metadata.Utils alias Pleroma.Web.Router.Helpers - import Phoenix.HTML + use Phoenix.HTML @media_types ["image", "audio", "video"] def emoji_for_user(%User{} = user) do - (user.source_data["tag"] || []) + user.source_data + |> Map.get("tag", []) |> Enum.filter(fn %{"type" => t} -> t == "Emoji" end) |> Enum.map(fn %{"icon" => %{"url" => url}, "name" => name} -> {String.trim(name, ":"), url} end) end - def fetch_media_type(url) do - Utils.fetch_media_type(@media_types, url["mediaType"]) + def fetch_media_type(%{"mediaType" => mediaType}) do + Utils.fetch_media_type(@media_types, mediaType) end end diff --git a/lib/pleroma/web/static_fe/user_representer.ex b/lib/pleroma/web/static_fe/user_representer.ex index 9d2f1eb85..26320ea69 100644 --- a/lib/pleroma/web/static_fe/user_representer.ex +++ b/lib/pleroma/web/static_fe/user_representer.ex @@ -24,12 +24,9 @@ defp set_timeline(data, %User{} = user) do end def represent(username_or_id) do - with %User{} = user <- User.get_cached_by_nickname_or_id(username_or_id), - data <- prepare_user(user) do - {:ok, data} - else - e -> - {:error, e} + case User.get_cached_by_nickname_or_id(username_or_id) do + %User{} = user -> {:ok, prepare_user(user)} + nil -> {:error, "User not found"} end end end diff --git a/lib/pleroma/web/templates/static_fe/static_fe/_notice.html.eex b/lib/pleroma/web/templates/static_fe/static_fe/_notice.html.eex index 90b5ef67c..ed43ae838 100644 --- a/lib/pleroma/web/templates/static_fe/static_fe/_notice.html.eex +++ b/lib/pleroma/web/templates/static_fe/static_fe/_notice.html.eex @@ -1,6 +1,7 @@
id="selected" <% end %>>

- <%= @data.published %>

+ <%= link @data.published, to: @data.link, class: "activity-link" %> +

<%= render("user_card.html", %{user: @data.user}) %>
<%= if @data.title != "" do %> From e944a2213dd5eaaf69b9e8d8f5e035dbba2fdab1 Mon Sep 17 00:00:00 2001 From: Phil Hagelberg Date: Tue, 29 Oct 2019 17:53:58 -0700 Subject: [PATCH 050/107] Use gettext for sensitive media warning. --- lib/pleroma/web/static_fe/static_fe_view.ex | 1 + lib/pleroma/web/templates/static_fe/static_fe/_notice.html.eex | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/web/static_fe/static_fe_view.ex b/lib/pleroma/web/static_fe/static_fe_view.ex index d633751dd..1194b7ecc 100644 --- a/lib/pleroma/web/static_fe/static_fe_view.ex +++ b/lib/pleroma/web/static_fe/static_fe_view.ex @@ -6,6 +6,7 @@ defmodule Pleroma.Web.StaticFE.StaticFEView do use Pleroma.Web, :view alias Pleroma.User + alias Pleroma.Web.Gettext alias Pleroma.Web.MediaProxy alias Pleroma.Formatter alias Pleroma.Web.Metadata.Utils diff --git a/lib/pleroma/web/templates/static_fe/static_fe/_notice.html.eex b/lib/pleroma/web/templates/static_fe/static_fe/_notice.html.eex index ed43ae838..c4cdb1029 100644 --- a/lib/pleroma/web/templates/static_fe/static_fe/_notice.html.eex +++ b/lib/pleroma/web/templates/static_fe/static_fe/_notice.html.eex @@ -15,7 +15,7 @@ <%= for %{"name" => name, "url" => [url | _]} <- @data.attachment do %> <%= if @data.sensitive do %>
- sensitive media + <%= Gettext.gettext("sensitive media") %>
<%= render("_attachment.html", %{name: name, url: url["href"], mediaType: fetch_media_type(url)}) %> From 41fde63defd332a84b6c4d4ca78848e623a9d122 Mon Sep 17 00:00:00 2001 From: Phil Hagelberg Date: Tue, 29 Oct 2019 19:27:42 -0700 Subject: [PATCH 051/107] Get rid of @data in views and use separate fields. --- .../web/static_fe/static_fe_controller.ex | 12 +++++------- .../static_fe/static_fe/_notice.html.eex | 18 +++++++++--------- .../static_fe/static_fe/conversation.html.eex | 4 ++-- .../static_fe/static_fe/profile.html.eex | 10 +++++----- 4 files changed, 21 insertions(+), 23 deletions(-) diff --git a/lib/pleroma/web/static_fe/static_fe_controller.ex b/lib/pleroma/web/static_fe/static_fe_controller.ex index 6e8d0d622..c77df8e7d 100644 --- a/lib/pleroma/web/static_fe/static_fe_controller.ex +++ b/lib/pleroma/web/static_fe/static_fe_controller.ex @@ -17,22 +17,20 @@ defmodule Pleroma.Web.StaticFE.StaticFEController do def show_notice(%{assigns: %{notice_id: notice_id}} = conn, _params) do with {:ok, data} <- ActivityRepresenter.represent(notice_id) do context = data.object.data["context"] - activities = ActivityPub.fetch_activities_for_context(context, %{}) - data = - for a <- Enum.reverse(activities) do + activities = + for a <- Enum.reverse(ActivityPub.fetch_activities_for_context(context, %{})) do ActivityRepresenter.prepare_activity(data.user, a) |> Map.put(:selected, a.object.id == data.object.id) end - render(conn, "conversation.html", data: data) + render(conn, "conversation.html", activities: activities) end end def show_user(%{assigns: %{username_or_id: username_or_id}} = conn, _params) do - with {:ok, data} <- UserRepresenter.represent(username_or_id) do - render(conn, "profile.html", data: data) - end + {:ok, data} = UserRepresenter.represent(username_or_id) + render(conn, "profile.html", %{user: data.user, timeline: data.timeline}) end def assign_id(%{path_info: ["notice", notice_id]} = conn, _opts), diff --git a/lib/pleroma/web/templates/static_fe/static_fe/_notice.html.eex b/lib/pleroma/web/templates/static_fe/static_fe/_notice.html.eex index c4cdb1029..b16d19a2c 100644 --- a/lib/pleroma/web/templates/static_fe/static_fe/_notice.html.eex +++ b/lib/pleroma/web/templates/static_fe/static_fe/_notice.html.eex @@ -1,19 +1,19 @@ -
id="selected" <% end %>> +
id="selected" <% end %>>

- <%= link @data.published, to: @data.link, class: "activity-link" %> + <%= link @published, to: @link, class: "activity-link" %>

- <%= render("user_card.html", %{user: @data.user}) %> + <%= render("user_card.html", %{user: @user}) %>
- <%= if @data.title != "" do %> + <%= if @title != "" do %>
- <%= raw @data.title %> -
<%= raw @data.content %>
+ <%= raw @title %> +
<%= raw @content %>
<% else %> -
<%= raw @data.content %>
+
<%= raw @content %>
<% end %> - <%= for %{"name" => name, "url" => [url | _]} <- @data.attachment do %> - <%= if @data.sensitive do %> + <%= for %{"name" => name, "url" => [url | _]} <- @attachment do %> + <%= if @sensitive do %>
<%= Gettext.gettext("sensitive media") %>
diff --git a/lib/pleroma/web/templates/static_fe/static_fe/conversation.html.eex b/lib/pleroma/web/templates/static_fe/static_fe/conversation.html.eex index 35c3c17cd..f0d3b5972 100644 --- a/lib/pleroma/web/templates/static_fe/static_fe/conversation.html.eex +++ b/lib/pleroma/web/templates/static_fe/static_fe/conversation.html.eex @@ -1,5 +1,5 @@
- <%= for notice <- @data do %> - <%= render("_notice.html", %{data: notice}) %> + <%= for activity <- @activities do %> + <%= render("_notice.html", activity) %> <% end %>
diff --git a/lib/pleroma/web/templates/static_fe/static_fe/profile.html.eex b/lib/pleroma/web/templates/static_fe/static_fe/profile.html.eex index 79bf5a729..da23be1e5 100644 --- a/lib/pleroma/web/templates/static_fe/static_fe/profile.html.eex +++ b/lib/pleroma/web/templates/static_fe/static_fe/profile.html.eex @@ -1,14 +1,14 @@

- +
- <%= raw (@data.user.name |> Formatter.emojify(emoji_for_user(@data.user))) %> + <%= raw (@user.name |> Formatter.emojify(emoji_for_user(@user))) %>

-

<%= raw @data.user.bio %>

+

<%= raw @user.bio %>

- <%= for activity <- @data.timeline do %> - <%= render("_notice.html", %{data: activity}) %> + <%= for activity <- @timeline do %> + <%= render("_notice.html", Map.put(activity, :selected, false)) %> <% end %>
From 33a26b61c30ad8084003f0f1c646bc997a8d88ac Mon Sep 17 00:00:00 2001 From: Phil Hagelberg Date: Tue, 29 Oct 2019 20:30:43 -0700 Subject: [PATCH 052/107] Remove activity/user representer; move logic to controller. --- .../web/static_fe/activity_representer.ex | 69 ------------------- .../web/static_fe/static_fe_controller.ex | 59 ++++++++++++---- lib/pleroma/web/static_fe/user_representer.ex | 32 --------- 3 files changed, 46 insertions(+), 114 deletions(-) delete mode 100644 lib/pleroma/web/static_fe/activity_representer.ex delete mode 100644 lib/pleroma/web/static_fe/user_representer.ex diff --git a/lib/pleroma/web/static_fe/activity_representer.ex b/lib/pleroma/web/static_fe/activity_representer.ex deleted file mode 100644 index 8a499195c..000000000 --- a/lib/pleroma/web/static_fe/activity_representer.ex +++ /dev/null @@ -1,69 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright ยฉ 2017-2019 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Web.StaticFE.ActivityRepresenter do - alias Pleroma.Activity - alias Pleroma.Object - alias Pleroma.User - alias Pleroma.Web.ActivityPub.Visibility - alias Pleroma.Web.Router.Helpers - - def prepare_activity(%User{} = user, %Activity{} = activity) do - object = Object.normalize(activity.data["object"]) - - %{} - |> set_user(user) - |> set_object(object) - |> set_title(object) - |> set_content(object) - |> set_link(activity.id) - |> set_published(object) - |> set_sensitive(object) - |> set_attachment(object.data["attachment"]) - |> set_attachments(object) - end - - defp set_user(data, %User{} = user), do: Map.put(data, :user, user) - - defp set_object(data, %Object{} = object), do: Map.put(data, :object, object) - - defp set_title(data, %Object{data: %{"name" => name}}) when is_binary(name), - do: Map.put(data, :title, name) - - defp set_title(data, %Object{data: %{"summary" => summary}}) when is_binary(summary), - do: Map.put(data, :title, summary) - - defp set_title(data, _), do: Map.put(data, :title, nil) - - defp set_content(data, %Object{data: %{"content" => content}}) when is_binary(content), - do: Map.put(data, :content, content) - - defp set_content(data, _), do: Map.put(data, :content, nil) - - defp set_attachment(data, attachment), do: Map.put(data, :attachment, attachment) - - defp set_link(data, activity_id), - do: Map.put(data, :link, Helpers.o_status_url(Pleroma.Web.Endpoint, :notice, activity_id)) - - defp set_published(data, %Object{data: %{"published" => published}}), - do: Map.put(data, :published, published) - - defp set_sensitive(data, %Object{data: %{"sensitive" => sensitive}}), - do: Map.put(data, :sensitive, sensitive) - - # TODO: attachments - defp set_attachments(data, _), do: Map.put(data, :attachments, []) - - def represent(activity_id) do - with %Activity{data: %{"type" => "Create"}} = activity <- - Activity.get_by_id_with_object(activity_id), - true <- Visibility.is_public?(activity), - {:ok, %User{} = user} <- User.get_or_fetch(activity.data["actor"]) do - {:ok, prepare_activity(user, activity)} - else - {:error, reason} -> {:error, reason} - _error -> {:error, "Not found"} - end - end -end diff --git a/lib/pleroma/web/static_fe/static_fe_controller.ex b/lib/pleroma/web/static_fe/static_fe_controller.ex index c77df8e7d..a5cb76167 100644 --- a/lib/pleroma/web/static_fe/static_fe_controller.ex +++ b/lib/pleroma/web/static_fe/static_fe_controller.ex @@ -5,32 +5,65 @@ defmodule Pleroma.Web.StaticFE.StaticFEController do use Pleroma.Web, :controller + alias Pleroma.Activity + alias Pleroma.Object + alias Pleroma.User alias Pleroma.Web.ActivityPub.ActivityPub - alias Pleroma.Web.StaticFE.ActivityRepresenter - alias Pleroma.Web.StaticFE.UserRepresenter + alias Pleroma.Web.Router.Helpers plug(:put_layout, :static_fe) plug(:put_view, Pleroma.Web.StaticFE.StaticFEView) plug(:assign_id) action_fallback(:not_found) + defp get_title(%Object{data: %{"name" => name}}) when is_binary(name), + do: name + + defp get_title(%Object{data: %{"summary" => summary}}) when is_binary(summary), + do: summary + + defp get_title(_), do: nil + + def represent(%Activity{} = activity, %User{} = user, selected) do + %{ + user: user, + title: get_title(activity.object), + content: activity.object.data["content"] || nil, + attachment: activity.object.data["attachment"], + link: Helpers.o_status_url(Pleroma.Web.Endpoint, :notice, activity.id), + published: activity.object.data["published"], + sensitive: activity.object.data["sensitive"], + selected: selected + } + end + + def represent(%Activity{} = activity, selected) do + {:ok, user} = User.get_or_fetch(activity.data["actor"]) + represent(activity, user, selected) + end + def show_notice(%{assigns: %{notice_id: notice_id}} = conn, _params) do - with {:ok, data} <- ActivityRepresenter.represent(notice_id) do - context = data.object.data["context"] + activity = Activity.get_by_id_with_object(notice_id) + context = activity.object.data["context"] + activities = ActivityPub.fetch_activities_for_context(context, %{}) - activities = - for a <- Enum.reverse(ActivityPub.fetch_activities_for_context(context, %{})) do - ActivityRepresenter.prepare_activity(data.user, a) - |> Map.put(:selected, a.object.id == data.object.id) - end + represented = + for a <- Enum.reverse(activities) do + represent(activity, a.object.id == activity.object.id) + end - render(conn, "conversation.html", activities: activities) - end + render(conn, "conversation.html", activities: represented) end def show_user(%{assigns: %{username_or_id: username_or_id}} = conn, _params) do - {:ok, data} = UserRepresenter.represent(username_or_id) - render(conn, "profile.html", %{user: data.user, timeline: data.timeline}) + %User{} = user = User.get_cached_by_nickname_or_id(username_or_id) + + timeline = + for activity <- ActivityPub.fetch_user_activities(user, nil, %{}) do + represent(activity, user, false) + end + + render(conn, "profile.html", %{user: user, timeline: timeline}) end def assign_id(%{path_info: ["notice", notice_id]} = conn, _opts), diff --git a/lib/pleroma/web/static_fe/user_representer.ex b/lib/pleroma/web/static_fe/user_representer.ex deleted file mode 100644 index 26320ea69..000000000 --- a/lib/pleroma/web/static_fe/user_representer.ex +++ /dev/null @@ -1,32 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright ยฉ 2017-2019 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Web.StaticFE.UserRepresenter do - alias Pleroma.User - alias Pleroma.Web.ActivityPub.ActivityPub - alias Pleroma.Web.StaticFE.ActivityRepresenter - - def prepare_user(%User{} = user) do - %{} - |> set_user(user) - |> set_timeline(user) - end - - defp set_user(data, %User{} = user), do: Map.put(data, :user, user) - - defp set_timeline(data, %User{} = user) do - activities = - ActivityPub.fetch_user_activities(user, nil, %{}) - |> Enum.map(fn activity -> ActivityRepresenter.prepare_activity(user, activity) end) - - Map.put(data, :timeline, activities) - end - - def represent(username_or_id) do - case User.get_cached_by_nickname_or_id(username_or_id) do - %User{} = user -> {:ok, prepare_user(user)} - nil -> {:error, "User not found"} - end - end -end From 918e1353f6bc7f6dfe317a87d942dfa2e53064af Mon Sep 17 00:00:00 2001 From: Phil Hagelberg Date: Tue, 29 Oct 2019 20:55:43 -0700 Subject: [PATCH 053/107] Add header to profile/notice pages linking to pleroma-fe. --- lib/pleroma/web/static_fe/static_fe_controller.ex | 6 ++++-- .../web/templates/static_fe/static_fe/_notice.html.eex | 2 +- .../static_fe/{user_card.html.eex => _user_card.html.eex} | 0 .../web/templates/static_fe/static_fe/conversation.html.eex | 2 ++ .../web/templates/static_fe/static_fe/profile.html.eex | 6 ++++-- 5 files changed, 11 insertions(+), 5 deletions(-) rename lib/pleroma/web/templates/static_fe/static_fe/{user_card.html.eex => _user_card.html.eex} (100%) diff --git a/lib/pleroma/web/static_fe/static_fe_controller.ex b/lib/pleroma/web/static_fe/static_fe_controller.ex index a5cb76167..fe2fb09c4 100644 --- a/lib/pleroma/web/static_fe/static_fe_controller.ex +++ b/lib/pleroma/web/static_fe/static_fe_controller.ex @@ -43,6 +43,7 @@ def represent(%Activity{} = activity, selected) do end def show_notice(%{assigns: %{notice_id: notice_id}} = conn, _params) do + instance_name = Pleroma.Config.get([:instance, :name], "Pleroma") activity = Activity.get_by_id_with_object(notice_id) context = activity.object.data["context"] activities = ActivityPub.fetch_activities_for_context(context, %{}) @@ -52,10 +53,11 @@ def show_notice(%{assigns: %{notice_id: notice_id}} = conn, _params) do represent(activity, a.object.id == activity.object.id) end - render(conn, "conversation.html", activities: represented) + render(conn, "conversation.html", %{activities: represented, instance_name: instance_name}) end def show_user(%{assigns: %{username_or_id: username_or_id}} = conn, _params) do + instance_name = Pleroma.Config.get([:instance, :name], "Pleroma") %User{} = user = User.get_cached_by_nickname_or_id(username_or_id) timeline = @@ -63,7 +65,7 @@ def show_user(%{assigns: %{username_or_id: username_or_id}} = conn, _params) do represent(activity, user, false) end - render(conn, "profile.html", %{user: user, timeline: timeline}) + render(conn, "profile.html", %{user: user, timeline: timeline, instance_name: instance_name}) end def assign_id(%{path_info: ["notice", notice_id]} = conn, _opts), diff --git a/lib/pleroma/web/templates/static_fe/static_fe/_notice.html.eex b/lib/pleroma/web/templates/static_fe/static_fe/_notice.html.eex index b16d19a2c..d1daa281c 100644 --- a/lib/pleroma/web/templates/static_fe/static_fe/_notice.html.eex +++ b/lib/pleroma/web/templates/static_fe/static_fe/_notice.html.eex @@ -2,7 +2,7 @@

<%= link @published, to: @link, class: "activity-link" %>

- <%= render("user_card.html", %{user: @user}) %> + <%= render("_user_card.html", %{user: @user}) %>
<%= if @title != "" do %>
diff --git a/lib/pleroma/web/templates/static_fe/static_fe/user_card.html.eex b/lib/pleroma/web/templates/static_fe/static_fe/_user_card.html.eex similarity index 100% rename from lib/pleroma/web/templates/static_fe/static_fe/user_card.html.eex rename to lib/pleroma/web/templates/static_fe/static_fe/_user_card.html.eex diff --git a/lib/pleroma/web/templates/static_fe/static_fe/conversation.html.eex b/lib/pleroma/web/templates/static_fe/static_fe/conversation.html.eex index f0d3b5972..3a1249df2 100644 --- a/lib/pleroma/web/templates/static_fe/static_fe/conversation.html.eex +++ b/lib/pleroma/web/templates/static_fe/static_fe/conversation.html.eex @@ -1,3 +1,5 @@ +

<%= link @instance_name, to: "/" %>

+
<%= for activity <- @activities do %> <%= render("_notice.html", activity) %> diff --git a/lib/pleroma/web/templates/static_fe/static_fe/profile.html.eex b/lib/pleroma/web/templates/static_fe/static_fe/profile.html.eex index da23be1e5..8f2c74627 100644 --- a/lib/pleroma/web/templates/static_fe/static_fe/profile.html.eex +++ b/lib/pleroma/web/templates/static_fe/static_fe/profile.html.eex @@ -1,11 +1,13 @@ -

+

<%= link @instance_name, to: "/" %>

+ +

<%= raw (@user.name |> Formatter.emojify(emoji_for_user(@user))) %> -

+

<%= raw @user.bio %>

<%= for activity <- @timeline do %> From 93e9c0cedf0e2b4ab5966832cc912369c7aaf3ad Mon Sep 17 00:00:00 2001 From: Phil Hagelberg Date: Tue, 29 Oct 2019 21:09:05 -0700 Subject: [PATCH 054/107] Format dates using CommonAPI utils. --- lib/pleroma/web/static_fe/static_fe_view.ex | 5 +++++ .../web/templates/static_fe/static_fe/_notice.html.eex | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/web/static_fe/static_fe_view.ex b/lib/pleroma/web/static_fe/static_fe_view.ex index 1194b7ecc..c19aa07e1 100644 --- a/lib/pleroma/web/static_fe/static_fe_view.ex +++ b/lib/pleroma/web/static_fe/static_fe_view.ex @@ -28,4 +28,9 @@ def emoji_for_user(%User{} = user) do def fetch_media_type(%{"mediaType" => mediaType}) do Utils.fetch_media_type(@media_types, mediaType) end + + def format_date(date) do + {:ok, date, _} = DateTime.from_iso8601(date) + Pleroma.Web.CommonAPI.Utils.format_asctime(date) + end end diff --git a/lib/pleroma/web/templates/static_fe/static_fe/_notice.html.eex b/lib/pleroma/web/templates/static_fe/static_fe/_notice.html.eex index d1daa281c..9841fcf84 100644 --- a/lib/pleroma/web/templates/static_fe/static_fe/_notice.html.eex +++ b/lib/pleroma/web/templates/static_fe/static_fe/_notice.html.eex @@ -1,6 +1,6 @@
id="selected" <% end %>>

- <%= link @published, to: @link, class: "activity-link" %> + <%= link format_date(@published), to: @link, class: "activity-link" %>

<%= render("_user_card.html", %{user: @user}) %>
From e4b9784c3938772edf45340107a34a58aeeea690 Mon Sep 17 00:00:00 2001 From: Phil Hagelberg Date: Tue, 29 Oct 2019 22:05:18 -0700 Subject: [PATCH 055/107] Show counts for replies, likes, and announces for selected notice. Using text instead of an icon, for now. --- lib/pleroma/web/static_fe/static_fe_controller.ex | 15 +++++++++++++-- .../web/templates/layout/static_fe.html.eex | 6 ++++++ .../static_fe/static_fe/_notice.html.eex | 7 +++++++ 3 files changed, 26 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/web/static_fe/static_fe_controller.ex b/lib/pleroma/web/static_fe/static_fe_controller.ex index fe2fb09c4..d2e72b476 100644 --- a/lib/pleroma/web/static_fe/static_fe_controller.ex +++ b/lib/pleroma/web/static_fe/static_fe_controller.ex @@ -24,6 +24,16 @@ defp get_title(%Object{data: %{"summary" => summary}}) when is_binary(summary), defp get_title(_), do: nil + def get_counts(%Activity{} = activity) do + %Object{data: data} = Object.normalize(activity) + + %{ + likes: data["like_count"] || 0, + replies: data["repliesCount"] || 0, + announces: data["announcement_count"] || 0 + } + end + def represent(%Activity{} = activity, %User{} = user, selected) do %{ user: user, @@ -33,7 +43,8 @@ def represent(%Activity{} = activity, %User{} = user, selected) do link: Helpers.o_status_url(Pleroma.Web.Endpoint, :notice, activity.id), published: activity.object.data["published"], sensitive: activity.object.data["sensitive"], - selected: selected + selected: selected, + counts: get_counts(activity) } end @@ -50,7 +61,7 @@ def show_notice(%{assigns: %{notice_id: notice_id}} = conn, _params) do represented = for a <- Enum.reverse(activities) do - represent(activity, a.object.id == activity.object.id) + represent(a, a.object.id == activity.object.id) end render(conn, "conversation.html", %{activities: represented, instance_name: instance_name}) diff --git a/lib/pleroma/web/templates/layout/static_fe.html.eex b/lib/pleroma/web/templates/layout/static_fe.html.eex index 9d7ee366a..e42047de9 100644 --- a/lib/pleroma/web/templates/layout/static_fe.html.eex +++ b/lib/pleroma/web/templates/layout/static_fe.html.eex @@ -24,6 +24,7 @@ .activity { padding: 1em; + padding-bottom: 2em; margin-bottom: 1em; } @@ -46,6 +47,11 @@ background-color: #1b2735; } + .counts dt, .counts dd { + float: left; + margin-left: 1em; + } + a { color: white; } diff --git a/lib/pleroma/web/templates/static_fe/static_fe/_notice.html.eex b/lib/pleroma/web/templates/static_fe/static_fe/_notice.html.eex index 9841fcf84..2a46dadb4 100644 --- a/lib/pleroma/web/templates/static_fe/static_fe/_notice.html.eex +++ b/lib/pleroma/web/templates/static_fe/static_fe/_notice.html.eex @@ -27,4 +27,11 @@ <% end %> <% end %>
+ <%= if @selected do %> +
+
<%= Gettext.gettext("replies") %>
<%= @counts.replies %>
+
<%= Gettext.gettext("announces") %>
<%= @counts.announces %>
+
<%= Gettext.gettext("likes") %>
<%= @counts.likes %>
+
+ <% end %>
From 1dc785b74be6dc790d2b24e833642060303ecee2 Mon Sep 17 00:00:00 2001 From: Phil Hagelberg Date: Wed, 30 Oct 2019 20:20:26 -0700 Subject: [PATCH 056/107] Move static-fe CSS to a separate file. --- .../web/templates/layout/static_fe.html.eex | 165 +---------------- priv/static/static/static-fe.css | 172 ++++++++++++++++++ 2 files changed, 173 insertions(+), 164 deletions(-) create mode 100644 priv/static/static/static-fe.css diff --git a/lib/pleroma/web/templates/layout/static_fe.html.eex b/lib/pleroma/web/templates/layout/static_fe.html.eex index e42047de9..62b59f17c 100644 --- a/lib/pleroma/web/templates/layout/static_fe.html.eex +++ b/lib/pleroma/web/templates/layout/static_fe.html.eex @@ -6,170 +6,7 @@ <%= Application.get_env(:pleroma, :instance)[:name] %> - +
diff --git a/priv/static/static/static-fe.css b/priv/static/static/static-fe.css new file mode 100644 index 000000000..d6bb58c33 --- /dev/null +++ b/priv/static/static/static-fe.css @@ -0,0 +1,172 @@ +body { + background-color: #282c37; + font-family: sans-serif; + color: white; +} + +.container { + margin: 50px auto; + max-width: 960px; + padding: 40px; + background-color: #313543; + border-radius: 4px; +} + +header { + border-bottom: 2em solid #282c37; +} + +.activity { + border-radius: 4px; + padding: 1em; + padding-bottom: 2em; + margin-bottom: 1em; +} + +.avatar { + cursor: pointer; +} + +.avatar img { + float: left; + border-radius: 4px; + margin-right: 4px; +} + +.activity-content img, video, audio { + padding: 1em; + max-width: 800px; + max-height: 800px; +} + +#selected { + background-color: #1b2735; +} + +.counts dt, .counts dd { + float: left; + margin-left: 1em; +} + +a { + color: white; +} + +.h-card { + min-height: 48px; + margin-bottom: 8px; +} + +.h-card a { + text-decoration: none; +} + +.h-card a:hover { + text-decoration: underline; +} + +.display-name { + padding-top: 4px; + display: block; + text-overflow: ellipsis; + overflow: hidden; + color: white; +} + +/* keep emoji from being hilariously huge */ +.display-name img { + max-height: 1em; +} + +.display-name .nickname { + padding-top: 4px; + display: block; +} + +.nickname:hover { + text-decoration: none; +} + +.pull-right { + float: right; +} + +.collapse { + margin: 0; + width: auto; +} + +h1 { + margin: 0; +} + +h2 { + color: #9baec8; + font-weight: normal; + font-size: 20px; + margin-bottom: 40px; +} + +form { + width: 100%; +} + +input { + box-sizing: border-box; + width: 100%; + padding: 10px; + margin-top: 20px; + background-color: rgba(0,0,0,.1); + color: white; + border: 0; + border-bottom: 2px solid #9baec8; + font-size: 14px; +} + +input:focus { + border-bottom: 2px solid #4b8ed8; +} + +input[type="checkbox"] { + width: auto; +} + +button { + box-sizing: border-box; + width: 100%; + color: white; + background-color: #419bdd; + border-radius: 4px; + border: none; + padding: 10px; + margin-top: 30px; + text-transform: uppercase; + font-weight: 500; + font-size: 16px; +} + +.alert-danger { + box-sizing: border-box; + width: 100%; + color: #D8000C; + background-color: #FFD2D2; + border-radius: 4px; + border: none; + padding: 10px; + margin-top: 20px; + font-weight: 500; + font-size: 16px; +} + +.alert-info { + box-sizing: border-box; + width: 100%; + color: #00529B; + background-color: #BDE5F8; + border-radius: 4px; + border: none; + padding: 10px; + margin-top: 20px; + font-weight: 500; + font-size: 16px; +} From 5d7c44266ba0355557e5a62ecca69428c5784d88 Mon Sep 17 00:00:00 2001 From: Phil Hagelberg Date: Wed, 30 Oct 2019 20:20:54 -0700 Subject: [PATCH 057/107] Change date formatting. --- lib/pleroma/web/static_fe/static_fe_view.ex | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/web/static_fe/static_fe_view.ex b/lib/pleroma/web/static_fe/static_fe_view.ex index c19aa07e1..6128b2497 100644 --- a/lib/pleroma/web/static_fe/static_fe_view.ex +++ b/lib/pleroma/web/static_fe/static_fe_view.ex @@ -5,6 +5,8 @@ defmodule Pleroma.Web.StaticFE.StaticFEView do use Pleroma.Web, :view + alias Calendar.Strftime + alias Pleroma.Emoji.Formatter alias Pleroma.User alias Pleroma.Web.Gettext alias Pleroma.Web.MediaProxy @@ -31,6 +33,6 @@ def fetch_media_type(%{"mediaType" => mediaType}) do def format_date(date) do {:ok, date, _} = DateTime.from_iso8601(date) - Pleroma.Web.CommonAPI.Utils.format_asctime(date) + Strftime.strftime!(date, "%Y/%m/%d %l:%M:%S %p UTC") end end From 2ac1ece652621df9adf591255f4506564a8ace68 Mon Sep 17 00:00:00 2001 From: Phil Hagelberg Date: Wed, 30 Oct 2019 20:21:10 -0700 Subject: [PATCH 058/107] Fix a bug where reblogs were displayed under the wrong user. --- lib/pleroma/web/static_fe/static_fe_controller.ex | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/lib/pleroma/web/static_fe/static_fe_controller.ex b/lib/pleroma/web/static_fe/static_fe_controller.ex index d2e72b476..9b565d07d 100644 --- a/lib/pleroma/web/static_fe/static_fe_controller.ex +++ b/lib/pleroma/web/static_fe/static_fe_controller.ex @@ -34,7 +34,9 @@ def get_counts(%Activity{} = activity) do } end - def represent(%Activity{} = activity, %User{} = user, selected) do + def represent(%Activity{} = activity, selected) do + {:ok, user} = User.get_or_fetch(activity.object.data["actor"]) + %{ user: user, title: get_title(activity.object), @@ -48,11 +50,6 @@ def represent(%Activity{} = activity, %User{} = user, selected) do } end - def represent(%Activity{} = activity, selected) do - {:ok, user} = User.get_or_fetch(activity.data["actor"]) - represent(activity, user, selected) - end - def show_notice(%{assigns: %{notice_id: notice_id}} = conn, _params) do instance_name = Pleroma.Config.get([:instance, :name], "Pleroma") activity = Activity.get_by_id_with_object(notice_id) @@ -73,7 +70,7 @@ def show_user(%{assigns: %{username_or_id: username_or_id}} = conn, _params) do timeline = for activity <- ActivityPub.fetch_user_activities(user, nil, %{}) do - represent(activity, user, false) + represent(activity, false) end render(conn, "profile.html", %{user: user, timeline: timeline, instance_name: instance_name}) From 274cc18e8a585bd72353f9135c18aec0cb8e7ce3 Mon Sep 17 00:00:00 2001 From: Phil Hagelberg Date: Thu, 31 Oct 2019 17:44:43 -0700 Subject: [PATCH 059/107] Visually separate header. --- .../web/static_fe/static_fe_controller.ex | 10 +++--- lib/pleroma/web/static_fe/static_fe_view.ex | 1 + .../static_fe/static_fe/conversation.html.eex | 16 +++++---- .../static_fe/static_fe/profile.html.eex | 36 +++++++++++-------- priv/static/static/static-fe.css | 12 ++++--- 5 files changed, 45 insertions(+), 30 deletions(-) diff --git a/lib/pleroma/web/static_fe/static_fe_controller.ex b/lib/pleroma/web/static_fe/static_fe_controller.ex index 9b565d07d..c35657d8e 100644 --- a/lib/pleroma/web/static_fe/static_fe_controller.ex +++ b/lib/pleroma/web/static_fe/static_fe_controller.ex @@ -34,17 +34,17 @@ def get_counts(%Activity{} = activity) do } end - def represent(%Activity{} = activity, selected) do + def represent(%Activity{object: %Object{data: data}} = activity, selected) do {:ok, user} = User.get_or_fetch(activity.object.data["actor"]) %{ user: user, title: get_title(activity.object), - content: activity.object.data["content"] || nil, - attachment: activity.object.data["attachment"], + content: data["content"] || nil, + attachment: data["attachment"], link: Helpers.o_status_url(Pleroma.Web.Endpoint, :notice, activity.id), - published: activity.object.data["published"], - sensitive: activity.object.data["sensitive"], + published: data["published"], + sensitive: data["sensitive"], selected: selected, counts: get_counts(activity) } diff --git a/lib/pleroma/web/static_fe/static_fe_view.ex b/lib/pleroma/web/static_fe/static_fe_view.ex index 6128b2497..160261af9 100644 --- a/lib/pleroma/web/static_fe/static_fe_view.ex +++ b/lib/pleroma/web/static_fe/static_fe_view.ex @@ -8,6 +8,7 @@ defmodule Pleroma.Web.StaticFE.StaticFEView do alias Calendar.Strftime alias Pleroma.Emoji.Formatter alias Pleroma.User + alias Pleroma.Web.Endpoint alias Pleroma.Web.Gettext alias Pleroma.Web.MediaProxy alias Pleroma.Formatter diff --git a/lib/pleroma/web/templates/static_fe/static_fe/conversation.html.eex b/lib/pleroma/web/templates/static_fe/static_fe/conversation.html.eex index 3a1249df2..7ac4a9e5f 100644 --- a/lib/pleroma/web/templates/static_fe/static_fe/conversation.html.eex +++ b/lib/pleroma/web/templates/static_fe/static_fe/conversation.html.eex @@ -1,7 +1,11 @@ -

<%= link @instance_name, to: "/" %>

+
+

<%= link @instance_name, to: "/" %>

+
-
- <%= for activity <- @activities do %> - <%= render("_notice.html", activity) %> - <% end %> -
+
+
+ <%= for activity <- @activities do %> + <%= render("_notice.html", activity) %> + <% end %> +
+
diff --git a/lib/pleroma/web/templates/static_fe/static_fe/profile.html.eex b/lib/pleroma/web/templates/static_fe/static_fe/profile.html.eex index 8f2c74627..9b3d0509e 100644 --- a/lib/pleroma/web/templates/static_fe/static_fe/profile.html.eex +++ b/lib/pleroma/web/templates/static_fe/static_fe/profile.html.eex @@ -1,16 +1,22 @@ -

<%= link @instance_name, to: "/" %>

+
+

<%= link @instance_name, to: "/" %>

-

-
- - - -
- <%= raw (@user.name |> Formatter.emojify(emoji_for_user(@user))) %> -

-

<%= raw @user.bio %>

-
- <%= for activity <- @timeline do %> - <%= render("_notice.html", Map.put(activity, :selected, false)) %> - <% end %> -
+

+
+ + + +
+ <%= raw Formatter.emojify(@user.name, emoji_for_user(@user)) %> | + <%= link "@#{@user.nickname}@#{Endpoint.host()}", to: User.profile_url(@user) %> +

+

<%= raw @user.bio %>

+
+ +
+
+ <%= for activity <- @timeline do %> + <%= render("_notice.html", Map.put(activity, :selected, false)) %> + <% end %> +
+
diff --git a/priv/static/static/static-fe.css b/priv/static/static/static-fe.css index d6bb58c33..19c56387b 100644 --- a/priv/static/static/static-fe.css +++ b/priv/static/static/static-fe.css @@ -4,7 +4,7 @@ body { color: white; } -.container { +main { margin: 50px auto; max-width: 960px; padding: 40px; @@ -13,7 +13,11 @@ .container { } header { - border-bottom: 2em solid #282c37; + margin: 50px auto; + max-width: 960px; + padding: 40px; + background-color: #313543; + border-radius: 4px; } .activity { @@ -57,11 +61,11 @@ .h-card { margin-bottom: 8px; } -.h-card a { +header a, .h-card a { text-decoration: none; } -.h-card a:hover { +header a:hover, .h-card a:hover { text-decoration: underline; } From c6c706161e462bb6190cb4471e81e5a8c3b66d20 Mon Sep 17 00:00:00 2001 From: Phil Hagelberg Date: Thu, 31 Oct 2019 18:26:34 -0700 Subject: [PATCH 060/107] Make sure notice link is remote if the post is remote. --- lib/pleroma/web/static_fe/static_fe_controller.ex | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/web/static_fe/static_fe_controller.ex b/lib/pleroma/web/static_fe/static_fe_controller.ex index c35657d8e..5f69218ce 100644 --- a/lib/pleroma/web/static_fe/static_fe_controller.ex +++ b/lib/pleroma/web/static_fe/static_fe_controller.ex @@ -37,12 +37,19 @@ def get_counts(%Activity{} = activity) do def represent(%Activity{object: %Object{data: data}} = activity, selected) do {:ok, user} = User.get_or_fetch(activity.object.data["actor"]) + link = + if user.local do + Helpers.o_status_url(Pleroma.Web.Endpoint, :notice, activity) + else + data["url"] || data["external_url"] || data["id"] + end + %{ user: user, title: get_title(activity.object), content: data["content"] || nil, attachment: data["attachment"], - link: Helpers.o_status_url(Pleroma.Web.Endpoint, :notice, activity.id), + link: link, published: data["published"], sensitive: data["sensitive"], selected: selected, From dc3b87d153415bee6a169b4c787f79dbee74c622 Mon Sep 17 00:00:00 2001 From: Phil Hagelberg Date: Fri, 1 Nov 2019 19:47:18 -0700 Subject: [PATCH 061/107] Move static FE routing into its own plug. Previously it was piggybacking on FallbackRedirectController for users and OStatusController for notices; now it's all in one place. --- lib/pleroma/plugs/static_fe_plug.ex | 14 +++++ lib/pleroma/web/ostatus/ostatus_controller.ex | 58 +++++++++---------- lib/pleroma/web/router.ex | 1 + .../web/static_fe/static_fe_controller.ex | 33 ++++------- 4 files changed, 53 insertions(+), 53 deletions(-) create mode 100644 lib/pleroma/plugs/static_fe_plug.ex diff --git a/lib/pleroma/plugs/static_fe_plug.ex b/lib/pleroma/plugs/static_fe_plug.ex new file mode 100644 index 000000000..d3abaf4cc --- /dev/null +++ b/lib/pleroma/plugs/static_fe_plug.ex @@ -0,0 +1,14 @@ +# Pleroma: A lightweight social networking server +# Copyright ยฉ 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Plugs.StaticFEPlug do + def init(options), do: options + + def call(conn, _) do + case Pleroma.Config.get([:instance, :static_fe], false) do + true -> Pleroma.Web.StaticFE.StaticFEController.call(conn, :show) + _ -> conn + end + end +end diff --git a/lib/pleroma/web/ostatus/ostatus_controller.ex b/lib/pleroma/web/ostatus/ostatus_controller.ex index be275977e..6958519de 100644 --- a/lib/pleroma/web/ostatus/ostatus_controller.ex +++ b/lib/pleroma/web/ostatus/ostatus_controller.ex @@ -76,41 +76,37 @@ def activity(%{assigns: %{format: format}} = conn, %{"uuid" => uuid}) do end def notice(%{assigns: %{format: format}} = conn, %{"id" => id}) do - if Pleroma.Config.get([:instance, :static_fe], false) do - Pleroma.Web.StaticFE.StaticFEController.call(conn, :show_notice) - else - with {_, %Activity{} = activity} <- {:activity, Activity.get_by_id_with_object(id)}, - {_, true} <- {:public?, Visibility.is_public?(activity)}, - %User{} = user <- User.get_cached_by_ap_id(activity.data["actor"]) do - cond do - format == "html" && activity.data["type"] == "Create" -> - %Object{} = object = Object.normalize(activity) + with {_, %Activity{} = activity} <- {:activity, Activity.get_by_id_with_object(id)}, + {_, true} <- {:public?, Visibility.is_public?(activity)}, + %User{} = user <- User.get_cached_by_ap_id(activity.data["actor"]) do + cond do + format == "html" && activity.data["type"] == "Create" -> + %Object{} = object = Object.normalize(activity) - RedirectController.redirector_with_meta( - conn, - %{ - activity_id: activity.id, - object: object, - url: Router.Helpers.o_status_url(Endpoint, :notice, activity.id), - user: user - } - ) + RedirectController.redirector_with_meta( + conn, + %{ + activity_id: activity.id, + object: object, + url: Router.Helpers.o_status_url(Endpoint, :notice, activity.id), + user: user + } + ) - format == "html" -> - RedirectController.redirector(conn, nil) + format == "html" -> + RedirectController.redirector(conn, nil) - true -> - represent_activity(conn, format, activity, user) - end - else - reason when reason in [{:public?, false}, {:activity, nil}] -> - conn - |> put_status(404) - |> RedirectController.redirector(nil, 404) - - e -> - e + true -> + represent_activity(conn, format, activity, user) end + else + reason when reason in [{:public?, false}, {:activity, nil}] -> + conn + |> put_status(404) + |> RedirectController.redirector(nil, 404) + + e -> + e end end diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 8fb4aec13..ecf5f744c 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -495,6 +495,7 @@ defmodule Pleroma.Web.Router do pipeline :ostatus do plug(:accepts, ["html", "xml", "atom", "activity+json", "json"]) + plug(Pleroma.Plugs.StaticFEPlug) end pipeline :oembed do diff --git a/lib/pleroma/web/static_fe/static_fe_controller.ex b/lib/pleroma/web/static_fe/static_fe_controller.ex index 5f69218ce..96e30f317 100644 --- a/lib/pleroma/web/static_fe/static_fe_controller.ex +++ b/lib/pleroma/web/static_fe/static_fe_controller.ex @@ -14,7 +14,6 @@ defmodule Pleroma.Web.StaticFE.StaticFEController do plug(:put_layout, :static_fe) plug(:put_view, Pleroma.Web.StaticFE.StaticFEView) plug(:assign_id) - action_fallback(:not_found) defp get_title(%Object{data: %{"name" => name}}) when is_binary(name), do: name @@ -34,14 +33,15 @@ def get_counts(%Activity{} = activity) do } end + def represent(%Activity{} = activity), do: represent(activity, false) + def represent(%Activity{object: %Object{data: data}} = activity, selected) do {:ok, user} = User.get_or_fetch(activity.object.data["actor"]) link = - if user.local do - Helpers.o_status_url(Pleroma.Web.Endpoint, :notice, activity) - else - data["url"] || data["external_url"] || data["id"] + case user.local do + true -> Helpers.o_status_url(Pleroma.Web.Endpoint, :notice, activity) + _ -> data["url"] || data["external_url"] || data["id"] end %{ @@ -57,28 +57,27 @@ def represent(%Activity{object: %Object{data: data}} = activity, selected) do } end - def show_notice(%{assigns: %{notice_id: notice_id}} = conn, _params) do + def show(%{assigns: %{notice_id: notice_id}} = conn, _params) do instance_name = Pleroma.Config.get([:instance, :name], "Pleroma") activity = Activity.get_by_id_with_object(notice_id) context = activity.object.data["context"] activities = ActivityPub.fetch_activities_for_context(context, %{}) - represented = + timeline = for a <- Enum.reverse(activities) do represent(a, a.object.id == activity.object.id) end - render(conn, "conversation.html", %{activities: represented, instance_name: instance_name}) + render(conn, "conversation.html", %{activities: timeline, instance_name: instance_name}) end - def show_user(%{assigns: %{username_or_id: username_or_id}} = conn, _params) do + def show(%{assigns: %{username_or_id: username_or_id}} = conn, _params) do instance_name = Pleroma.Config.get([:instance, :name], "Pleroma") %User{} = user = User.get_cached_by_nickname_or_id(username_or_id) timeline = - for activity <- ActivityPub.fetch_user_activities(user, nil, %{}) do - represent(activity, false) - end + ActivityPub.fetch_user_activities(user, nil, %{}) + |> Enum.map(&represent/1) render(conn, "profile.html", %{user: user, timeline: timeline, instance_name: instance_name}) end @@ -89,15 +88,5 @@ def assign_id(%{path_info: ["notice", notice_id]} = conn, _opts), def assign_id(%{path_info: ["users", user_id]} = conn, _opts), do: assign(conn, :username_or_id, user_id) - def assign_id(%{path_info: [user_id]} = conn, _opts), - do: assign(conn, :username_or_id, user_id) - def assign_id(conn, _opts), do: conn - - # Fallback for unhandled types - def not_found(conn, _opts) do - conn - |> put_status(404) - |> text("Not found") - end end From e8bee35578fbbc442657baa4dee0047906b247a9 Mon Sep 17 00:00:00 2001 From: Phil Hagelberg Date: Sun, 3 Nov 2019 12:29:17 -0800 Subject: [PATCH 062/107] Static FE plug should only respond to text/html requests. --- lib/pleroma/plugs/static_fe_plug.ex | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/plugs/static_fe_plug.ex b/lib/pleroma/plugs/static_fe_plug.ex index d3abaf4cc..dcbabc9df 100644 --- a/lib/pleroma/plugs/static_fe_plug.ex +++ b/lib/pleroma/plugs/static_fe_plug.ex @@ -5,9 +5,14 @@ defmodule Pleroma.Plugs.StaticFEPlug do def init(options), do: options + def accepts_html?({"accept", a}), do: String.contains?(a, "text/html") + def accepts_html?({_, _}), do: false + def call(conn, _) do - case Pleroma.Config.get([:instance, :static_fe], false) do - true -> Pleroma.Web.StaticFE.StaticFEController.call(conn, :show) + with true <- Pleroma.Config.get([:instance, :static_fe], false), + {_, _} <- Enum.find(conn.req_headers, &accepts_html?/1) do + Pleroma.Web.StaticFE.StaticFEController.call(conn, :show) + else _ -> conn end end From 8969c5522d0ff4b95705ce8dd1249aa76414fe0e Mon Sep 17 00:00:00 2001 From: Phil Hagelberg Date: Mon, 4 Nov 2019 22:02:10 -0800 Subject: [PATCH 063/107] Make many of the improvements suggested in review. --- lib/pleroma/plugs/static_fe_plug.ex | 21 ++++++++++++------- .../web/static_fe/static_fe_controller.ex | 16 ++++++-------- lib/pleroma/web/static_fe/static_fe_view.ex | 2 ++ .../web/templates/layout/static_fe.html.eex | 2 +- .../static_fe/static_fe/conversation.html.eex | 2 +- .../static_fe/static_fe/profile.html.eex | 2 +- 6 files changed, 25 insertions(+), 20 deletions(-) diff --git a/lib/pleroma/plugs/static_fe_plug.ex b/lib/pleroma/plugs/static_fe_plug.ex index dcbabc9df..2af45e52a 100644 --- a/lib/pleroma/plugs/static_fe_plug.ex +++ b/lib/pleroma/plugs/static_fe_plug.ex @@ -3,17 +3,24 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Plugs.StaticFEPlug do + import Plug.Conn + alias Pleroma.Web.StaticFE.StaticFEController + def init(options), do: options - def accepts_html?({"accept", a}), do: String.contains?(a, "text/html") - def accepts_html?({_, _}), do: false - def call(conn, _) do - with true <- Pleroma.Config.get([:instance, :static_fe], false), - {_, _} <- Enum.find(conn.req_headers, &accepts_html?/1) do - Pleroma.Web.StaticFE.StaticFEController.call(conn, :show) + if enabled?() and accepts_html?(conn) do + conn + |> StaticFEController.call(:show) + |> halt() else - _ -> conn + conn end end + + defp enabled?, do: Pleroma.Config.get([:instance, :static_fe], false) + + defp accepts_html?(conn) do + conn |> get_req_header("accept") |> List.first() |> String.contains?("text/html") + end end diff --git a/lib/pleroma/web/static_fe/static_fe_controller.ex b/lib/pleroma/web/static_fe/static_fe_controller.ex index 96e30f317..a00c6db4f 100644 --- a/lib/pleroma/web/static_fe/static_fe_controller.ex +++ b/lib/pleroma/web/static_fe/static_fe_controller.ex @@ -58,28 +58,24 @@ def represent(%Activity{object: %Object{data: data}} = activity, selected) do end def show(%{assigns: %{notice_id: notice_id}} = conn, _params) do - instance_name = Pleroma.Config.get([:instance, :name], "Pleroma") activity = Activity.get_by_id_with_object(notice_id) - context = activity.object.data["context"] - activities = ActivityPub.fetch_activities_for_context(context, %{}) - timeline = - for a <- Enum.reverse(activities) do - represent(a, a.object.id == activity.object.id) - end + activity.object.data["context"] + |> ActivityPub.fetch_activities_for_context(%{}) + |> Enum.reverse() + |> Enum.map(&represent(&1, &1.object.id == activity.object.id)) - render(conn, "conversation.html", %{activities: timeline, instance_name: instance_name}) + render(conn, "conversation.html", %{activities: timeline}) end def show(%{assigns: %{username_or_id: username_or_id}} = conn, _params) do - instance_name = Pleroma.Config.get([:instance, :name], "Pleroma") %User{} = user = User.get_cached_by_nickname_or_id(username_or_id) timeline = ActivityPub.fetch_user_activities(user, nil, %{}) |> Enum.map(&represent/1) - render(conn, "profile.html", %{user: user, timeline: timeline, instance_name: instance_name}) + render(conn, "profile.html", %{user: user, timeline: timeline}) end def assign_id(%{path_info: ["notice", notice_id]} = conn, _opts), diff --git a/lib/pleroma/web/static_fe/static_fe_view.ex b/lib/pleroma/web/static_fe/static_fe_view.ex index 160261af9..5612a06bb 100644 --- a/lib/pleroma/web/static_fe/static_fe_view.ex +++ b/lib/pleroma/web/static_fe/static_fe_view.ex @@ -36,4 +36,6 @@ def format_date(date) do {:ok, date, _} = DateTime.from_iso8601(date) Strftime.strftime!(date, "%Y/%m/%d %l:%M:%S %p UTC") end + + def instance_name, do: Pleroma.Config.get([:instance, :name], "Pleroma") end diff --git a/lib/pleroma/web/templates/layout/static_fe.html.eex b/lib/pleroma/web/templates/layout/static_fe.html.eex index 62b59f17c..4b889bb19 100644 --- a/lib/pleroma/web/templates/layout/static_fe.html.eex +++ b/lib/pleroma/web/templates/layout/static_fe.html.eex @@ -4,7 +4,7 @@ - <%= Application.get_env(:pleroma, :instance)[:name] %> + <%= Pleroma.Config.get([:instance, :name]) %> diff --git a/lib/pleroma/web/templates/static_fe/static_fe/conversation.html.eex b/lib/pleroma/web/templates/static_fe/static_fe/conversation.html.eex index 7ac4a9e5f..2acd84828 100644 --- a/lib/pleroma/web/templates/static_fe/static_fe/conversation.html.eex +++ b/lib/pleroma/web/templates/static_fe/static_fe/conversation.html.eex @@ -1,5 +1,5 @@
-

<%= link @instance_name, to: "/" %>

+

<%= link instance_name(), to: "/" %>

diff --git a/lib/pleroma/web/templates/static_fe/static_fe/profile.html.eex b/lib/pleroma/web/templates/static_fe/static_fe/profile.html.eex index 9b3d0509e..fa3df3b4e 100644 --- a/lib/pleroma/web/templates/static_fe/static_fe/profile.html.eex +++ b/lib/pleroma/web/templates/static_fe/static_fe/profile.html.eex @@ -1,5 +1,5 @@
-

<%= link @instance_name, to: "/" %>

+

<%= link instance_name(), to: "/" %>

From df2f59be911acd4626886befbc0c6bcd75752080 Mon Sep 17 00:00:00 2001 From: Phil Hagelberg Date: Mon, 4 Nov 2019 22:49:05 -0800 Subject: [PATCH 064/107] Pagination for user profiles. --- .../web/static_fe/static_fe_controller.ex | 22 +++++++++++++++---- .../static_fe/static_fe/profile.html.eex | 9 ++++++++ 2 files changed, 27 insertions(+), 4 deletions(-) diff --git a/lib/pleroma/web/static_fe/static_fe_controller.ex b/lib/pleroma/web/static_fe/static_fe_controller.ex index a00c6db4f..9f4eeaa36 100644 --- a/lib/pleroma/web/static_fe/static_fe_controller.ex +++ b/lib/pleroma/web/static_fe/static_fe_controller.ex @@ -15,6 +15,8 @@ defmodule Pleroma.Web.StaticFE.StaticFEController do plug(:put_view, Pleroma.Web.StaticFE.StaticFEView) plug(:assign_id) + @page_keys ["max_id", "min_id", "limit", "since_id", "order"] + defp get_title(%Object{data: %{"name" => name}}) when is_binary(name), do: name @@ -53,7 +55,8 @@ def represent(%Activity{object: %Object{data: data}} = activity, selected) do published: data["published"], sensitive: data["sensitive"], selected: selected, - counts: get_counts(activity) + counts: get_counts(activity), + id: activity.id } end @@ -68,14 +71,25 @@ def show(%{assigns: %{notice_id: notice_id}} = conn, _params) do render(conn, "conversation.html", %{activities: timeline}) end - def show(%{assigns: %{username_or_id: username_or_id}} = conn, _params) do + def show(%{assigns: %{username_or_id: username_or_id}} = conn, params) do %User{} = user = User.get_cached_by_nickname_or_id(username_or_id) timeline = - ActivityPub.fetch_user_activities(user, nil, %{}) + ActivityPub.fetch_user_activities(user, nil, Map.take(params, @page_keys)) |> Enum.map(&represent/1) - render(conn, "profile.html", %{user: user, timeline: timeline}) + prev_page_id = + (params["min_id"] || params["max_id"]) && + List.first(timeline) && List.first(timeline).id + + next_page_id = List.last(timeline) && List.last(timeline).id + + render(conn, "profile.html", %{ + user: user, + timeline: timeline, + prev_page_id: prev_page_id, + next_page_id: next_page_id + }) end def assign_id(%{path_info: ["notice", notice_id]} = conn, _opts), diff --git a/lib/pleroma/web/templates/static_fe/static_fe/profile.html.eex b/lib/pleroma/web/templates/static_fe/static_fe/profile.html.eex index fa3df3b4e..94063c92d 100644 --- a/lib/pleroma/web/templates/static_fe/static_fe/profile.html.eex +++ b/lib/pleroma/web/templates/static_fe/static_fe/profile.html.eex @@ -18,5 +18,14 @@ <%= for activity <- @timeline do %> <%= render("_notice.html", Map.put(activity, :selected, false)) %> <% end %> +

+ <%= if @prev_page_id do %> + <%= link "ยซ", to: "?min_id=" <> @prev_page_id %> + <% end %> + <%= if @prev_page_id && @next_page_id, do: " | " %> + <%= if @next_page_id do %> + <%= link "ยป", to: "?max_id=" <> @next_page_id %> + <% end %> +

From 828259fb6517d35b5f950e07601bab0bdc5b5efd Mon Sep 17 00:00:00 2001 From: Phil Hagelberg Date: Mon, 4 Nov 2019 22:56:51 -0800 Subject: [PATCH 065/107] Catch 404s. --- .../web/static_fe/static_fe_controller.ex | 55 ++++++++++++------- 1 file changed, 34 insertions(+), 21 deletions(-) diff --git a/lib/pleroma/web/static_fe/static_fe_controller.ex b/lib/pleroma/web/static_fe/static_fe_controller.ex index 9f4eeaa36..4798cad24 100644 --- a/lib/pleroma/web/static_fe/static_fe_controller.ex +++ b/lib/pleroma/web/static_fe/static_fe_controller.ex @@ -61,35 +61,48 @@ def represent(%Activity{object: %Object{data: data}} = activity, selected) do end def show(%{assigns: %{notice_id: notice_id}} = conn, _params) do - activity = Activity.get_by_id_with_object(notice_id) - timeline = - activity.object.data["context"] - |> ActivityPub.fetch_activities_for_context(%{}) - |> Enum.reverse() - |> Enum.map(&represent(&1, &1.object.id == activity.object.id)) + case Activity.get_by_id_with_object(notice_id) do + %Activity{} = activity -> + timeline = + activity.object.data["context"] + |> ActivityPub.fetch_activities_for_context(%{}) + |> Enum.reverse() + |> Enum.map(&represent(&1, &1.object.id == activity.object.id)) - render(conn, "conversation.html", %{activities: timeline}) + render(conn, "conversation.html", %{activities: timeline}) + + _ -> + conn + |> put_status(404) + |> render_error(:not_found, "Notice not found") + end end def show(%{assigns: %{username_or_id: username_or_id}} = conn, params) do - %User{} = user = User.get_cached_by_nickname_or_id(username_or_id) + case User.get_cached_by_nickname_or_id(username_or_id) do + %User{} = user -> + timeline = + ActivityPub.fetch_user_activities(user, nil, Map.take(params, @page_keys)) + |> Enum.map(&represent/1) - timeline = - ActivityPub.fetch_user_activities(user, nil, Map.take(params, @page_keys)) - |> Enum.map(&represent/1) + prev_page_id = + (params["min_id"] || params["max_id"]) && + List.first(timeline) && List.first(timeline).id - prev_page_id = - (params["min_id"] || params["max_id"]) && - List.first(timeline) && List.first(timeline).id + next_page_id = List.last(timeline) && List.last(timeline).id - next_page_id = List.last(timeline) && List.last(timeline).id + render(conn, "profile.html", %{ + user: user, + timeline: timeline, + prev_page_id: prev_page_id, + next_page_id: next_page_id + }) - render(conn, "profile.html", %{ - user: user, - timeline: timeline, - prev_page_id: prev_page_id, - next_page_id: next_page_id - }) + _ -> + conn + |> put_status(404) + |> render_error(:not_found, "User not found") + end end def assign_id(%{path_info: ["notice", notice_id]} = conn, _opts), From bfd5d798262f0ecc7ebc260d92c766d39c0766de Mon Sep 17 00:00:00 2001 From: Phil Hagelberg Date: Tue, 5 Nov 2019 21:28:36 -0800 Subject: [PATCH 066/107] Include metadata in static FE conversations and profiles. --- lib/pleroma/web/static_fe/static_fe_controller.ex | 11 +++++++++-- lib/pleroma/web/templates/layout/static_fe.html.eex | 5 ++--- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/lib/pleroma/web/static_fe/static_fe_controller.ex b/lib/pleroma/web/static_fe/static_fe_controller.ex index 4798cad24..10bd3fecd 100644 --- a/lib/pleroma/web/static_fe/static_fe_controller.ex +++ b/lib/pleroma/web/static_fe/static_fe_controller.ex @@ -9,6 +9,7 @@ defmodule Pleroma.Web.StaticFE.StaticFEController do alias Pleroma.Object alias Pleroma.User alias Pleroma.Web.ActivityPub.ActivityPub + alias Pleroma.Web.Metadata alias Pleroma.Web.Router.Helpers plug(:put_layout, :static_fe) @@ -63,13 +64,16 @@ def represent(%Activity{object: %Object{data: data}} = activity, selected) do def show(%{assigns: %{notice_id: notice_id}} = conn, _params) do case Activity.get_by_id_with_object(notice_id) do %Activity{} = activity -> + %User{} = user = User.get_by_ap_id(activity.object.data["actor"]) + meta = Metadata.build_tags(%{activity_id: notice_id, object: activity.object, user: user}) + timeline = activity.object.data["context"] |> ActivityPub.fetch_activities_for_context(%{}) |> Enum.reverse() |> Enum.map(&represent(&1, &1.object.id == activity.object.id)) - render(conn, "conversation.html", %{activities: timeline}) + render(conn, "conversation.html", %{activities: timeline, meta: meta}) _ -> conn @@ -81,6 +85,8 @@ def show(%{assigns: %{notice_id: notice_id}} = conn, _params) do def show(%{assigns: %{username_or_id: username_or_id}} = conn, params) do case User.get_cached_by_nickname_or_id(username_or_id) do %User{} = user -> + meta = Metadata.build_tags(%{user: user}) + timeline = ActivityPub.fetch_user_activities(user, nil, Map.take(params, @page_keys)) |> Enum.map(&represent/1) @@ -95,7 +101,8 @@ def show(%{assigns: %{username_or_id: username_or_id}} = conn, params) do user: user, timeline: timeline, prev_page_id: prev_page_id, - next_page_id: next_page_id + next_page_id: next_page_id, + meta: meta }) _ -> diff --git a/lib/pleroma/web/templates/layout/static_fe.html.eex b/lib/pleroma/web/templates/layout/static_fe.html.eex index 4b889bb19..5d820bb4b 100644 --- a/lib/pleroma/web/templates/layout/static_fe.html.eex +++ b/lib/pleroma/web/templates/layout/static_fe.html.eex @@ -3,9 +3,8 @@ - - <%= Pleroma.Config.get([:instance, :name]) %> - + <%= Pleroma.Config.get([:instance, :name]) %> + <%= Phoenix.HTML.raw(@meta || "") %> From e27c61218d292c5fbf268f27e81dbe22f93ba90f Mon Sep 17 00:00:00 2001 From: Phil Hagelberg Date: Tue, 5 Nov 2019 21:37:46 -0800 Subject: [PATCH 067/107] Expand subject content automatically when config is set. --- lib/pleroma/web/static_fe/static_fe_view.ex | 7 +++++++ .../web/templates/static_fe/static_fe/_notice.html.eex | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/web/static_fe/static_fe_view.ex b/lib/pleroma/web/static_fe/static_fe_view.ex index 5612a06bb..72e667728 100644 --- a/lib/pleroma/web/static_fe/static_fe_view.ex +++ b/lib/pleroma/web/static_fe/static_fe_view.ex @@ -38,4 +38,11 @@ def format_date(date) do end def instance_name, do: Pleroma.Config.get([:instance, :name], "Pleroma") + + def open_content? do + Pleroma.Config.get( + [:frontend_configurations, :collapse_message_with_subjects], + true + ) + end end diff --git a/lib/pleroma/web/templates/static_fe/static_fe/_notice.html.eex b/lib/pleroma/web/templates/static_fe/static_fe/_notice.html.eex index 2a46dadb4..df5e5eedd 100644 --- a/lib/pleroma/web/templates/static_fe/static_fe/_notice.html.eex +++ b/lib/pleroma/web/templates/static_fe/static_fe/_notice.html.eex @@ -5,7 +5,7 @@ <%= render("_user_card.html", %{user: @user}) %>
<%= if @title != "" do %> -
+
open<% end %>> <%= raw @title %>
<%= raw @content %>
From b0080fa73010cda34215baeee230481b5c56dbca Mon Sep 17 00:00:00 2001 From: Phil Hagelberg Date: Tue, 5 Nov 2019 22:00:19 -0800 Subject: [PATCH 068/107] Render errors in HTML, not with JS. --- lib/pleroma/web/static_fe/static_fe_controller.ex | 4 ++-- lib/pleroma/web/templates/layout/static_fe.html.eex | 2 +- .../web/templates/static_fe/static_fe/error.html.eex | 7 +++++++ 3 files changed, 10 insertions(+), 3 deletions(-) create mode 100644 lib/pleroma/web/templates/static_fe/static_fe/error.html.eex diff --git a/lib/pleroma/web/static_fe/static_fe_controller.ex b/lib/pleroma/web/static_fe/static_fe_controller.ex index 10bd3fecd..0be47d6b3 100644 --- a/lib/pleroma/web/static_fe/static_fe_controller.ex +++ b/lib/pleroma/web/static_fe/static_fe_controller.ex @@ -78,7 +78,7 @@ def show(%{assigns: %{notice_id: notice_id}} = conn, _params) do _ -> conn |> put_status(404) - |> render_error(:not_found, "Notice not found") + |> render("error.html", %{message: "Post not found.", meta: ""}) end end @@ -108,7 +108,7 @@ def show(%{assigns: %{username_or_id: username_or_id}} = conn, params) do _ -> conn |> put_status(404) - |> render_error(:not_found, "User not found") + |> render("error.html", %{message: "User not found.", meta: ""}) end end diff --git a/lib/pleroma/web/templates/layout/static_fe.html.eex b/lib/pleroma/web/templates/layout/static_fe.html.eex index 5d820bb4b..819632cec 100644 --- a/lib/pleroma/web/templates/layout/static_fe.html.eex +++ b/lib/pleroma/web/templates/layout/static_fe.html.eex @@ -4,7 +4,7 @@ <%= Pleroma.Config.get([:instance, :name]) %> - <%= Phoenix.HTML.raw(@meta || "") %> + <%= Phoenix.HTML.raw(assigns[:meta] || "") %> diff --git a/lib/pleroma/web/templates/static_fe/static_fe/error.html.eex b/lib/pleroma/web/templates/static_fe/static_fe/error.html.eex new file mode 100644 index 000000000..d98a1eba7 --- /dev/null +++ b/lib/pleroma/web/templates/static_fe/static_fe/error.html.eex @@ -0,0 +1,7 @@ +
+

<%= gettext("Oops") %>

+
+ +
+

<%= @message %>

+
From 886a07ba573a7dc566f51cfb44e69dac49f401cd Mon Sep 17 00:00:00 2001 From: Phil Hagelberg Date: Thu, 7 Nov 2019 19:31:28 -0800 Subject: [PATCH 069/107] Move static_fe config to its own section instead of in :instance. --- config/config.exs | 2 ++ lib/pleroma/plugs/static_fe_plug.ex | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/config/config.exs b/config/config.exs index 787809b27..685c18380 100644 --- a/config/config.exs +++ b/config/config.exs @@ -599,6 +599,8 @@ config :pleroma, Pleroma.Plugs.RemoteIp, enabled: false +config :pleroma, :static_fe, enabled: false + config :pleroma, :web_cache_ttl, activity_pub: nil, activity_pub_question: 30_000 diff --git a/lib/pleroma/plugs/static_fe_plug.ex b/lib/pleroma/plugs/static_fe_plug.ex index 2af45e52a..b3fb3c582 100644 --- a/lib/pleroma/plugs/static_fe_plug.ex +++ b/lib/pleroma/plugs/static_fe_plug.ex @@ -18,7 +18,7 @@ def call(conn, _) do end end - defp enabled?, do: Pleroma.Config.get([:instance, :static_fe], false) + defp enabled?, do: Pleroma.Config.get([:static_fe, :enabled], false) defp accepts_html?(conn) do conn |> get_req_header("accept") |> List.first() |> String.contains?("text/html") From 4729027f91852a921cf74f507fbc1ac8761a07f0 Mon Sep 17 00:00:00 2001 From: Phil Hagelberg Date: Thu, 7 Nov 2019 21:43:21 -0800 Subject: [PATCH 070/107] Prevent non-local notices from rendering. --- lib/pleroma/web/static_fe/static_fe_controller.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/static_fe/static_fe_controller.ex b/lib/pleroma/web/static_fe/static_fe_controller.ex index 0be47d6b3..66d2d0367 100644 --- a/lib/pleroma/web/static_fe/static_fe_controller.ex +++ b/lib/pleroma/web/static_fe/static_fe_controller.ex @@ -63,7 +63,7 @@ def represent(%Activity{object: %Object{data: data}} = activity, selected) do def show(%{assigns: %{notice_id: notice_id}} = conn, _params) do case Activity.get_by_id_with_object(notice_id) do - %Activity{} = activity -> + %Activity{local: true} = activity -> %User{} = user = User.get_by_ap_id(activity.object.data["actor"]) meta = Metadata.build_tags(%{activity_id: notice_id, object: activity.object, user: user}) From 2bf592f5dc16752bf640da94c169c9cd2d7a5ebb Mon Sep 17 00:00:00 2001 From: Phil Hagelberg Date: Thu, 7 Nov 2019 22:29:46 -0800 Subject: [PATCH 071/107] Add tests for static_fe controller. --- .../static_fe/static_fe_controller_test.exs | 91 +++++++++++++++++++ 1 file changed, 91 insertions(+) create mode 100644 test/web/static_fe/static_fe_controller_test.exs diff --git a/test/web/static_fe/static_fe_controller_test.exs b/test/web/static_fe/static_fe_controller_test.exs new file mode 100644 index 000000000..9099540bd --- /dev/null +++ b/test/web/static_fe/static_fe_controller_test.exs @@ -0,0 +1,91 @@ +defmodule Pleroma.Web.StaticFE.StaticFEControllerTest do + use Pleroma.Web.ConnCase + alias Pleroma.Web.CommonAPI + import Pleroma.Factory + + clear_config_all([:static_fe, :enabled]) do + Pleroma.Config.put([:static_fe, :enabled], true) + end + + describe "user profile page" do + test "just the profile as HTML", %{conn: conn} do + user = insert(:user) + conn = conn + |> put_req_header("accept", "text/html") + |> get("/users/#{user.nickname}") + + assert html_response(conn, 200) =~ user.nickname + end + + test "renders json unless there's an html accept header", %{conn: conn} do + user = insert(:user) + conn = conn + |> put_req_header("accept", "application/json") + |> get("/users/#{user.nickname}") + + assert json_response(conn, 200) + end + + test "404 when user not found", %{conn: conn} do + conn = conn + |> put_req_header("accept", "text/html") + |> get("/users/limpopo") + + assert html_response(conn, 404) =~ "not found" + end + + test "pagination", %{conn: conn} do + user = insert(:user) + Enum.map(1..30, fn i -> CommonAPI.post(user, %{"status" => "test#{i}"}) end) + conn = conn + |> put_req_header("accept", "text/html") + |> get("/users/#{user.nickname}") + html = html_response(conn, 200) + + assert html =~ ">test30<" + assert html =~ ">test11<" + refute html =~ ">test10<" + refute html =~ ">test1<" + end + + test "pagination, page 2", %{conn: conn} do + user = insert(:user) + activities = + Enum.map(1..30, fn i -> CommonAPI.post(user, %{"status" => "test#{i}"}) end) + {:ok, a11} = Enum.at(activities, 11) + conn = conn + |> put_req_header("accept", "text/html") + |> get("/users/#{user.nickname}?max_id=#{a11.id}") + html = html_response(conn, 200) + + assert html =~ ">test1<" + assert html =~ ">test10<" + refute html =~ ">test20<" + refute html =~ ">test29<" + end + end + + describe "notice rendering" do + test "single notice page", %{conn: conn} do + user = insert(:user) + {:ok, activity} = CommonAPI.post(user, %{"status" => "testing a thing!"}) + + conn = conn + |> put_req_header("accept", "text/html") + |> get("/notice/#{activity.id}") + + html = html_response(conn, 200) + assert html =~ "
" + assert html =~ user.nickname + assert html =~ "testing a thing!" + end + + test "404 when notice not found", %{conn: conn} do + conn = conn + |> put_req_header("accept", "text/html") + |> get("/notice/88c9c317") + + assert html_response(conn, 404) =~ "not found" + end + end +end From ef7c3bdc7a5d4047eca15b8469e1f7d7ab3bd39e Mon Sep 17 00:00:00 2001 From: Phil Hagelberg Date: Fri, 8 Nov 2019 08:55:32 -0800 Subject: [PATCH 072/107] Add some further test cases. Including like ... private visibility, cos that's super important. --- .../web/static_fe/static_fe_controller.ex | 24 ++-- .../static_fe/static_fe_controller_test.exs | 136 +++++++++++++++--- 2 files changed, 126 insertions(+), 34 deletions(-) diff --git a/lib/pleroma/web/static_fe/static_fe_controller.ex b/lib/pleroma/web/static_fe/static_fe_controller.ex index 66d2d0367..5e60c82b0 100644 --- a/lib/pleroma/web/static_fe/static_fe_controller.ex +++ b/lib/pleroma/web/static_fe/static_fe_controller.ex @@ -9,6 +9,7 @@ defmodule Pleroma.Web.StaticFE.StaticFEController do alias Pleroma.Object alias Pleroma.User alias Pleroma.Web.ActivityPub.ActivityPub + alias Pleroma.Web.ActivityPub.Visibility alias Pleroma.Web.Metadata alias Pleroma.Web.Router.Helpers @@ -62,19 +63,20 @@ def represent(%Activity{object: %Object{data: data}} = activity, selected) do end def show(%{assigns: %{notice_id: notice_id}} = conn, _params) do - case Activity.get_by_id_with_object(notice_id) do - %Activity{local: true} = activity -> - %User{} = user = User.get_by_ap_id(activity.object.data["actor"]) - meta = Metadata.build_tags(%{activity_id: notice_id, object: activity.object, user: user}) + with %Activity{local: true} = activity <- + Activity.get_by_id_with_object(notice_id), + true <- Visibility.is_public?(activity.object), + %User{} = user <- User.get_by_ap_id(activity.object.data["actor"]) do + meta = Metadata.build_tags(%{activity_id: notice_id, object: activity.object, user: user}) - timeline = - activity.object.data["context"] - |> ActivityPub.fetch_activities_for_context(%{}) - |> Enum.reverse() - |> Enum.map(&represent(&1, &1.object.id == activity.object.id)) - - render(conn, "conversation.html", %{activities: timeline, meta: meta}) + timeline = + activity.object.data["context"] + |> ActivityPub.fetch_activities_for_context(%{}) + |> Enum.reverse() + |> Enum.map(&represent(&1, &1.object.id == activity.object.id)) + render(conn, "conversation.html", %{activities: timeline, meta: meta}) + else _ -> conn |> put_status(404) diff --git a/test/web/static_fe/static_fe_controller_test.exs b/test/web/static_fe/static_fe_controller_test.exs index 9099540bd..e4bb78b01 100644 --- a/test/web/static_fe/static_fe_controller_test.exs +++ b/test/web/static_fe/static_fe_controller_test.exs @@ -1,6 +1,8 @@ defmodule Pleroma.Web.StaticFE.StaticFEControllerTest do use Pleroma.Web.ConnCase alias Pleroma.Web.CommonAPI + alias Pleroma.Web.ActivityPub.Transmogrifier + import Pleroma.Factory clear_config_all([:static_fe, :enabled]) do @@ -10,36 +12,60 @@ defmodule Pleroma.Web.StaticFE.StaticFEControllerTest do describe "user profile page" do test "just the profile as HTML", %{conn: conn} do user = insert(:user) - conn = conn - |> put_req_header("accept", "text/html") - |> get("/users/#{user.nickname}") + + conn = + conn + |> put_req_header("accept", "text/html") + |> get("/users/#{user.nickname}") assert html_response(conn, 200) =~ user.nickname end test "renders json unless there's an html accept header", %{conn: conn} do user = insert(:user) - conn = conn - |> put_req_header("accept", "application/json") - |> get("/users/#{user.nickname}") + + conn = + conn + |> put_req_header("accept", "application/json") + |> get("/users/#{user.nickname}") assert json_response(conn, 200) end test "404 when user not found", %{conn: conn} do - conn = conn - |> put_req_header("accept", "text/html") - |> get("/users/limpopo") + conn = + conn + |> put_req_header("accept", "text/html") + |> get("/users/limpopo") assert html_response(conn, 404) =~ "not found" end + test "profile does not include private messages", %{conn: conn} do + user = insert(:user) + CommonAPI.post(user, %{"status" => "public"}) + CommonAPI.post(user, %{"status" => "private", "visibility" => "private"}) + + conn = + conn + |> put_req_header("accept", "text/html") + |> get("/users/#{user.nickname}") + + html = html_response(conn, 200) + + assert html =~ ">public<" + refute html =~ ">private<" + end + test "pagination", %{conn: conn} do user = insert(:user) Enum.map(1..30, fn i -> CommonAPI.post(user, %{"status" => "test#{i}"}) end) - conn = conn - |> put_req_header("accept", "text/html") - |> get("/users/#{user.nickname}") + + conn = + conn + |> put_req_header("accept", "text/html") + |> get("/users/#{user.nickname}") + html = html_response(conn, 200) assert html =~ ">test30<" @@ -50,12 +76,14 @@ test "pagination", %{conn: conn} do test "pagination, page 2", %{conn: conn} do user = insert(:user) - activities = - Enum.map(1..30, fn i -> CommonAPI.post(user, %{"status" => "test#{i}"}) end) + activities = Enum.map(1..30, fn i -> CommonAPI.post(user, %{"status" => "test#{i}"}) end) {:ok, a11} = Enum.at(activities, 11) - conn = conn - |> put_req_header("accept", "text/html") - |> get("/users/#{user.nickname}?max_id=#{a11.id}") + + conn = + conn + |> put_req_header("accept", "text/html") + |> get("/users/#{user.nickname}?max_id=#{a11.id}") + html = html_response(conn, 200) assert html =~ ">test1<" @@ -70,9 +98,10 @@ test "single notice page", %{conn: conn} do user = insert(:user) {:ok, activity} = CommonAPI.post(user, %{"status" => "testing a thing!"}) - conn = conn - |> put_req_header("accept", "text/html") - |> get("/notice/#{activity.id}") + conn = + conn + |> put_req_header("accept", "text/html") + |> get("/notice/#{activity.id}") html = html_response(conn, 200) assert html =~ "
" @@ -80,10 +109,71 @@ test "single notice page", %{conn: conn} do assert html =~ "testing a thing!" end + test "shows the whole thread", %{conn: conn} do + user = insert(:user) + {:ok, activity} = CommonAPI.post(user, %{"status" => "space: the final frontier"}) + + CommonAPI.post(user, %{ + "status" => "these are the voyages or something", + "in_reply_to_status_id" => activity.id + }) + + conn = + conn + |> put_req_header("accept", "text/html") + |> get("/notice/#{activity.id}") + + html = html_response(conn, 200) + assert html =~ "the final frontier" + assert html =~ "voyages" + end + test "404 when notice not found", %{conn: conn} do - conn = conn - |> put_req_header("accept", "text/html") - |> get("/notice/88c9c317") + conn = + conn + |> put_req_header("accept", "text/html") + |> get("/notice/88c9c317") + + assert html_response(conn, 404) =~ "not found" + end + + test "404 for private status", %{conn: conn} do + user = insert(:user) + + {:ok, activity} = + CommonAPI.post(user, %{"status" => "don't show me!", "visibility" => "private"}) + + conn = + conn + |> put_req_header("accept", "text/html") + |> get("/notice/#{activity.id}") + + assert html_response(conn, 404) =~ "not found" + end + + test "404 for remote cached status", %{conn: conn} do + user = insert(:user) + + message = %{ + "@context" => "https://www.w3.org/ns/activitystreams", + "to" => user.follower_address, + "cc" => "https://www.w3.org/ns/activitystreams#Public", + "type" => "Create", + "object" => %{ + "content" => "blah blah blah", + "type" => "Note", + "attributedTo" => user.ap_id, + "inReplyTo" => nil + }, + "actor" => user.ap_id + } + + assert {:ok, activity} = Transmogrifier.handle_incoming(message) + + conn = + conn + |> put_req_header("accept", "text/html") + |> get("/notice/#{activity.id}") assert html_response(conn, 404) =~ "not found" end From 6ef804966461ff4351e6c8d3c867131c2af5d26a Mon Sep 17 00:00:00 2001 From: Phil Hagelberg Date: Sat, 9 Nov 2019 09:50:45 -0800 Subject: [PATCH 073/107] Add changelog entry, cheatsheet docs, and alphabetize. --- CHANGELOG.md | 1 + docs/configuration/cheatsheet.md | 7 +++++++ test/web/static_fe/static_fe_controller_test.exs | 2 +- 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b33d61819..7ae52a28b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -37,6 +37,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ### Added - Refreshing poll results for remote polls - Authentication: Added rate limit for password-authorized actions / login existence checks +- Static Frontend: Add the ability to render user profiles and notices server-side without requiring JS app. - Mix task to re-count statuses for all users (`mix pleroma.count_statuses`) - Support for `X-Forwarded-For` and similar HTTP headers which used by reverse proxies to pass a real user IP address to the backend. Must not be enabled unless your instance is behind at least one reverse proxy (such as Nginx, Apache HTTPD or Varnish Cache).
diff --git a/docs/configuration/cheatsheet.md b/docs/configuration/cheatsheet.md index 8f609fcfd..dddd379d3 100644 --- a/docs/configuration/cheatsheet.md +++ b/docs/configuration/cheatsheet.md @@ -794,3 +794,10 @@ config :auto_linker, ] ``` +## :static_fe + +Render profiles and posts using server-generated HTML that is viewable without using JavaScript. + +Available options: + +* `enabled` - Enables the rendering of static HTML. Defaults to `false`. diff --git a/test/web/static_fe/static_fe_controller_test.exs b/test/web/static_fe/static_fe_controller_test.exs index e4bb78b01..effdfbeb3 100644 --- a/test/web/static_fe/static_fe_controller_test.exs +++ b/test/web/static_fe/static_fe_controller_test.exs @@ -1,7 +1,7 @@ defmodule Pleroma.Web.StaticFE.StaticFEControllerTest do use Pleroma.Web.ConnCase - alias Pleroma.Web.CommonAPI alias Pleroma.Web.ActivityPub.Transmogrifier + alias Pleroma.Web.CommonAPI import Pleroma.Factory From 3cc49cdb78bf14897030c476b00fb07064f2d74e Mon Sep 17 00:00:00 2001 From: Phil Hagelberg Date: Sat, 9 Nov 2019 18:26:19 -0800 Subject: [PATCH 074/107] Formatter moved to new module. --- lib/pleroma/web/static_fe/static_fe_view.ex | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/pleroma/web/static_fe/static_fe_view.ex b/lib/pleroma/web/static_fe/static_fe_view.ex index 72e667728..821ece9a9 100644 --- a/lib/pleroma/web/static_fe/static_fe_view.ex +++ b/lib/pleroma/web/static_fe/static_fe_view.ex @@ -11,7 +11,6 @@ defmodule Pleroma.Web.StaticFE.StaticFEView do alias Pleroma.Web.Endpoint alias Pleroma.Web.Gettext alias Pleroma.Web.MediaProxy - alias Pleroma.Formatter alias Pleroma.Web.Metadata.Utils alias Pleroma.Web.Router.Helpers From 4499a7a0751ec992a78a1023e8e34300f06e14b1 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Sun, 10 Nov 2019 15:02:47 +0300 Subject: [PATCH 075/107] Disable attachment links by default Closes #1394 --- CHANGELOG.md | 1 + config/config.exs | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1eb7d14cd..4ec084dbd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ### Changed - **Breaking:** Elixir >=1.8 is now required (was >= 1.7) +- **Breaking:** attachment links (`config :pleroma, :instance, no_attachment_links` and `config :pleroma, Pleroma.Upload, link_name`) disabled by default - Replaced [pleroma_job_queue](https://git.pleroma.social/pleroma/pleroma_job_queue) and `Pleroma.Web.Federator.RetryQueue` with [Oban](https://github.com/sorentwo/oban) (see [`docs/config.md`](docs/config.md) on migrating customized worker / retry settings) - Introduced [quantum](https://github.com/quantum-elixir/quantum-core) job scheduler - Enabled `:instance, extended_nickname_format` in the default config diff --git a/config/config.exs b/config/config.exs index 7e1fe2a81..54de8fa9f 100644 --- a/config/config.exs +++ b/config/config.exs @@ -90,7 +90,7 @@ config :pleroma, Pleroma.Upload, uploader: Pleroma.Uploaders.Local, filters: [Pleroma.Upload.Filter.Dedupe], - link_name: true, + link_name: false, proxy_remote: false, proxy_opts: [ redirect_on_failure: false, @@ -257,7 +257,7 @@ mrf_transparency_exclusions: [], autofollowed_nicknames: [], max_pinned_statuses: 1, - no_attachment_links: false, + no_attachment_links: true, welcome_user_nickname: nil, welcome_message: nil, max_report_comment_size: 1000, From 6a4201e0b444748318845caddf0e972d0fac87d7 Mon Sep 17 00:00:00 2001 From: Alexander Date: Sun, 10 Nov 2019 22:54:37 +0300 Subject: [PATCH 076/107] fix for migrate task --- lib/mix/tasks/pleroma/config.ex | 2 +- lib/pleroma/docs/json.ex | 2 +- test/web/admin_api/admin_api_controller_test.exs | 4 ++++ 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/lib/mix/tasks/pleroma/config.ex b/lib/mix/tasks/pleroma/config.ex index 11e4fde43..0e21408b2 100644 --- a/lib/mix/tasks/pleroma/config.ex +++ b/lib/mix/tasks/pleroma/config.ex @@ -45,7 +45,7 @@ def run(["migrate_from_db", env, delete?]) do if Pleroma.Config.get([:instance, :dynamic_configuration]) do config_path = "config/#{env}.exported_from_db.secret.exs" - {:ok, file} = File.open(config_path, [:write]) + {:ok, file} = File.open(config_path, [:write, :utf8]) IO.write(file, "use Mix.Config\r\n") Repo.all(Config) diff --git a/lib/pleroma/docs/json.ex b/lib/pleroma/docs/json.ex index 18ba01d58..f2a56d845 100644 --- a/lib/pleroma/docs/json.ex +++ b/lib/pleroma/docs/json.ex @@ -5,7 +5,7 @@ defmodule Pleroma.Docs.JSON do def process(descriptions) do config_path = "docs/generate_config.json" - with {:ok, file} <- File.open(config_path, [:write]), + with {:ok, file} <- File.open(config_path, [:write, :utf8]), json <- generate_json(descriptions), :ok <- IO.write(file, json), :ok <- File.close(file) do diff --git a/test/web/admin_api/admin_api_controller_test.exs b/test/web/admin_api/admin_api_controller_test.exs index 2a9e4f5a0..bc9235309 100644 --- a/test/web/admin_api/admin_api_controller_test.exs +++ b/test/web/admin_api/admin_api_controller_test.exs @@ -2269,6 +2269,10 @@ test "delete part of settings by atom subkeys", %{conn: conn} do Pleroma.Config.put([:instance, :dynamic_configuration], true) end + clear_config([:feed, :post_title]) do + Pleroma.Config.put([:feed, :post_title], %{max_length: 100, omission: "โ€ฆ"}) + end + test "transfer settings to DB and to file", %{conn: conn, admin: admin} do assert Pleroma.Repo.all(Pleroma.Web.AdminAPI.Config) == [] conn = get(conn, "/api/pleroma/admin/config/migrate_to_db") From b2e8371e6a366a1b75c520c617a23edfff3f1274 Mon Sep 17 00:00:00 2001 From: Maxim Filippov Date: Mon, 11 Nov 2019 09:55:00 +0000 Subject: [PATCH 077/107] Apply suggestion to CHANGELOG.md --- CHANGELOG.md | 1 - 1 file changed, 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8e03ee6b5..b33d61819 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -39,7 +39,6 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Authentication: Added rate limit for password-authorized actions / login existence checks - Mix task to re-count statuses for all users (`mix pleroma.count_statuses`) - Support for `X-Forwarded-For` and similar HTTP headers which used by reverse proxies to pass a real user IP address to the backend. Must not be enabled unless your instance is behind at least one reverse proxy (such as Nginx, Apache HTTPD or Varnish Cache). -- Admin API: Add ability to fetch reports, grouped by status `GET /api/pleroma/admin/grouped_reports`
API Changes From 31343e4321a3c4053b66a1d6dc3da0e42dbdd972 Mon Sep 17 00:00:00 2001 From: Maxim Filippov Date: Mon, 11 Nov 2019 19:06:09 +0900 Subject: [PATCH 078/107] Code style fixes --- CHANGELOG.md | 1 + docs/API/admin_api.md | 2 -- lib/pleroma/activity.ex | 7 +------ mix.lock | 2 +- 4 files changed, 3 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b33d61819..5442bfc5e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -43,6 +43,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). API Changes - Job queue stats to the healthcheck page +- Admin API: Add ability to fetch reports, grouped by status `GET /api/pleroma/admin/grouped_reports` - Admin API: Add ability to require password reset - Mastodon API: Account entities now include `follow_requests_count` (planned Mastodon 3.x addition) - Pleroma API: `GET /api/v1/pleroma/accounts/:id/scrobbles` to get a list of recently scrobbled items diff --git a/docs/API/admin_api.md b/docs/API/admin_api.md index ce70b5122..9d914c9a6 100644 --- a/docs/API/admin_api.md +++ b/docs/API/admin_api.md @@ -58,7 +58,6 @@ Authentication is required and the user must be an admin. ### Remove a user -- Method `DELETE` - Params: - `nicknames` - Response: Array of user nicknames @@ -735,7 +734,6 @@ Copy settings on key `:pleroma` to DB. Copy all settings from DB to `config/prod.exported_from_db.secret.exs` with deletion from DB. -- Method `GET` - Params: none - Response: diff --git a/lib/pleroma/activity.ex b/lib/pleroma/activity.ex index 7b77f72c2..7e283df32 100644 --- a/lib/pleroma/activity.ex +++ b/lib/pleroma/activity.ex @@ -92,12 +92,7 @@ def with_preloaded_object(query, join_type \\ :inner) do def with_joined_user_actor(query, join_type \\ :inner) do join(query, join_type, [activity], u in User, - on: - fragment( - "? = ?->>'actor'", - u.ap_id, - activity.data - ), + on: u.ap_id == activity.actor, as: :user_actor ) end diff --git a/mix.lock b/mix.lock index c707667b2..4529506a8 100644 --- a/mix.lock +++ b/mix.lock @@ -36,7 +36,7 @@ "ex_rated": {:hex, :ex_rated, "1.3.3", "30ecbdabe91f7eaa9d37fa4e81c85ba420f371babeb9d1910adbcd79ec798d27", [:mix], [{:ex2ms, "~> 1.5", [hex: :ex2ms, repo: "hexpm", optional: false]}], "hexpm"}, "ex_syslogger": {:git, "https://github.com/slashmili/ex_syslogger.git", "f3963399047af17e038897c69e20d552e6899e1d", [tag: "1.4.0"]}, "excoveralls": {:hex, :excoveralls, "0.11.2", "0c6f2c8db7683b0caa9d490fb8125709c54580b4255ffa7ad35f3264b075a643", [:mix], [{:hackney, "~> 1.0", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm"}, - "fast_html": {:hex, :fast_html, "0.99.0", "ea740358b15c7da6085b421b775f22d4f2c6928a28a15ebb5ad4e8a2ce00350b", [:make, :mix], [], "hexpm"}, + "fast_html": {:hex, :fast_html, "0.99.3", "e7ce6245fed0635f4719a31cc409091ed17b2091165a4a1cffbf2ceac77abbf4", [:make, :mix], [], "hexpm"}, "fast_sanitize": {:hex, :fast_sanitize, "0.1.1", "a403c3c09369e23423d3e6beb14068ad07be82741d10b293c71abac445dcc636", [:mix], [{:fast_html, "~> 0.99", [hex: :fast_html, repo: "hexpm", optional: false]}, {:plug, "~> 1.8", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"}, "flake_id": {:hex, :flake_id, "0.1.0", "7716b086d2e405d09b647121a166498a0d93d1a623bead243e1f74216079ccb3", [:mix], [{:base62, "~> 1.2", [hex: :base62, repo: "hexpm", optional: false]}, {:ecto, ">= 2.0.0", [hex: :ecto, repo: "hexpm", optional: true]}], "hexpm"}, "floki": {:hex, :floki, "0.23.0", "956ab6dba828c96e732454809fb0bd8d43ce0979b75f34de6322e73d4c917829", [:mix], [{:html_entities, "~> 0.4.0", [hex: :html_entities, repo: "hexpm", optional: false]}], "hexpm"}, From 1649d6f6894bbb2c36095d34eddd17d2e5f8d9df Mon Sep 17 00:00:00 2001 From: Maxim Filippov Date: Mon, 11 Nov 2019 19:16:04 +0900 Subject: [PATCH 079/107] Add "/api/pleroma/admin/reports/:id" -> "/api/pleroma/admin/reports" changelog entry --- CHANGELOG.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5442bfc5e..411b0b46d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - **Breaking** Admin API: `PATCH /api/pleroma/admin/users/:nickname/force_password_reset` is now `PATCH /api/pleroma/admin/users/force_password_reset` (accepts `nicknames` array in the request body) - **Breaking:** Admin API: Return link alongside with token on password reset +- **Breaking:** Admin API: `PUT /api/pleroma/admin/reports/:id` is now `PATCH /api/pleroma/admin/reports`, see admin_api.md for details - **Breaking:** `/api/pleroma/admin/users/invite_token` now uses `POST`, changed accepted params and returns full invite in json instead of only token string. - Admin API: Return `total` when querying for reports - Mastodon API: Return `pleroma.direct_conversation_id` when creating a direct message (`POST /api/v1/statuses`) @@ -54,7 +55,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Metadata Link: Atom syndication Feed - Mastodon API: Add `exclude_visibilities` parameter to the timeline and notification endpoints - Admin API: `/users/:nickname/toggle_activation` endpoint is now deprecated in favor of: `/users/activate`, `/users/deactivate`, both accept `nicknames` array -- Admin API: `POST/DELETE /api/pleroma/admin/users/:nickname/permission_group/:permission_group` are deprecated in favor of: `POST/DELETE /api/pleroma/admin/users/permission_group/:permission_group` (both accept `nicknames` array), `DELETE /api/pleroma/admin/users` (`nickname` query param or `nickname` sent in JSON body) is deprecated in favor of: `DELETE /api/pleroma/admin/users` (`nicknames` query array param or `nicknames` sent in JSON body). +- Admin API: Multiple endpoints now require `nicknames` array, instead of singe `nickname`: + - `POST/DELETE /api/pleroma/admin/users/:nickname/permission_group/:permission_group` are deprecated in favor of: `POST/DELETE /api/pleroma/admin/users/permission_group/:permission_group` + - `DELETE /api/pleroma/admin/users` (`nickname` query param or `nickname` sent in JSON body) is deprecated in favor of: `DELETE /api/pleroma/admin/users` (`nicknames` query array param or `nicknames` sent in JSON body) - Admin API: Add `GET /api/pleroma/admin/relay` endpoint - lists all followed relays - Pleroma API: `POST /api/v1/pleroma/conversations/read` to mark all conversations as read - Mastodon API: Add `/api/v1/markers` for managing timeline read markers From 8521553ad92981e9939ce6ce2208db685ecd068c Mon Sep 17 00:00:00 2001 From: lain Date: Mon, 11 Nov 2019 12:37:13 +0100 Subject: [PATCH 080/107] User: Don't let deactivated users authenticate. --- lib/pleroma/user.ex | 3 +++ test/user_test.exs | 7 +++++++ 2 files changed, 10 insertions(+) diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index f8c2db1e1..fcb1d5143 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -124,6 +124,9 @@ defmodule Pleroma.User do timestamps() end + @doc "Returns if the user should be allowed to authenticate" + def auth_active?(%User{deactivated: true}), do: false + def auth_active?(%User{confirmation_pending: true}), do: !Pleroma.Config.get([:instance, :account_activation_required]) diff --git a/test/user_test.exs b/test/user_test.exs index 6b1b24ce5..8fdb6b25f 100644 --- a/test/user_test.exs +++ b/test/user_test.exs @@ -1195,6 +1195,13 @@ test "auth_active?/1 works correctly" do refute User.auth_active?(local_user) assert User.auth_active?(confirmed_user) assert User.auth_active?(remote_user) + + # also shows unactive for deactivated users + + deactivated_but_confirmed = + insert(:user, local: true, confirmation_pending: false, deactivated: true) + + refute User.auth_active?(deactivated_but_confirmed) end describe "superuser?/1" do From f6056e9c9cdb280238845e4c9a0d2a1fb82cab78 Mon Sep 17 00:00:00 2001 From: lain Date: Mon, 11 Nov 2019 12:43:46 +0100 Subject: [PATCH 081/107] UserEnabledPlug: Don't authenticate unconfirmed users. --- lib/pleroma/plugs/user_enabled_plug.ex | 10 +++++++--- test/plugs/user_enabled_plug_test.exs | 17 +++++++++++++++++ 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/plugs/user_enabled_plug.ex b/lib/pleroma/plugs/user_enabled_plug.ex index fbb4bf115..8d102ee5b 100644 --- a/lib/pleroma/plugs/user_enabled_plug.ex +++ b/lib/pleroma/plugs/user_enabled_plug.ex @@ -10,9 +10,13 @@ def init(options) do options end - def call(%{assigns: %{user: %User{deactivated: true}}} = conn, _) do - conn - |> assign(:user, nil) + def call(%{assigns: %{user: %User{} = user}} = conn, _) do + if User.auth_active?(user) do + conn + else + conn + |> assign(:user, nil) + end end def call(conn, _) do diff --git a/test/plugs/user_enabled_plug_test.exs b/test/plugs/user_enabled_plug_test.exs index 996a7d77b..a4035bf0e 100644 --- a/test/plugs/user_enabled_plug_test.exs +++ b/test/plugs/user_enabled_plug_test.exs @@ -16,6 +16,23 @@ test "doesn't do anything if the user isn't set", %{conn: conn} do assert ret_conn == conn end + test "with a user that's not confirmed and a config requiring confirmation, it removes that user", + %{conn: conn} do + old = Pleroma.Config.get([:instance, :account_activation_required]) + Pleroma.Config.put([:instance, :account_activation_required], true) + + user = insert(:user, confirmation_pending: true) + + conn = + conn + |> assign(:user, user) + |> UserEnabledPlug.call(%{}) + + assert conn.assigns.user == nil + + Pleroma.Config.put([:instance, :account_activation_required], old) + end + test "with a user that is deactivated, it removes that user", %{conn: conn} do user = insert(:user, deactivated: true) From 94627baa5cca524fe0c4f7043e25d71ed245626a Mon Sep 17 00:00:00 2001 From: Steven Fuchs Date: Mon, 11 Nov 2019 12:13:06 +0000 Subject: [PATCH 082/107] New rate limiter --- lib/pleroma/application.ex | 3 +- lib/pleroma/plugs/rate_limiter.ex | 131 -------- .../plugs/rate_limiter/limiter_supervisor.ex | 44 +++ .../plugs/rate_limiter/rate_limiter.ex | 227 ++++++++++++++ lib/pleroma/plugs/rate_limiter/supervisor.ex | 16 + .../controllers/account_controller.ex | 6 +- .../controllers/auth_controller.ex | 2 +- .../controllers/search_controller.ex | 2 +- .../controllers/status_controller.ex | 6 +- .../web/mongooseim/mongoose_im_controller.ex | 4 +- lib/pleroma/web/oauth/oauth_controller.ex | 3 +- lib/pleroma/web/ostatus/ostatus_controller.ex | 5 +- .../controllers/account_controller.ex | 2 +- mix.exs | 1 - mix.lock | 1 - test/plugs/rate_limiter_test.exs | 285 ++++++++++-------- 16 files changed, 464 insertions(+), 274 deletions(-) delete mode 100644 lib/pleroma/plugs/rate_limiter.ex create mode 100644 lib/pleroma/plugs/rate_limiter/limiter_supervisor.ex create mode 100644 lib/pleroma/plugs/rate_limiter/rate_limiter.ex create mode 100644 lib/pleroma/plugs/rate_limiter/supervisor.ex diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index d681eecc8..2b6a55f98 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -36,7 +36,8 @@ def start(_type, _args) do Pleroma.Emoji, Pleroma.Captcha, Pleroma.Daemons.ScheduledActivityDaemon, - Pleroma.Daemons.ActivityExpirationDaemon + Pleroma.Daemons.ActivityExpirationDaemon, + Pleroma.Plugs.RateLimiter.Supervisor ] ++ cachex_children() ++ hackney_pool_children() ++ diff --git a/lib/pleroma/plugs/rate_limiter.ex b/lib/pleroma/plugs/rate_limiter.ex deleted file mode 100644 index 31388f574..000000000 --- a/lib/pleroma/plugs/rate_limiter.ex +++ /dev/null @@ -1,131 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright ยฉ 2017-2019 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Plugs.RateLimiter do - @moduledoc """ - - ## Configuration - - A keyword list of rate limiters where a key is a limiter name and value is the limiter configuration. The basic configuration is a tuple where: - - * The first element: `scale` (Integer). The time scale in milliseconds. - * The second element: `limit` (Integer). How many requests to limit in the time scale provided. - - It is also possible to have different limits for unauthenticated and authenticated users: the keyword value must be a list of two tuples where the first one is a config for unauthenticated users and the second one is for authenticated. - - To disable a limiter set its value to `nil`. - - ### Example - - config :pleroma, :rate_limit, - one: {1000, 10}, - two: [{10_000, 10}, {10_000, 50}], - foobar: nil - - Here we have three limiters: - - * `one` which is not over 10req/1s - * `two` which has two limits: 10req/10s for unauthenticated users and 50req/10s for authenticated users - * `foobar` which is disabled - - ## Usage - - AllowedSyntax: - - plug(Pleroma.Plugs.RateLimiter, :limiter_name) - plug(Pleroma.Plugs.RateLimiter, {:limiter_name, options}) - - Allowed options: - - * `bucket_name` overrides bucket name (e.g. to have a separate limit for a set of actions) - * `params` appends values of specified request params (e.g. ["id"]) to bucket name - - Inside a controller: - - plug(Pleroma.Plugs.RateLimiter, :one when action == :one) - plug(Pleroma.Plugs.RateLimiter, :two when action in [:two, :three]) - - plug( - Pleroma.Plugs.RateLimiter, - {:status_id_action, bucket_name: "status_id_action:fav_unfav", params: ["id"]} - when action in ~w(fav_status unfav_status)a - ) - - or inside a router pipeline: - - pipeline :api do - ... - plug(Pleroma.Plugs.RateLimiter, :one) - ... - end - """ - import Pleroma.Web.TranslationHelpers - import Plug.Conn - - alias Pleroma.User - - def init(limiter_name) when is_atom(limiter_name) do - init({limiter_name, []}) - end - - def init({limiter_name, opts}) do - case Pleroma.Config.get([:rate_limit, limiter_name]) do - nil -> nil - config -> {limiter_name, config, opts} - end - end - - # Do not limit if there is no limiter configuration - def call(conn, nil), do: conn - - def call(conn, settings) do - case check_rate(conn, settings) do - {:ok, _count} -> - conn - - {:error, _count} -> - render_throttled_error(conn) - end - end - - defp bucket_name(conn, limiter_name, opts) do - bucket_name = opts[:bucket_name] || limiter_name - - if params_names = opts[:params] do - params_values = for p <- Enum.sort(params_names), do: conn.params[p] - Enum.join([bucket_name] ++ params_values, ":") - else - bucket_name - end - end - - defp check_rate( - %{assigns: %{user: %User{id: user_id}}} = conn, - {limiter_name, [_, {scale, limit}], opts} - ) do - bucket_name = bucket_name(conn, limiter_name, opts) - ExRated.check_rate("#{bucket_name}:#{user_id}", scale, limit) - end - - defp check_rate(conn, {limiter_name, [{scale, limit} | _], opts}) do - bucket_name = bucket_name(conn, limiter_name, opts) - ExRated.check_rate("#{bucket_name}:#{ip(conn)}", scale, limit) - end - - defp check_rate(conn, {limiter_name, {scale, limit}, opts}) do - check_rate(conn, {limiter_name, [{scale, limit}, {scale, limit}], opts}) - end - - def ip(%{remote_ip: remote_ip}) do - remote_ip - |> Tuple.to_list() - |> Enum.join(".") - end - - defp render_throttled_error(conn) do - conn - |> render_error(:too_many_requests, "Throttled") - |> halt() - end -end diff --git a/lib/pleroma/plugs/rate_limiter/limiter_supervisor.ex b/lib/pleroma/plugs/rate_limiter/limiter_supervisor.ex new file mode 100644 index 000000000..187582ede --- /dev/null +++ b/lib/pleroma/plugs/rate_limiter/limiter_supervisor.ex @@ -0,0 +1,44 @@ +defmodule Pleroma.Plugs.RateLimiter.LimiterSupervisor do + use DynamicSupervisor + + import Cachex.Spec + + def start_link(init_arg) do + DynamicSupervisor.start_link(__MODULE__, init_arg, name: __MODULE__) + end + + def add_limiter(limiter_name, expiration) do + {:ok, _pid} = + DynamicSupervisor.start_child( + __MODULE__, + %{ + id: String.to_atom("rl_#{limiter_name}"), + start: + {Cachex, :start_link, + [ + limiter_name, + [ + expiration: + expiration( + default: expiration, + interval: check_interval(expiration), + lazy: true + ) + ] + ]} + } + ) + end + + @impl true + def init(_init_arg) do + DynamicSupervisor.init(strategy: :one_for_one) + end + + defp check_interval(exp) do + (exp / 2) + |> Kernel.trunc() + |> Kernel.min(5000) + |> Kernel.max(1) + end +end diff --git a/lib/pleroma/plugs/rate_limiter/rate_limiter.ex b/lib/pleroma/plugs/rate_limiter/rate_limiter.ex new file mode 100644 index 000000000..d720508c8 --- /dev/null +++ b/lib/pleroma/plugs/rate_limiter/rate_limiter.ex @@ -0,0 +1,227 @@ +# Pleroma: A lightweight social networking server +# Copyright ยฉ 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Plugs.RateLimiter do + @moduledoc """ + + ## Configuration + + A keyword list of rate limiters where a key is a limiter name and value is the limiter configuration. The basic configuration is a tuple where: + + * The first element: `scale` (Integer). The time scale in milliseconds. + * The second element: `limit` (Integer). How many requests to limit in the time scale provided. + + It is also possible to have different limits for unauthenticated and authenticated users: the keyword value must be a list of two tuples where the first one is a config for unauthenticated users and the second one is for authenticated. + + To disable a limiter set its value to `nil`. + + ### Example + + config :pleroma, :rate_limit, + one: {1000, 10}, + two: [{10_000, 10}, {10_000, 50}], + foobar: nil + + Here we have three limiters: + + * `one` which is not over 10req/1s + * `two` which has two limits: 10req/10s for unauthenticated users and 50req/10s for authenticated users + * `foobar` which is disabled + + ## Usage + + AllowedSyntax: + + plug(Pleroma.Plugs.RateLimiter, name: :limiter_name) + plug(Pleroma.Plugs.RateLimiter, options) # :name is a required option + + Allowed options: + + * `name` required, always used to fetch the limit values from the config + * `bucket_name` overrides name for counting purposes (e.g. to have a separate limit for a set of actions) + * `params` appends values of specified request params (e.g. ["id"]) to bucket name + + Inside a controller: + + plug(Pleroma.Plugs.RateLimiter, [name: :one] when action == :one) + plug(Pleroma.Plugs.RateLimiter, [name: :two] when action in [:two, :three]) + + plug( + Pleroma.Plugs.RateLimiter, + [name: :status_id_action, bucket_name: "status_id_action:fav_unfav", params: ["id"]] + when action in ~w(fav_status unfav_status)a + ) + + or inside a router pipeline: + + pipeline :api do + ... + plug(Pleroma.Plugs.RateLimiter, name: :one) + ... + end + """ + import Pleroma.Web.TranslationHelpers + import Plug.Conn + + alias Pleroma.Plugs.RateLimiter.LimiterSupervisor + alias Pleroma.User + + def init(opts) do + limiter_name = Keyword.get(opts, :name) + + case Pleroma.Config.get([:rate_limit, limiter_name]) do + nil -> + nil + + config -> + name_root = Keyword.get(opts, :bucket_name, limiter_name) + + %{ + name: name_root, + limits: config, + opts: opts + } + end + end + + # Do not limit if there is no limiter configuration + def call(conn, nil), do: conn + + def call(conn, settings) do + settings + |> incorporate_conn_info(conn) + |> check_rate() + |> case do + {:ok, _count} -> + conn + + {:error, _count} -> + render_throttled_error(conn) + end + end + + def inspect_bucket(conn, name_root, settings) do + settings = + settings + |> incorporate_conn_info(conn) + + bucket_name = make_bucket_name(%{settings | name: name_root}) + key_name = make_key_name(settings) + limit = get_limits(settings) + + case Cachex.get(bucket_name, key_name) do + {:error, :no_cache} -> + {:err, :not_found} + + {:ok, nil} -> + {0, limit} + + {:ok, value} -> + {value, limit - value} + end + end + + defp check_rate(settings) do + bucket_name = make_bucket_name(settings) + key_name = make_key_name(settings) + limit = get_limits(settings) + + case Cachex.get_and_update(bucket_name, key_name, &increment_value(&1, limit)) do + {:commit, value} -> + {:ok, value} + + {:ignore, value} -> + {:error, value} + + {:error, :no_cache} -> + initialize_buckets(settings) + check_rate(settings) + end + end + + defp increment_value(nil, _limit), do: {:commit, 1} + + defp increment_value(val, limit) when val >= limit, do: {:ignore, val} + + defp increment_value(val, _limit), do: {:commit, val + 1} + + defp incorporate_conn_info(settings, %{assigns: %{user: %User{id: user_id}}, params: params}) do + Map.merge(settings, %{ + mode: :user, + conn_params: params, + conn_info: "#{user_id}" + }) + end + + defp incorporate_conn_info(settings, %{params: params} = conn) do + Map.merge(settings, %{ + mode: :anon, + conn_params: params, + conn_info: "#{ip(conn)}" + }) + end + + defp ip(%{remote_ip: remote_ip}) do + remote_ip + |> Tuple.to_list() + |> Enum.join(".") + end + + defp render_throttled_error(conn) do + conn + |> render_error(:too_many_requests, "Throttled") + |> halt() + end + + defp make_key_name(settings) do + "" + |> attach_params(settings) + |> attach_identity(settings) + end + + defp get_scale(_, {scale, _}), do: scale + + defp get_scale(:anon, [{scale, _}, {_, _}]), do: scale + + defp get_scale(:user, [{_, _}, {scale, _}]), do: scale + + defp get_limits(%{limits: {_scale, limit}}), do: limit + + defp get_limits(%{mode: :user, limits: [_, {_, limit}]}), do: limit + + defp get_limits(%{limits: [{_, limit}, _]}), do: limit + + defp make_bucket_name(%{mode: :user, name: name_root}), + do: user_bucket_name(name_root) + + defp make_bucket_name(%{mode: :anon, name: name_root}), + do: anon_bucket_name(name_root) + + defp attach_params(input, %{conn_params: conn_params, opts: opts}) do + param_string = + opts + |> Keyword.get(:params, []) + |> Enum.sort() + |> Enum.map(&Map.get(conn_params, &1, "")) + |> Enum.join(":") + + "#{input}#{param_string}" + end + + defp initialize_buckets(%{name: _name, limits: nil}), do: :ok + + defp initialize_buckets(%{name: name, limits: limits}) do + LimiterSupervisor.add_limiter(anon_bucket_name(name), get_scale(:anon, limits)) + LimiterSupervisor.add_limiter(user_bucket_name(name), get_scale(:user, limits)) + end + + defp attach_identity(base, %{mode: :user, conn_info: conn_info}), + do: "user:#{base}:#{conn_info}" + + defp attach_identity(base, %{mode: :anon, conn_info: conn_info}), + do: "ip:#{base}:#{conn_info}" + + defp user_bucket_name(name_root), do: "user:#{name_root}" |> String.to_atom() + defp anon_bucket_name(name_root), do: "anon:#{name_root}" |> String.to_atom() +end diff --git a/lib/pleroma/plugs/rate_limiter/supervisor.ex b/lib/pleroma/plugs/rate_limiter/supervisor.ex new file mode 100644 index 000000000..9672f7876 --- /dev/null +++ b/lib/pleroma/plugs/rate_limiter/supervisor.ex @@ -0,0 +1,16 @@ +defmodule Pleroma.Plugs.RateLimiter.Supervisor do + use Supervisor + + def start_link(opts) do + Supervisor.start_link(__MODULE__, opts, name: __MODULE__) + end + + def init(_args) do + children = [ + Pleroma.Plugs.RateLimiter.LimiterSupervisor + ] + + opts = [strategy: :one_for_one, name: Pleroma.Web.Streamer.Supervisor] + Supervisor.init(children, opts) + end +end diff --git a/lib/pleroma/web/mastodon_api/controllers/account_controller.ex b/lib/pleroma/web/mastodon_api/controllers/account_controller.ex index 73fad519e..5b01b964b 100644 --- a/lib/pleroma/web/mastodon_api/controllers/account_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/account_controller.ex @@ -66,9 +66,9 @@ defmodule Pleroma.Web.MastodonAPI.AccountController do @relations [:follow, :unfollow] @needs_account ~W(followers following lists follow unfollow mute unmute block unblock)a - plug(RateLimiter, {:relations_id_action, params: ["id", "uri"]} when action in @relations) - plug(RateLimiter, :relations_actions when action in @relations) - plug(RateLimiter, :app_account_creation when action == :create) + plug(RateLimiter, [name: :relations_id_action, params: ["id", "uri"]] when action in @relations) + plug(RateLimiter, [name: :relations_actions] when action in @relations) + plug(RateLimiter, [name: :app_account_creation] when action == :create) plug(:assign_account_by_id when action in @needs_account) action_fallback(Pleroma.Web.MastodonAPI.FallbackController) diff --git a/lib/pleroma/web/mastodon_api/controllers/auth_controller.ex b/lib/pleroma/web/mastodon_api/controllers/auth_controller.ex index bfd5120ba..d9e51de7f 100644 --- a/lib/pleroma/web/mastodon_api/controllers/auth_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/auth_controller.ex @@ -15,7 +15,7 @@ defmodule Pleroma.Web.MastodonAPI.AuthController do @local_mastodon_name "Mastodon-Local" - plug(Pleroma.Plugs.RateLimiter, :password_reset when action == :password_reset) + plug(Pleroma.Plugs.RateLimiter, [name: :password_reset] when action == :password_reset) @doc "GET /web/login" def login(%{assigns: %{user: %User{}}} = conn, _params) do diff --git a/lib/pleroma/web/mastodon_api/controllers/search_controller.ex b/lib/pleroma/web/mastodon_api/controllers/search_controller.ex index 6cfd68a84..0a929f55b 100644 --- a/lib/pleroma/web/mastodon_api/controllers/search_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/search_controller.ex @@ -22,7 +22,7 @@ defmodule Pleroma.Web.MastodonAPI.SearchController do plug(Pleroma.Plugs.EnsurePublicOrAuthenticatedPlug) - plug(RateLimiter, :search when action in [:search, :search2, :account_search]) + plug(RateLimiter, [name: :search] when action in [:search, :search2, :account_search]) def account_search(%{assigns: %{user: user}} = conn, %{"q" => query} = params) do accounts = User.search(query, search_options(params, user)) diff --git a/lib/pleroma/web/mastodon_api/controllers/status_controller.ex b/lib/pleroma/web/mastodon_api/controllers/status_controller.ex index e5d016f63..74b223cf4 100644 --- a/lib/pleroma/web/mastodon_api/controllers/status_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/status_controller.ex @@ -82,17 +82,17 @@ defmodule Pleroma.Web.MastodonAPI.StatusController do plug( RateLimiter, - {:status_id_action, bucket_name: "status_id_action:reblog_unreblog", params: ["id"]} + [name: :status_id_action, bucket_name: "status_id_action:reblog_unreblog", params: ["id"]] when action in ~w(reblog unreblog)a ) plug( RateLimiter, - {:status_id_action, bucket_name: "status_id_action:fav_unfav", params: ["id"]} + [name: :status_id_action, bucket_name: "status_id_action:fav_unfav", params: ["id"]] when action in ~w(favourite unfavourite)a ) - plug(RateLimiter, :statuses_actions when action in @rate_limited_status_actions) + plug(RateLimiter, [name: :statuses_actions] when action in @rate_limited_status_actions) action_fallback(Pleroma.Web.MastodonAPI.FallbackController) diff --git a/lib/pleroma/web/mongooseim/mongoose_im_controller.ex b/lib/pleroma/web/mongooseim/mongoose_im_controller.ex index 6ed181cff..358600e7d 100644 --- a/lib/pleroma/web/mongooseim/mongoose_im_controller.ex +++ b/lib/pleroma/web/mongooseim/mongoose_im_controller.ex @@ -10,8 +10,8 @@ defmodule Pleroma.Web.MongooseIM.MongooseIMController do alias Pleroma.Repo alias Pleroma.User - plug(RateLimiter, :authentication when action in [:user_exists, :check_password]) - plug(RateLimiter, {:authentication, params: ["user"]} when action == :check_password) + plug(RateLimiter, [name: :authentication] when action in [:user_exists, :check_password]) + plug(RateLimiter, [name: :authentication, params: ["user"]] when action == :check_password) def user_exists(conn, %{"user" => username}) do with %User{} <- Repo.get_by(User, nickname: username, local: true) do diff --git a/lib/pleroma/web/oauth/oauth_controller.ex b/lib/pleroma/web/oauth/oauth_controller.ex index fe71aca8c..1b1394787 100644 --- a/lib/pleroma/web/oauth/oauth_controller.ex +++ b/lib/pleroma/web/oauth/oauth_controller.ex @@ -6,6 +6,7 @@ defmodule Pleroma.Web.OAuth.OAuthController do use Pleroma.Web, :controller alias Pleroma.Helpers.UriHelper + alias Pleroma.Plugs.RateLimiter alias Pleroma.Registration alias Pleroma.Repo alias Pleroma.User @@ -24,7 +25,7 @@ defmodule Pleroma.Web.OAuth.OAuthController do plug(:fetch_session) plug(:fetch_flash) - plug(Pleroma.Plugs.RateLimiter, :authentication when action == :create_authorization) + plug(RateLimiter, [name: :authentication] when action == :create_authorization) action_fallback(Pleroma.Web.OAuth.FallbackController) diff --git a/lib/pleroma/web/ostatus/ostatus_controller.ex b/lib/pleroma/web/ostatus/ostatus_controller.ex index 6958519de..12a7c2365 100644 --- a/lib/pleroma/web/ostatus/ostatus_controller.ex +++ b/lib/pleroma/web/ostatus/ostatus_controller.ex @@ -8,6 +8,7 @@ defmodule Pleroma.Web.OStatus.OStatusController do alias Fallback.RedirectController alias Pleroma.Activity alias Pleroma.Object + alias Pleroma.Plugs.RateLimiter alias Pleroma.User alias Pleroma.Web.ActivityPub.ActivityPubController alias Pleroma.Web.ActivityPub.ObjectView @@ -17,8 +18,8 @@ defmodule Pleroma.Web.OStatus.OStatusController do alias Pleroma.Web.Router plug( - Pleroma.Plugs.RateLimiter, - {:ap_routes, params: ["uuid"]} when action in [:object, :activity] + RateLimiter, + [name: :ap_routes, params: ["uuid"]] when action in [:object, :activity] ) plug( diff --git a/lib/pleroma/web/pleroma_api/controllers/account_controller.ex b/lib/pleroma/web/pleroma_api/controllers/account_controller.ex index db6faac83..bc2f1017c 100644 --- a/lib/pleroma/web/pleroma_api/controllers/account_controller.ex +++ b/lib/pleroma/web/pleroma_api/controllers/account_controller.ex @@ -42,7 +42,7 @@ defmodule Pleroma.Web.PleromaAPI.AccountController do when action != :confirmation_resend ) - plug(RateLimiter, :account_confirmation_resend when action == :confirmation_resend) + plug(RateLimiter, [name: :account_confirmation_resend] when action == :confirmation_resend) plug(:assign_account_by_id when action in [:favourites, :subscribe, :unsubscribe]) plug(:put_view, Pleroma.Web.MastodonAPI.AccountView) diff --git a/mix.exs b/mix.exs index dd7c7e979..81ce4f25c 100644 --- a/mix.exs +++ b/mix.exs @@ -155,7 +155,6 @@ defp deps do {:joken, "~> 2.0"}, {:benchee, "~> 1.0"}, {:esshd, "~> 0.1.0", runtime: Application.get_env(:esshd, :enabled, false)}, - {:ex_rated, "~> 1.3"}, {:ex_const, "~> 0.2"}, {:plug_static_index_html, "~> 1.0.0"}, {:excoveralls, "~> 0.11.1", only: :test}, diff --git a/mix.lock b/mix.lock index 5b471fe3d..d4a80df77 100644 --- a/mix.lock +++ b/mix.lock @@ -33,7 +33,6 @@ "ex_const": {:hex, :ex_const, "0.2.4", "d06e540c9d834865b012a17407761455efa71d0ce91e5831e86881b9c9d82448", [:mix], [], "hexpm"}, "ex_doc": {:hex, :ex_doc, "0.21.2", "caca5bc28ed7b3bdc0b662f8afe2bee1eedb5c3cf7b322feeeb7c6ebbde089d6", [:mix], [{:earmark, "~> 1.3.3 or ~> 1.4", [hex: :earmark, repo: "hexpm", optional: false]}, {:makeup_elixir, "~> 0.14", [hex: :makeup_elixir, repo: "hexpm", optional: false]}], "hexpm"}, "ex_machina": {:hex, :ex_machina, "2.3.0", "92a5ad0a8b10ea6314b876a99c8c9e3f25f4dde71a2a835845b136b9adaf199a", [:mix], [{:ecto, "~> 2.2 or ~> 3.0", [hex: :ecto, repo: "hexpm", optional: true]}, {:ecto_sql, "~> 3.0", [hex: :ecto_sql, repo: "hexpm", optional: true]}], "hexpm"}, - "ex_rated": {:hex, :ex_rated, "1.3.3", "30ecbdabe91f7eaa9d37fa4e81c85ba420f371babeb9d1910adbcd79ec798d27", [:mix], [{:ex2ms, "~> 1.5", [hex: :ex2ms, repo: "hexpm", optional: false]}], "hexpm"}, "ex_syslogger": {:git, "https://github.com/slashmili/ex_syslogger.git", "f3963399047af17e038897c69e20d552e6899e1d", [tag: "1.4.0"]}, "excoveralls": {:hex, :excoveralls, "0.11.2", "0c6f2c8db7683b0caa9d490fb8125709c54580b4255ffa7ad35f3264b075a643", [:mix], [{:hackney, "~> 1.0", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm"}, "fast_html": {:hex, :fast_html, "0.99.3", "e7ce6245fed0635f4719a31cc409091ed17b2091165a4a1cffbf2ceac77abbf4", [:make, :mix], [], "hexpm"}, diff --git a/test/plugs/rate_limiter_test.exs b/test/plugs/rate_limiter_test.exs index 395095079..bacd621e1 100644 --- a/test/plugs/rate_limiter_test.exs +++ b/test/plugs/rate_limiter_test.exs @@ -12,163 +12,196 @@ defmodule Pleroma.Plugs.RateLimiterTest do # Note: each example must work with separate buckets in order to prevent concurrency issues - test "init/1" do - limiter_name = :test_init - Pleroma.Config.put([:rate_limit, limiter_name], {1, 1}) + describe "config" do + test "config is required for plug to work" do + limiter_name = :test_init + Pleroma.Config.put([:rate_limit, limiter_name], {1, 1}) - assert {limiter_name, {1, 1}, []} == RateLimiter.init(limiter_name) - assert nil == RateLimiter.init(:foo) + assert %{limits: {1, 1}, name: :test_init, opts: [name: :test_init]} == + RateLimiter.init(name: limiter_name) + + assert nil == RateLimiter.init(name: :foo) + end + + test "it restricts based on config values" do + limiter_name = :test_opts + scale = 60 + limit = 5 + + Pleroma.Config.put([:rate_limit, limiter_name], {scale, limit}) + + opts = RateLimiter.init(name: limiter_name) + conn = conn(:get, "/") + + for i <- 1..5 do + conn = RateLimiter.call(conn, opts) + assert {^i, _} = RateLimiter.inspect_bucket(conn, limiter_name, opts) + Process.sleep(10) + end + + conn = RateLimiter.call(conn, opts) + assert %{"error" => "Throttled"} = Phoenix.ConnTest.json_response(conn, :too_many_requests) + assert conn.halted + + Process.sleep(50) + + conn = conn(:get, "/") + + conn = RateLimiter.call(conn, opts) + assert {1, 4} = RateLimiter.inspect_bucket(conn, limiter_name, opts) + + refute conn.status == Plug.Conn.Status.code(:too_many_requests) + refute conn.resp_body + refute conn.halted + end end - test "ip/1" do - assert "127.0.0.1" == RateLimiter.ip(%{remote_ip: {127, 0, 0, 1}}) + describe "options" do + test "`bucket_name` option overrides default bucket name" do + limiter_name = :test_bucket_name + + Pleroma.Config.put([:rate_limit, limiter_name], {1000, 5}) + + base_bucket_name = "#{limiter_name}:group1" + opts = RateLimiter.init(name: limiter_name, bucket_name: base_bucket_name) + + conn = conn(:get, "/") + + RateLimiter.call(conn, opts) + assert {1, 4} = RateLimiter.inspect_bucket(conn, base_bucket_name, opts) + assert {:err, :not_found} = RateLimiter.inspect_bucket(conn, limiter_name, opts) + end + + test "`params` option allows different queries to be tracked independently" do + limiter_name = :test_params + Pleroma.Config.put([:rate_limit, limiter_name], {1000, 5}) + + opts = RateLimiter.init(name: limiter_name, params: ["id"]) + + conn = conn(:get, "/?id=1") + conn = Plug.Conn.fetch_query_params(conn) + conn_2 = conn(:get, "/?id=2") + + RateLimiter.call(conn, opts) + assert {1, 4} = RateLimiter.inspect_bucket(conn, limiter_name, opts) + assert {0, 5} = RateLimiter.inspect_bucket(conn_2, limiter_name, opts) + end + + test "it supports combination of options modifying bucket name" do + limiter_name = :test_options_combo + Pleroma.Config.put([:rate_limit, limiter_name], {1000, 5}) + + base_bucket_name = "#{limiter_name}:group1" + opts = RateLimiter.init(name: limiter_name, bucket_name: base_bucket_name, params: ["id"]) + id = "100" + + conn = conn(:get, "/?id=#{id}") + conn = Plug.Conn.fetch_query_params(conn) + conn_2 = conn(:get, "/?id=#{101}") + + RateLimiter.call(conn, opts) + assert {1, 4} = RateLimiter.inspect_bucket(conn, base_bucket_name, opts) + assert {0, 5} = RateLimiter.inspect_bucket(conn_2, base_bucket_name, opts) + end end - test "it restricts by opts" do - limiter_name = :test_opts - scale = 1000 - limit = 5 + describe "unauthenticated users" do + test "are restricted based on remote IP" do + limiter_name = :test_unauthenticated + Pleroma.Config.put([:rate_limit, limiter_name], [{1000, 5}, {1, 10}]) - Pleroma.Config.put([:rate_limit, limiter_name], {scale, limit}) + opts = RateLimiter.init(name: limiter_name) - opts = RateLimiter.init(limiter_name) - conn = conn(:get, "/") - bucket_name = "#{limiter_name}:#{RateLimiter.ip(conn)}" + conn = %{conn(:get, "/") | remote_ip: {127, 0, 0, 2}} + conn_2 = %{conn(:get, "/") | remote_ip: {127, 0, 0, 3}} - conn = RateLimiter.call(conn, opts) - assert {1, 4, _, _, _} = ExRated.inspect_bucket(bucket_name, scale, limit) + for i <- 1..5 do + conn = RateLimiter.call(conn, opts) + assert {^i, _} = RateLimiter.inspect_bucket(conn, limiter_name, opts) + refute conn.halted + end - conn = RateLimiter.call(conn, opts) - assert {2, 3, _, _, _} = ExRated.inspect_bucket(bucket_name, scale, limit) + conn = RateLimiter.call(conn, opts) - conn = RateLimiter.call(conn, opts) - assert {3, 2, _, _, _} = ExRated.inspect_bucket(bucket_name, scale, limit) + assert %{"error" => "Throttled"} = Phoenix.ConnTest.json_response(conn, :too_many_requests) + assert conn.halted - conn = RateLimiter.call(conn, opts) - assert {4, 1, _, _, _} = ExRated.inspect_bucket(bucket_name, scale, limit) + conn_2 = RateLimiter.call(conn_2, opts) + assert {1, 4} = RateLimiter.inspect_bucket(conn_2, limiter_name, opts) - conn = RateLimiter.call(conn, opts) - assert {5, 0, to_reset, _, _} = ExRated.inspect_bucket(bucket_name, scale, limit) - - conn = RateLimiter.call(conn, opts) - - assert %{"error" => "Throttled"} = Phoenix.ConnTest.json_response(conn, :too_many_requests) - assert conn.halted - - Process.sleep(to_reset) - - conn = conn(:get, "/") - - conn = RateLimiter.call(conn, opts) - assert {1, 4, _, _, _} = ExRated.inspect_bucket(bucket_name, scale, limit) - - refute conn.status == Plug.Conn.Status.code(:too_many_requests) - refute conn.resp_body - refute conn.halted + refute conn_2.status == Plug.Conn.Status.code(:too_many_requests) + refute conn_2.resp_body + refute conn_2.halted + end end - test "`bucket_name` option overrides default bucket name" do - limiter_name = :test_bucket_name - scale = 1000 - limit = 5 + describe "authenticated users" do + setup do + Ecto.Adapters.SQL.Sandbox.checkout(Pleroma.Repo) - Pleroma.Config.put([:rate_limit, limiter_name], {scale, limit}) - base_bucket_name = "#{limiter_name}:group1" - opts = RateLimiter.init({limiter_name, bucket_name: base_bucket_name}) + :ok + end - conn = conn(:get, "/") - default_bucket_name = "#{limiter_name}:#{RateLimiter.ip(conn)}" - customized_bucket_name = "#{base_bucket_name}:#{RateLimiter.ip(conn)}" + test "can have limits seperate from unauthenticated connections" do + limiter_name = :test_authenticated - RateLimiter.call(conn, opts) - assert {1, 4, _, _, _} = ExRated.inspect_bucket(customized_bucket_name, scale, limit) - assert {0, 5, _, _, _} = ExRated.inspect_bucket(default_bucket_name, scale, limit) - end + scale = 1000 + limit = 5 + Pleroma.Config.put([:rate_limit, limiter_name], [{1, 10}, {scale, limit}]) - test "`params` option appends specified params' values to bucket name" do - limiter_name = :test_params - scale = 1000 - limit = 5 + opts = RateLimiter.init(name: limiter_name) - Pleroma.Config.put([:rate_limit, limiter_name], {scale, limit}) - opts = RateLimiter.init({limiter_name, params: ["id"]}) - id = "1" + user = insert(:user) + conn = conn(:get, "/") |> assign(:user, user) - conn = conn(:get, "/?id=#{id}") - conn = Plug.Conn.fetch_query_params(conn) + for i <- 1..5 do + conn = RateLimiter.call(conn, opts) + assert {^i, _} = RateLimiter.inspect_bucket(conn, limiter_name, opts) + refute conn.halted + end - default_bucket_name = "#{limiter_name}:#{RateLimiter.ip(conn)}" - parametrized_bucket_name = "#{limiter_name}:#{id}:#{RateLimiter.ip(conn)}" + conn = RateLimiter.call(conn, opts) - RateLimiter.call(conn, opts) - assert {1, 4, _, _, _} = ExRated.inspect_bucket(parametrized_bucket_name, scale, limit) - assert {0, 5, _, _, _} = ExRated.inspect_bucket(default_bucket_name, scale, limit) - end + assert %{"error" => "Throttled"} = Phoenix.ConnTest.json_response(conn, :too_many_requests) + assert conn.halted - test "it supports combination of options modifying bucket name" do - limiter_name = :test_options_combo - scale = 1000 - limit = 5 + Process.sleep(1550) - Pleroma.Config.put([:rate_limit, limiter_name], {scale, limit}) - base_bucket_name = "#{limiter_name}:group1" - opts = RateLimiter.init({limiter_name, bucket_name: base_bucket_name, params: ["id"]}) - id = "100" + conn = conn(:get, "/") |> assign(:user, user) + conn = RateLimiter.call(conn, opts) + assert {1, 4} = RateLimiter.inspect_bucket(conn, limiter_name, opts) - conn = conn(:get, "/?id=#{id}") - conn = Plug.Conn.fetch_query_params(conn) + refute conn.status == Plug.Conn.Status.code(:too_many_requests) + refute conn.resp_body + refute conn.halted + end - default_bucket_name = "#{limiter_name}:#{RateLimiter.ip(conn)}" - parametrized_bucket_name = "#{base_bucket_name}:#{id}:#{RateLimiter.ip(conn)}" + test "diffrerent users are counted independently" do + limiter_name = :test_authenticated + Pleroma.Config.put([:rate_limit, limiter_name], [{1, 10}, {1000, 5}]) - RateLimiter.call(conn, opts) - assert {1, 4, _, _, _} = ExRated.inspect_bucket(parametrized_bucket_name, scale, limit) - assert {0, 5, _, _, _} = ExRated.inspect_bucket(default_bucket_name, scale, limit) - end + opts = RateLimiter.init(name: limiter_name) - test "optional limits for authenticated users" do - limiter_name = :test_authenticated - Ecto.Adapters.SQL.Sandbox.checkout(Pleroma.Repo) + user = insert(:user) + conn = conn(:get, "/") |> assign(:user, user) - scale = 1000 - limit = 5 - Pleroma.Config.put([:rate_limit, limiter_name], [{1, 10}, {scale, limit}]) + user_2 = insert(:user) + conn_2 = conn(:get, "/") |> assign(:user, user_2) - opts = RateLimiter.init(limiter_name) + for i <- 1..5 do + conn = RateLimiter.call(conn, opts) + assert {^i, _} = RateLimiter.inspect_bucket(conn, limiter_name, opts) + end - user = insert(:user) - conn = conn(:get, "/") |> assign(:user, user) - bucket_name = "#{limiter_name}:#{user.id}" + conn = RateLimiter.call(conn, opts) + assert %{"error" => "Throttled"} = Phoenix.ConnTest.json_response(conn, :too_many_requests) + assert conn.halted - conn = RateLimiter.call(conn, opts) - assert {1, 4, _, _, _} = ExRated.inspect_bucket(bucket_name, scale, limit) - - conn = RateLimiter.call(conn, opts) - assert {2, 3, _, _, _} = ExRated.inspect_bucket(bucket_name, scale, limit) - - conn = RateLimiter.call(conn, opts) - assert {3, 2, _, _, _} = ExRated.inspect_bucket(bucket_name, scale, limit) - - conn = RateLimiter.call(conn, opts) - assert {4, 1, _, _, _} = ExRated.inspect_bucket(bucket_name, scale, limit) - - conn = RateLimiter.call(conn, opts) - assert {5, 0, to_reset, _, _} = ExRated.inspect_bucket(bucket_name, scale, limit) - - conn = RateLimiter.call(conn, opts) - - assert %{"error" => "Throttled"} = Phoenix.ConnTest.json_response(conn, :too_many_requests) - assert conn.halted - - Process.sleep(to_reset) - - conn = conn(:get, "/") |> assign(:user, user) - - conn = RateLimiter.call(conn, opts) - assert {1, 4, _, _, _} = ExRated.inspect_bucket(bucket_name, scale, limit) - - refute conn.status == Plug.Conn.Status.code(:too_many_requests) - refute conn.resp_body - refute conn.halted + conn_2 = RateLimiter.call(conn_2, opts) + assert {1, 4} = RateLimiter.inspect_bucket(conn_2, limiter_name, opts) + refute conn_2.status == Plug.Conn.Status.code(:too_many_requests) + refute conn_2.resp_body + refute conn_2.halted + end end end From ab2e61238ee0dd5df0744c8bd8a7ffb089c4a0f8 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Mon, 11 Nov 2019 19:13:07 +0700 Subject: [PATCH 083/107] Add a warning about Pg version to the RUM related docs --- docs/configuration/cheatsheet.md | 6 +++++- docs/installation/otp_en.md | 8 ++++++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/docs/configuration/cheatsheet.md b/docs/configuration/cheatsheet.md index 8f609fcfd..61783cf3f 100644 --- a/docs/configuration/cheatsheet.md +++ b/docs/configuration/cheatsheet.md @@ -523,7 +523,7 @@ config :pleroma, :workers, Configuration for [Quantum](https://github.com/quantum-elixir/quantum-core) jobs scheduler. -See [Quantum readme](https://github.com/quantum-elixir/quantum-core#usage) for the list of supported options. +See [Quantum readme](https://github.com/quantum-elixir/quantum-core#usage) for the list of supported options. Example: @@ -593,6 +593,10 @@ See the [Quack Github](https://github.com/azohra/quack) for more details ## Database options ### RUM indexing for full text search + +!!! warning + It is recommended to use PostgreSQL v11 or newer. We have seen some minor issues with lower PostgreSQL versions. + * `rum_enabled`: If RUM indexes should be used. Defaults to `false`. RUM indexes are an alternative indexing scheme that is not included in PostgreSQL by default. While they may eventually be mainlined, for now they have to be installed as a PostgreSQL extension from https://github.com/postgrespro/rum. diff --git a/docs/installation/otp_en.md b/docs/installation/otp_en.md index c028f4229..965e30e2a 100644 --- a/docs/installation/otp_en.md +++ b/docs/installation/otp_en.md @@ -42,6 +42,10 @@ apk add curl unzip ncurses postgresql postgresql-contrib nginx certbot ## Setup ### Configuring PostgreSQL #### (Optional) Installing RUM indexes + +!!! warning + It is recommended to use PostgreSQL v11 or newer. We have seen some minor issues with lower PostgreSQL versions. + RUM indexes are an alternative indexing scheme that is not included in PostgreSQL by default. You can read more about them on the [Configuration page](../configuration/cheatsheet.md#rum-indexing-for-full-text-search). They are completely optional and most of the time are not worth it, especially if you are running a single user instance (unless you absolutely need ordered search results). Debian/Ubuntu (available only on Buster/19.04): @@ -74,7 +78,7 @@ rc-service postgresql restart # Create the Pleroma user adduser --system --shell /bin/false --home /opt/pleroma pleroma -# Set the flavour environment variable to the string you got in Detecting flavour section. +# Set the flavour environment variable to the string you got in Detecting flavour section. # For example if the flavour is `arm64-musl` the command will be export FLAVOUR="arm64-musl" @@ -180,7 +184,7 @@ rc-service pleroma start rc-update add pleroma ``` -If everything worked, you should see Pleroma-FE when visiting your domain. If that didn't happen, try reviewing the installation steps, starting Pleroma in the foreground and seeing if there are any errrors. +If everything worked, you should see Pleroma-FE when visiting your domain. If that didn't happen, try reviewing the installation steps, starting Pleroma in the foreground and seeing if there are any errrors. Still doesn't work? Feel free to contact us on [#pleroma on freenode](https://webchat.freenode.net/?channels=%23pleroma) or via matrix at , you can also [file an issue on our Gitlab](https://git.pleroma.social/pleroma/pleroma/issues/new) From 86d821a96eb9da10455c90c0638cc0a3c594ad7f Mon Sep 17 00:00:00 2001 From: lain Date: Mon, 11 Nov 2019 13:37:56 +0100 Subject: [PATCH 084/107] Dokku deploys: Keep the git dir so version number generation works. --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0f8a0659b..4f448a784 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -113,6 +113,7 @@ review_app: - echo "$SSH_PRIVATE_KEY" | tr -d '\r' | ssh-add - - ssh-keyscan -H "pleroma.online" >> ~/.ssh/known_hosts - (ssh -t dokku@pleroma.online -- apps:create "$CI_ENVIRONMENT_SLUG") || true + - (ssh -t dokku@pleroma.online -- git:set "$CI_ENVIRONMENT_SLUG" keep-git-dir true) || true - ssh -t dokku@pleroma.online -- config:set "$CI_ENVIRONMENT_SLUG" APP_NAME="$CI_ENVIRONMENT_SLUG" APP_HOST="$CI_ENVIRONMENT_SLUG.pleroma.online" MIX_ENV=dokku - (ssh -t dokku@pleroma.online -- postgres:create $(echo $CI_ENVIRONMENT_SLUG | sed -e 's/-/_/g')_db) || true - (ssh -t dokku@pleroma.online -- postgres:link $(echo $CI_ENVIRONMENT_SLUG | sed -e 's/-/_/g')_db "$CI_ENVIRONMENT_SLUG") || true From 7ba30cf8b6ee86297562d6af50b267ec0967a63b Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Mon, 11 Nov 2019 19:47:33 +0700 Subject: [PATCH 085/107] Use PG12 in CI for the RUM pipeline --- .gitlab-ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0f8a0659b..b801f28c4 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -34,7 +34,7 @@ benchmark: variables: MIX_ENV: benchmark services: - - name: lainsoykaf/postgres-with-rum + - name: postgres:9.6 alias: postgres command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"] script: @@ -46,7 +46,7 @@ benchmark: unit-testing: stage: test services: - - name: lainsoykaf/postgres-with-rum + - name: postgres:9.6 alias: postgres command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"] script: @@ -58,7 +58,7 @@ unit-testing: unit-testing-rum: stage: test services: - - name: lainsoykaf/postgres-with-rum + - name: minibikini/postgres-with-rum:12 alias: postgres command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"] variables: @@ -138,7 +138,7 @@ stop_review_app: - ssh -t dokku@pleroma.online -- --force postgres:destroy $(echo $CI_ENVIRONMENT_SLUG | sed -e 's/-/_/g')_db amd64: - stage: release + stage: release # TODO: Replace with upstream image when 1.9.0 comes out image: rinpatch/elixir:1.9.0-rc.0 only: &release-only From 72cc92259ea2f9d299943b845f7a339255cf99fe Mon Sep 17 00:00:00 2001 From: lain Date: Mon, 11 Nov 2019 12:49:18 +0000 Subject: [PATCH 086/107] Default config: Use extended nickname format --- config/config.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/config.exs b/config/config.exs index 54de8fa9f..17d15256f 100644 --- a/config/config.exs +++ b/config/config.exs @@ -274,7 +274,7 @@ account_field_name_length: 512, account_field_value_length: 2048, external_user_synchronization: true, - extended_nickname_format: false + extended_nickname_format: true config :pleroma, :feed, post_title: %{ From 827b938502627e048a486fa6e17a181acaf0b508 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Mon, 11 Nov 2019 17:05:30 +0300 Subject: [PATCH 087/107] Add a changelog entry for !1940 --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4ec084dbd..727dde9be 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -64,6 +64,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ### Fixed - Report emails now include functional links to profiles of remote user accounts +- Not being able to log in to some third-party apps when logged in to MastoFE
API Changes From b39b49cc146945ab86db272ae2cd1fe8fad3d9d5 Mon Sep 17 00:00:00 2001 From: href Date: Mon, 11 Nov 2019 19:03:43 +0100 Subject: [PATCH 088/107] report federating status in nodeinfo --- .../web/nodeinfo/nodeinfo_controller.ex | 1 + test/web/node_info_test.exs | 24 +++++++++++++++++++ 2 files changed, 25 insertions(+) diff --git a/lib/pleroma/web/nodeinfo/nodeinfo_controller.ex b/lib/pleroma/web/nodeinfo/nodeinfo_controller.ex index d7ae503f6..486b9f6a4 100644 --- a/lib/pleroma/web/nodeinfo/nodeinfo_controller.ex +++ b/lib/pleroma/web/nodeinfo/nodeinfo_controller.ex @@ -46,6 +46,7 @@ def raw_nodeinfo do data |> Map.merge(%{quarantined_instances: quarantined}) + |> Map.put(:enabled, Config.get([:instance, :federating])) else %{} end diff --git a/test/web/node_info_test.exs b/test/web/node_info_test.exs index a3281b25b..6cc876602 100644 --- a/test/web/node_info_test.exs +++ b/test/web/node_info_test.exs @@ -84,6 +84,30 @@ test "it returns the safe_dm_mentions feature if enabled", %{conn: conn} do Pleroma.Config.put([:instance, :safe_dm_mentions], option) end + test "it shows if federation is enabled/disabled", %{conn: conn} do + original = Pleroma.Config.get([:instance, :federating]) + + Pleroma.Config.put([:instance, :federating], true) + + response = + conn + |> get("/nodeinfo/2.1.json") + |> json_response(:ok) + + assert response["metadata"]["federation"]["enabled"] == true + + Pleroma.Config.put([:instance, :federating], false) + + response = + conn + |> get("/nodeinfo/2.1.json") + |> json_response(:ok) + + assert response["metadata"]["federation"]["enabled"] == false + + Pleroma.Config.put([:instance, :federating], original) + end + test "it shows MRF transparency data if enabled", %{conn: conn} do config = Pleroma.Config.get([:instance, :rewrite_policy]) Pleroma.Config.put([:instance, :rewrite_policy], [Pleroma.Web.ActivityPub.MRF.SimplePolicy]) From 04473677ccd70281172b8295b44feebb01168386 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Mon, 11 Nov 2019 23:24:15 +0300 Subject: [PATCH 089/107] docs: move static-fe docs under a proper category --- docs/configuration/cheatsheet.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/configuration/cheatsheet.md b/docs/configuration/cheatsheet.md index 6c7f60203..7832f6962 100644 --- a/docs/configuration/cheatsheet.md +++ b/docs/configuration/cheatsheet.md @@ -180,6 +180,14 @@ config :pleroma, :frontend_configurations, These settings **need to be complete**, they will override the defaults. +### :static_fe + +Render profiles and posts using server-generated HTML that is viewable without using JavaScript. + +Available options: + +* `enabled` - Enables the rendering of static HTML. Defaults to `false`. + ### :assets This section configures assets to be used with various frontends. Currently the only option @@ -797,11 +805,3 @@ config :auto_linker, rel: "ugc" ] ``` - -## :static_fe - -Render profiles and posts using server-generated HTML that is viewable without using JavaScript. - -Available options: - -* `enabled` - Enables the rendering of static HTML. Defaults to `false`. From 7d101bc9c5f8ffc1d78c8d5e22f630ad0a7d7e1b Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Mon, 11 Nov 2019 18:29:55 -0600 Subject: [PATCH 090/107] Fix rendering conversations when there's a malformed status --- lib/pleroma/web/mastodon_api/views/conversation_view.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/mastodon_api/views/conversation_view.ex b/lib/pleroma/web/mastodon_api/views/conversation_view.ex index c5998e661..51d6c0898 100644 --- a/lib/pleroma/web/mastodon_api/views/conversation_view.ex +++ b/lib/pleroma/web/mastodon_api/views/conversation_view.ex @@ -12,7 +12,7 @@ defmodule Pleroma.Web.MastodonAPI.ConversationView do alias Pleroma.Web.MastodonAPI.StatusView def render("participations.json", %{participations: participations, for: user}) do - render_many(participations, __MODULE__, "participation.json", as: :participation, for: user) + safe_render_many(participations, __MODULE__, "participation.json", %{as: :participation, for: user}) end def render("participation.json", %{participation: participation, for: user}) do From e835cd97f6988522dae8f60a0381f0f93c6abb2d Mon Sep 17 00:00:00 2001 From: lain Date: Tue, 12 Nov 2019 12:07:17 +0100 Subject: [PATCH 091/107] Containment: Add a catch-all clause to contain_origin. --- lib/pleroma/object/containment.ex | 2 ++ test/object/containment_test.exs | 10 ++++++++++ 2 files changed, 12 insertions(+) diff --git a/lib/pleroma/object/containment.ex b/lib/pleroma/object/containment.ex index a1f9c1250..25aa32f60 100644 --- a/lib/pleroma/object/containment.ex +++ b/lib/pleroma/object/containment.ex @@ -64,6 +64,8 @@ def contain_origin(id, %{"actor" => _actor} = params) do def contain_origin(id, %{"attributedTo" => actor} = params), do: contain_origin(id, Map.put(params, "actor", actor)) + def contain_origin(_id, _data), do: :error + def contain_origin_from_id(id, %{"id" => other_id} = _params) when is_binary(other_id) do id_uri = URI.parse(id) other_uri = URI.parse(other_id) diff --git a/test/object/containment_test.exs b/test/object/containment_test.exs index 71fe5204c..7636803a6 100644 --- a/test/object/containment_test.exs +++ b/test/object/containment_test.exs @@ -17,6 +17,16 @@ defmodule Pleroma.Object.ContainmentTest do end describe "general origin containment" do + test "works for completely actorless posts" do + assert :error == + Containment.contain_origin("https://glaceon.social/users/monorail", %{ + "deleted" => "2019-10-30T05:48:50.249606Z", + "formerType" => "Note", + "id" => "https://glaceon.social/users/monorail/statuses/103049757364029187", + "type" => "Tombstone" + }) + end + test "contain_origin_from_id() catches obvious spoofing attempts" do data = %{ "id" => "http://example.com/~alyssa/activities/1234.json" From 62f3a93049649dee0ccd7b883887be2fd343fb3e Mon Sep 17 00:00:00 2001 From: Phil Hagelberg Date: Mon, 11 Nov 2019 17:16:44 -0800 Subject: [PATCH 092/107] For remote notices, redirect to the original instead of 404. We shouldn't treat these like local statuses, but I don't think a 404 is the right choice either here, because within pleroma-fe, these are valid URLs. So with remote notices you have the awkward situation where clicking a link will behave differently depending on whether you open it in a new tab or not; the new tab will 404 if it hits static-fe. This new redirecting behavior should improve that situation. --- lib/pleroma/web/static_fe/static_fe_controller.ex | 5 +++++ test/web/static_fe/static_fe_controller_test.exs | 4 ++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/web/static_fe/static_fe_controller.ex b/lib/pleroma/web/static_fe/static_fe_controller.ex index 5e60c82b0..ba44b8a4f 100644 --- a/lib/pleroma/web/static_fe/static_fe_controller.ex +++ b/lib/pleroma/web/static_fe/static_fe_controller.ex @@ -77,6 +77,11 @@ def show(%{assigns: %{notice_id: notice_id}} = conn, _params) do render(conn, "conversation.html", %{activities: timeline, meta: meta}) else + %Activity{object: %Object{data: data}} -> + conn + |> put_status(:found) + |> redirect(external: data["url"] || data["external_url"] || data["id"]) + _ -> conn |> put_status(404) diff --git a/test/web/static_fe/static_fe_controller_test.exs b/test/web/static_fe/static_fe_controller_test.exs index effdfbeb3..b8fb67b22 100644 --- a/test/web/static_fe/static_fe_controller_test.exs +++ b/test/web/static_fe/static_fe_controller_test.exs @@ -151,7 +151,7 @@ test "404 for private status", %{conn: conn} do assert html_response(conn, 404) =~ "not found" end - test "404 for remote cached status", %{conn: conn} do + test "302 for remote cached status", %{conn: conn} do user = insert(:user) message = %{ @@ -175,7 +175,7 @@ test "404 for remote cached status", %{conn: conn} do |> put_req_header("accept", "text/html") |> get("/notice/#{activity.id}") - assert html_response(conn, 404) =~ "not found" + assert html_response(conn, 302) =~ "redirected" end end end From 72cf6a76f4064f226552802b201ba0902084c52a Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Wed, 13 Nov 2019 18:07:53 +0700 Subject: [PATCH 093/107] Fix random fails of the rate limiter tests --- test/plugs/rate_limiter_test.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/plugs/rate_limiter_test.exs b/test/plugs/rate_limiter_test.exs index bacd621e1..49f63c424 100644 --- a/test/plugs/rate_limiter_test.exs +++ b/test/plugs/rate_limiter_test.exs @@ -25,7 +25,7 @@ test "config is required for plug to work" do test "it restricts based on config values" do limiter_name = :test_opts - scale = 60 + scale = 80 limit = 5 Pleroma.Config.put([:rate_limit, limiter_name], {scale, limit}) From 3350cd8d965f45b51257ab6b9d5073fc81bdaa06 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Wed, 13 Nov 2019 18:48:53 +0700 Subject: [PATCH 094/107] Fix formatting in OpenBSD install manual --- docs/installation/openbsd_en.md | 38 +++++++++++++++++++++------------ 1 file changed, 24 insertions(+), 14 deletions(-) diff --git a/docs/installation/openbsd_en.md b/docs/installation/openbsd_en.md index 3585a326b..45602bd75 100644 --- a/docs/installation/openbsd_en.md +++ b/docs/installation/openbsd_en.md @@ -1,9 +1,13 @@ # Installing on OpenBSD + This guide describes the installation and configuration of pleroma (and the required software to run it) on a single OpenBSD 6.4 server. + For any additional information regarding commands and configuration files mentioned here, check the man pages [online](https://man.openbsd.org/) or directly on your server with the man command. #### Required software + The following packages need to be installed: + * elixir * gmake * ImageMagick @@ -11,8 +15,11 @@ The following packages need to be installed: * postgresql-server * postgresql-contrib -To install them, run the following command (with doas or as root): -`pkg_add elixir gmake ImageMagick git postgresql-server postgresql-contrib` +To install them, run the following command (with doas or as root): + +``` +pkg_add elixir gmake ImageMagick git postgresql-server postgresql-contrib +``` Pleroma requires a reverse proxy, OpenBSD has relayd in base (and is used in this guide) and packages/ports are available for nginx (www/nginx) and apache (www/apache-httpd). Independently of the reverse proxy, [acme-client(1)](https://man.openbsd.org/acme-client) can be used to get a certificate from Let's Encrypt. @@ -31,8 +38,8 @@ Create the \_pleroma user, assign it the pleroma login class and create its home #### Clone pleroma's directory Enter a shell as the \_pleroma user. As root, run `su _pleroma -;cd`. Then clone the repository with `git clone -b stable https://git.pleroma.social/pleroma/pleroma.git`. Pleroma is now installed in /home/\_pleroma/pleroma/, it will be configured and started at the end of this guide. -#### Postgresql -Start a shell as the \_postgresql user (as root run `su _postgresql -` then run the `initdb` command to initialize postgresql: +#### PostgreSQL +Start a shell as the \_postgresql user (as root run `su _postgresql -` then run the `initdb` command to initialize postgresql: If you wish to not use the default location for postgresql's data (/var/postgresql/data), add the following switch at the end of the command: `-D ` and modify the `datadir` variable in the /etc/rc.d/postgresql script. When this is done, enable postgresql so that it starts on boot and start it. As root, run: @@ -44,6 +51,7 @@ To check that it started properly and didn't fail right after starting, you can #### httpd httpd will have three fuctions: + * redirect requests trying to reach the instance over http to the https URL * serve a robots.txt file * get Let's Encrypt certificates, with acme-client @@ -76,9 +84,9 @@ types { include "/usr/share/misc/mime.types" } ``` -Do not forget to change *\* to your server's address(es). If httpd should only listen on one protocol family, comment one of the two first *listen* options. +Do not forget to change ** to your server's address(es). If httpd should only listen on one protocol family, comment one of the two first *listen* options. -Create the /var/www/htdocs/local/ folder and write the content of your robots.txt in /var/www/htdocs/local/robots.txt. +Create the /var/www/htdocs/local/ folder and write the content of your robots.txt in /var/www/htdocs/local/robots.txt. Check the configuration with `httpd -n`, if it is OK enable and start httpd (as root): ``` rcctl enable httpd @@ -86,7 +94,7 @@ rcctl start httpd ``` #### acme-client -acme-client is used to get SSL/TLS certificates from Let's Encrypt. +acme-client is used to get SSL/TLS certificates from Let's Encrypt. Insert the following configuration in /etc/acme-client.conf: ``` # @@ -107,7 +115,7 @@ domain { challengedir "/var/www/acme/" } ``` -Replace *\* by the domain name you'll use for your instance. As root, run `acme-client -n` to check the config, then `acme-client -ADv ` to create account and domain keys, and request a certificate for the first time. +Replace ** by the domain name you'll use for your instance. As root, run `acme-client -n` to check the config, then `acme-client -ADv ` to create account and domain keys, and request a certificate for the first time. Make acme-client run everyday by adding it in /etc/daily.local. As root, run the following command: `echo "acme-client " >> /etc/daily.local`. Relayd will look for certificates and keys based on the address it listens on (see next part), the easiest way to make them available to relayd is to create a link, as root run: @@ -118,7 +126,7 @@ ln -s /etc/ssl/private/.key /etc/ssl/private/.key This will have to be done for each IPv4 and IPv6 address relayd listens on. #### relayd -relayd will be used as the reverse proxy sitting in front of pleroma. +relayd will be used as the reverse proxy sitting in front of pleroma. Insert the following configuration in /etc/relayd.conf: ``` # $OpenBSD: relayd.conf,v 1.4 2018/03/23 09:55:06 claudio Exp $ @@ -169,7 +177,7 @@ relay wwwtls { forward to port 80 check http "/robots.txt" code 200 } ``` -Again, change *\* to your server's address(es) and comment one of the two *listen* options if needed. Also change *wss://CHANGEME.tld* to *wss://\*. +Again, change ** to your server's address(es) and comment one of the two *listen* options if needed. Also change *wss://CHANGEME.tld* to *wss://*. Check the configuration with `relayd -n`, if it is OK enable and start relayd (as root): ``` rcctl enable relayd @@ -177,7 +185,7 @@ rcctl start relayd ``` #### pf -Enabling and configuring pf is highly recommended. +Enabling and configuring pf is highly recommended. In /etc/pf.conf, insert the following configuration: ``` # Macros @@ -202,20 +210,22 @@ pass in quick on $if inet6 proto icmp6 to ($if) icmp6-type { echoreq unreach par pass in quick on $if proto tcp to ($if) port { http https } # relayd/httpd pass in quick on $if proto tcp from $authorized_ssh_clients to ($if) port ssh ``` -Replace *\* by your server's network interface name (which you can get with ifconfig). Consider replacing the content of the authorized\_ssh\_clients macro by, for exemple, your home IP address, to avoid SSH connection attempts from bots. +Replace ** by your server's network interface name (which you can get with ifconfig). Consider replacing the content of the authorized\_ssh\_clients macro by, for exemple, your home IP address, to avoid SSH connection attempts from bots. Check pf's configuration by running `pfctl -nf /etc/pf.conf`, load it with `pfctl -f /etc/pf.conf` and enable pf at boot with `rcctl enable pf`. #### Configure and start pleroma -Enter a shell as \_pleroma (as root `su _pleroma -`) and enter pleroma's installation directory (`cd ~/pleroma/`). +Enter a shell as \_pleroma (as root `su _pleroma -`) and enter pleroma's installation directory (`cd ~/pleroma/`). + Then follow the main installation guide: + * run `mix deps.get` * run `mix pleroma.instance gen` and enter your instance's information when asked * copy config/generated\_config.exs to config/prod.secret.exs. The default values should be sufficient but you should edit it and check that everything seems OK. * exit your current shell back to a root one and run `psql -U postgres -f /home/_pleroma/config/setup_db.psql` to setup the database. * return to a \_pleroma shell into pleroma's installation directory (`su _pleroma -;cd ~/pleroma`) and run `MIX_ENV=prod mix ecto.migrate` -As \_pleroma in /home/\_pleroma/pleroma, you can now run `LC_ALL=en_US.UTF-8 MIX_ENV=prod mix phx.server` to start your instance. +As \_pleroma in /home/\_pleroma/pleroma, you can now run `LC_ALL=en_US.UTF-8 MIX_ENV=prod mix phx.server` to start your instance. In another SSH session/tmux window, check that it is working properly by running `ftp -MVo - http://127.0.0.1:4000/api/v1/instance`, you should get json output. Double-check that *uri*'s value is your instance's domain name. ##### Starting pleroma at boot From 0867cb083eb469ae10cd48d424a51efb2fae4018 Mon Sep 17 00:00:00 2001 From: Phil Hagelberg Date: Tue, 12 Nov 2019 17:19:46 -0800 Subject: [PATCH 095/107] Support redirecting by object ID in static FE. This matches the behavior of pleroma-fe better. Fixes #1412. --- lib/pleroma/web/static_fe/static_fe_controller.ex | 15 +++++++++++++++ test/web/static_fe/static_fe_controller_test.exs | 15 +++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/lib/pleroma/web/static_fe/static_fe_controller.ex b/lib/pleroma/web/static_fe/static_fe_controller.ex index ba44b8a4f..b45d82c2d 100644 --- a/lib/pleroma/web/static_fe/static_fe_controller.ex +++ b/lib/pleroma/web/static_fe/static_fe_controller.ex @@ -119,11 +119,26 @@ def show(%{assigns: %{username_or_id: username_or_id}} = conn, params) do end end + def show(%{assigns: %{object_id: _}} = conn, _params) do + url = Helpers.url(conn) <> conn.request_path + case Activity.get_create_by_object_ap_id_with_object(url) do + %Activity{} = activity -> + redirect(conn, to: "/notice/#{activity.id}") + _ -> + conn + |> put_status(404) + |> render("error.html", %{message: "Post not found.", meta: ""}) + end + end + def assign_id(%{path_info: ["notice", notice_id]} = conn, _opts), do: assign(conn, :notice_id, notice_id) def assign_id(%{path_info: ["users", user_id]} = conn, _opts), do: assign(conn, :username_or_id, user_id) + def assign_id(%{path_info: ["objects", object_id]} = conn, _opts), + do: assign(conn, :object_id, object_id) + def assign_id(conn, _opts), do: conn end diff --git a/test/web/static_fe/static_fe_controller_test.exs b/test/web/static_fe/static_fe_controller_test.exs index b8fb67b22..6ea8cea34 100644 --- a/test/web/static_fe/static_fe_controller_test.exs +++ b/test/web/static_fe/static_fe_controller_test.exs @@ -1,5 +1,6 @@ defmodule Pleroma.Web.StaticFE.StaticFEControllerTest do use Pleroma.Web.ConnCase + alias Pleroma.Activity alias Pleroma.Web.ActivityPub.Transmogrifier alias Pleroma.Web.CommonAPI @@ -128,6 +129,20 @@ test "shows the whole thread", %{conn: conn} do assert html =~ "voyages" end + test "redirect by AP object ID", %{conn: conn} do + user = insert(:user) + + {:ok, %Activity{data: %{"object" => object_url}}} = + CommonAPI.post(user, %{"status" => "beam me up"}) + + conn = + conn + |> put_req_header("accept", "text/html") + |> get(URI.parse(object_url).path) + + assert html_response(conn, 302) =~ "redirected" + end + test "404 when notice not found", %{conn: conn} do conn = conn From 3c60adbc1f773c732458d68b4becaf9bb36d7062 Mon Sep 17 00:00:00 2001 From: Phil Hagelberg Date: Tue, 12 Nov 2019 17:33:54 -0800 Subject: [PATCH 096/107] Support redirecting by activity UUID in static FE as well. --- .../web/static_fe/static_fe_controller.ex | 41 ++++++++++++++----- .../static_fe/static_fe_controller_test.exs | 14 +++++++ 2 files changed, 44 insertions(+), 11 deletions(-) diff --git a/lib/pleroma/web/static_fe/static_fe_controller.ex b/lib/pleroma/web/static_fe/static_fe_controller.ex index b45d82c2d..8ccf15f4b 100644 --- a/lib/pleroma/web/static_fe/static_fe_controller.ex +++ b/lib/pleroma/web/static_fe/static_fe_controller.ex @@ -27,6 +27,12 @@ defp get_title(%Object{data: %{"summary" => summary}}) when is_binary(summary), defp get_title(_), do: nil + defp not_found(conn, message) do + conn + |> put_status(404) + |> render("error.html", %{message: message, meta: ""}) + end + def get_counts(%Activity{} = activity) do %Object{data: data} = Object.normalize(activity) @@ -83,9 +89,7 @@ def show(%{assigns: %{notice_id: notice_id}} = conn, _params) do |> redirect(external: data["url"] || data["external_url"] || data["id"]) _ -> - conn - |> put_status(404) - |> render("error.html", %{message: "Post not found.", meta: ""}) + not_found(conn, "Post not found.") end end @@ -113,21 +117,33 @@ def show(%{assigns: %{username_or_id: username_or_id}} = conn, params) do }) _ -> - conn - |> put_status(404) - |> render("error.html", %{message: "User not found.", meta: ""}) + not_found(conn, "User not found.") end end def show(%{assigns: %{object_id: _}} = conn, _params) do url = Helpers.url(conn) <> conn.request_path + case Activity.get_create_by_object_ap_id_with_object(url) do %Activity{} = activity -> - redirect(conn, to: "/notice/#{activity.id}") - _ -> - conn - |> put_status(404) - |> render("error.html", %{message: "Post not found.", meta: ""}) + to = Helpers.o_status_path(Pleroma.Web.Endpoint, :notice, activity) + redirect(conn, to: to) + + _ -> + not_found(conn, "Post not found.") + end + end + + def show(%{assigns: %{activity_id: _}} = conn, _params) do + url = Helpers.url(conn) <> conn.request_path + + case Activity.get_by_ap_id(url) do + %Activity{} = activity -> + to = Helpers.o_status_path(Pleroma.Web.Endpoint, :notice, activity) + redirect(conn, to: to) + + _ -> + not_found(conn, "Post not found.") end end @@ -140,5 +156,8 @@ def assign_id(%{path_info: ["users", user_id]} = conn, _opts), def assign_id(%{path_info: ["objects", object_id]} = conn, _opts), do: assign(conn, :object_id, object_id) + def assign_id(%{path_info: ["activities", activity_id]} = conn, _opts), + do: assign(conn, :activity_id, activity_id) + def assign_id(conn, _opts), do: conn end diff --git a/test/web/static_fe/static_fe_controller_test.exs b/test/web/static_fe/static_fe_controller_test.exs index 6ea8cea34..2ce8f9fa3 100644 --- a/test/web/static_fe/static_fe_controller_test.exs +++ b/test/web/static_fe/static_fe_controller_test.exs @@ -143,6 +143,20 @@ test "redirect by AP object ID", %{conn: conn} do assert html_response(conn, 302) =~ "redirected" end + test "redirect by activity ID", %{conn: conn} do + user = insert(:user) + + {:ok, %Activity{data: %{"id" => id}}} = + CommonAPI.post(user, %{"status" => "I'm a doctor, not a devops!"}) + + conn = + conn + |> put_req_header("accept", "text/html") + |> get(URI.parse(id).path) + + assert html_response(conn, 302) =~ "redirected" + end + test "404 when notice not found", %{conn: conn} do conn = conn From 7a322713c33e8ef2fdc326f9e35d1fcbe7590c93 Mon Sep 17 00:00:00 2001 From: lain Date: Thu, 14 Nov 2019 08:21:06 +0000 Subject: [PATCH 097/107] Update CHANGELOG.md --- CHANGELOG.md | 1 - 1 file changed, 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c9b447d09..0464f4571 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -59,7 +59,6 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Pleroma API: `POST /api/v1/pleroma/conversations/read` to mark all conversations as read - Mastodon API: Add `/api/v1/markers` for managing timeline read markers - Mastodon API: Add the `recipients` parameter to `GET /api/v1/conversations` -- Pleroma API: Add Emoji reactions - Configuration: `feed` option for user atom feed. - Pleroma API: Add Emoji reactions
From 2861f97e4616ea7a708215b440cf568c0124d2b3 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Thu, 14 Nov 2019 16:06:08 +0300 Subject: [PATCH 098/107] ci: disable --trace for unit tests it is mostly useless, but makes failures harder to find --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d915ebae9..ab62c8827 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -53,7 +53,7 @@ unit-testing: - mix deps.get - mix ecto.create - mix ecto.migrate - - mix coveralls --trace --preload-modules + - mix coveralls --preload-modules unit-testing-rum: stage: test @@ -68,7 +68,7 @@ unit-testing-rum: - mix ecto.create - mix ecto.migrate - "mix ecto.migrate --migrations-path priv/repo/optional_migrations/rum_indexing/" - - mix test --trace --preload-modules + - mix test --preload-modules lint: stage: test From 94f1cfced872924592eff39b00781989a9e2f96f Mon Sep 17 00:00:00 2001 From: rinpatch Date: Thu, 14 Nov 2019 17:26:59 +0300 Subject: [PATCH 099/107] format the code --- lib/pleroma/web/mastodon_api/views/conversation_view.ex | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/web/mastodon_api/views/conversation_view.ex b/lib/pleroma/web/mastodon_api/views/conversation_view.ex index 51d6c0898..2220fbcb1 100644 --- a/lib/pleroma/web/mastodon_api/views/conversation_view.ex +++ b/lib/pleroma/web/mastodon_api/views/conversation_view.ex @@ -12,7 +12,10 @@ defmodule Pleroma.Web.MastodonAPI.ConversationView do alias Pleroma.Web.MastodonAPI.StatusView def render("participations.json", %{participations: participations, for: user}) do - safe_render_many(participations, __MODULE__, "participation.json", %{as: :participation, for: user}) + safe_render_many(participations, __MODULE__, "participation.json", %{ + as: :participation, + for: user + }) end def render("participation.json", %{participation: participation, for: user}) do From 1d33eeca7224b3b1db97444920b3414b5f65fe69 Mon Sep 17 00:00:00 2001 From: Ariadne Conill Date: Thu, 14 Nov 2019 18:26:06 -0600 Subject: [PATCH 100/107] config: add configuration for MRF ObjectAgePolicy --- config/config.exs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/config/config.exs b/config/config.exs index 75f463797..bf2b3f6e2 100644 --- a/config/config.exs +++ b/config/config.exs @@ -381,6 +381,10 @@ accept: [], reject: [] +config :pleroma, :mrf_object_age, + threshold: 172_800, + actions: [:delist, :strip_followers] + config :pleroma, :rich_media, enabled: true, ignore_hosts: [], From 29f49bf5fa9d033d88ddd2440db703681a25cc4e Mon Sep 17 00:00:00 2001 From: Ariadne Conill Date: Thu, 14 Nov 2019 18:31:30 -0600 Subject: [PATCH 101/107] docs: document MRF ObjectAgePolicy --- docs/configuration/cheatsheet.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/configuration/cheatsheet.md b/docs/configuration/cheatsheet.md index 7832f6962..d798bd692 100644 --- a/docs/configuration/cheatsheet.md +++ b/docs/configuration/cheatsheet.md @@ -41,6 +41,7 @@ You shouldn't edit the base config directly to avoid breakages and merge conflic * `Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy`: Crawls attachments using their MediaProxy URLs so that the MediaProxy cache is primed. * `Pleroma.Web.ActivityPub.MRF.MentionPolicy`: Drops posts mentioning configurable users. (See [`:mrf_mention`](#mrf_mention)). * `Pleroma.Web.ActivityPub.MRF.VocabularyPolicy`: Restricts activities to a configured set of vocabulary. (See [`:mrf_vocabulary`](#mrf_vocabulary)). + * `Pleroma.Web.ActivityPub.MRF.ObjectAgePolicy`: Rejects or delists posts based on their age when received. (See [`:mrf_object_age`](#mrf_object_age)). * `public`: Makes the client API in authentificated mode-only except for user-profiles. Useful for disabling the Local Timeline and The Whole Known Network. * `quarantined_instances`: List of ActivityPub instances where private(DMs, followers-only) activities will not be send. * `managed_config`: Whenether the config for pleroma-fe is configured in [:frontend_configurations](#frontend_configurations) or in ``static/config.json``. @@ -137,6 +138,13 @@ config :pleroma, :mrf_user_allowlist, "example.org": ["https://example.org/users/admin"] ``` +#### :mrf_object_age +* `threshold`: Required age (in seconds) of a post before actions are taken. +* `actions`: A list of actions to apply to the post: + * `:delist` removes the post from public timelines + * `:strip_followers` removes followers from the ActivityPub recipient list, ensuring they won't be delivered to home timelines + * `:reject` rejects the message entirely + ### :activitypub * ``unfollow_blocked``: Whether blocks result in people getting unfollowed * ``outgoing_blocks``: Whether to federate blocks to other instances From 5705cf0e3e675c142442a6183d5613ae936f3276 Mon Sep 17 00:00:00 2001 From: Ariadne Conill Date: Thu, 14 Nov 2019 19:48:10 -0600 Subject: [PATCH 102/107] MRF: add ObjectAgePolicy which deals with old posts being imported --- .../web/activity_pub/mrf/object_age_policy.ex | 103 +++++++++++++++++ .../mrf/object_age_policy_test.exs | 105 ++++++++++++++++++ 2 files changed, 208 insertions(+) create mode 100644 lib/pleroma/web/activity_pub/mrf/object_age_policy.ex create mode 100644 test/web/activity_pub/mrf/object_age_policy_test.exs diff --git a/lib/pleroma/web/activity_pub/mrf/object_age_policy.ex b/lib/pleroma/web/activity_pub/mrf/object_age_policy.ex new file mode 100644 index 000000000..f6c6f31cb --- /dev/null +++ b/lib/pleroma/web/activity_pub/mrf/object_age_policy.ex @@ -0,0 +1,103 @@ +# Pleroma: A lightweight social networking server +# Copyright ยฉ 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ActivityPub.MRF.ObjectAgePolicy do + alias Pleroma.Config + alias Pleroma.User + alias Pleroma.Web.ActivityPub.MRF + + require Logger + require Pleroma.Constants + + @moduledoc "Filter activities depending on their age" + @behaviour MRF + + defp check_date(%{"published" => published} = message) do + with %DateTime{} = now <- DateTime.utc_now(), + {:ok, %DateTime{} = then, _} <- DateTime.from_iso8601(published), + max_ttl <- Config.get([:mrf_object_age, :threshold]), + {:ttl, false} <- {:ttl, DateTime.diff(now, then) > max_ttl} do + {:ok, message} + else + {:ttl, true} -> + {:reject, nil} + + e -> + {:error, e} + end + end + + defp check_reject(message, actions) do + if :reject in actions do + {:reject, nil} + else + {:ok, message} + end + end + + defp check_delist(message, actions) do + if :delist in actions do + with %User{} = user <- User.get_cached_by_ap_id(message["actor"]) do + to = List.delete(message["to"], Pleroma.Constants.as_public()) ++ [user.follower_address] + cc = List.delete(message["cc"], user.follower_address) ++ [Pleroma.Constants.as_public()] + + message = + message + |> Map.put("to", to) + |> Map.put("cc", cc) + + {:ok, message} + else + # Unhandleable error: somebody is messing around, just drop the message. + e -> + Logger.error("ERROR: #{inspect(e)}") + {:reject, nil} + end + else + {:ok, message} + end + end + + defp check_strip_followers(message, actions) do + if :strip_followers in actions do + with %User{} = user <- User.get_cached_by_ap_id(message["actor"]) do + to = List.delete(message["to"], user.follower_address) + cc = List.delete(message["cc"], user.follower_address) + + message = + message + |> Map.put("to", to) + |> Map.put("cc", cc) + + {:ok, message} + else + # Unhandleable error: somebody is messing around, just drop the message. + _e -> + {:reject, nil} + end + else + {:ok, message} + end + end + + @impl true + def filter(%{"type" => "Create", "published" => _} = message) do + with actions <- Config.get([:mrf_object_age, :actions]), + {:reject, _} <- check_date(message), + {:ok, message} <- check_reject(message, actions), + {:ok, message} <- check_delist(message, actions), + {:ok, message} <- check_strip_followers(message, actions) do + {:ok, message} + else + # check_date() is allowed to short-circuit the pipeline + e -> e + end + end + + @impl true + def filter(message), do: {:ok, message} + + @impl true + def describe, do: {:ok, %{}} +end diff --git a/test/web/activity_pub/mrf/object_age_policy_test.exs b/test/web/activity_pub/mrf/object_age_policy_test.exs new file mode 100644 index 000000000..3ea00d768 --- /dev/null +++ b/test/web/activity_pub/mrf/object_age_policy_test.exs @@ -0,0 +1,105 @@ +# Pleroma: A lightweight social networking server +# Copyright ยฉ 2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ActivityPub.MRF.ObjectAgePolicyTest do + use Pleroma.DataCase + alias Pleroma.Config + alias Pleroma.User + alias Pleroma.Web.ActivityPub.Visibility + alias Pleroma.Web.ActivityPub.MRF.ObjectAgePolicy + + clear_config([:mrf_object_age]) do + Config.put(:mrf_object_age, + threshold: 172_800, + actions: [:delist, :strip_followers] + ) + end + + setup_all do + Tesla.Mock.mock_global(fn env -> apply(HttpRequestMock, :request, [env]) end) + :ok + end + + describe "with reject action" do + test "it rejects an old post" do + Config.put([:mrf_object_age, :actions], [:reject]) + + data = + File.read!("test/fixtures/mastodon-post-activity.json") + |> Poison.decode!() + + {:reject, _} = ObjectAgePolicy.filter(data) + end + + test "it allows a new post" do + Config.put([:mrf_object_age, :actions], [:reject]) + + data = + File.read!("test/fixtures/mastodon-post-activity.json") + |> Poison.decode!() + |> Map.put("published", DateTime.utc_now() |> DateTime.to_iso8601()) + + {:ok, _} = ObjectAgePolicy.filter(data) + end + end + + describe "with delist action" do + test "it delists an old post" do + Config.put([:mrf_object_age, :actions], [:delist]) + + data = + File.read!("test/fixtures/mastodon-post-activity.json") + |> Poison.decode!() + + {:ok, _u} = User.get_or_fetch_by_ap_id(data["actor"]) + + {:ok, data} = ObjectAgePolicy.filter(data) + + assert Visibility.get_visibility(%{data: data}) == "unlisted" + end + + test "it allows a new post" do + Config.put([:mrf_object_age, :actions], [:delist]) + + data = + File.read!("test/fixtures/mastodon-post-activity.json") + |> Poison.decode!() + |> Map.put("published", DateTime.utc_now() |> DateTime.to_iso8601()) + + {:ok, _user} = User.get_or_fetch_by_ap_id(data["actor"]) + + {:ok, ^data} = ObjectAgePolicy.filter(data) + end + end + + describe "with strip_followers action" do + test "it strips followers collections from an old post" do + Config.put([:mrf_object_age, :actions], [:strip_followers]) + + data = + File.read!("test/fixtures/mastodon-post-activity.json") + |> Poison.decode!() + + {:ok, user} = User.get_or_fetch_by_ap_id(data["actor"]) + + {:ok, data} = ObjectAgePolicy.filter(data) + + refute user.follower_address in data["to"] + refute user.follower_address in data["cc"] + end + + test "it allows a new post" do + Config.put([:mrf_object_age, :actions], [:strip_followers]) + + data = + File.read!("test/fixtures/mastodon-post-activity.json") + |> Poison.decode!() + |> Map.put("published", DateTime.utc_now() |> DateTime.to_iso8601()) + + {:ok, _u} = User.get_or_fetch_by_ap_id(data["actor"]) + + {:ok, ^data} = ObjectAgePolicy.filter(data) + end + end +end From 2469880a2bc61729f4518084ec341775a51deda5 Mon Sep 17 00:00:00 2001 From: Ariadne Conill Date: Thu, 14 Nov 2019 19:49:25 -0600 Subject: [PATCH 103/107] add changelog entry for MRF ObjectAgePolicy --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b4ad91b0d..920830039 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -42,6 +42,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Static Frontend: Add the ability to render user profiles and notices server-side without requiring JS app. - Mix task to re-count statuses for all users (`mix pleroma.count_statuses`) - Support for `X-Forwarded-For` and similar HTTP headers which used by reverse proxies to pass a real user IP address to the backend. Must not be enabled unless your instance is behind at least one reverse proxy (such as Nginx, Apache HTTPD or Varnish Cache). +- MRF: New module which handles incoming posts based on their age.
API Changes From eecd64cc0786a22d1ba90214e6c6bd5fb5829ec0 Mon Sep 17 00:00:00 2001 From: Ariadne Conill Date: Thu, 14 Nov 2019 19:56:14 -0600 Subject: [PATCH 104/107] object age policy: remove debug logging --- lib/pleroma/web/activity_pub/mrf/object_age_policy.ex | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/lib/pleroma/web/activity_pub/mrf/object_age_policy.ex b/lib/pleroma/web/activity_pub/mrf/object_age_policy.ex index f6c6f31cb..8b36c1021 100644 --- a/lib/pleroma/web/activity_pub/mrf/object_age_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/object_age_policy.ex @@ -7,7 +7,6 @@ defmodule Pleroma.Web.ActivityPub.MRF.ObjectAgePolicy do alias Pleroma.User alias Pleroma.Web.ActivityPub.MRF - require Logger require Pleroma.Constants @moduledoc "Filter activities depending on their age" @@ -50,8 +49,7 @@ defp check_delist(message, actions) do {:ok, message} else # Unhandleable error: somebody is messing around, just drop the message. - e -> - Logger.error("ERROR: #{inspect(e)}") + _e -> {:reject, nil} end else From 7c59bc9ef95d42ad155177e514408ceb56160aa6 Mon Sep 17 00:00:00 2001 From: Ariadne Conill Date: Thu, 14 Nov 2019 20:18:45 -0600 Subject: [PATCH 105/107] fix credo --- test/web/activity_pub/mrf/object_age_policy_test.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/web/activity_pub/mrf/object_age_policy_test.exs b/test/web/activity_pub/mrf/object_age_policy_test.exs index 3ea00d768..643609da4 100644 --- a/test/web/activity_pub/mrf/object_age_policy_test.exs +++ b/test/web/activity_pub/mrf/object_age_policy_test.exs @@ -6,8 +6,8 @@ defmodule Pleroma.Web.ActivityPub.MRF.ObjectAgePolicyTest do use Pleroma.DataCase alias Pleroma.Config alias Pleroma.User - alias Pleroma.Web.ActivityPub.Visibility alias Pleroma.Web.ActivityPub.MRF.ObjectAgePolicy + alias Pleroma.Web.ActivityPub.Visibility clear_config([:mrf_object_age]) do Config.put(:mrf_object_age, From 075789c442501edc10cf20dc54cf011ddcc5bc14 Mon Sep 17 00:00:00 2001 From: lain Date: Fri, 15 Nov 2019 12:31:09 +0000 Subject: [PATCH 106/107] Apply suggestion to CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 920830039..a675fc426 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -42,7 +42,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Static Frontend: Add the ability to render user profiles and notices server-side without requiring JS app. - Mix task to re-count statuses for all users (`mix pleroma.count_statuses`) - Support for `X-Forwarded-For` and similar HTTP headers which used by reverse proxies to pass a real user IP address to the backend. Must not be enabled unless your instance is behind at least one reverse proxy (such as Nginx, Apache HTTPD or Varnish Cache). -- MRF: New module which handles incoming posts based on their age. +- MRF: New module which handles incoming posts based on their age. By default, all incoming posts that are older than 2 days will be unlisted and not shown to their followers.
API Changes From f17e0f8e4f8f6249d1de9ad8a21953cca4963045 Mon Sep 17 00:00:00 2001 From: lain Date: Fri, 15 Nov 2019 14:13:21 +0100 Subject: [PATCH 107/107] OAuthPlug, Router: Handle deactivated users in the UserEnabledPlug --- lib/pleroma/plugs/oauth_plug.ex | 2 +- lib/pleroma/web/router.ex | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/plugs/oauth_plug.ex b/lib/pleroma/plugs/oauth_plug.ex index fd004fcd2..11a5b7642 100644 --- a/lib/pleroma/plugs/oauth_plug.ex +++ b/lib/pleroma/plugs/oauth_plug.ex @@ -71,7 +71,7 @@ defp fetch_user_and_token(token) do ) # credo:disable-for-next-line Credo.Check.Readability.MaxLineLength - with %Token{user: %{deactivated: false} = user} = token_record <- Repo.one(query) do + with %Token{user: user} = token_record <- Repo.one(query) do {:ok, user, token_record} end end diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 8fb4aec13..af7c0e289 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -13,6 +13,7 @@ defmodule Pleroma.Web.Router do pipeline :oauth do plug(:fetch_session) plug(Pleroma.Plugs.OAuthPlug) + plug(Pleroma.Plugs.UserEnabledPlug) end pipeline :api do