Merge branch 'activity-expiration' into 'develop'

Activity expiration

See merge request pleroma/pleroma!1595

CHANGELOG.md

@@ -50,6 +50,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - ActivityPub: Deactivated user deletion
 
 ### Added
+- Expiring/ephemeral activites. All activities can have expires_at value set, which controls when they should be deleted automatically.
+- Mastodon API: in post_status, the expires_in parameter lets you set the number of seconds until an activity expires. It must be at least one hour.
+- Mastodon API: all status JSON responses contain a `pleroma.expires_at` item which states when an activity will expire. The value is only shown to the user who created the activity. To everyone else it's empty.
+- Configuration: `ActivityExpiration.enabled` controls whether expired activites will get deleted at the appropriate time. Enabled by default.
 - Conversations: Add Pleroma-specific conversation endpoints and status posting extensions. Run the `bump_all_conversations` task again to create the necessary data.
 - **Breaking:** MRF describe API, which adds support for exposing configuration information about MRF policies to NodeInfo.
   Custom modules will need to be updated by adding, at the very least, `def describe, do: {:ok, %{}}` to the MRF policy modules.

config/config.exs

@@ -456,6 +456,7 @@
   max_retries: 5
 
 config :pleroma_job_queue, :queues,
+  activity_expiration: 10,
   federator_incoming: 50,
   federator_outgoing: 50,
   web_push: 50,
@@ -566,6 +567,8 @@
   account_confirmation_resend: {8_640_000, 5},
   ap_routes: {60_000, 15}
 
+config :pleroma, Pleroma.ActivityExpiration, enabled: true
+
 # Import environment specific config. This must remain at the bottom
 # of this file so it overrides the configuration defined above.
 import_config "#{Mix.env()}.exs"

config/test.exs

@@ -86,11 +86,10 @@
 
 config :pleroma, Pleroma.ReverseProxy.Client, Pleroma.ReverseProxy.ClientMock
 
-try do
+if File.exists?("./config/test.secret.exs") do
   import_config "test.secret.exs"
-rescue
+else
-  _ ->
+  IO.puts(
-    IO.puts(
+    "You may want to create test.secret.exs to declare custom database connection parameters."
-      "You may want to create test.secret.exs to declare custom database connection parameters."
+  )
-    )
 end

docs/api/differences_in_mastoapi_responses.md

@@ -25,6 +25,7 @@ Has these additional fields under the `pleroma` object:
 - `in_reply_to_account_acct`: the `acct` property of User entity for replied user (if any)
 - `content`: a map consisting of alternate representations of the `content` property with the key being it's mimetype. Currently the only alternate representation supported is `text/plain`
 - `spoiler_text`: a map consisting of alternate representations of the `spoiler_text` property with the key being it's mimetype. Currently the only alternate representation supported is `text/plain`
+- `expires_at`: a datetime (iso8601) that states when the post will expire (be deleted automatically), or empty if the post won't expire
 
 ## Attachments
 
@@ -86,6 +87,7 @@ Additional parameters can be added to the JSON body/Form data:
 - `content_type`: string, contain the MIME type of the status, it is transformed into HTML by the backend. You can get the list of the supported MIME types with the nodeinfo endpoint.
 - `to`: A list of nicknames (like `lain@soykaf.club` or `lain` on the local server) that will be used to determine who is going to be addressed by this post. Using this will disable the implicit addressing by mentioned names in the `status` body, only the people in the `to` list will be addressed. The normal rules for for post visibility are not affected by this and will still apply.
 - `visibility`: string, besides standard MastoAPI values (`direct`, `private`, `unlisted` or `public`) it can be used to address a List by setting it to `list:LIST_ID`.
+- `expires_in`: The number of seconds the posted activity should expire in. When a posted activity expires it will be deleted from the server, and a delete request for it will be federated. This needs to be longer than an hour.
 - `in_reply_to_conversation_id`: Will reply to a given conversation, addressing only the people who are part of the recipient set of that conversation. Sets the visibility to `direct`.
 
 ## PATCH `/api/v1/update_credentials`

docs/config.md

@@ -495,6 +495,10 @@ config :auto_linker,
 * `total_user_limit`: the number of scheduled activities a user is allowed to create in total (Default: `300`)
 * `enabled`: whether scheduled activities are sent to the job queue to be executed
 
+## Pleroma.ActivityExpiration
+
+# `enabled`: whether expired activities will be sent to the job queue to be deleted
+
 ## Pleroma.Web.Auth.Authenticator
 
 * `Pleroma.Web.Auth.PleromaAuthenticator`: default database authenticator

lib/pleroma/activity.ex

@@ -6,6 +6,7 @@ defmodule Pleroma.Activity do
   use Ecto.Schema
 
   alias Pleroma.Activity
+  alias Pleroma.ActivityExpiration
   alias Pleroma.Bookmark
   alias Pleroma.Notification
   alias Pleroma.Object
@@ -59,6 +60,8 @@ defmodule Pleroma.Activity do
     # typical case.
     has_one(:object, Object, on_delete: :nothing, foreign_key: :id)
 
+    has_one(:expiration, ActivityExpiration, on_delete: :delete_all)
+
     timestamps()
   end
 

lib/pleroma/activity_expiration.ex

@@ -0,0 +1,68 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.ActivityExpiration do
+  use Ecto.Schema
+
+  alias Pleroma.Activity
+  alias Pleroma.ActivityExpiration
+  alias Pleroma.FlakeId
+  alias Pleroma.Repo
+
+  import Ecto.Changeset
+  import Ecto.Query
+
+  @type t :: %__MODULE__{}
+  @min_activity_lifetime :timer.hours(1)
+
+  schema "activity_expirations" do
+    belongs_to(:activity, Activity, type: FlakeId)
+    field(:scheduled_at, :naive_datetime)
+  end
+
+  def changeset(%ActivityExpiration{} = expiration, attrs) do
+    expiration
+    |> cast(attrs, [:scheduled_at])
+    |> validate_required([:scheduled_at])
+    |> validate_scheduled_at()
+  end
+
+  def get_by_activity_id(activity_id) do
+    ActivityExpiration
+    |> where([exp], exp.activity_id == ^activity_id)
+    |> Repo.one()
+  end
+
+  def create(%Activity{} = activity, scheduled_at) do
+    %ActivityExpiration{activity_id: activity.id}
+    |> changeset(%{scheduled_at: scheduled_at})
+    |> Repo.insert()
+  end
+
+  def due_expirations(offset \\ 0) do
+    naive_datetime =
+      NaiveDateTime.utc_now()
+      |> NaiveDateTime.add(offset, :millisecond)
+
+    ActivityExpiration
+    |> where([exp], exp.scheduled_at < ^naive_datetime)
+    |> Repo.all()
+  end
+
+  def validate_scheduled_at(changeset) do
+    validate_change(changeset, :scheduled_at, fn _, scheduled_at ->
+      if not expires_late_enough?(scheduled_at) do
+        [scheduled_at: "an ephemeral activity must live for at least one hour"]
+      else
+        []
+      end
+    end)
+  end
+
+  def expires_late_enough?(scheduled_at) do
+    now = NaiveDateTime.utc_now()
+    diff = NaiveDateTime.diff(scheduled_at, now, :millisecond)
+    diff >= @min_activity_lifetime
+  end
+end

lib/pleroma/activity_expiration_worker.ex

@@ -0,0 +1,62 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.ActivityExpirationWorker do
+  alias Pleroma.Activity
+  alias Pleroma.ActivityExpiration
+  alias Pleroma.Config
+  alias Pleroma.Repo
+  alias Pleroma.User
+  alias Pleroma.Web.CommonAPI
+  require Logger
+  use GenServer
+  import Ecto.Query
+
+  @schedule_interval :timer.minutes(1)
+
+  def start_link(_) do
+    GenServer.start_link(__MODULE__, nil)
+  end
+
+  @impl true
+  def init(_) do
+    if Config.get([ActivityExpiration, :enabled]) do
+      schedule_next()
+      {:ok, nil}
+    else
+      :ignore
+    end
+  end
+
+  def perform(:execute, expiration_id) do
+    try do
+      expiration =
+        ActivityExpiration
+        |> where([e], e.id == ^expiration_id)
+        |> Repo.one!()
+
+      activity = Activity.get_by_id_with_object(expiration.activity_id)
+      user = User.get_by_ap_id(activity.object.data["actor"])
+      CommonAPI.delete(activity.id, user)
+    rescue
+      error ->
+        Logger.error("#{__MODULE__} Couldn't delete expired activity: #{inspect(error)}")
+    end
+  end
+
+  @impl true
+  def handle_info(:perform, state) do
+    ActivityExpiration.due_expirations(@schedule_interval)
+    |> Enum.each(fn expiration ->
+      PleromaJobQueue.enqueue(:activity_expiration, __MODULE__, [:execute, expiration.id])
+    end)
+
+    schedule_next()
+    {:noreply, state}
+  end
+
+  defp schedule_next do
+    Process.send_after(self(), :perform, @schedule_interval)
+  end
+end

lib/pleroma/application.ex

@@ -35,7 +35,8 @@ def start(_type, _args) do
         Pleroma.Emoji,
         Pleroma.Captcha,
         Pleroma.FlakeId,
-        Pleroma.ScheduledActivityWorker
+        Pleroma.ScheduledActivityWorker,
+        Pleroma.ActivityExpirationWorker
       ] ++
         cachex_children() ++
         hackney_pool_children() ++

lib/pleroma/web/common_api/common_api.ex

@@ -4,6 +4,7 @@
 
 defmodule Pleroma.Web.CommonAPI do
   alias Pleroma.Activity
+  alias Pleroma.ActivityExpiration
   alias Pleroma.Conversation.Participation
   alias Pleroma.Formatter
   alias Pleroma.Object
@@ -200,6 +201,23 @@ def get_replied_to_visibility(activity) do
     end
   end
 
+  defp check_expiry_date({:ok, nil} = res), do: res
+
+  defp check_expiry_date({:ok, in_seconds}) do
+    expiry = NaiveDateTime.utc_now() |> NaiveDateTime.add(in_seconds)
+
+    if ActivityExpiration.expires_late_enough?(expiry) do
+      {:ok, expiry}
+    else
+      {:error, "Expiry date is too soon"}
+    end
+  end
+
+  defp check_expiry_date(expiry_str) do
+    Ecto.Type.cast(:integer, expiry_str)
+    |> check_expiry_date()
+  end
+
   def post(user, %{"status" => status} = data) do
     limit = Pleroma.Config.get([:instance, :limit])
 
@@ -226,6 +244,7 @@ def post(user, %{"status" => status} = data) do
          context <- make_context(in_reply_to, in_reply_to_conversation),
          cw <- data["spoiler_text"] || "",
          sensitive <- data["sensitive"] || Enum.member?(tags, {"#nsfw", "nsfw"}),
+         {:ok, expires_at} <- check_expiry_date(data["expires_in"]),
          full_payload <- String.trim(status <> cw),
          :ok <- validate_character_limit(full_payload, attachments, limit),
          object <-
@@ -251,15 +270,24 @@ def post(user, %{"status" => status} = data) do
       preview? = Pleroma.Web.ControllerHelper.truthy_param?(data["preview"]) || false
       direct? = visibility == "direct"
 
-      %{
+      result =
-        to: to,
+        %{
-        actor: user,
+          to: to,
-        context: context,
+          actor: user,
-        object: object,
+          context: context,
-        additional: %{"cc" => cc, "directMessage" => direct?}
+          object: object,
-      }
+          additional: %{"cc" => cc, "directMessage" => direct?}
-      |> maybe_add_list_data(user, visibility)
+        }
-      |> ActivityPub.create(preview?)
+        |> maybe_add_list_data(user, visibility)
+        |> ActivityPub.create(preview?)
+
+      if expires_at do
+        with {:ok, activity} <- result do
+          {:ok, _} = ActivityExpiration.create(activity, expires_at)
+        end
+      end
+
+      result
     else
       {:private_to_public, true} ->
         {:error, dgettext("errors", "The message visibility must be direct")}

lib/pleroma/web/mastodon_api/views/status_view.ex

@@ -8,6 +8,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do
   require Pleroma.Constants
 
   alias Pleroma.Activity
+  alias Pleroma.ActivityExpiration
   alias Pleroma.Conversation
   alias Pleroma.Conversation.Participation
   alias Pleroma.HTML
@@ -177,6 +178,15 @@ def render("status.json", %{activity: %{data: %{"object" => _object}} = activity
 
     bookmarked = Activity.get_bookmark(activity, opts[:for]) != nil
 
+    client_posted_this_activity = opts[:for] && user.id == opts[:for].id
+
+    expires_at =
+      with true <- client_posted_this_activity,
+           expiration when not is_nil(expiration) <-
+             ActivityExpiration.get_by_activity_id(activity.id) do
+        expiration.scheduled_at
+      end
+
     thread_muted? =
       case activity.thread_muted? do
         thread_muted? when is_boolean(thread_muted?) -> thread_muted?
@@ -288,6 +298,7 @@ def render("status.json", %{activity: %{data: %{"object" => _object}} = activity
         in_reply_to_account_acct: reply_to_user && reply_to_user.nickname,
         content: %{"text/plain" => content_plaintext},
         spoiler_text: %{"text/plain" => summary_plaintext},
+        expires_at: expires_at,
         direct_conversation_id: direct_conversation_id
       }
     }

priv/repo/migrations/20190716100804_add_expirations_table.exs

@@ -0,0 +1,10 @@
+defmodule Pleroma.Repo.Migrations.AddExpirationsTable do
+  use Ecto.Migration
+
+  def change do
+    create_if_not_exists table(:activity_expirations) do
+      add(:activity_id, references(:activities, type: :uuid, on_delete: :delete_all))
+      add(:scheduled_at, :naive_datetime, null: false)
+    end
+  end
+end

test/activity_expiration_test.exs

@@ -0,0 +1,27 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.ActivityExpirationTest do
+  use Pleroma.DataCase
+  alias Pleroma.ActivityExpiration
+  import Pleroma.Factory
+
+  test "finds activities due to be deleted only" do
+    activity = insert(:note_activity)
+    expiration_due = insert(:expiration_in_the_past, %{activity_id: activity.id})
+    activity2 = insert(:note_activity)
+    insert(:expiration_in_the_future, %{activity_id: activity2.id})
+
+    expirations = ActivityExpiration.due_expirations()
+
+    assert length(expirations) == 1
+    assert hd(expirations) == expiration_due
+  end
+
+  test "denies expirations that don't live long enough" do
+    activity = insert(:note_activity)
+    now = NaiveDateTime.utc_now()
+    assert {:error, _} = ActivityExpiration.create(activity, now)
+  end
+end

test/activity_expiration_worker_test.exs

@@ -0,0 +1,17 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.ActivityExpirationWorkerTest do
+  use Pleroma.DataCase
+  alias Pleroma.Activity
+  import Pleroma.Factory
+
+  test "deletes an activity" do
+    activity = insert(:note_activity)
+    expiration = insert(:expiration_in_the_past, %{activity_id: activity.id})
+    Pleroma.ActivityExpirationWorker.perform(:execute, expiration.id)
+
+    refute Repo.get(Activity, activity.id)
+  end
+end

test/activity_test.exs

@@ -164,4 +164,13 @@ test "find all statuses for unauthenticated users when `limit_to_local_content`
       Pleroma.Config.put([:instance, :limit_to_local_content], :unauthenticated)
     end
   end
+
+  test "add an activity with an expiration" do
+    activity = insert(:note_activity)
+    insert(:expiration_in_the_future, %{activity_id: activity.id})
+
+    Pleroma.ActivityExpiration
+    |> where([a], a.activity_id == ^activity.id)
+    |> Repo.one!()
+  end
 end

test/support/factory.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Factory do
@@ -143,6 +143,25 @@ def note_activity_factory(attrs \\ %{}) do
     |> Map.merge(attrs)
   end
 
+  defp expiration_offset_by_minutes(attrs, minutes) do
+    scheduled_at =
+      NaiveDateTime.utc_now()
+      |> NaiveDateTime.add(:timer.minutes(minutes), :millisecond)
+      |> NaiveDateTime.truncate(:second)
+
+    %Pleroma.ActivityExpiration{}
+    |> Map.merge(attrs)
+    |> Map.put(:scheduled_at, scheduled_at)
+  end
+
+  def expiration_in_the_past_factory(attrs \\ %{}) do
+    expiration_offset_by_minutes(attrs, -60)
+  end
+
+  def expiration_in_the_future_factory(attrs \\ %{}) do
+    expiration_offset_by_minutes(attrs, 61)
+  end
+
   def article_activity_factory do
     article = insert(:article)
 

test/web/common_api/common_api_test.exs

@@ -204,6 +204,21 @@ test "it returns error when character limit is exceeded" do
       assert {:error, "The status is over the character limit"} =
                CommonAPI.post(user, %{"status" => "foobar"})
     end
+
+    test "it can handle activities that expire" do
+      user = insert(:user)
+
+      expires_at =
+        NaiveDateTime.utc_now()
+        |> NaiveDateTime.truncate(:second)
+        |> NaiveDateTime.add(1_000_000, :second)
+
+      assert {:ok, activity} =
+               CommonAPI.post(user, %{"status" => "chai", "expires_in" => 1_000_000})
+
+      assert expiration = Pleroma.ActivityExpiration.get_by_activity_id(activity.id)
+      assert expiration.scheduled_at == expires_at
+    end
   end
 
   describe "reactions" do

test/web/mastodon_api/mastodon_api_controller_test.exs

@@ -7,6 +7,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIControllerTest do
 
   alias Ecto.Changeset
   alias Pleroma.Activity
+  alias Pleroma.ActivityExpiration
   alias Pleroma.Config
   alias Pleroma.Notification
   alias Pleroma.Object
@@ -150,6 +151,32 @@ test "posting a status", %{conn: conn} do
 
       assert %{"id" => third_id} = json_response(conn_three, 200)
       refute id == third_id
+
+      # An activity that will expire:
+      # 2 hours
+      expires_in = 120 * 60
+
+      conn_four =
+        conn
+        |> post("api/v1/statuses", %{
+          "status" => "oolong",
+          "expires_in" => expires_in
+        })
+
+      assert fourth_response = %{"id" => fourth_id} = json_response(conn_four, 200)
+      assert activity = Activity.get_by_id(fourth_id)
+      assert expiration = ActivityExpiration.get_by_activity_id(fourth_id)
+
+      estimated_expires_at =
+        NaiveDateTime.utc_now()
+        |> NaiveDateTime.add(expires_in)
+        |> NaiveDateTime.truncate(:second)
+
+      # This assert will fail if the test takes longer than a minute. I sure hope it never does:
+      assert abs(NaiveDateTime.diff(expiration.scheduled_at, estimated_expires_at, :second)) < 60
+
+      assert fourth_response["pleroma"]["expires_at"] ==
+               NaiveDateTime.to_iso8601(expiration.scheduled_at)
     end
 
     test "replying to a status", %{conn: conn} do
@@ -403,7 +430,7 @@ test "direct timeline", %{conn: conn} do
     assert %{"visibility" => "direct"} = status
     assert status["url"] != direct.data["id"]
 
-    # User should be able to see his own direct message
+    # User should be able to see their own direct message
     res_conn =
       build_conn()
       |> assign(:user, user_one)

test/web/mastodon_api/status_view_test.exs

@@ -149,6 +149,7 @@ test "a note activity" do
         in_reply_to_account_acct: nil,
         content: %{"text/plain" => HtmlSanitizeEx.strip_tags(object_data["content"])},
         spoiler_text: %{"text/plain" => HtmlSanitizeEx.strip_tags(object_data["summary"])},
+        expires_at: nil,
         direct_conversation_id: nil
       }
     }