Merge branch 'feature/make_attachments_cleanup_optional' into 'develop'

Make attachments cleanup optional

See merge request pleroma/pleroma!2170

CHANGELOG.md

@@ -12,7 +12,6 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
 ### Changed
 - **Breaking:** Pleroma won't start if it detects unapplied migrations
-- **Breaking:** attachments are removed along with statuses. Does not affect duplicate files and attachments without status.
 - **Breaking:** Elixir >=1.8 is now required (was >= 1.7)
 - **Breaking:** `Pleroma.Plugs.RemoteIp` and `:rate_limiter` enabled by default. Please ensure your reverse proxy forwards the real IP!
 - **Breaking:** attachment links (`config :pleroma, :instance, no_attachment_links` and `config :pleroma, Pleroma.Upload, link_name`) disabled by default
@@ -54,6 +53,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
 ### Added
 - `:chat_limit` option to limit chat characters.
+- `cleanup_attachments` option to remove attachments along with statuses. Does not affect duplicate files and attachments without status. Enabling this will increase load to database when deleting statuses on larger instances.
 - Refreshing poll results for remote polls
 - Authentication: Added rate limit for password-authorized actions / login existence checks
 - Static Frontend: Add the ability to render user profiles and notices server-side without requiring JS app.

config/config.exs

@@ -271,7 +271,8 @@
   account_field_name_length: 512,
   account_field_value_length: 2048,
   external_user_synchronization: true,
-  extended_nickname_format: true
+  extended_nickname_format: true,
+  cleanup_attachments: false
 
 config :pleroma, :feed,
   post_title: %{

config/description.exs

@@ -764,6 +764,15 @@
           "Set to `true` to use extended local nicknames format (allows underscores/dashes)." <>
             " This will break federation with older software for theses nicknames."
       },
+      %{
+        key: :cleanup_attachments,
+        type: :boolean,
+        description: """
+        "Set to `true` to remove associated attachments when status is removed.
+        This will not affect duplicates and attachments without status.
+        Enabling this will increase load to database when deleting statuses on larger instances.
+        """
+      },
       %{
         key: :max_pinned_statuses,
         type: :integer,

docs/configuration/cheatsheet.md

@@ -69,6 +69,7 @@ You shouldn't edit the base config directly to avoid breakages and merge conflic
 * `account_field_name_length`: An account field name maximum length (default: `512`).
 * `account_field_value_length`: An account field value maximum length (default: `2048`).
 * `external_user_synchronization`: Enabling following/followers counters synchronization for external users.
+* `cleanup_attachments`: Remove attachments along with statuses. Does not affect duplicate files and attachments without status. Enabling this will increase load to database when deleting statuses on larger instances.
 
 ## Federation
 ### MRF policies

lib/pleroma/object.ex

@@ -184,11 +184,14 @@ def delete(%Object{data: %{"id" => id}} = object) do
     with {:ok, _obj} = swap_object_with_tombstone(object),
          deleted_activity = Activity.delete_all_by_object_ap_id(id),
          {:ok, true} <- Cachex.del(:object_cache, "object:#{id}"),
-         {:ok, _} <- Cachex.del(:web_resp_cache, URI.parse(id).path),
-         {:ok, _} <-
-           Pleroma.Workers.AttachmentsCleanupWorker.enqueue("cleanup_attachments", %{
-             "object" => object
-           }) do
+         {:ok, _} <- Cachex.del(:web_resp_cache, URI.parse(id).path) do
+      with true <- Pleroma.Config.get([:instance, :cleanup_attachments]) do
+        {:ok, _} =
+          Pleroma.Workers.AttachmentsCleanupWorker.enqueue("cleanup_attachments", %{
+            "object" => object
+          })
+      end
+
       {:ok, object, deleted_activity}
     end
   end

test/object_test.exs

@@ -76,8 +76,9 @@ test "ensures cache is cleared for the object" do
   describe "delete attachments" do
     clear_config([Pleroma.Upload])
 
-    test "in subdirectories" do
+    test "Disabled via config" do
       Pleroma.Config.put([Pleroma.Upload, :uploader], Pleroma.Uploaders.Local)
+      Pleroma.Config.put([:instance, :cleanup_attachments], false)
 
       file = %Plug.Upload{
         content_type: "image/jpg",
@@ -103,6 +104,41 @@ test "in subdirectories" do
 
       ObanHelpers.perform(all_enqueued(worker: Pleroma.Workers.AttachmentsCleanupWorker))
 
+      assert Object.get_by_id(note.id).data["deleted"]
+      refute Object.get_by_id(attachment.id) == nil
+
+      assert {:ok, ["an_image.jpg"]} == File.ls("#{uploads_dir}/#{path}")
+    end
+
+    test "in subdirectories" do
+      Pleroma.Config.put([Pleroma.Upload, :uploader], Pleroma.Uploaders.Local)
+      Pleroma.Config.put([:instance, :cleanup_attachments], true)
+
+      file = %Plug.Upload{
+        content_type: "image/jpg",
+        path: Path.absname("test/fixtures/image.jpg"),
+        filename: "an_image.jpg"
+      }
+
+      user = insert(:user)
+
+      {:ok, %Object{} = attachment} =
+        Pleroma.Web.ActivityPub.ActivityPub.upload(file, actor: user.ap_id)
+
+      %{data: %{"attachment" => [%{"url" => [%{"href" => href}]}]}} =
+        note = insert(:note, %{user: user, data: %{"attachment" => [attachment.data]}})
+
+      uploads_dir = Pleroma.Config.get!([Pleroma.Uploaders.Local, :uploads])
+
+      path = href |> Path.dirname() |> Path.basename()
+
+      assert {:ok, ["an_image.jpg"]} == File.ls("#{uploads_dir}/#{path}")
+
+      Object.delete(note)
+
+      ObanHelpers.perform(all_enqueued(worker: Pleroma.Workers.AttachmentsCleanupWorker))
+
+      assert Object.get_by_id(note.id).data["deleted"]
       assert Object.get_by_id(attachment.id) == nil
 
       assert {:ok, []} == File.ls("#{uploads_dir}/#{path}")
@@ -111,6 +147,7 @@ test "in subdirectories" do
     test "with dedupe enabled" do
       Pleroma.Config.put([Pleroma.Upload, :uploader], Pleroma.Uploaders.Local)
       Pleroma.Config.put([Pleroma.Upload, :filters], [Pleroma.Upload.Filter.Dedupe])
+      Pleroma.Config.put([:instance, :cleanup_attachments], true)
 
       uploads_dir = Pleroma.Config.get!([Pleroma.Uploaders.Local, :uploads])
 
@@ -139,6 +176,7 @@ test "with dedupe enabled" do
 
       ObanHelpers.perform(all_enqueued(worker: Pleroma.Workers.AttachmentsCleanupWorker))
 
+      assert Object.get_by_id(note.id).data["deleted"]
       assert Object.get_by_id(attachment.id) == nil
       assert {:ok, files} = File.ls(uploads_dir)
       refute filename in files
@@ -146,6 +184,7 @@ test "with dedupe enabled" do
 
     test "with objects that have legacy data.url attribute" do
       Pleroma.Config.put([Pleroma.Upload, :uploader], Pleroma.Uploaders.Local)
+      Pleroma.Config.put([:instance, :cleanup_attachments], true)
 
       file = %Plug.Upload{
         content_type: "image/jpg",
@@ -173,6 +212,7 @@ test "with objects that have legacy data.url attribute" do
 
       ObanHelpers.perform(all_enqueued(worker: Pleroma.Workers.AttachmentsCleanupWorker))
 
+      assert Object.get_by_id(note.id).data["deleted"]
       assert Object.get_by_id(attachment.id) == nil
 
       assert {:ok, []} == File.ls("#{uploads_dir}/#{path}")
@@ -181,6 +221,7 @@ test "with objects that have legacy data.url attribute" do
     test "With custom base_url" do
       Pleroma.Config.put([Pleroma.Upload, :uploader], Pleroma.Uploaders.Local)
       Pleroma.Config.put([Pleroma.Upload, :base_url], "https://sub.domain.tld/dir/")
+      Pleroma.Config.put([:instance, :cleanup_attachments], true)
 
       file = %Plug.Upload{
         content_type: "image/jpg",
@@ -206,6 +247,7 @@ test "With custom base_url" do
 
       ObanHelpers.perform(all_enqueued(worker: Pleroma.Workers.AttachmentsCleanupWorker))
 
+      assert Object.get_by_id(note.id).data["deleted"]
       assert Object.get_by_id(attachment.id) == nil
 
       assert {:ok, []} == File.ls("#{uploads_dir}/#{path}")