mirror of
https://github.com/paperless-ngx/paperless-ngx.git
synced 2025-12-13 18:17:02 +01:00
Enhancement: Limit excessively long content length when computing suggestions (#10656)
This helps prevent excessive processing times on very large documents by limiting the text analyzed during date parsing, tag prediction, and correspondent matching. If the document exceeds 1.2M chars, crop to 1M char.
This commit is contained in:
parent
84d85d7a23
commit
8adc26e09d
3 changed files with 65 additions and 5 deletions
|
|
@ -6,6 +6,7 @@ from unittest import mock
|
|||
|
||||
from django.test import TestCase
|
||||
from django.test import override_settings
|
||||
from faker import Faker
|
||||
|
||||
from documents.models import Correspondent
|
||||
from documents.models import Document
|
||||
|
|
@ -105,3 +106,27 @@ class TestDocument(TestCase):
|
|||
created=date(2020, 12, 25),
|
||||
)
|
||||
self.assertEqual(doc.get_public_filename(), "2020-12-25 test")
|
||||
|
||||
|
||||
def test_suggestion_content():
|
||||
"""
|
||||
Check that the document for suggestion is cropped, only if it exceeds the length limit.
|
||||
"""
|
||||
fake_text = Faker().text(max_nb_chars=1201000)
|
||||
|
||||
# Do not crop content under 1.2M chars
|
||||
content_under_limit = fake_text[:1200000]
|
||||
doc = Document(
|
||||
title="test",
|
||||
created=date(2025, 6, 1),
|
||||
content=content_under_limit,
|
||||
)
|
||||
assert doc.suggestion_content == content_under_limit
|
||||
|
||||
# If over the limit, crop to 1M char (800K from the beginning, 200K from the end)
|
||||
content_over_limit = fake_text[:1200001]
|
||||
expected_cropped_content = (
|
||||
content_over_limit[:800000] + " " + content_over_limit[-200000:]
|
||||
)
|
||||
doc.content = content_over_limit
|
||||
assert doc.suggestion_content == expected_cropped_content
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue