diff --git a/docs/development.md b/docs/development.md index 71ca4d930..f724b551f 100644 --- a/docs/development.md +++ b/docs/development.md @@ -152,6 +152,19 @@ $ ng build --configuration production configuration. This is not ideal. But for now, make sure no settings except for DEBUG are overridden when testing. +- Testing under docker is possible (but generally unsupported) if a + bare-metal environment cannot be setup. To run tests under docker, use + this command: + + ```bash + # paperless-ngx/ + + $ docker run --rm -i -t \ + -v $PWD:/usr/src/paperless \ + --entrypoint=uv paperlessngx/paperless-ngx:latest \ + run pytest + ``` + !!! note The line length rule E501 is generally useful for getting multiple diff --git a/src/documents/management/commands/document_consumer.py b/src/documents/management/commands/document_consumer.py index 97027e02d..a541f66c3 100644 --- a/src/documents/management/commands/document_consumer.py +++ b/src/documents/management/commands/document_consumer.py @@ -250,10 +250,22 @@ class Command(BaseCommand): if options["oneshot"]: return + inotify = None if settings.CONSUMER_POLLING == 0 and INotify: - self.handle_inotify(directory, recursive, is_testing=options["testing"]) + try: + inotify = INotify() + except OSError: + logger.exception("inotify failed to instantiate") + + if inotify: + self.handle_inotify( + inotify, + directory, + recursive, + is_testing=options["testing"], + ) else: - if INotify is None and settings.CONSUMER_POLLING == 0: # pragma: no cover + if inotify is None and settings.CONSUMER_POLLING == 0: # pragma: no cover logger.warning("Using polling as INotify import failed") self.handle_polling(directory, recursive, is_testing=options["testing"]) @@ -286,7 +298,7 @@ class Command(BaseCommand): observer.stop() observer.join() - def handle_inotify(self, directory, recursive, *, is_testing: bool): + def handle_inotify(self, inotify, directory, recursive, *, is_testing: bool): logger.info(f"Using inotify to watch directory for changes: {directory}") timeout_ms = None @@ -294,7 +306,6 @@ class Command(BaseCommand): timeout_ms = self.testing_timeout_ms logger.debug(f"Configuring timeout to {timeout_ms}ms") - inotify = INotify() inotify_flags = flags.CLOSE_WRITE | flags.MOVED_TO | flags.MODIFY if recursive: inotify.add_watch_recursive(directory, inotify_flags) diff --git a/src/documents/tests/test_api_status.py b/src/documents/tests/test_api_status.py index 9b7bf37ad..f80512a96 100644 --- a/src/documents/tests/test_api_status.py +++ b/src/documents/tests/test_api_status.py @@ -34,7 +34,7 @@ class TestSystemStatus(APITestCase): self.assertEqual(response.status_code, status.HTTP_200_OK) self.assertEqual(response.data["pngx_version"], version.__full_version_str__) self.assertIsNotNone(response.data["server_os"]) - self.assertEqual(response.data["install_type"], "bare-metal") + self.assertIn(response.data["install_type"], ["bare-metal", "docker"]) self.assertIsNotNone(response.data["storage"]["total"]) self.assertIsNotNone(response.data["storage"]["available"]) self.assertEqual(response.data["database"]["type"], "sqlite") diff --git a/src/documents/tests/test_file_handling.py b/src/documents/tests/test_file_handling.py index c0070aa81..00df33a5b 100644 --- a/src/documents/tests/test_file_handling.py +++ b/src/documents/tests/test_file_handling.py @@ -26,6 +26,7 @@ from documents.tasks import empty_trash from documents.tests.factories import DocumentFactory from documents.tests.utils import DirectoriesMixin from documents.tests.utils import FileSystemAssertsMixin +from documents.tests.utils import skip_if_root class TestFileHandling(DirectoriesMixin, FileSystemAssertsMixin, TestCase): @@ -89,6 +90,7 @@ class TestFileHandling(DirectoriesMixin, FileSystemAssertsMixin, TestCase): settings.ORIGINALS_DIR / "test" / "test.pdf.gpg", ) + @skip_if_root @override_settings(FILENAME_FORMAT="{correspondent}/{correspondent}") def test_file_renaming_missing_permissions(self): document = Document() diff --git a/src/documents/tests/test_management_consumer.py b/src/documents/tests/test_management_consumer.py index 38b9eadda..447de4926 100644 --- a/src/documents/tests/test_management_consumer.py +++ b/src/documents/tests/test_management_consumer.py @@ -5,6 +5,7 @@ from threading import Thread from time import sleep from unittest import mock +import pytest from django.conf import settings from django.core.management import CommandError from django.core.management import call_command @@ -18,6 +19,14 @@ from documents.models import Tag from documents.tests.utils import DirectoriesMixin from documents.tests.utils import DocumentConsumeDelayMixin +try: + from inotifyrecursive import INotify + + INotify() + SKIP_INOTIFY_TESTS = False +except (ImportError, OSError): + SKIP_INOTIFY_TESTS = True + class ConsumerThread(Thread): def __init__(self): @@ -108,10 +117,10 @@ class ConsumerThreadMixin(DocumentConsumeDelayMixin): print("file completed.") # noqa: T201 -@override_settings( - CONSUMER_INOTIFY_DELAY=0.01, -) -class TestConsumer(DirectoriesMixin, ConsumerThreadMixin, TransactionTestCase): +class _TestConsumerBase(DirectoriesMixin, ConsumerThreadMixin): + # Note: adding TransactionTestCase as a base makes pytest discover this + # class. Only add it to the actual test classes. + def test_consume_file(self): self.t_start() @@ -365,6 +374,14 @@ class TestConsumer(DirectoriesMixin, ConsumerThreadMixin, TransactionTestCase): self.consume_file_mock.assert_not_called() +@override_settings( + CONSUMER_INOTIFY_DELAY=0.01, +) +@pytest.mark.skipif(SKIP_INOTIFY_TESTS, reason="no inotify") +class TestConsumer(_TestConsumerBase, TransactionTestCase): + pass + + @override_settings( CONSUMER_POLLING=1, # please leave the delay here and down below @@ -372,13 +389,14 @@ class TestConsumer(DirectoriesMixin, ConsumerThreadMixin, TransactionTestCase): CONSUMER_POLLING_DELAY=3, CONSUMER_POLLING_RETRY_COUNT=20, ) -class TestConsumerPolling(TestConsumer): +class TestConsumerPolling(_TestConsumerBase, TransactionTestCase): # just do all the tests with polling pass @override_settings(CONSUMER_INOTIFY_DELAY=0.01, CONSUMER_RECURSIVE=True) -class TestConsumerRecursive(TestConsumer): +@pytest.mark.skipif(SKIP_INOTIFY_TESTS, reason="no inotify") +class TestConsumerRecursive(_TestConsumerBase, TransactionTestCase): # just do all the tests with recursive pass @@ -389,14 +407,14 @@ class TestConsumerRecursive(TestConsumer): CONSUMER_POLLING_DELAY=3, CONSUMER_POLLING_RETRY_COUNT=20, ) -class TestConsumerRecursivePolling(TestConsumer): +class TestConsumerRecursivePolling(_TestConsumerBase, TransactionTestCase): # just do all the tests with polling and recursive pass +@override_settings(CONSUMER_RECURSIVE=True, CONSUMER_SUBDIRS_AS_TAGS=True) class TestConsumerTags(DirectoriesMixin, ConsumerThreadMixin, TransactionTestCase): - @override_settings(CONSUMER_RECURSIVE=True, CONSUMER_SUBDIRS_AS_TAGS=True) - def test_consume_file_with_path_tags(self): + def _test_consume_file_with_path_tags(self): tag_names = ("existingTag", "Space Tag") # Create a Tag prior to consuming a file using it in path tag_ids = [ @@ -429,10 +447,14 @@ class TestConsumerTags(DirectoriesMixin, ConsumerThreadMixin, TransactionTestCas # their order. self.assertCountEqual(overrides.tag_ids, tag_ids) + @pytest.mark.skipif(SKIP_INOTIFY_TESTS, reason="no inotify") + def test_consume_file_with_path_tags(self): + self._test_consume_file_with_path_tags() + @override_settings( CONSUMER_POLLING=1, CONSUMER_POLLING_DELAY=3, CONSUMER_POLLING_RETRY_COUNT=20, ) def test_consume_file_with_path_tags_polling(self): - self.test_consume_file_with_path_tags() + self._test_consume_file_with_path_tags() diff --git a/src/documents/tests/test_management_exporter.py b/src/documents/tests/test_management_exporter.py index a67e5e8c5..6d9ffabd0 100644 --- a/src/documents/tests/test_management_exporter.py +++ b/src/documents/tests/test_management_exporter.py @@ -42,6 +42,7 @@ from documents.tests.utils import DirectoriesMixin from documents.tests.utils import FileSystemAssertsMixin from documents.tests.utils import SampleDirMixin from documents.tests.utils import paperless_environment +from documents.tests.utils import skip_if_root from paperless_mail.models import MailAccount @@ -571,7 +572,7 @@ class TestExportImport( with self.assertRaises(CommandError) as e: call_command(*args) - self.assertEqual("That path isn't a directory", str(e)) + self.assertEqual("That path doesn't exist", str(e.exception)) def test_export_target_exists_but_is_file(self): """ @@ -589,8 +590,9 @@ class TestExportImport( with self.assertRaises(CommandError) as e: call_command(*args) - self.assertEqual("That path isn't a directory", str(e)) + self.assertEqual("That path isn't a directory", str(e.exception)) + @skip_if_root def test_export_target_not_writable(self): """ GIVEN: @@ -608,7 +610,10 @@ class TestExportImport( with self.assertRaises(CommandError) as e: call_command(*args) - self.assertEqual("That path doesn't appear to be writable", str(e)) + self.assertEqual( + "That path doesn't appear to be writable", + str(e.exception), + ) def test_no_archive(self): """ diff --git a/src/documents/tests/test_management_fuzzy.py b/src/documents/tests/test_management_fuzzy.py index 453a86082..2a4f28025 100644 --- a/src/documents/tests/test_management_fuzzy.py +++ b/src/documents/tests/test_management_fuzzy.py @@ -34,7 +34,7 @@ class TestFuzzyMatchCommand(TestCase): """ with self.assertRaises(CommandError) as e: self.call_command("--ratio", "-1") - self.assertIn("The ratio must be between 0 and 100", str(e)) + self.assertIn("The ratio must be between 0 and 100", str(e.exception)) def test_invalid_ratio_upper_limit(self): """ @@ -47,7 +47,7 @@ class TestFuzzyMatchCommand(TestCase): """ with self.assertRaises(CommandError) as e: self.call_command("--ratio", "101") - self.assertIn("The ratio must be between 0 and 100", str(e)) + self.assertIn("The ratio must be between 0 and 100", str(e.exception)) def test_invalid_process_count(self): """ @@ -60,7 +60,7 @@ class TestFuzzyMatchCommand(TestCase): """ with self.assertRaises(CommandError) as e: self.call_command("--processes", "0") - self.assertIn("There must be at least 1 process", str(e)) + self.assertIn("There must be at least 1 process", str(e.exception)) def test_no_matches(self): """ diff --git a/src/documents/tests/test_management_importer.py b/src/documents/tests/test_management_importer.py index e700ecdc9..2ae1834a5 100644 --- a/src/documents/tests/test_management_importer.py +++ b/src/documents/tests/test_management_importer.py @@ -16,6 +16,7 @@ from documents.settings import EXPORTER_FILE_NAME from documents.tests.utils import DirectoriesMixin from documents.tests.utils import FileSystemAssertsMixin from documents.tests.utils import SampleDirMixin +from documents.tests.utils import skip_if_root class TestCommandImport( @@ -40,10 +41,10 @@ class TestCommandImport( "--no-progress-bar", str(self.dirs.scratch_dir), ) - self.assertIn( - "That directory doesn't appear to contain a manifest.json file.", - str(e), - ) + self.assertIn( + "That directory doesn't appear to contain a manifest.json file.", + str(e.exception), + ) def test_check_manifest_malformed(self): """ @@ -66,10 +67,10 @@ class TestCommandImport( "--no-progress-bar", str(self.dirs.scratch_dir), ) - self.assertIn( - "The manifest file contains a record which does not refer to an actual document file.", - str(e), - ) + self.assertIn( + "The manifest file contains a record which does not refer to an actual document file.", + str(e.exception), + ) def test_check_manifest_file_not_found(self): """ @@ -95,8 +96,9 @@ class TestCommandImport( "--no-progress-bar", str(self.dirs.scratch_dir), ) - self.assertIn('The manifest file refers to "noexist.pdf"', str(e)) + self.assertIn('The manifest file refers to "noexist.pdf"', str(e.exception)) + @skip_if_root def test_import_permission_error(self): """ GIVEN: @@ -129,14 +131,14 @@ class TestCommandImport( cmd.data_only = False with self.assertRaises(CommandError) as cm: cmd.check_manifest_validity() - self.assertInt("Failed to read from original file", str(cm.exception)) + self.assertIn("Failed to read from original file", str(cm.exception)) original_path.chmod(0o444) archive_path.chmod(0o222) with self.assertRaises(CommandError) as cm: cmd.check_manifest_validity() - self.assertInt("Failed to read from archive file", str(cm.exception)) + self.assertIn("Failed to read from archive file", str(cm.exception)) def test_import_source_not_existing(self): """ @@ -149,8 +151,9 @@ class TestCommandImport( """ with self.assertRaises(CommandError) as cm: call_command("document_importer", Path("/tmp/notapath")) - self.assertInt("That path doesn't exist", str(cm.exception)) + self.assertIn("That path doesn't exist", str(cm.exception)) + @skip_if_root def test_import_source_not_readable(self): """ GIVEN: @@ -165,10 +168,10 @@ class TestCommandImport( path.chmod(0o222) with self.assertRaises(CommandError) as cm: call_command("document_importer", path) - self.assertInt( - "That path doesn't appear to be readable", - str(cm.exception), - ) + self.assertIn( + "That path doesn't appear to be readable", + str(cm.exception), + ) def test_import_source_does_not_exist(self): """ @@ -185,8 +188,7 @@ class TestCommandImport( with self.assertRaises(CommandError) as e: call_command("document_importer", "--no-progress-bar", str(path)) - - self.assertIn("That path doesn't exist", str(e)) + self.assertIn("That path doesn't exist", str(e.exception)) def test_import_files_exist(self): """ diff --git a/src/documents/tests/test_sanity_check.py b/src/documents/tests/test_sanity_check.py index fff5f2528..dc9c881f8 100644 --- a/src/documents/tests/test_sanity_check.py +++ b/src/documents/tests/test_sanity_check.py @@ -10,6 +10,7 @@ from django.test import override_settings from documents.models import Document from documents.sanity_checker import check_sanity from documents.tests.utils import DirectoriesMixin +from documents.tests.utils import skip_if_root class TestSanityCheck(DirectoriesMixin, TestCase): @@ -95,6 +96,7 @@ class TestSanityCheck(DirectoriesMixin, TestCase): Path(doc.thumbnail_path).unlink() self.assertSanityError(doc, "Thumbnail of document does not exist") + @skip_if_root def test_thumbnail_no_access(self): doc = self.make_test_data() Path(doc.thumbnail_path).chmod(0o000) @@ -106,6 +108,7 @@ class TestSanityCheck(DirectoriesMixin, TestCase): Path(doc.source_path).unlink() self.assertSanityError(doc, "Original of document does not exist.") + @skip_if_root def test_original_no_access(self): doc = self.make_test_data() Path(doc.source_path).chmod(0o000) @@ -123,6 +126,7 @@ class TestSanityCheck(DirectoriesMixin, TestCase): Path(doc.archive_path).unlink() self.assertSanityError(doc, "Archived version of document does not exist.") + @skip_if_root def test_archive_no_access(self): doc = self.make_test_data() Path(doc.archive_path).chmod(0o000) diff --git a/src/documents/tests/utils.py b/src/documents/tests/utils.py index 88dddc557..9ae06eac7 100644 --- a/src/documents/tests/utils.py +++ b/src/documents/tests/utils.py @@ -1,3 +1,4 @@ +import os import shutil import tempfile import time @@ -28,6 +29,8 @@ from documents.data_models import DocumentSource from documents.parsers import ParseError from documents.plugins.helpers import ProgressStatusOptions +skip_if_root = pytest.mark.skipif(os.getuid() == 0, reason="running as root") + def setup_directories(): dirs = namedtuple("Dirs", ()) diff --git a/src/paperless/tests/test_checks.py b/src/paperless/tests/test_checks.py index 781956ff6..f6a404938 100644 --- a/src/paperless/tests/test_checks.py +++ b/src/paperless/tests/test_checks.py @@ -7,6 +7,7 @@ from django.test import override_settings from documents.tests.utils import DirectoriesMixin from documents.tests.utils import FileSystemAssertsMixin +from documents.tests.utils import skip_if_root from paperless.checks import audit_log_check from paperless.checks import binaries_check from paperless.checks import debug_mode_check @@ -37,6 +38,7 @@ class TestChecks(DirectoriesMixin, TestCase): for msg in msgs: self.assertTrue(msg.msg.endswith("is set but doesn't exist.")) + @skip_if_root def test_paths_check_no_access(self): Path(self.dirs.data_dir).chmod(0o000) Path(self.dirs.media_dir).chmod(0o000)