docker image with qreader, 2nd image without qreader

- organize imports
- add qreader pytest.mark
- relaxed mode for pytest
- run tests in docker
- more tests

Dockerfile

@@ -1,10 +1,11 @@
-FROM python:3.11-alpine
+FROM python:3.11-slim-bullseye
 
 WORKDIR /extract
 
 COPY . .
 
-RUN pip install -r requirements.txt
+RUN apt-get update && apt-get install -y libzbar0 python3-opencv \
+    && pip install -r requirements.txt
 
 WORKDIR /files
 

Dockerfile_no_qr_reader

@@ -0,0 +1,11 @@
+FROM python:3.11-alpine
+
+WORKDIR /extract
+
+COPY . .
+
+RUN pip install protobuf qrcode Pillow
+
+WORKDIR /files
+
+ENTRYPOINT [ "python", "/extract/extract_otp_secret_keys.py" ]

README.md

@@ -303,10 +303,22 @@ Install [Docker](https://docs.docker.com/get-docker/).
 Build and run the app within the container:
 
 ```bash
-docker build . -t extract_otp
-docker run --rm -v "$(pwd)":/files:ro extract_otp -p example_export.txt
+docker build . -t extract_otp_secret_keys --pull
+docker run --rm -v "$(pwd)":/files:ro extract_otp_secret_keys example_export.txt
+docker run --rm -v "$(pwd)":/files:ro extract_otp_secret_keys example_export.png
 ```
 
+docker run --rm -v "$(pwd)":/files:ro -i extract_otp_secret_keys = < example_export.png
+docker run --entrypoint /bin/bash -it --rm -v "$(pwd)":/files:ro extract_otp_secret_keys
+docker run --entrypoint /extract/run_pytest.sh --rm -v "$(pwd)":/files:ro extract_otp_secret_keys
+
+docker build . -t extract_otp_secret_keys_no_qr_reader -f Dockerfile_no_qr_reader --pull
+docker run --entrypoint /extract/run_pytest.sh --rm -v "$(pwd)":/files:ro extract_otp_secret_keys_no_qr_reader
+docker run --entrypoint /extract/run_pytest.sh --rm -v "$(pwd)":/files:ro extract_otp_secret_keys_no_qr_reader test_extract_otp_secret_keys_pytest.py -k "not qreader"
+docker run --rm -v "$(pwd)":/files:ro extract_otp_secret_keys_no_qr_reader example_export.txt
+docker run --rm -v "$(pwd)":/files:ro -i extract_otp_secret_keys_no_qr_reader - < example_export.txt
+docker build . -t extract_otp_secret_keys_no_qr_reader -f Dockerfile_no_qr_reader --pull && docker run --entrypoint /extract/run_pytest.sh --rm -v "$(pwd)":/files:ro extract_otp_secret_keys_no_qr_reader test_extract_otp_secret_keys_pytest.py -k "not qreader" -vvv --relaxed -s
+
 ## Tests
 
 ### PyTest

conftest.py

@@ -0,0 +1,10 @@
+import pytest
+
+
+def pytest_addoption(parser):
+    parser.addoption( "--relaxed", action='store_true', help="run tests in relaxed mode")
+
+
+@pytest.fixture
+def relaxed(request):
+    return request.config.getoption("--relaxed")

extract_otp_secret_keys.py

@@ -51,11 +51,12 @@ import re
 import sys
 import urllib.parse as urlparse
 
-import cv2
-import numpy
-
 import protobuf_generated_python.google_auth_pb2
 
+# These dynamic import are below:
+# import cv2
+# import numpy
+# from qreader import QReader
 
 def sys_main():
     main(sys.argv[1:])
@@ -66,6 +67,7 @@ def main(sys_args):
 
     # allow to use sys.stdout with with (avoid closing)
     sys.stdout.close = lambda: None
+    # sys.stdout.reconfigure(encoding='utf-8')
 
     args = parse_args(sys_args)
     verbose = args.verbose if args.verbose else 0
@@ -147,7 +149,7 @@ def get_lines_from_file(filename):
     if filename != '=':
         check_file_exists(filename)
         lines = read_lines_from_text_file(filename)
-        if lines:
+        if lines or filename == '-':
             return lines
 
     # could not process text file, try reading as image
@@ -166,6 +168,8 @@ def read_lines_from_text_file(filename):
                 abort('\nBinary input was given in stdin, please use = instead of - as infile argument for images.')
             # unfortunately yield line leads to random test fails
             lines.append(line)
+        if not lines:
+            eprint("WARN: {} is empty".format(filename.replace('-', 'stdin')))
         return lines
     except UnicodeDecodeError:
         if filename == '-':
@@ -178,6 +182,12 @@ def read_lines_from_text_file(filename):
 
 
 def convert_img_to_line(filename):
+    try:
+        import cv2
+        import numpy
+    except Exception as e:
+        eprint("WARNING: No cv2 or numpy module installed. Exception: {}".format(str(e)))
+        return []
     if verbose: print('Reading image {}'.format(filename))
     try:
         if filename != '=':
@@ -188,11 +198,15 @@ def convert_img_to_line(filename):
             except AttributeError:
                 # Workaround for pytest, since pytest cannot monkeypatch sys.stdin.buffer
                 stdin = sys.stdin.read()
+            if not stdin:
+                eprint("WARN: stdin is empty")
             try:
-                array = numpy.frombuffer(stdin, dtype='uint8')
+                img_array = numpy.frombuffer(stdin, dtype='uint8')
             except TypeError as e:
                 abort('\nERROR: Cannot read binary stdin buffer. Exception: {}'.format(str(e)))
-            image = cv2.imdecode(array, cv2.IMREAD_UNCHANGED)
+            if not img_array.size:
+                return []
+            image = cv2.imdecode(img_array, cv2.IMREAD_UNCHANGED)
 
         if image is None:
             abort('\nERROR: Unable to open file for reading.\ninput file: {}'.format(filename))

pytest.ini

@@ -0,0 +1,3 @@
+[pytest]
+markers =
+    qreader: QR image reader tests

run_pytest.sh

@@ -0,0 +1,3 @@
+#!/bin/sh
+cd /extract
+pip install -U pytest && pytest "$@"

test_extract_otp_secret_keys_pytest.py

@@ -20,14 +20,13 @@
 
 import io
 import os
+import re
 import sys
 
-from pytest import mark, raises
+import pytest
 
 import extract_otp_secret_keys
-from utils import (file_exits, read_binary_file_as_stream, read_csv,
-                   read_csv_str, read_file_to_str, read_json, read_json_str,
-                   remove_dir_with_files, remove_files)
+from utils import *
 
 
 def test_extract_stdout(capsys):
@@ -41,6 +40,7 @@ def test_extract_stdout(capsys):
     assert captured.err == ''
 
 
+@pytest.mark.qreader
 def test_extract_multiple_files_and_mixed(capsys):
     # Act
     extract_otp_secret_keys.main([
@@ -58,7 +58,7 @@ def test_extract_multiple_files_and_mixed(capsys):
 
 def test_extract_non_existent_file(capsys):
     # Act
-    with raises(SystemExit) as e:
+    with pytest.raises(SystemExit) as e:
         extract_otp_secret_keys.main(['test/non_existent_file.txt'])
 
     # Assert
@@ -86,12 +86,58 @@ def test_extract_stdin_stdout(capsys, monkeypatch):
     assert captured.err == ''
 
 
+def test_extract_stdin_empty(capsys, monkeypatch):
+    # Arrange
+    monkeypatch.setattr('sys.stdin', io.StringIO())
+
+    # Act
+    extract_otp_secret_keys.main(['-'])
+
+    # Assert
+    captured = capsys.readouterr()
+
+    assert captured.out == ''
+    assert captured.err == 'WARN: stdin is empty\n'
+
+
+def test_extract_empty_file(capsys):
+    # Act
+    with pytest.raises(SystemExit) as e:
+        extract_otp_secret_keys.main(['test/empty_file.txt'])
+
+    # Assert
+    captured = capsys.readouterr()
+
+    expected_stderr = 'WARN: test/empty_file.txt is empty\n\nERROR: Unable to open file for reading.\ninput file: test/empty_file.txt\n'
+
+    assert captured.err == expected_stderr
+    assert captured.out == ''
+    assert e.value.code == 1
+    assert e.type == SystemExit
+
+
+@pytest.mark.qreader
+def test_extract_stdin_img_empty(capsys, monkeypatch):
+    # Arrange
+    monkeypatch.setattr('sys.stdin', io.BytesIO())
+
+    # Act
+    extract_otp_secret_keys.main(['='])
+
+    # Assert
+    captured = capsys.readouterr()
+
+    assert captured.out == ''
+    assert captured.err == 'WARN: stdin is empty\n'
+
+
+@pytest.mark.qreader
 def test_extract_stdin_stdout_wrong_symbol(capsys, monkeypatch):
     # Arrange
     monkeypatch.setattr('sys.stdin', io.StringIO(read_file_to_str('example_export.txt')))
 
     # Act
-    with raises(SystemExit) as e:
+    with pytest.raises(SystemExit) as e:
         extract_otp_secret_keys.main(['='])
 
     # Assert
@@ -359,8 +405,11 @@ def test_extract_saveqr(capsys):
     cleanup()
 
 
-@mark.skipif(sys.implementation.name == 'pypy', reason="Encoding problems in verbose mode in pypy.")
-def test_extract_verbose(capsys):
+def test_normalize_bytes():
+    assert replace_escaped_octal_utf8_bytes_with_str('Before\\\\302\\\\277\\\\303\nname: enc: \\302\\277\\303\\244\\303\\204\\303\\251\\303\\211?\nAfter') == 'Before\\\\302\\\\277\\\\303\nname: enc: ¿äÄéÉ?\nAfter'
+
+
+def test_extract_verbose(capsys, relaxed):
     # Act
     extract_otp_secret_keys.main(['-v', 'example_export.txt'])
 
@@ -369,7 +418,13 @@ def test_extract_verbose(capsys):
 
     expected_stdout = read_file_to_str('test/print_verbose_output.txt')
 
-    assert captured.out == expected_stdout
+    if relaxed or sys.implementation.name == 'pypy':
+        print('\nRelaxed mode\n')
+
+        assert replace_escaped_octal_utf8_bytes_with_str(captured.out) == replace_escaped_octal_utf8_bytes_with_str(expected_stdout)
+        assert quick_and_dirty_workaround_encoding_problem(captured.out) == quick_and_dirty_workaround_encoding_problem(expected_stdout)
+    else:
+        assert captured.out == expected_stdout
     assert captured.err == ''
 
 
@@ -388,7 +443,7 @@ def test_extract_debug(capsys):
 
 
 def test_extract_help(capsys):
-    with raises(SystemExit) as e:
+    with pytest.raises(SystemExit) as e:
         # Act
         extract_otp_secret_keys.main(['-h'])
 
@@ -404,7 +459,7 @@ def test_extract_help(capsys):
 
 def test_extract_no_arguments(capsys):
     # Act
-    with raises(SystemExit) as e:
+    with pytest.raises(SystemExit) as e:
         extract_otp_secret_keys.main([])
 
     # Assert
@@ -419,7 +474,7 @@ def test_extract_no_arguments(capsys):
 
 
 def test_verbose_and_quiet(capsys):
-    with raises(SystemExit) as e:
+    with pytest.raises(SystemExit) as e:
         # Act
         extract_otp_secret_keys.main(['-v', '-q', 'example_export.txt'])
 
@@ -434,7 +489,7 @@ def test_verbose_and_quiet(capsys):
 
 
 def test_wrong_data(capsys):
-    with raises(SystemExit) as e:
+    with pytest.raises(SystemExit) as e:
         # Act
         extract_otp_secret_keys.main(['test/test_export_wrong_data.txt'])
 
@@ -453,7 +508,7 @@ data=XXXX
 
 
 def test_wrong_content(capsys):
-    with raises(SystemExit) as e:
+    with pytest.raises(SystemExit) as e:
         # Act
         extract_otp_secret_keys.main(['test/test_export_wrong_content.txt'])
 
@@ -509,6 +564,7 @@ def test_add_pre_suffix(capsys):
     assert extract_otp_secret_keys.add_pre_suffix("name", "totp") == "name.totp"
 
 
+@pytest.mark.qreader
 def test_img_qr_reader_from_file_happy_path(capsys):
     # Act
     extract_otp_secret_keys.main(['test/test_googleauth_export.png'])
@@ -519,7 +575,7 @@ def test_img_qr_reader_from_file_happy_path(capsys):
     assert captured.out == EXPECTED_STDOUT_FROM_EXAMPLE_EXPORT_PNG
     assert captured.err == ''
 
-
+@pytest.mark.qreader
 def test_img_qr_reader_from_stdin(capsys, monkeypatch):
     # Arrange
     # sys.stdin.buffer should be monkey patched, but it does not work
@@ -553,13 +609,14 @@ Type:    totp
     assert captured.err == ''
 
 
+@pytest.mark.qreader
 def test_img_qr_reader_from_stdin_wrong_symbol(capsys, monkeypatch):
     # Arrange
     # sys.stdin.buffer should be monkey patched, but it does not work
     monkeypatch.setattr('sys.stdin', read_binary_file_as_stream('test/test_googleauth_export.png'))
 
     # Act
-    with raises(SystemExit) as e:
+    with pytest.raises(SystemExit) as e:
         extract_otp_secret_keys.main(['-'])
 
     # Assert
@@ -573,9 +630,10 @@ def test_img_qr_reader_from_stdin_wrong_symbol(capsys, monkeypatch):
     assert e.type == SystemExit
 
 
+@pytest.mark.qreader
 def test_img_qr_reader_no_qr_code_in_image(capsys):
     # Act
-    with raises(SystemExit) as e:
+    with pytest.raises(SystemExit) as e:
         extract_otp_secret_keys.main(['test/lena_std.tif'])
 
     # Assert
@@ -589,9 +647,10 @@ def test_img_qr_reader_no_qr_code_in_image(capsys):
     assert e.type == SystemExit
 
 
+@pytest.mark.qreader
 def test_img_qr_reader_nonexistent_file(capsys):
     # Act
-    with raises(SystemExit) as e:
+    with pytest.raises(SystemExit) as e:
         extract_otp_secret_keys.main(['test/nonexistent.bmp'])
 
     # Assert
@@ -607,7 +666,7 @@ def test_img_qr_reader_nonexistent_file(capsys):
 
 def test_non_image_file(capsys):
     # Act
-    with raises(SystemExit) as e:
+    with pytest.raises(SystemExit) as e:
         extract_otp_secret_keys.main(['test/text_masquerading_as_image.jpeg'])
 
     # Assert

utils.py

@@ -14,12 +14,13 @@
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
 import csv
+import glob
+import io
 import json
 import os
+import re
 import shutil
-import io
 import sys
-import glob
 
 
 # Ref. https://stackoverflow.com/a/16571630
@@ -107,3 +108,17 @@ def read_binary_file_as_stream(filename):
     """Returns binary file content."""
     with open(filename, "rb",) as infile:
         return io.BytesIO(infile.read())
+
+def replace_escaped_octal_utf8_bytes_with_str(str):
+    encoded_name_strings = re.findall(r'name: .*$', str, flags=re.MULTILINE)
+    for encoded_name_string in encoded_name_strings:
+        escaped_bytes = re.findall(r'((?:\\[0-9]+)+)', encoded_name_string)
+        for byte_sequence in escaped_bytes:
+            unicode_str = b''.join([int(byte, 8).to_bytes(1) for byte in byte_sequence.split('\\') if byte]).decode('utf-8')
+            print("Replace '{}' by '{}'".format(byte_sequence, unicode_str))
+            str = str.replace(byte_sequence, unicode_str)
+    return str
+
+
+def quick_and_dirty_workaround_encoding_problem(str):
+    return re.sub(r'name: "encoding: .*$', '', str, flags=re.MULTILINE)