paperless-ngx/.github/workflows/ci.yml

name: ci

on:
  push:
    tags:
      # https://semver.org/#spec-item-2
      - 'v[0-9]+.[0-9]+.[0-9]+'
      # https://semver.org/#spec-item-9
      - 'v[0-9]+.[0-9]+.[0-9]+-beta.rc[0-9]+'
    branches-ignore:
      - 'translations**'
  pull_request:
    branches-ignore:
      - 'translations**'

jobs:
  pre-commit:
    name: Linting Checks
    runs-on: ubuntu-latest
    steps:
      -
        name: Checkout repository
        uses: actions/checkout@v3

      -
        name: Install tools
        uses: actions/setup-python@v4
        with:
          python-version: "3.9"

      -
        name: Check files
        uses: pre-commit/action@v3.0.0

  documentation:
    name: "Build Documentation"
    runs-on: ubuntu-20.04
    needs:
      - pre-commit
    steps:
      -
        name: Checkout
        uses: actions/checkout@v3
      -
        name: Install pipenv
        run: |
          pipx install pipenv==2022.10.12
      -
        name: Set up Python
        uses: actions/setup-python@v4
        with:
          python-version: 3.8
          cache: "pipenv"
          cache-dependency-path: 'Pipfile.lock'
      -
        name: Install dependencies
        run: |
          pipenv sync --dev
      -
        name: List installed Python dependencies
        run: |
          pipenv run pip list
      -
        name: Make documentation
        run: |
          pipenv run mkdocs build --config-file ./mkdocs.yml
      -
        name: Upload artifact
        uses: actions/upload-artifact@v3
        with:
          name: documentation
          path: site/

  documentation-deploy:
    name: "Deploy Documentation"
    runs-on: ubuntu-20.04
    if: github.event_name == 'push' && github.ref == 'refs/heads/main'
    needs:
      - documentation
    steps:
      -
        name: Checkout
        uses: actions/checkout@v3
      -
        name: Deploy docs
        uses: mhausenblas/mkdocs-deploy-gh-pages@master
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          CUSTOM_DOMAIN: docs.paperless-ngx.com
          CONFIG_FILE: mkdocs.yml
          EXTRA_PACKAGES: build-base

  tests-backend:
    name: "Tests (${{ matrix.python-version }})"
    runs-on: ubuntu-20.04
    needs:
      - pre-commit
    strategy:
      matrix:
        python-version: ['3.8', '3.9', '3.10']
      fail-fast: false
    env:
      # Enable Tika end to end testing
      TIKA_LIVE: 1
      # Enable paperless_mail testing against real server
      PAPERLESS_MAIL_TEST_HOST: ${{ secrets.TEST_MAIL_HOST }}
      PAPERLESS_MAIL_TEST_USER: ${{ secrets.TEST_MAIL_USER }}
      PAPERLESS_MAIL_TEST_PASSWD: ${{ secrets.TEST_MAIL_PASSWD }}
      # Skip Tests which require convert
      PAPERLESS_TEST_SKIP_CONVERT: 1
      # Enable Gotenberg end to end testing
      GOTENBERG_LIVE: 1
    steps:
      -
        name: Checkout
        uses: actions/checkout@v3
        with:
          fetch-depth: 0
      -
        name: Start containers
        run: |
          docker compose --file ${GITHUB_WORKSPACE}/docker/compose/docker-compose.ci-test.yml pull --quiet
          docker compose --file ${GITHUB_WORKSPACE}/docker/compose/docker-compose.ci-test.yml up --detach
      -
        name: Install pipenv
        run: |
          pipx install pipenv==2022.10.12
      -
        name: Set up Python
        uses: actions/setup-python@v4
        with:
          python-version: "${{ matrix.python-version }}"
          cache: "pipenv"
          cache-dependency-path: 'Pipfile.lock'
      -
        name: Install system dependencies
        run: |
          sudo apt-get update -qq
          sudo apt-get install -qq --no-install-recommends unpaper tesseract-ocr imagemagick ghostscript libzbar0 poppler-utils
      -
        name: Install Python dependencies
        run: |
          pipenv sync --dev
      -
        name: List installed Python dependencies
        run: |
          pipenv run pip list
      -
        name: Tests
        run: |
          cd src/
          pipenv run pytest -rfEp
      -
        name: Get changed files
        id: changed-files-specific
        uses: tj-actions/changed-files@v34
        with:
          files: |
            src/**
      -
        name: List all changed files
        run: |
          for file in ${{ steps.changed-files-specific.outputs.all_changed_files }}; do
            echo "${file} was changed"
          done
      -
        name: Publish coverage results
        if: matrix.python-version == '3.9' && steps.changed-files-specific.outputs.any_changed == 'true'
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        # https://github.com/coveralls-clients/coveralls-python/issues/251
        run: |
          cd src/
          pipenv run coveralls --service=github
      -
        name: Stop containers
        if: always()
        run: |
          docker compose --file ${GITHUB_WORKSPACE}/docker/compose/docker-compose.ci-test.yml logs
          docker compose --file ${GITHUB_WORKSPACE}/docker/compose/docker-compose.ci-test.yml down

  tests-frontend:
    name: "Tests Frontend"
    runs-on: ubuntu-20.04
    needs:
      - pre-commit
    strategy:
      matrix:
        node-version: [16.x]
    steps:
      - uses: actions/checkout@v3
      -
        name: Use Node.js ${{ matrix.node-version }}
        uses: actions/setup-node@v3
        with:
          node-version: ${{ matrix.node-version }}
      - run: cd src-ui && npm ci
      - run: cd src-ui && npm run test
      - run: cd src-ui && npm run e2e:ci

  prepare-docker-build:
    name: Prepare Docker Pipeline Data
    if: github.event_name == 'push' && (startsWith(github.ref, 'refs/heads/feature-') || github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/beta' || contains(github.ref, 'beta.rc') || startsWith(github.ref, 'refs/tags/v'))
    runs-on: ubuntu-20.04
    # If the push triggered the installer library workflow, wait for it to
    # complete here.  This ensures the required versions for the final
    # image have been built, while not waiting at all if the versions haven't changed
    concurrency:
      group: build-installer-library
      cancel-in-progress: false
    needs:
      - documentation
      - tests-backend
      - tests-frontend
    steps:
      -
        name: Set ghcr repository name
        id: set-ghcr-repository
        run: |
          ghcr_name=$(echo "${GITHUB_REPOSITORY}" | awk '{ print tolower($0) }')
          echo "repository=${ghcr_name}" >> $GITHUB_OUTPUT
      -
        name: Checkout
        uses: actions/checkout@v3
      -
        name: Set up Python
        uses: actions/setup-python@v4
        with:
          python-version: "3.9"
      -
        name: Setup qpdf image
        id: qpdf-setup
        run: |
          build_json=$(python ${GITHUB_WORKSPACE}/.github/scripts/get-build-json.py qpdf)

          echo ${build_json}

          echo "qpdf-json=${build_json}" >> $GITHUB_OUTPUT
      -
        name: Setup psycopg2 image
        id: psycopg2-setup
        run: |
          build_json=$(python ${GITHUB_WORKSPACE}/.github/scripts/get-build-json.py psycopg2)

          echo ${build_json}

          echo "psycopg2-json=${build_json}" >> $GITHUB_OUTPUT
      -
        name: Setup pikepdf image
        id: pikepdf-setup
        run: |
          build_json=$(python ${GITHUB_WORKSPACE}/.github/scripts/get-build-json.py pikepdf)

          echo ${build_json}

          echo "pikepdf-json=${build_json}" >> $GITHUB_OUTPUT
      -
        name: Setup jbig2enc image
        id: jbig2enc-setup
        run: |
          build_json=$(python ${GITHUB_WORKSPACE}/.github/scripts/get-build-json.py jbig2enc)

          echo ${build_json}

          echo "jbig2enc-json=${build_json}" >> $GITHUB_OUTPUT

    outputs:

      ghcr-repository: ${{ steps.set-ghcr-repository.outputs.repository }}

      qpdf-json: ${{ steps.qpdf-setup.outputs.qpdf-json }}

      pikepdf-json: ${{ steps.pikepdf-setup.outputs.pikepdf-json }}

      psycopg2-json: ${{ steps.psycopg2-setup.outputs.psycopg2-json }}

      jbig2enc-json: ${{ steps.jbig2enc-setup.outputs.jbig2enc-json}}

  # build and push image to docker hub.
  build-docker-image:
    runs-on: ubuntu-20.04
    concurrency:
      group: ${{ github.workflow }}-build-docker-image-${{ github.ref_name }}
      cancel-in-progress: true
    needs:
      - prepare-docker-build
    steps:
      -
        name: Check pushing to Docker Hub
        id: docker-hub
        # Only push to Dockerhub from the main repo AND the ref is either:
        #  main
        #  dev
        #  beta
        #  a tag
        # Otherwise forks would require a Docker Hub account and secrets setup
        run: |
          if [[ ${{ needs.prepare-docker-build.outputs.ghcr-repository }} == "paperless-ngx/paperless-ngx" && ( ${{ github.ref_name }} == "main" || ${{ github.ref_name }} == "dev" || ${{ github.ref_name }} == "beta" || ${{ startsWith(github.ref, 'refs/tags/v') }} == "true" ) ]] ; then
            echo "Enabling DockerHub image push"
            echo "enable=true" >> $GITHUB_OUTPUT
          else
            echo "Not pushing to DockerHub"
            echo "enable=false" >> $GITHUB_OUTPUT
          fi
      -
        name: Gather Docker metadata
        id: docker-meta
        uses: docker/metadata-action@v4
        with:
          images: |
            ghcr.io/${{ needs.prepare-docker-build.outputs.ghcr-repository }}
            name=paperlessngx/paperless-ngx,enable=${{ steps.docker-hub.outputs.enable }}
          tags: |
            # Tag branches with branch name
            type=ref,event=branch
            # Process semver tags
            # For a tag x.y.z or vX.Y.Z, output an x.y.z and x.y image tag
            type=semver,pattern={{version}}
            type=semver,pattern={{major}}.{{minor}}
      -
        name: Checkout
        uses: actions/checkout@v3
      -
        name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v2
      -
        name: Set up QEMU
        uses: docker/setup-qemu-action@v2
      -
        name: Login to Github Container Registry
        uses: docker/login-action@v2
        with:
          registry: ghcr.io
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}
      -
        name: Login to Docker Hub
        uses: docker/login-action@v2
        # Don't attempt to login is not pushing to Docker Hub
        if: steps.docker-hub.outputs.enable == 'true'
        with:
          username: ${{ secrets.DOCKERHUB_USERNAME }}
          password: ${{ secrets.DOCKERHUB_TOKEN }}
      -
        name: Build and push
        uses: docker/build-push-action@v3
        with:
          context: .
          file: ./Dockerfile
          platforms: linux/amd64,linux/arm/v7,linux/arm64
          push: ${{ github.event_name != 'pull_request' }}
          tags: ${{ steps.docker-meta.outputs.tags }}
          labels: ${{ steps.docker-meta.outputs.labels }}
          build-args: |
            JBIG2ENC_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.jbig2enc-json).version }}
            QPDF_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.qpdf-json).version }}
            PIKEPDF_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.pikepdf-json).version }}
            PSYCOPG2_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.psycopg2-json).version }}
          # Get cache layers from this branch, then dev, then main
          # This allows new branches to get at least some cache benefits, generally from dev
          cache-from: |
            type=registry,ref=ghcr.io/${{ needs.prepare-docker-build.outputs.ghcr-repository }}/builder/cache/app:${{ github.ref_name }}
            type=registry,ref=ghcr.io/${{ needs.prepare-docker-build.outputs.ghcr-repository }}/builder/cache/app:dev
            type=registry,ref=ghcr.io/${{ needs.prepare-docker-build.outputs.ghcr-repository }}/builder/cache/app:main
          cache-to: |
            type=registry,mode=max,ref=ghcr.io/${{ needs.prepare-docker-build.outputs.ghcr-repository }}/builder/cache/app:${{ github.ref_name }}
      -
        name: Inspect image
        run: |
          docker buildx imagetools inspect ${{ fromJSON(steps.docker-meta.outputs.json).tags[0] }}
      -
        name: Export frontend artifact from docker
        run: |
          docker create --name frontend-extract ${{ fromJSON(steps.docker-meta.outputs.json).tags[0] }}
          docker cp frontend-extract:/usr/src/paperless/src/documents/static/frontend src/documents/static/frontend/
      -
        name: Upload frontend artifact
        uses: actions/upload-artifact@v3
        with:
          name: frontend-compiled
          path: src/documents/static/frontend/

  build-release:
    needs:
      - build-docker-image
    runs-on: ubuntu-20.04
    steps:
      -
        name: Checkout
        uses: actions/checkout@v3
      -
        name: Install pipenv
        run: |
          pip3 install --upgrade pip setuptools wheel pipx
          pipx install pipenv
      -
        name: Set up Python
        uses: actions/setup-python@v4
        with:
          python-version: 3.9
          cache: "pipenv"
          cache-dependency-path: 'Pipfile.lock'
      -
        name: Install Python dependencies
        run: |
          pipenv sync --dev
      -
        name: Install system dependencies
        run: |
          sudo apt-get update -qq
          sudo apt-get install -qq --no-install-recommends gettext liblept5
      -
        name: Download frontend artifact
        uses: actions/download-artifact@v3
        with:
          name: frontend-compiled
          path: src/documents/static/frontend/
      -
        name: Download documentation artifact
        uses: actions/download-artifact@v3
        with:
          name: documentation
          path: docs/_build/html/
      -
        name: Generate requirements file
        run: |
          pipenv requirements > requirements.txt
      -
        name: Compile messages
        run: |
          cd src/
          pipenv run python3 manage.py compilemessages
      -
        name: Collect static files
        run: |
          cd src/
          pipenv run python3 manage.py collectstatic --no-input
      -
        name: Move files
        run: |
          mkdir dist
          mkdir dist/paperless-ngx
          mkdir dist/paperless-ngx/scripts
          cp .dockerignore .env Dockerfile Pipfile Pipfile.lock requirements.txt LICENSE README.md dist/paperless-ngx/
          cp paperless.conf.example dist/paperless-ngx/paperless.conf
          cp gunicorn.conf.py dist/paperless-ngx/gunicorn.conf.py
          cp -r docker/ dist/paperless-ngx/docker
          cp scripts/*.service scripts/*.sh dist/paperless-ngx/scripts/
          cp -r src/ dist/paperless-ngx/src
          cp -r docs/_build/html/ dist/paperless-ngx/docs
          mv static dist/paperless-ngx
      -
        name: Make release package
        run: |
          cd dist
          tar -cJf paperless-ngx.tar.xz paperless-ngx/
      -
        name: Upload release artifact
        uses: actions/upload-artifact@v3
        with:
          name: release
          path: dist/paperless-ngx.tar.xz

  publish-release:
    runs-on: ubuntu-20.04
    outputs:
      prerelease: ${{ steps.get_version.outputs.prerelease }}
      changelog: ${{ steps.create-release.outputs.body }}
      version: ${{ steps.get_version.outputs.version }}
    needs:
      - build-release
    if: github.ref_type == 'tag' && (startsWith(github.ref_name, 'v') || contains(github.ref_name, '-beta.rc'))
    steps:
      -
        name: Download release artifact
        uses: actions/download-artifact@v3
        with:
          name: release
          path: ./
      -
        name: Get version
        id: get_version
        run: |
          echo "version=${{ github.ref_name }}" >> $GITHUB_OUTPUT
          if [[ ${{ contains(github.ref_name, '-beta.rc') }} == 'true' ]]; then
            echo "prerelease=true" >> $GITHUB_OUTPUT
          else
            echo "prerelease=false" >> $GITHUB_OUTPUT
          fi
      -
        name: Create Release and Changelog
        id: create-release
        uses: paperless-ngx/release-drafter@master
        with:
          name: Paperless-ngx ${{ steps.get_version.outputs.version }}
          tag: ${{ steps.get_version.outputs.version }}
          version: ${{ steps.get_version.outputs.version }}
          prerelease: ${{ steps.get_version.outputs.prerelease }}
          publish: true # ensures release is not marked as draft
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
      -
        name: Upload release archive
        id: upload-release-asset
        uses: shogo82148/actions-upload-release-asset@v1
        with:
          github_token: ${{ secrets.GITHUB_TOKEN }}
          upload_url: ${{ steps.create-release.outputs.upload_url }}
          asset_path: ./paperless-ngx.tar.xz
          asset_name: paperless-ngx-${{ steps.get_version.outputs.version }}.tar.xz
          asset_content_type: application/x-xz

  append-changelog:
    runs-on: ubuntu-20.04
    needs:
      - publish-release
    if: needs.publish-release.outputs.prerelease == 'false'
    steps:
      -
        name: Checkout
        uses: actions/checkout@v3
        with:
          ref: main
      -
        name: Install pipenv
        run: |
          pip3 install --upgrade pip setuptools wheel pipx
          pipx install pipenv
      -
        name: Set up Python
        uses: actions/setup-python@v4
        with:
          python-version: 3.9
          cache: "pipenv"
          cache-dependency-path: 'Pipfile.lock'
      -
        name: Append Changelog to docs
        id: append-Changelog
        working-directory: docs
        run: |
          git branch ${{ needs.publish-release.outputs.version }}-changelog
          git checkout ${{ needs.publish-release.outputs.version }}-changelog
          echo -e "# Changelog\n\n${{ needs.publish-release.outputs.changelog }}\n" > changelog-new.md
          echo "Manually linking usernames"
          sed -i -r 's|@(.+?) \(\[#|[@\1](https://github.com/\1) ([#|ig' changelog-new.md
          CURRENT_CHANGELOG=`tail --lines +2 changelog.md`
          echo -e "$CURRENT_CHANGELOG" >> changelog-new.md
          mv changelog-new.md changelog.md
          pipenv run pre-commit run --files changelog.md || true
          git config --global user.name "github-actions"
          git config --global user.email "41898282+github-actions[bot]@users.noreply.github.com"
          git commit -am "Changelog ${{ needs.publish-release.outputs.version }} - GHA"
          git push origin ${{ needs.publish-release.outputs.version }}-changelog
      -
        name: Create Pull Request
        uses: actions/github-script@v6
        with:
          script: |
            const { repo, owner } = context.repo;
            const result = await github.rest.pulls.create({
              title: '[Documentation] Add ${{ needs.publish-release.outputs.version }} changelog',
              owner,
              repo,
              head: '${{ needs.publish-release.outputs.version }}-changelog',
              base: 'main',
              body: 'This PR is auto-generated by CI.'
            });
            github.rest.issues.addLabels({
              owner,
              repo,
              issue_number: result.data.number,
              labels: ['documentation']
            });