diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml
index cb37a49..55eaada 100644
--- a/.github/workflows/docs.yml
+++ b/.github/workflows/docs.yml
@@ -6,7 +6,6 @@ on:
       - main
     tags:
       - "[0-9]+.[0-9]+.[0-9]+"
-  pull_request:
 
 env:
   COLORTERM: 'yes'
@@ -71,7 +70,9 @@ jobs:
             fi
           done
           antsibull-changelog lint -vvv
-          antsibull-changelog generate
+          if [[ "${{github.ref}}" == "refs/heads/main" ]]; then
+            antsibull-changelog release --version "prerelease-$(grep version galaxy.yml | awk -F'"' '{ print $2 }')"
+          fi
         working-directory: ansible_collections/middleware_automation/keycloak
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index aafcfb1..30e480f 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -1,30 +1,86 @@
 ---
 name: Release collection
 on:
-  push:
-    tags:
-      - "[0-9]+.[0-9]+.[0-9]+"
+  workflow_dispatch:
 
 jobs:
   release:
     runs-on: ubuntu-latest
+    if: github.repository == 'ansible-middleware/keycloak'
+    permissions:
+      actions: write
+      checks: write
+      contents: write
+      deployments: write
+      packages: write
+      pages: write
     steps:
       - name: Checkout code
         uses: actions/checkout@v2
+      - name: Fetch tags
+        run: git fetch --depth=1 origin +refs/tags/*:refs/tags/*
       - name: Set up Python
         uses: actions/setup-python@v1
         with:
           python-version: "3.x"
-      - name: Get Tag Version
+      - name: Get current version
         id: get_version
-        run: echo ::set-output name=TAG_VERSION::${GITHUB_REF#refs/tags/}
+        run: echo "::set-output name=TAG_VERSION::$(grep version galaxy.yml | awk -F'"' '{ print $2 }')"
+      - name: Check if tag exists
+        id: check_tag
+        run: echo "::set-output name=TAG_EXISTS::$(git tag | grep ${{ steps.get_version.outputs.TAG_VERSION }})"
+      - name: Fail if tag exists
+        if: ${{ steps.get_version.outputs.TAG_VERSION == steps.check_tag.outputs.TAG_EXISTS }}
+        uses: actions/github-script@v3
+        with:
+          script: |
+              core.setFailed('Release tag already exists')
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
           pip install ansible-core
+          sudo apt install -y sed hub
       - name: Build collection
         run: |
           ansible-galaxy collection build .
+      - name: Scan PR merges from latest tag
+        run: |
+          TYPES=("minor_changes" "major_changes" "bugfixes" "deprecated_features" "removed_features")
+          TAG=$(git describe --abbrev=0 --tags)
+          PRS=($(comm -12 <(git log --oneline ${TAG}.. --format="tformat:%H" | sort ) <(hub pr list -s all -f '%sm%n' --color=never | sort )))
+          IFS=$'\n' FRAGMENTS=($(hub pr list -s all -f '%sm~%I~%L~%t~%n' --color=never| grep -P "$(echo "^(${PRS[@]})" | tr ' ' '|')"))
+          for frag in "${FRAGMENTS[@]}"; do
+            PR=$(echo $frag|cut -d~ -f2)
+            type="$(echo $frag|cut -d~ -f3)"
+            msg="$(echo $frag|cut -d~ -f4|sed 's/`/``/g')"
+            if [[ "${TYPES[*]}" =~ "${type}" ]]; then
+              echo -e "$type:\n  - >\n    $msg \`#${PR}<https://github.com/ansible-middleware/keycloak/pull/${PR}>\`" \
+                > changelogs/fragments/${PR}.yaml
+            fi
+          done
+          antsibull-changelog lint -vvv
+          antsibull-changelog generate
+          antsibull-changelog release
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      - name: Commit changelogs
+        run: |
+          git config user.name github-actions
+          git config user.email github-actions@github.com
+          git add CHANGELOG.rst changelogs/changelog.yaml
+          git commit -m "Update changelog for release ${{ steps.get_version.outputs.TAG_VERSION }}" || true
+          git push origin
+      - name: Publish collection
+        env:
+          ANSIBLE_GALAXY_API_KEY: ${{ secrets.ANSIBLE_GALAXY_API_KEY }}
+        run: |
+          ansible-galaxy collection publish *.tar.gz --api-key $ANSIBLE_GALAXY_API_KEY
+      - name: Create release tag
+        run: |
+          git config user.name github-actions
+          git config user.email github-actions@github.com
+          git tag -a ${{ steps.get_version.outputs.TAG_VERSION }}" -m "Release v${{ steps.get_version.outputs.TAG_VERSION }}" || true
+          git push origin --tags
       - name: Publish Release
         uses: softprops/action-gh-release@v1
         env:
@@ -32,11 +88,6 @@ jobs:
         with:
           files: "*.tar.gz"
           body: "Release v${{ steps.get_version.outputs.TAG_VERSION }}"
-      - name: Publish collection
-        env:
-          ANSIBLE_GALAXY_API_KEY: ${{ secrets.ANSIBLE_GALAXY_API_KEY }}
-        run: |
-          ansible-galaxy collection publish *.tar.gz --api-key $ANSIBLE_GALAXY_API_KEY
   dispatch:
     needs: release
     strategy:
@@ -49,5 +100,5 @@ jobs:
         with:
           token: ${{ secrets.TRIGGERING_PAT }}
           repository: ${{ matrix.repo }}
-          event-type: "Dependency released - Keycloak"
+          event-type: "Dependency released - Keycloak v${{ steps.get_version.outputs.TAG_VERSION }}"
           client-payload: '{ "github": ${{toJson(github)}} }'