diff --git a/.github/workflows/post-release-mergeback.yml b/.github/workflows/post-release-mergeback.yml
index 58a64d2cf..1a38dfd74 100644
--- a/.github/workflows/post-release-mergeback.yml
+++ b/.github/workflows/post-release-mergeback.yml
@@ -130,7 +130,7 @@ jobs:
           npm version patch --no-git-tag-version
 
           # Update the changelog
-          perl -i -pe 's/^/## \[UNRELEASED\]\n\nNo user facing changes.\n\n/ if($.==3)' CHANGELOG.md
+          perl -i -pe 's/^/## \[UNRELEASED\]\n\nNo user facing changes.\n\n/ if($.==5)' CHANGELOG.md
           git add .
           git commit -m "Update changelog and version after ${VERSION}"
 
diff --git a/CHANGELOG.md b/CHANGELOG.md
index ae61e8bb7..7503ba985 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,11 +1,11 @@
 # CodeQL Action Changelog
 
+See the [releases page](https://github.com/github/codeql-action/releases) for the relevant changes to the CodeQL CLI and language packs.
+
 ## [UNRELEASED]
 
 No user facing changes.
 
-See the [releases page](https://github.com/github/codeql-action/releases) for the relevant changes to the CodeQL CLI and language packs.
-
 ## 2.21.0 - 19 Jul 2023
 
 - CodeQL Action now requires CodeQL CLI 2.9.4 or later. For more information, see the corresponding changelog entry for CodeQL Action version 2.20.4. [#1724](https://github.com/github/codeql-action/pull/1724)