diff --git a/.github/workflows/create-releases.yml b/.github/workflows/create-releases.yml
new file mode 100644
index 00000000..17809767
--- /dev/null
+++ b/.github/workflows/create-releases.yml
@@ -0,0 +1,34 @@
+name: release
+on:
+  workflow_dispatch:
+env:
+  # This special value tells pypi that the user identity is supplied within the token
+  TWINE_USERNAME: __token__
+  # Note, the PYPI_API_TOKEN is for the OpenLLM PyPI user, on
+  # https://github.com/bentoml/openllm/settings/secrets/actions
+  TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }}
+  GIT_AUTHOR_NAME: github-actions[bot]
+  GIT_AUTHOR_EMAIL: github-actions[bot]@users.noreply.github.com
+# https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#defaultsrun
+defaults:
+  run:
+    shell: bash --noprofile --norc -exo pipefail {0}
+jobs:
+  release:
+    if: github.repository_owner == 'bentoml'
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - name: Setup CI
+        uses: ./.github/actions/setup-repo
+      - name: Install jq and curl
+        run: sudo apt-get install -y jq curl
+      - name: Create a release
+        run: |
+          git config user.name "${{ env.GIT_AUTHOR_NAME }}"
+          git config user.email "${{ env.GIT_AUTHOR_EMAIL }}"
+          ./tools/release
diff --git a/.github/workflows/release.yml b/.github/workflows/release-notes.yml
similarity index 93%
rename from .github/workflows/release.yml
rename to .github/workflows/release-notes.yml
index b73dae66..eada6332 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release-notes.yml
@@ -1,9 +1,8 @@
-name: release
+name: release-notes
 on:
-  workflow_dispatch:
   push:
     tags:
-      - '*.*.*'
+      - 'v*.*.*'
 env:
   # This special value tells pypi that the user identity is supplied within the token
   TWINE_USERNAME: __token__
@@ -20,8 +19,6 @@ jobs:
   release:
     if: github.repository_owner == 'bentoml'
     runs-on: ubuntu-latest
-    permissions:
-      contents: write
     steps:
       - uses: actions/checkout@v3
         with: