diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 28f382788a..a2de238535 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -425,6 +425,13 @@ jobs:
           token: ${{ steps.token.outputs.token }}
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0
+      - name: Compute server cache key
+        run: |
+          BUILD_ARGS=$'DEVICE=cpu\n'
+          HASH=$(sha256sum <<< "${BUILD_ARGS}" | cut -d' ' -f1)
+          ARCH=$(echo "${{ runner.arch }}" | tr '[:upper:]' '[:lower:]')
+          [[ "$ARCH" == "x64" ]] && ARCH="amd64"
+          echo "SERVER_CACHE_KEY=linux-${ARCH}-${HASH}-main" >> $GITHUB_ENV
       - name: Build Docker images from cache
         run: docker compose -f docker-compose.yml -f docker-compose.ci.yml --profile test build
       - name: Start Docker Compose
@@ -470,6 +477,13 @@ jobs:
           token: ${{ steps.token.outputs.token }}
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0
+      - name: Compute server cache key
+        run: |
+          BUILD_ARGS=$'DEVICE=cpu\n'
+          HASH=$(sha256sum <<< "${BUILD_ARGS}" | cut -d' ' -f1)
+          ARCH=$(echo "${{ runner.arch }}" | tr '[:upper:]' '[:lower:]')
+          [[ "$ARCH" == "x64" ]] && ARCH="amd64"
+          echo "SERVER_CACHE_KEY=linux-${ARCH}-${HASH}-main" >> $GITHUB_ENV
       - name: Build Docker images from cache
         run: docker compose -f docker-compose.yml -f docker-compose.ci.yml --profile test build
       - name: Start Docker Compose
@@ -515,6 +529,13 @@ jobs:
           token: ${{ steps.token.outputs.token }}
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0
+      - name: Compute server cache key
+        run: |
+          BUILD_ARGS=$'DEVICE=cpu\n'
+          HASH=$(sha256sum <<< "${BUILD_ARGS}" | cut -d' ' -f1)
+          ARCH=$(echo "${{ runner.arch }}" | tr '[:upper:]' '[:lower:]')
+          [[ "$ARCH" == "x64" ]] && ARCH="amd64"
+          echo "SERVER_CACHE_KEY=linux-${ARCH}-${HASH}-main" >> $GITHUB_ENV
       - name: Build Docker images from cache
         run: docker compose -f docker-compose.yml -f docker-compose.ci.yml --profile test build
       - name: Start Docker Compose
@@ -566,6 +587,13 @@ jobs:
           token: ${{ steps.token.outputs.token }}
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0
+      - name: Compute server cache key
+        run: |
+          BUILD_ARGS=$'DEVICE=cpu\n'
+          HASH=$(sha256sum <<< "${BUILD_ARGS}" | cut -d' ' -f1)
+          ARCH=$(echo "${{ runner.arch }}" | tr '[:upper:]' '[:lower:]')
+          [[ "$ARCH" == "x64" ]] && ARCH="amd64"
+          echo "SERVER_CACHE_KEY=linux-${ARCH}-${HASH}-main" >> $GITHUB_ENV
       - name: Build Docker images from cache
         run: docker compose -f docker-compose.yml -f docker-compose.ci.yml --profile test build
       - name: Start Docker Compose
@@ -617,6 +645,13 @@ jobs:
           token: ${{ steps.token.outputs.token }}
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0
+      - name: Compute server cache key
+        run: |
+          BUILD_ARGS=$'DEVICE=cpu\n'
+          HASH=$(sha256sum <<< "${BUILD_ARGS}" | cut -d' ' -f1)
+          ARCH=$(echo "${{ runner.arch }}" | tr '[:upper:]' '[:lower:]')
+          [[ "$ARCH" == "x64" ]] && ARCH="amd64"
+          echo "SERVER_CACHE_KEY=linux-${ARCH}-${HASH}-main" >> $GITHUB_ENV
       - name: Build Docker images from cache
         run: docker compose -f docker-compose.yml -f docker-compose.ci.yml --profile test build
       - name: Start Docker Compose
diff --git a/e2e/docker-compose.ci.yml b/e2e/docker-compose.ci.yml
index efcc690180..73a491025a 100644
--- a/e2e/docker-compose.ci.yml
+++ b/e2e/docker-compose.ci.yml
@@ -1,13 +1,6 @@
 name: immich-e2e
 
 services:
-  immich-server:
-    build:
-      cache_from:
-        - type=gha,scope=e2e-server-${RUNNER_ARCH:-X64}
-      cache_to:
-        - type=gha,mode=max,scope=e2e-server-${RUNNER_ARCH:-X64}
-
   e2e-auth-server:
     build:
       cache_from:
diff --git a/e2e/docker-compose.yml b/e2e/docker-compose.yml
index 32a61e048e..c4eb4b8d9f 100644
--- a/e2e/docker-compose.yml
+++ b/e2e/docker-compose.yml
@@ -17,8 +17,7 @@ services:
       context: ../
       dockerfile: server/Dockerfile
       cache_from:
-        - type=registry,ref=ghcr.io/immich-app/immich-server-build-cache:linux-amd64-cc099f297acd18c924b35ece3245215b53d106eb2518e3af6415931d055746cd-main
-        - type=registry,ref=ghcr.io/immich-app/immich-server-build-cache:linux-arm64-cc099f297acd18c924b35ece3245215b53d106eb2518e3af6415931d055746cd-main
+        - type=registry,ref=ghcr.io/immich-app/immich-server-build-cache:${SERVER_CACHE_KEY:-linux-amd64-cc099f297acd18c924b35ece3245215b53d106eb2518e3af6415931d055746cd-main}
       args:
         - BUILD_ID=1234567890
         - BUILD_IMAGE=e2e