diff --git a/.github/workflows/build-container.yml b/.github/workflows/build-container.yml
index c661087aae..96b1bdba35 100644
--- a/.github/workflows/build-container.yml
+++ b/.github/workflows/build-container.yml
@@ -21,11 +21,6 @@ jobs:
         uses: docker/setup-qemu-action@v2
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v2
-      - name: Cache Docker layers
-        uses: actions/cache@v3
-        with:
-          path: /tmp/.buildx-cache
-          key: buildx-${{ hashFiles('docker-build/Dockerfile') }}
       - name: Build container
         uses: docker/build-push-action@v3
         with:
@@ -34,8 +29,6 @@ jobs:
           platforms: linux/amd64
           push: false
           tags: ${{ env.dockertag }}:latest
-          cache-from: type=local,src=/tmp/.buildx-cache
-          cache-to: type=local,dest=/tmp/.buildx-cache
           build-args: |
             invokeai_git=${{ github.repository }}
             invokeai_branch=${{ github.ref_name }}