viktor/wrongmove
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Replace deployment-level rollout check with pod-level verify-deploy

Browse source 
Check for a pod younger than 60s that is ready and running the exact
expected image tag, instead of polling deployment status fields.
This commit is contained in:
Viktor Barzin 2026-02-09 23:21:07 +00:00
parent dbcd30679a
commit b816f695f0
No known key found for this signature in database
GPG key ID: 0EB088298288D958
1 changed files with 65 additions and 44 deletions
Download patch file Download diff file Expand all files Collapse all files

109
.drone.yml
View file

@ -51,34 +51,40 @@ steps:
- apk add curl - apk add curl
- 'curl -s -X PATCH "https://kubernetes:6443/apis/apps/v1/namespaces/realestate-crawler/deployments/realestate-crawler-ui" -H "Authorization: Bearer $(cat /var/run/secrets/kubernetes.io/serviceaccount/token)" -H "Content-Type: application/json-patch+json" -k -d ''[{"op":"replace","path":"/spec/template/spec/containers/0/image","value":"viktorbarzin/immoweb:''"$DRONE_BUILD_NUMBER"''"}]'' | head' - 'curl -s -X PATCH "https://kubernetes:6443/apis/apps/v1/namespaces/realestate-crawler/deployments/realestate-crawler-ui" -H "Authorization: Bearer $(cat /var/run/secrets/kubernetes.io/serviceaccount/token)" -H "Content-Type: application/json-patch+json" -k -d ''[{"op":"replace","path":"/spec/template/spec/containers/0/image","value":"viktorbarzin/immoweb:''"$DRONE_BUILD_NUMBER"''"}]'' | head'
- name: Wait for rollout - name: verify-deploy
image: alpine image: alpine
commands: commands:
- apk add --no-cache curl jq - apk add --no-cache curl jq
- >- - |
TOKEN=$(cat /var/run/secrets/kubernetes.io/serviceaccount/token); TOKEN=$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)
API="https://kubernetes:6443/apis/apps/v1/namespaces/realestate-crawler/deployments"; EXPECTED_IMAGE="viktorbarzin/immoweb:${DRONE_BUILD_NUMBER}"
for DEPLOY in realestate-crawler-ui; do PODS_API="https://kubernetes:6443/api/v1/namespaces/realestate-crawler/pods?labelSelector=app%3Drealestate-crawler-ui"
echo "Waiting for $DEPLOY rollout...";
for i in $(seq 1 60); do for i in $(seq 1 60); do
S=$(curl -sk -H "Authorization: Bearer $TOKEN" "$API/$DEPLOY"); RESULT=$(curl -sfk "$PODS_API" \
DESIRED=$(echo "$S" | jq '.spec.replicas'); -H "Authorization: Bearer $TOKEN" \
UPDATED=$(echo "$S" | jq '.status.updatedReplicas // 0'); -H "Accept: application/json" | \
READY=$(echo "$S" | jq '.status.readyReplicas // 0'); jq --arg img "$EXPECTED_IMAGE" '[.items[] | select(
GEN=$(echo "$S" | jq '.metadata.generation'); (now - (.status.startTime | fromdateiso8601)) < 60 and
OBS=$(echo "$S" | jq '.status.observedGeneration // 0'); (.status.containerStatuses[]? | .ready == true) and
if [ "$OBS" -ge "$GEN" ] && [ "$UPDATED" -eq "$DESIRED" ] && [ "$READY" -eq "$DESIRED" ]; then (.spec.containers[]? | .image == $img)
echo "$DEPLOY rollout complete ($READY/$DESIRED ready)"; ) | {name: .metadata.name, age: (now - (.status.startTime | fromdateiso8601) | floor), image: .spec.containers[0].image, started: .status.startTime}]')
break;
fi; COUNT=$(echo "$RESULT" | jq 'length')
echo " $DEPLOY: $READY/$DESIRED ready, $UPDATED updated (attempt $i/60)"; echo "Attempt $i/60: $COUNT pod(s) younger than 60s, ready, running $EXPECTED_IMAGE"
sleep 5;
done; if [ "$COUNT" -gt 0 ]; then
if [ "$i" -eq 60 ] && { [ "$READY" -ne "$DESIRED" ] || [ "$UPDATED" -ne "$DESIRED" ]; }; then echo "$RESULT" | jq -r '.[] | " \(.name) age=\(.age)s image=\(.image) started=\(.started)"'
echo "ERROR: $DEPLOY rollout timed out after 300s"; exit 1; echo "New pod is live!"
fi; exit 0
fi
sleep 5
done done
echo "ERROR: No new ready pod with image $EXPECTED_IMAGE appeared within 5 minutes"
exit 1
--- ---
kind: pipeline kind: pipeline
type: kubernetes type: kubernetes
@ -135,30 +141,45 @@ steps:
- 'curl -s -X PATCH "https://kubernetes:6443/apis/apps/v1/namespaces/realestate-crawler/deployments/realestate-crawler-celery" -H "Authorization: Bearer $(cat /var/run/secrets/kubernetes.io/serviceaccount/token)" -H "Content-Type: application/json-patch+json" -k -d ''[{"op":"replace","path":"/spec/template/spec/containers/0/image","value":"viktorbarzin/realestatecrawler:''"$DRONE_BUILD_NUMBER"''"}]'' | head' - 'curl -s -X PATCH "https://kubernetes:6443/apis/apps/v1/namespaces/realestate-crawler/deployments/realestate-crawler-celery" -H "Authorization: Bearer $(cat /var/run/secrets/kubernetes.io/serviceaccount/token)" -H "Content-Type: application/json-patch+json" -k -d ''[{"op":"replace","path":"/spec/template/spec/containers/0/image","value":"viktorbarzin/realestatecrawler:''"$DRONE_BUILD_NUMBER"''"}]'' | head'
- 'curl -s -X PATCH "https://kubernetes:6443/apis/apps/v1/namespaces/realestate-crawler/deployments/realestate-crawler-celery-beat" -H "Authorization: Bearer $(cat /var/run/secrets/kubernetes.io/serviceaccount/token)" -H "Content-Type: application/json-patch+json" -k -d ''[{"op":"replace","path":"/spec/template/spec/containers/0/image","value":"viktorbarzin/realestatecrawler:''"$DRONE_BUILD_NUMBER"''"}]'' | head' - 'curl -s -X PATCH "https://kubernetes:6443/apis/apps/v1/namespaces/realestate-crawler/deployments/realestate-crawler-celery-beat" -H "Authorization: Bearer $(cat /var/run/secrets/kubernetes.io/serviceaccount/token)" -H "Content-Type: application/json-patch+json" -k -d ''[{"op":"replace","path":"/spec/template/spec/containers/0/image","value":"viktorbarzin/realestatecrawler:''"$DRONE_BUILD_NUMBER"''"}]'' | head'
- name: Wait for rollout - name: verify-deploy
image: alpine image: alpine
commands: commands:
- apk add --no-cache curl jq - apk add --no-cache curl jq
- >- - |
TOKEN=$(cat /var/run/secrets/kubernetes.io/serviceaccount/token); TOKEN=$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)
API="https://kubernetes:6443/apis/apps/v1/namespaces/realestate-crawler/deployments"; EXPECTED_IMAGE="viktorbarzin/realestatecrawler:${DRONE_BUILD_NUMBER}"
BASE_API="https://kubernetes:6443/api/v1/namespaces/realestate-crawler/pods"
for DEPLOY in realestate-crawler-api realestate-crawler-celery realestate-crawler-celery-beat; do for DEPLOY in realestate-crawler-api realestate-crawler-celery realestate-crawler-celery-beat; do
echo "Waiting for $DEPLOY rollout..."; echo "Verifying $DEPLOY..."
PODS_API="${BASE_API}?labelSelector=app%3D${DEPLOY}"
FOUND=0
for i in $(seq 1 60); do for i in $(seq 1 60); do
S=$(curl -sk -H "Authorization: Bearer $TOKEN" "$API/$DEPLOY"); RESULT=$(curl -sfk "$PODS_API" \
DESIRED=$(echo "$S" | jq '.spec.replicas'); -H "Authorization: Bearer $TOKEN" \
UPDATED=$(echo "$S" | jq '.status.updatedReplicas // 0'); -H "Accept: application/json" | \
READY=$(echo "$S" | jq '.status.readyReplicas // 0'); jq --arg img "$EXPECTED_IMAGE" '[.items[] | select(
GEN=$(echo "$S" | jq '.metadata.generation'); (now - (.status.startTime | fromdateiso8601)) < 60 and
OBS=$(echo "$S" | jq '.status.observedGeneration // 0'); (.status.containerStatuses[]? | .ready == true) and
if [ "$OBS" -ge "$GEN" ] && [ "$UPDATED" -eq "$DESIRED" ] && [ "$READY" -eq "$DESIRED" ]; then (.spec.containers[]? | .image == $img)
echo "$DEPLOY rollout complete ($READY/$DESIRED ready)"; ) | {name: .metadata.name, age: (now - (.status.startTime | fromdateiso8601) | floor), image: .spec.containers[0].image, started: .status.startTime}]')
break;
fi; COUNT=$(echo "$RESULT" | jq 'length')
echo " $DEPLOY: $READY/$DESIRED ready, $UPDATED updated (attempt $i/60)"; echo " Attempt $i/60: $COUNT pod(s) younger than 60s, ready, running $EXPECTED_IMAGE"
sleep 5;
done; if [ "$COUNT" -gt 0 ]; then
if [ "$i" -eq 60 ] && { [ "$READY" -ne "$DESIRED" ] || [ "$UPDATED" -ne "$DESIRED" ]; }; then echo "$RESULT" | jq -r '.[] | " \(.name) age=\(.age)s image=\(.image) started=\(.started)"'
echo "ERROR: $DEPLOY rollout timed out after 300s"; exit 1; echo "$DEPLOY is live!"
fi; FOUND=1
break
fi
sleep 5
done
if [ "$FOUND" -ne 1 ]; then
echo "ERROR: No new ready pod for $DEPLOY with image $EXPECTED_IMAGE appeared within 5 minutes"
exit 1
fi
done done