Replace deployment-level rollout check with pod-level verify-deploy
Check for a pod younger than 60s that is ready and running the exact expected image tag, instead of polling deployment status fields.
This commit is contained in:
Viktor Barzin
parent
dbcd30679a
commit
b816f695f0
1 changed files with 65 additions and 44 deletions
109
.drone.yml
109
.drone.yml
|
|
@ -51,34 +51,40 @@ steps:
|
|||
- apk add curl
|
||||
- 'curl -s -X PATCH "https://kubernetes:6443/apis/apps/v1/namespaces/realestate-crawler/deployments/realestate-crawler-ui" -H "Authorization: Bearer $(cat /var/run/secrets/kubernetes.io/serviceaccount/token)" -H "Content-Type: application/json-patch+json" -k -d ''[{"op":"replace","path":"/spec/template/spec/containers/0/image","value":"viktorbarzin/immoweb:''"$DRONE_BUILD_NUMBER"''"}]'' | head'
|
||||
|
||||
- name: Wait for rollout
|
||||
- name: verify-deploy
|
||||
image: alpine
|
||||
commands:
|
||||
- apk add --no-cache curl jq
|
||||
- >-
|
||||
TOKEN=$(cat /var/run/secrets/kubernetes.io/serviceaccount/token);
|
||||
API="https://kubernetes:6443/apis/apps/v1/namespaces/realestate-crawler/deployments";
|
||||
for DEPLOY in realestate-crawler-ui; do
|
||||
echo "Waiting for $DEPLOY rollout...";
|
||||
for i in $(seq 1 60); do
|
||||
S=$(curl -sk -H "Authorization: Bearer $TOKEN" "$API/$DEPLOY");
|
||||
DESIRED=$(echo "$S" | jq '.spec.replicas');
|
||||
UPDATED=$(echo "$S" | jq '.status.updatedReplicas // 0');
|
||||
READY=$(echo "$S" | jq '.status.readyReplicas // 0');
|
||||
GEN=$(echo "$S" | jq '.metadata.generation');
|
||||
OBS=$(echo "$S" | jq '.status.observedGeneration // 0');
|
||||
if [ "$OBS" -ge "$GEN" ] && [ "$UPDATED" -eq "$DESIRED" ] && [ "$READY" -eq "$DESIRED" ]; then
|
||||
echo "$DEPLOY rollout complete ($READY/$DESIRED ready)";
|
||||
break;
|
||||
fi;
|
||||
echo " $DEPLOY: $READY/$DESIRED ready, $UPDATED updated (attempt $i/60)";
|
||||
sleep 5;
|
||||
done;
|
||||
if [ "$i" -eq 60 ] && { [ "$READY" -ne "$DESIRED" ] || [ "$UPDATED" -ne "$DESIRED" ]; }; then
|
||||
echo "ERROR: $DEPLOY rollout timed out after 300s"; exit 1;
|
||||
fi;
|
||||
- |
|
||||
TOKEN=$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)
|
||||
EXPECTED_IMAGE="viktorbarzin/immoweb:${DRONE_BUILD_NUMBER}"
|
||||
PODS_API="https://kubernetes:6443/api/v1/namespaces/realestate-crawler/pods?labelSelector=app%3Drealestate-crawler-ui"
|
||||
|
||||
for i in $(seq 1 60); do
|
||||
RESULT=$(curl -sfk "$PODS_API" \
|
||||
-H "Authorization: Bearer $TOKEN" \
|
||||
-H "Accept: application/json" | \
|
||||
jq --arg img "$EXPECTED_IMAGE" '[.items[] | select(
|
||||
(now - (.status.startTime | fromdateiso8601)) < 60 and
|
||||
(.status.containerStatuses[]? | .ready == true) and
|
||||
(.spec.containers[]? | .image == $img)
|
||||
) | {name: .metadata.name, age: (now - (.status.startTime | fromdateiso8601) | floor), image: .spec.containers[0].image, started: .status.startTime}]')
|
||||
|
||||
COUNT=$(echo "$RESULT" | jq 'length')
|
||||
echo "Attempt $i/60: $COUNT pod(s) younger than 60s, ready, running $EXPECTED_IMAGE"
|
||||
|
||||
if [ "$COUNT" -gt 0 ]; then
|
||||
echo "$RESULT" | jq -r '.[] | " \(.name) age=\(.age)s image=\(.image) started=\(.started)"'
|
||||
echo "New pod is live!"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
sleep 5
|
||||
done
|
||||
|
||||
echo "ERROR: No new ready pod with image $EXPECTED_IMAGE appeared within 5 minutes"
|
||||
exit 1
|
||||
|
||||
---
|
||||
kind: pipeline
|
||||
type: kubernetes
|
||||
|
|
@ -135,30 +141,45 @@ steps:
|
|||
- 'curl -s -X PATCH "https://kubernetes:6443/apis/apps/v1/namespaces/realestate-crawler/deployments/realestate-crawler-celery" -H "Authorization: Bearer $(cat /var/run/secrets/kubernetes.io/serviceaccount/token)" -H "Content-Type: application/json-patch+json" -k -d ''[{"op":"replace","path":"/spec/template/spec/containers/0/image","value":"viktorbarzin/realestatecrawler:''"$DRONE_BUILD_NUMBER"''"}]'' | head'
|
||||
- 'curl -s -X PATCH "https://kubernetes:6443/apis/apps/v1/namespaces/realestate-crawler/deployments/realestate-crawler-celery-beat" -H "Authorization: Bearer $(cat /var/run/secrets/kubernetes.io/serviceaccount/token)" -H "Content-Type: application/json-patch+json" -k -d ''[{"op":"replace","path":"/spec/template/spec/containers/0/image","value":"viktorbarzin/realestatecrawler:''"$DRONE_BUILD_NUMBER"''"}]'' | head'
|
||||
|
||||
- name: Wait for rollout
|
||||
- name: verify-deploy
|
||||
image: alpine
|
||||
commands:
|
||||
- apk add --no-cache curl jq
|
||||
- >-
|
||||
TOKEN=$(cat /var/run/secrets/kubernetes.io/serviceaccount/token);
|
||||
API="https://kubernetes:6443/apis/apps/v1/namespaces/realestate-crawler/deployments";
|
||||
- |
|
||||
TOKEN=$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)
|
||||
EXPECTED_IMAGE="viktorbarzin/realestatecrawler:${DRONE_BUILD_NUMBER}"
|
||||
BASE_API="https://kubernetes:6443/api/v1/namespaces/realestate-crawler/pods"
|
||||
|
||||
for DEPLOY in realestate-crawler-api realestate-crawler-celery realestate-crawler-celery-beat; do
|
||||
echo "Waiting for $DEPLOY rollout...";
|
||||
echo "Verifying $DEPLOY..."
|
||||
PODS_API="${BASE_API}?labelSelector=app%3D${DEPLOY}"
|
||||
|
||||
FOUND=0
|
||||
for i in $(seq 1 60); do
|
||||
S=$(curl -sk -H "Authorization: Bearer $TOKEN" "$API/$DEPLOY");
|
||||
DESIRED=$(echo "$S" | jq '.spec.replicas');
|
||||
UPDATED=$(echo "$S" | jq '.status.updatedReplicas // 0');
|
||||
READY=$(echo "$S" | jq '.status.readyReplicas // 0');
|
||||
GEN=$(echo "$S" | jq '.metadata.generation');
|
||||
OBS=$(echo "$S" | jq '.status.observedGeneration // 0');
|
||||
if [ "$OBS" -ge "$GEN" ] && [ "$UPDATED" -eq "$DESIRED" ] && [ "$READY" -eq "$DESIRED" ]; then
|
||||
echo "$DEPLOY rollout complete ($READY/$DESIRED ready)";
|
||||
break;
|
||||
fi;
|
||||
echo " $DEPLOY: $READY/$DESIRED ready, $UPDATED updated (attempt $i/60)";
|
||||
sleep 5;
|
||||
done;
|
||||
if [ "$i" -eq 60 ] && { [ "$READY" -ne "$DESIRED" ] || [ "$UPDATED" -ne "$DESIRED" ]; }; then
|
||||
echo "ERROR: $DEPLOY rollout timed out after 300s"; exit 1;
|
||||
fi;
|
||||
RESULT=$(curl -sfk "$PODS_API" \
|
||||
-H "Authorization: Bearer $TOKEN" \
|
||||
-H "Accept: application/json" | \
|
||||
jq --arg img "$EXPECTED_IMAGE" '[.items[] | select(
|
||||
(now - (.status.startTime | fromdateiso8601)) < 60 and
|
||||
(.status.containerStatuses[]? | .ready == true) and
|
||||
(.spec.containers[]? | .image == $img)
|
||||
) | {name: .metadata.name, age: (now - (.status.startTime | fromdateiso8601) | floor), image: .spec.containers[0].image, started: .status.startTime}]')
|
||||
|
||||
COUNT=$(echo "$RESULT" | jq 'length')
|
||||
echo " Attempt $i/60: $COUNT pod(s) younger than 60s, ready, running $EXPECTED_IMAGE"
|
||||
|
||||
if [ "$COUNT" -gt 0 ]; then
|
||||
echo "$RESULT" | jq -r '.[] | " \(.name) age=\(.age)s image=\(.image) started=\(.started)"'
|
||||
echo "$DEPLOY is live!"
|
||||
FOUND=1
|
||||
break
|
||||
fi
|
||||
|
||||
sleep 5
|
||||
done
|
||||
|
||||
if [ "$FOUND" -ne 1 ]; then
|
||||
echo "ERROR: No new ready pod for $DEPLOY with image $EXPECTED_IMAGE appeared within 5 minutes"
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue