diff --git a/.drone.yml b/.drone.yml index 9cb50f2..d37af44 100644 --- a/.drone.yml +++ b/.drone.yml @@ -51,34 +51,40 @@ steps: - apk add curl - 'curl -s -X PATCH "https://kubernetes:6443/apis/apps/v1/namespaces/realestate-crawler/deployments/realestate-crawler-ui" -H "Authorization: Bearer $(cat /var/run/secrets/kubernetes.io/serviceaccount/token)" -H "Content-Type: application/json-patch+json" -k -d ''[{"op":"replace","path":"/spec/template/spec/containers/0/image","value":"viktorbarzin/immoweb:''"$DRONE_BUILD_NUMBER"''"}]'' | head' - - name: Wait for rollout + - name: verify-deploy image: alpine commands: - apk add --no-cache curl jq - - >- - TOKEN=$(cat /var/run/secrets/kubernetes.io/serviceaccount/token); - API="https://kubernetes:6443/apis/apps/v1/namespaces/realestate-crawler/deployments"; - for DEPLOY in realestate-crawler-ui; do - echo "Waiting for $DEPLOY rollout..."; - for i in $(seq 1 60); do - S=$(curl -sk -H "Authorization: Bearer $TOKEN" "$API/$DEPLOY"); - DESIRED=$(echo "$S" | jq '.spec.replicas'); - UPDATED=$(echo "$S" | jq '.status.updatedReplicas // 0'); - READY=$(echo "$S" | jq '.status.readyReplicas // 0'); - GEN=$(echo "$S" | jq '.metadata.generation'); - OBS=$(echo "$S" | jq '.status.observedGeneration // 0'); - if [ "$OBS" -ge "$GEN" ] && [ "$UPDATED" -eq "$DESIRED" ] && [ "$READY" -eq "$DESIRED" ]; then - echo "$DEPLOY rollout complete ($READY/$DESIRED ready)"; - break; - fi; - echo " $DEPLOY: $READY/$DESIRED ready, $UPDATED updated (attempt $i/60)"; - sleep 5; - done; - if [ "$i" -eq 60 ] && { [ "$READY" -ne "$DESIRED" ] || [ "$UPDATED" -ne "$DESIRED" ]; }; then - echo "ERROR: $DEPLOY rollout timed out after 300s"; exit 1; - fi; + - | + TOKEN=$(cat /var/run/secrets/kubernetes.io/serviceaccount/token) + EXPECTED_IMAGE="viktorbarzin/immoweb:${DRONE_BUILD_NUMBER}" + PODS_API="https://kubernetes:6443/api/v1/namespaces/realestate-crawler/pods?labelSelector=app%3Drealestate-crawler-ui" + + for i in $(seq 1 60); do + RESULT=$(curl -sfk "$PODS_API" \ + -H "Authorization: Bearer $TOKEN" \ + -H "Accept: application/json" | \ + jq --arg img "$EXPECTED_IMAGE" '[.items[] | select( + (now - (.status.startTime | fromdateiso8601)) < 60 and + (.status.containerStatuses[]? | .ready == true) and + (.spec.containers[]? | .image == $img) + ) | {name: .metadata.name, age: (now - (.status.startTime | fromdateiso8601) | floor), image: .spec.containers[0].image, started: .status.startTime}]') + + COUNT=$(echo "$RESULT" | jq 'length') + echo "Attempt $i/60: $COUNT pod(s) younger than 60s, ready, running $EXPECTED_IMAGE" + + if [ "$COUNT" -gt 0 ]; then + echo "$RESULT" | jq -r '.[] | " \(.name) age=\(.age)s image=\(.image) started=\(.started)"' + echo "New pod is live!" + exit 0 + fi + + sleep 5 done + echo "ERROR: No new ready pod with image $EXPECTED_IMAGE appeared within 5 minutes" + exit 1 + --- kind: pipeline type: kubernetes @@ -135,30 +141,45 @@ steps: - 'curl -s -X PATCH "https://kubernetes:6443/apis/apps/v1/namespaces/realestate-crawler/deployments/realestate-crawler-celery" -H "Authorization: Bearer $(cat /var/run/secrets/kubernetes.io/serviceaccount/token)" -H "Content-Type: application/json-patch+json" -k -d ''[{"op":"replace","path":"/spec/template/spec/containers/0/image","value":"viktorbarzin/realestatecrawler:''"$DRONE_BUILD_NUMBER"''"}]'' | head' - 'curl -s -X PATCH "https://kubernetes:6443/apis/apps/v1/namespaces/realestate-crawler/deployments/realestate-crawler-celery-beat" -H "Authorization: Bearer $(cat /var/run/secrets/kubernetes.io/serviceaccount/token)" -H "Content-Type: application/json-patch+json" -k -d ''[{"op":"replace","path":"/spec/template/spec/containers/0/image","value":"viktorbarzin/realestatecrawler:''"$DRONE_BUILD_NUMBER"''"}]'' | head' - - name: Wait for rollout + - name: verify-deploy image: alpine commands: - apk add --no-cache curl jq - - >- - TOKEN=$(cat /var/run/secrets/kubernetes.io/serviceaccount/token); - API="https://kubernetes:6443/apis/apps/v1/namespaces/realestate-crawler/deployments"; + - | + TOKEN=$(cat /var/run/secrets/kubernetes.io/serviceaccount/token) + EXPECTED_IMAGE="viktorbarzin/realestatecrawler:${DRONE_BUILD_NUMBER}" + BASE_API="https://kubernetes:6443/api/v1/namespaces/realestate-crawler/pods" + for DEPLOY in realestate-crawler-api realestate-crawler-celery realestate-crawler-celery-beat; do - echo "Waiting for $DEPLOY rollout..."; + echo "Verifying $DEPLOY..." + PODS_API="${BASE_API}?labelSelector=app%3D${DEPLOY}" + + FOUND=0 for i in $(seq 1 60); do - S=$(curl -sk -H "Authorization: Bearer $TOKEN" "$API/$DEPLOY"); - DESIRED=$(echo "$S" | jq '.spec.replicas'); - UPDATED=$(echo "$S" | jq '.status.updatedReplicas // 0'); - READY=$(echo "$S" | jq '.status.readyReplicas // 0'); - GEN=$(echo "$S" | jq '.metadata.generation'); - OBS=$(echo "$S" | jq '.status.observedGeneration // 0'); - if [ "$OBS" -ge "$GEN" ] && [ "$UPDATED" -eq "$DESIRED" ] && [ "$READY" -eq "$DESIRED" ]; then - echo "$DEPLOY rollout complete ($READY/$DESIRED ready)"; - break; - fi; - echo " $DEPLOY: $READY/$DESIRED ready, $UPDATED updated (attempt $i/60)"; - sleep 5; - done; - if [ "$i" -eq 60 ] && { [ "$READY" -ne "$DESIRED" ] || [ "$UPDATED" -ne "$DESIRED" ]; }; then - echo "ERROR: $DEPLOY rollout timed out after 300s"; exit 1; - fi; + RESULT=$(curl -sfk "$PODS_API" \ + -H "Authorization: Bearer $TOKEN" \ + -H "Accept: application/json" | \ + jq --arg img "$EXPECTED_IMAGE" '[.items[] | select( + (now - (.status.startTime | fromdateiso8601)) < 60 and + (.status.containerStatuses[]? | .ready == true) and + (.spec.containers[]? | .image == $img) + ) | {name: .metadata.name, age: (now - (.status.startTime | fromdateiso8601) | floor), image: .spec.containers[0].image, started: .status.startTime}]') + + COUNT=$(echo "$RESULT" | jq 'length') + echo " Attempt $i/60: $COUNT pod(s) younger than 60s, ready, running $EXPECTED_IMAGE" + + if [ "$COUNT" -gt 0 ]; then + echo "$RESULT" | jq -r '.[] | " \(.name) age=\(.age)s image=\(.image) started=\(.started)"' + echo "$DEPLOY is live!" + FOUND=1 + break + fi + + sleep 5 + done + + if [ "$FOUND" -ne 1 ]; then + echo "ERROR: No new ready pod for $DEPLOY with image $EXPECTED_IMAGE appeared within 5 minutes" + exit 1 + fi done