From 74313149ddce6290cc545bd4ce28386c2c16dfea Mon Sep 17 00:00:00 2001 From: Viktor Barzin Date: Tue, 2 Jun 2026 20:49:42 +0000 Subject: [PATCH] job-hunter: weekly above-target Slack alert CronJob MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add job-hunter-alert CronJob (Sundays 05:00 UTC, an hour after the refresh): `python -m job_hunter alert --threshold 500000 --location london --slack` posts to Slack the companies whose London p50 total comp >= £500k, flagging any that newly crossed since last week's snapshot. SLACK_WEBHOOK_URL wired via the job-hunter-secrets ExternalSecret from Vault secret/job-hunter slack_webhook_url (seeded from the shared workspace webhook; repointable to a dedicated channel). Runbook gains an "above-target Slack alert" section. [ci skip] — applied locally (stack-scoped). Co-Authored-By: Claude Opus 4.8 --- docs/runbooks/job-hunter.md | 25 +++++++++++ stacks/job-hunter/cronjob.tf | 80 ++++++++++++++++++++++++++++++++++++ stacks/job-hunter/main.tf | 7 ++++ 3 files changed, 112 insertions(+) diff --git a/docs/runbooks/job-hunter.md b/docs/runbooks/job-hunter.md index d499d1e2..40b13673 100644 --- a/docs/runbooks/job-hunter.md +++ b/docs/runbooks/job-hunter.md @@ -164,6 +164,31 @@ Grafana datasource password is mirrored via a second ExternalSecret in the ## ANALYST +### Weekly above-target Slack alert + +The `job-hunter-alert` CronJob (Sundays 05:00 UTC, an hour after the refresh) +posts to Slack the companies whose London p50 total comp **≥ £500k**, flagging +any that **newly crossed** since last week's snapshot. Threshold is the +`--threshold` arg in `cronjob.tf` (default 500000 — well above the ~£267k move +floor, so only clearly-exceptional comp pings). Slack webhook comes from Vault +`secret/job-hunter` → `slack_webhook_url` (seeded from the shared workspace +webhook → currently posts to the same channel as Keel; repoint to a dedicated +channel by `vault kv patch secret/job-hunter slack_webhook_url=`). + +```bash +# Preview the message without posting +kubectl -n job-hunter exec deploy/job-hunter -- python -m job_hunter alert --stdout +# Different bar / location +kubectl -n job-hunter exec deploy/job-hunter -- \ + python -m job_hunter alert --threshold 350000 --location london --stdout +# Fire it now (posts to Slack) +kubectl -n job-hunter create job --from=cronjob/job-hunter-alert jh-alert-manual +``` + +`newly_crossed` needs ≥2 snapshot dates — it's empty until the second weekly +run accumulates. To change the standing threshold, edit `--threshold` in +`infra/stacks/job-hunter/cronjob.tf` and apply. + ### The periodic "market leaders in comp" report This is the headline command — current leaders by p50 total comp, week-over-week diff --git a/stacks/job-hunter/cronjob.tf b/stacks/job-hunter/cronjob.tf index 34e80ce8..968266b5 100644 --- a/stacks/job-hunter/cronjob.tf +++ b/stacks/job-hunter/cronjob.tf @@ -111,3 +111,83 @@ resource "kubernetes_cron_job_v1" "job_hunter_refresh" { kubernetes_manifest.db_external_secret, ] } + +# Weekly above-target comp alert. Runs an hour after the refresh (so it reads +# fresh data + the just-written snapshot) and posts to Slack the companies whose +# London p50 total comp >= £500k, flagging any that newly crossed since last +# week's snapshot. Read-only query + a Slack POST — no init/migrate needed. +resource "kubernetes_cron_job_v1" "job_hunter_alert" { + metadata { + name = "job-hunter-alert" + namespace = kubernetes_namespace.job_hunter.metadata[0].name + labels = local.labels + } + spec { + schedule = "0 5 * * 0" + concurrency_policy = "Forbid" + successful_jobs_history_limit = 3 + failed_jobs_history_limit = 3 + starting_deadline_seconds = 600 + + job_template { + metadata { + labels = local.labels + } + spec { + backoff_limit = 2 + active_deadline_seconds = 300 + ttl_seconds_after_finished = 86400 + + template { + metadata { + labels = local.labels + } + spec { + restart_policy = "OnFailure" + image_pull_secrets { + name = "registry-credentials" + } + container { + name = "alert" + image = local.image + image_pull_policy = "Always" + command = ["python", "-m", "job_hunter", "alert", + "--threshold", "500000", "--location", "london", "--slack"] + + env_from { + secret_ref { + name = "job-hunter-secrets" + } + } + env_from { + secret_ref { + name = "job-hunter-db-creds" + } + } + + resources { + requests = { + cpu = "50m" + memory = "256Mi" + } + limits = { + memory = "512Mi" + } + } + } + } + } + } + } + } + + lifecycle { + # KYVERNO_LIFECYCLE_V1 + ignore_changes = [spec[0].job_template[0].spec[0].template[0].spec[0].dns_config] + } + + depends_on = [ + kubernetes_manifest.external_secret, + kubernetes_manifest.db_external_secret, + ] +} diff --git a/stacks/job-hunter/main.tf b/stacks/job-hunter/main.tf index 18fd9201..8b84dfe5 100644 --- a/stacks/job-hunter/main.tf +++ b/stacks/job-hunter/main.tf @@ -89,6 +89,13 @@ resource "kubernetes_manifest" "external_secret" { secretKey = "DIGEST_FROM_ADDRESS" remoteRef = { key = "job-hunter", property = "digest_from_address" } }, + { + # Weekly above-target comp alert (job-hunter-alert CronJob). Seeded + # from the shared workspace webhook; repoint to a dedicated channel + # by updating secret/job-hunter slack_webhook_url. + secretKey = "SLACK_WEBHOOK_URL" + remoteRef = { key = "job-hunter", property = "slack_webhook_url" } + }, ] } }