job-hunter: weekly refresh CronJob + ops/analyst runbook
All checks were successful
ci/woodpecker/push/default Pipeline was successful
ci/woodpecker/push/build-cli Pipeline was successful

Add kubernetes_cron_job_v1.job_hunter_refresh — Sundays 04:00 UTC, runs
`refresh --source ats --source hn --source levels_fyi`, which upserts roles/
comp AND appends the dated comp_snapshots/roles_snapshots series consumed by
`job-hunter analyze`. Mirrors the Deployment's alembic-migrate init container
so a refresh never runs against an un-migrated DB; concurrency Forbid,
backoff 1, 30m activeDeadline, KYVERNO_LIFECYCLE_V1 dns_config ignore.

Add docs/runbooks/job-hunter.md: ops (health checks, manual refresh, add an
ATS company / CDIO watch, secret bag + rotation, failure table, TF apply) and
analyst (the analyze report, query recipes, SQL trend queries against the
snapshot tables, interpretation caveats) sections.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Viktor Barzin 2026-06-02 19:37:57 +00:00
parent 87f1dcb72d
commit cda858d560
2 changed files with 326 additions and 0 deletions

View file

@ -0,0 +1,109 @@
# Weekly market scrape. Runs ats + hn + levels_fyi, which upsert into
# comp_points/roles AND append dated rows into comp_snapshots/roles_snapshots
# (the trend series consumed by `job-hunter analyze`). Sundays 04:00 UTC
# low-traffic window, polite to levels.fyi (~25 companies, 3s jitter each).
#
# The alembic-migrate init container mirrors the Deployment so the CronJob can
# never run a refresh against an un-migrated DB (snapshot inserts would fail).
# Image is :latest (Keel-managed for the Deployment); the CronJob pulls the
# current latest at each run, so it always executes the newest code.
resource "kubernetes_cron_job_v1" "job_hunter_refresh" {
metadata {
name = "job-hunter-refresh"
namespace = kubernetes_namespace.job_hunter.metadata[0].name
labels = local.labels
}
spec {
schedule = "0 4 * * 0"
concurrency_policy = "Forbid"
successful_jobs_history_limit = 3
failed_jobs_history_limit = 3
starting_deadline_seconds = 600
job_template {
metadata {
labels = local.labels
}
spec {
backoff_limit = 1
active_deadline_seconds = 1800 # cap a hung scrape at 30m
ttl_seconds_after_finished = 86400
template {
metadata {
labels = local.labels
}
spec {
restart_policy = "OnFailure"
image_pull_secrets {
name = "registry-credentials"
}
init_container {
name = "alembic-migrate"
image = local.image
command = ["python", "-m", "job_hunter", "migrate"]
env_from {
secret_ref {
name = "job-hunter-secrets"
}
}
env_from {
secret_ref {
name = "job-hunter-db-creds"
}
}
resources {
requests = {
cpu = "50m"
memory = "256Mi"
}
limits = {
memory = "512Mi"
}
}
}
container {
name = "refresh"
image = local.image
command = ["python", "-m", "job_hunter", "refresh",
"--source", "ats", "--source", "hn", "--source", "levels_fyi"]
env_from {
secret_ref {
name = "job-hunter-secrets"
}
}
env_from {
secret_ref {
name = "job-hunter-db-creds"
}
}
resources {
requests = {
cpu = "100m"
memory = "512Mi"
}
limits = {
memory = "1Gi"
}
}
}
}
}
}
}
}
lifecycle {
# KYVERNO_LIFECYCLE_V1
ignore_changes = [spec[0].job_template[0].spec[0].template[0].spec[0].dns_config]
}
depends_on = [
kubernetes_manifest.external_secret,
kubernetes_manifest.db_external_secret,
]
}