job-hunter: weekly refresh CronJob + ops/analyst runbook
Add kubernetes_cron_job_v1.job_hunter_refresh — Sundays 04:00 UTC, runs `refresh --source ats --source hn --source levels_fyi`, which upserts roles/ comp AND appends the dated comp_snapshots/roles_snapshots series consumed by `job-hunter analyze`. Mirrors the Deployment's alembic-migrate init container so a refresh never runs against an un-migrated DB; concurrency Forbid, backoff 1, 30m activeDeadline, KYVERNO_LIFECYCLE_V1 dns_config ignore. Add docs/runbooks/job-hunter.md: ops (health checks, manual refresh, add an ATS company / CDIO watch, secret bag + rotation, failure table, TF apply) and analyst (the analyze report, query recipes, SQL trend queries against the snapshot tables, interpretation caveats) sections. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
parent
87f1dcb72d
commit
cda858d560
2 changed files with 326 additions and 0 deletions
109
stacks/job-hunter/cronjob.tf
Normal file
109
stacks/job-hunter/cronjob.tf
Normal file
|
|
@ -0,0 +1,109 @@
|
|||
# Weekly market scrape. Runs ats + hn + levels_fyi, which upsert into
|
||||
# comp_points/roles AND append dated rows into comp_snapshots/roles_snapshots
|
||||
# (the trend series consumed by `job-hunter analyze`). Sundays 04:00 UTC —
|
||||
# low-traffic window, polite to levels.fyi (~25 companies, 3s jitter each).
|
||||
#
|
||||
# The alembic-migrate init container mirrors the Deployment so the CronJob can
|
||||
# never run a refresh against an un-migrated DB (snapshot inserts would fail).
|
||||
# Image is :latest (Keel-managed for the Deployment); the CronJob pulls the
|
||||
# current latest at each run, so it always executes the newest code.
|
||||
resource "kubernetes_cron_job_v1" "job_hunter_refresh" {
|
||||
metadata {
|
||||
name = "job-hunter-refresh"
|
||||
namespace = kubernetes_namespace.job_hunter.metadata[0].name
|
||||
labels = local.labels
|
||||
}
|
||||
spec {
|
||||
schedule = "0 4 * * 0"
|
||||
concurrency_policy = "Forbid"
|
||||
successful_jobs_history_limit = 3
|
||||
failed_jobs_history_limit = 3
|
||||
starting_deadline_seconds = 600
|
||||
|
||||
job_template {
|
||||
metadata {
|
||||
labels = local.labels
|
||||
}
|
||||
spec {
|
||||
backoff_limit = 1
|
||||
active_deadline_seconds = 1800 # cap a hung scrape at 30m
|
||||
ttl_seconds_after_finished = 86400
|
||||
|
||||
template {
|
||||
metadata {
|
||||
labels = local.labels
|
||||
}
|
||||
spec {
|
||||
restart_policy = "OnFailure"
|
||||
image_pull_secrets {
|
||||
name = "registry-credentials"
|
||||
}
|
||||
|
||||
init_container {
|
||||
name = "alembic-migrate"
|
||||
image = local.image
|
||||
command = ["python", "-m", "job_hunter", "migrate"]
|
||||
env_from {
|
||||
secret_ref {
|
||||
name = "job-hunter-secrets"
|
||||
}
|
||||
}
|
||||
env_from {
|
||||
secret_ref {
|
||||
name = "job-hunter-db-creds"
|
||||
}
|
||||
}
|
||||
resources {
|
||||
requests = {
|
||||
cpu = "50m"
|
||||
memory = "256Mi"
|
||||
}
|
||||
limits = {
|
||||
memory = "512Mi"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
container {
|
||||
name = "refresh"
|
||||
image = local.image
|
||||
command = ["python", "-m", "job_hunter", "refresh",
|
||||
"--source", "ats", "--source", "hn", "--source", "levels_fyi"]
|
||||
|
||||
env_from {
|
||||
secret_ref {
|
||||
name = "job-hunter-secrets"
|
||||
}
|
||||
}
|
||||
env_from {
|
||||
secret_ref {
|
||||
name = "job-hunter-db-creds"
|
||||
}
|
||||
}
|
||||
|
||||
resources {
|
||||
requests = {
|
||||
cpu = "100m"
|
||||
memory = "512Mi"
|
||||
}
|
||||
limits = {
|
||||
memory = "1Gi"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
lifecycle {
|
||||
# KYVERNO_LIFECYCLE_V1
|
||||
ignore_changes = [spec[0].job_template[0].spec[0].template[0].spec[0].dns_config]
|
||||
}
|
||||
|
||||
depends_on = [
|
||||
kubernetes_manifest.external_secret,
|
||||
kubernetes_manifest.db_external_secret,
|
||||
]
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue