Add crawl job progress drawer with phase tracking and live logs
- Add phase-aware progress reporting across all crawl phases (splitting, fetching, filtering, processing) with per-step counters - Add TaskProgressDrawer component with phase timeline stepper, detail counters, progress bar with ETA, and live worker log viewer - Add on_step_complete callback to ListingProcessor for granular tracking of details/images/OCR steps - Extend QuerySplitter on_progress callback with structured counter data - Capture celery worker logs via ring buffer handler and inject into task state updates for frontend display - Guard taskResult updates with phase presence check to prevent drawer from blanking during state transitions
This commit is contained in:
Viktor Barzin
parent
4018503723
commit
b4837e1603
6 changed files with 617 additions and 24 deletions
|
|
@ -7,6 +7,7 @@ import { useEffect, useState } from 'react';
|
|||
import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from './ui/tooltip';
|
||||
import { Button } from './ui/button';
|
||||
import { Loader2, CheckCircle2, XCircle, X, Trash2 } from 'lucide-react';
|
||||
import { TaskProgressDrawer } from './TaskProgressDrawer';
|
||||
|
||||
interface TaskIndicatorProps {
|
||||
taskID: string | null;
|
||||
|
|
@ -19,8 +20,10 @@ export function TaskIndicator({ taskID, onTaskCancelled }: TaskIndicatorProps) {
|
|||
const [processed, setProcessed] = useState<number | null>(null);
|
||||
const [total, setTotal] = useState<number | null>(null);
|
||||
const [taskStatus, setTaskStatus] = useState<TaskStatus | null>(null);
|
||||
const [taskResult, setTaskResult] = useState<TaskResult | null>(null);
|
||||
const [isCancelling, setIsCancelling] = useState(false);
|
||||
const [isClearing, setIsClearing] = useState(false);
|
||||
const [drawerOpen, setDrawerOpen] = useState(false);
|
||||
|
||||
useEffect(() => {
|
||||
getUser().then(setUser);
|
||||
|
|
@ -29,6 +32,7 @@ export function TaskIndicator({ taskID, onTaskCancelled }: TaskIndicatorProps) {
|
|||
useEffect(() => {
|
||||
if (!user || !taskID) {
|
||||
setTaskStatus(null);
|
||||
setTaskResult(null);
|
||||
return;
|
||||
}
|
||||
|
||||
|
|
@ -37,6 +41,7 @@ export function TaskIndicator({ taskID, onTaskCancelled }: TaskIndicatorProps) {
|
|||
setProgressPercentage(0);
|
||||
setProcessed(null);
|
||||
setTotal(null);
|
||||
setTaskResult(null);
|
||||
|
||||
const pollTaskStatus = async () => {
|
||||
try {
|
||||
|
|
@ -46,6 +51,20 @@ export function TaskIndicator({ taskID, onTaskCancelled }: TaskIndicatorProps) {
|
|||
|
||||
if (status === TaskStatus.SUCCESS) {
|
||||
setProgressPercentage(100);
|
||||
// Parse final result for the drawer to show completed state.
|
||||
// Only update taskResult if the new result has phase info;
|
||||
// otherwise keep the last in-progress result which has richer data
|
||||
// than the bare SUCCESS return value.
|
||||
if (data.result) {
|
||||
try {
|
||||
const parsedResult: TaskResult = JSON.parse(data.result);
|
||||
if (parsedResult.phase) {
|
||||
setTaskResult(parsedResult);
|
||||
}
|
||||
} catch {
|
||||
// Ignore parsing errors
|
||||
}
|
||||
}
|
||||
return true; // Stop polling
|
||||
}
|
||||
|
||||
|
|
@ -57,7 +76,18 @@ export function TaskIndicator({ taskID, onTaskCancelled }: TaskIndicatorProps) {
|
|||
if (data.result) {
|
||||
try {
|
||||
const parsedResult: TaskResult = JSON.parse(data.result);
|
||||
setProgressPercentage(parsedResult.progress * 100);
|
||||
// Only update taskResult if the parsed data has a phase field.
|
||||
// This prevents blanking the drawer when the backend sends a
|
||||
// state update without phase info (e.g. during brief transitions).
|
||||
if (parsedResult.phase) {
|
||||
setTaskResult(parsedResult);
|
||||
}
|
||||
// Only update progress/processed/total when the fields are
|
||||
// actually present — otherwise keep the previous values so
|
||||
// the UI doesn't flash back to 0 during phase transitions.
|
||||
if (parsedResult.progress !== undefined) {
|
||||
setProgressPercentage(parsedResult.progress * 100);
|
||||
}
|
||||
if (parsedResult.processed !== undefined) {
|
||||
setProcessed(parsedResult.processed);
|
||||
}
|
||||
|
|
@ -113,6 +143,7 @@ export function TaskIndicator({ taskID, onTaskCancelled }: TaskIndicatorProps) {
|
|||
const result = await clearAllTasks(user);
|
||||
if (result.success) {
|
||||
setTaskStatus(null);
|
||||
setTaskResult(null);
|
||||
onTaskCancelled?.();
|
||||
}
|
||||
} catch {
|
||||
|
|
@ -144,18 +175,27 @@ export function TaskIndicator({ taskID, onTaskCancelled }: TaskIndicatorProps) {
|
|||
if (processed !== null && total !== null && total > 0) {
|
||||
return `${processed} / ${total}`;
|
||||
}
|
||||
if (taskResult?.phase && taskResult.phase !== 'processing') {
|
||||
const phaseLabels: Record<string, string> = {
|
||||
splitting: 'Splitting',
|
||||
splitting_complete: 'Split done',
|
||||
fetching: 'Fetching',
|
||||
filtering: 'Filtering',
|
||||
};
|
||||
return phaseLabels[taskResult.phase] ?? `${Math.round(progressPercentage)}%`;
|
||||
}
|
||||
return `${Math.round(progressPercentage)}%`;
|
||||
};
|
||||
|
||||
const getTooltipContent = () => {
|
||||
if (isInProgress) {
|
||||
if (processed !== null && total !== null && total > 0) {
|
||||
return `Processing: ${processed} / ${total} listings (${Math.round(progressPercentage)}%)`;
|
||||
return `Processing: ${processed} / ${total} listings (${Math.round(progressPercentage)}%) — click for details`;
|
||||
}
|
||||
return `Task running: ${Math.round(progressPercentage)}%`;
|
||||
return `Task running: ${getProgressText()} — click for details`;
|
||||
}
|
||||
if (taskStatus === TaskStatus.SUCCESS) {
|
||||
return 'Task completed successfully';
|
||||
return 'Task completed successfully — click for details';
|
||||
}
|
||||
if (taskStatus === TaskStatus.REVOKED) {
|
||||
return 'Task was cancelled';
|
||||
|
|
@ -168,7 +208,10 @@ export function TaskIndicator({ taskID, onTaskCancelled }: TaskIndicatorProps) {
|
|||
<div className="flex items-center gap-2">
|
||||
<Tooltip>
|
||||
<TooltipTrigger asChild>
|
||||
<div className="flex items-center gap-2 cursor-default">
|
||||
<div
|
||||
className="flex items-center gap-2 cursor-pointer"
|
||||
onClick={() => setDrawerOpen(true)}
|
||||
>
|
||||
{getStatusIcon()}
|
||||
{isInProgress && (
|
||||
<div className="flex items-center gap-2">
|
||||
|
|
@ -230,6 +273,15 @@ export function TaskIndicator({ taskID, onTaskCancelled }: TaskIndicatorProps) {
|
|||
</TooltipContent>
|
||||
</Tooltip>
|
||||
</div>
|
||||
<TaskProgressDrawer
|
||||
open={drawerOpen}
|
||||
onOpenChange={setDrawerOpen}
|
||||
taskResult={taskResult}
|
||||
taskStatus={taskStatus}
|
||||
taskID={taskID}
|
||||
onCancel={handleCancel}
|
||||
isCancelling={isCancelling}
|
||||
/>
|
||||
</TooltipProvider>
|
||||
);
|
||||
}
|
||||
|
|
|
|||
363
crawler/frontend/src/components/TaskProgressDrawer.tsx
Normal file
363
crawler/frontend/src/components/TaskProgressDrawer.tsx
Normal file
|
|
@ -0,0 +1,363 @@
|
|||
import { TaskStatus, type TaskPhase, type TaskResult } from '@/types';
|
||||
import {
|
||||
Sheet,
|
||||
SheetContent,
|
||||
SheetHeader,
|
||||
SheetTitle,
|
||||
SheetDescription,
|
||||
SheetFooter,
|
||||
} from './ui/sheet';
|
||||
import { Button } from './ui/button';
|
||||
import { CheckCircle2, Circle, Loader2, XCircle } from 'lucide-react';
|
||||
import { useEffect, useRef } from 'react';
|
||||
|
||||
interface TaskProgressDrawerProps {
|
||||
open: boolean;
|
||||
onOpenChange: (open: boolean) => void;
|
||||
taskResult: TaskResult | null;
|
||||
taskStatus: TaskStatus | null;
|
||||
taskID: string | null;
|
||||
onCancel: () => void;
|
||||
isCancelling: boolean;
|
||||
}
|
||||
|
||||
const PHASES: { key: TaskPhase; label: string }[] = [
|
||||
{ key: 'splitting', label: 'Splitting queries' },
|
||||
{ key: 'fetching', label: 'Fetching listings' },
|
||||
{ key: 'filtering', label: 'Filtering results' },
|
||||
{ key: 'processing', label: 'Processing listings' },
|
||||
];
|
||||
|
||||
function getPhaseIndex(phase: TaskPhase | undefined): number {
|
||||
if (!phase) return -1;
|
||||
if (phase === 'splitting_complete') return 1; // splitting done, fetching is next
|
||||
if (phase === 'completed') return PHASES.length;
|
||||
return PHASES.findIndex((p) => p.key === phase);
|
||||
}
|
||||
|
||||
function formatEta(seconds: number | undefined): string {
|
||||
if (seconds === undefined || seconds <= 0) return '';
|
||||
const mins = Math.floor(seconds / 60);
|
||||
const secs = Math.round(seconds % 60);
|
||||
if (mins > 0) {
|
||||
return `~${mins}m ${secs}s remaining`;
|
||||
}
|
||||
return `~${secs}s remaining`;
|
||||
}
|
||||
|
||||
function StatusBadge({ status }: { status: TaskStatus | null }) {
|
||||
if (!status) return null;
|
||||
|
||||
const isInProgress =
|
||||
status !== TaskStatus.SUCCESS &&
|
||||
status !== TaskStatus.FAILURE &&
|
||||
status !== TaskStatus.REVOKED;
|
||||
|
||||
if (isInProgress) {
|
||||
return (
|
||||
<span className="inline-flex items-center gap-1 rounded-full bg-blue-100 px-2 py-0.5 text-xs font-medium text-blue-700">
|
||||
<Loader2 className="h-3 w-3 animate-spin" />
|
||||
Running
|
||||
</span>
|
||||
);
|
||||
}
|
||||
if (status === TaskStatus.SUCCESS) {
|
||||
return (
|
||||
<span className="inline-flex items-center gap-1 rounded-full bg-green-100 px-2 py-0.5 text-xs font-medium text-green-700">
|
||||
<CheckCircle2 className="h-3 w-3" />
|
||||
Complete
|
||||
</span>
|
||||
);
|
||||
}
|
||||
if (status === TaskStatus.REVOKED) {
|
||||
return (
|
||||
<span className="inline-flex items-center gap-1 rounded-full bg-yellow-100 px-2 py-0.5 text-xs font-medium text-yellow-700">
|
||||
<XCircle className="h-3 w-3" />
|
||||
Cancelled
|
||||
</span>
|
||||
);
|
||||
}
|
||||
return (
|
||||
<span className="inline-flex items-center gap-1 rounded-full bg-red-100 px-2 py-0.5 text-xs font-medium text-red-700">
|
||||
<XCircle className="h-3 w-3" />
|
||||
Failed
|
||||
</span>
|
||||
);
|
||||
}
|
||||
|
||||
function PhaseTimeline({
|
||||
currentPhase,
|
||||
taskStatus,
|
||||
}: {
|
||||
currentPhase: TaskPhase | undefined;
|
||||
taskStatus: TaskStatus | null;
|
||||
}) {
|
||||
const isTerminal =
|
||||
taskStatus === TaskStatus.SUCCESS ||
|
||||
taskStatus === TaskStatus.FAILURE ||
|
||||
taskStatus === TaskStatus.REVOKED;
|
||||
const activeIdx = isTerminal ? PHASES.length : getPhaseIndex(currentPhase);
|
||||
|
||||
return (
|
||||
<div className="flex flex-col gap-1">
|
||||
{PHASES.map((phase, idx) => {
|
||||
const isCompleted = idx < activeIdx;
|
||||
const isActive = idx === activeIdx && !isTerminal;
|
||||
const isFuture = idx > activeIdx;
|
||||
|
||||
return (
|
||||
<div key={phase.key} className="flex items-center gap-2">
|
||||
{isCompleted && (
|
||||
<CheckCircle2 className="h-4 w-4 text-green-500 shrink-0" />
|
||||
)}
|
||||
{isActive && (
|
||||
<Loader2 className="h-4 w-4 animate-spin text-blue-500 shrink-0" />
|
||||
)}
|
||||
{isFuture && (
|
||||
<Circle className="h-4 w-4 text-muted-foreground/40 shrink-0" />
|
||||
)}
|
||||
<span
|
||||
className={
|
||||
isActive
|
||||
? 'text-sm font-medium text-foreground'
|
||||
: isCompleted
|
||||
? 'text-sm text-muted-foreground'
|
||||
: 'text-sm text-muted-foreground/40'
|
||||
}
|
||||
>
|
||||
{phase.label}
|
||||
</span>
|
||||
</div>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function CounterRow({ label, value, total }: { label: string; value?: number; total?: number }) {
|
||||
if (value === undefined) return null;
|
||||
return (
|
||||
<div className="flex justify-between text-sm">
|
||||
<span className="text-muted-foreground">{label}</span>
|
||||
<span className="font-mono tabular-nums">
|
||||
{value}
|
||||
{total !== undefined && ` / ${total}`}
|
||||
</span>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function PhaseDetails({ result }: { result: TaskResult }) {
|
||||
const phase = result.phase;
|
||||
|
||||
if (phase === 'splitting' || phase === 'splitting_complete') {
|
||||
return (
|
||||
<div className="rounded-md border p-3 space-y-1">
|
||||
<p className="text-xs font-medium text-muted-foreground uppercase tracking-wide mb-2">
|
||||
Splitting
|
||||
</p>
|
||||
<CounterRow
|
||||
label="Subqueries probed"
|
||||
value={result.subqueries_probed}
|
||||
total={result.subqueries_initial}
|
||||
/>
|
||||
{result.subqueries_total !== undefined && (
|
||||
<CounterRow label="Final subqueries" value={result.subqueries_total} />
|
||||
)}
|
||||
{result.estimated_results !== undefined && (
|
||||
<CounterRow label="Estimated results" value={result.estimated_results} />
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
if (phase === 'fetching') {
|
||||
return (
|
||||
<div className="rounded-md border p-3 space-y-1">
|
||||
<p className="text-xs font-medium text-muted-foreground uppercase tracking-wide mb-2">
|
||||
Fetching
|
||||
</p>
|
||||
<CounterRow
|
||||
label="Subqueries completed"
|
||||
value={result.subqueries_completed}
|
||||
total={result.subqueries_total}
|
||||
/>
|
||||
<CounterRow label="IDs collected" value={result.ids_collected} />
|
||||
<CounterRow label="Pages fetched" value={result.pages_fetched} />
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
if (phase === 'filtering') {
|
||||
return (
|
||||
<div className="rounded-md border p-3 space-y-1">
|
||||
<p className="text-xs font-medium text-muted-foreground uppercase tracking-wide mb-2">
|
||||
Filtering
|
||||
</p>
|
||||
<CounterRow label="Total from API" value={result.total_found} />
|
||||
<CounterRow label="Already in DB" value={result.existing_in_db} />
|
||||
<CounterRow label="New to process" value={result.new_listings} />
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
if (phase === 'processing') {
|
||||
return (
|
||||
<div className="rounded-md border p-3 space-y-1">
|
||||
<p className="text-xs font-medium text-muted-foreground uppercase tracking-wide mb-2">
|
||||
Processing
|
||||
</p>
|
||||
<CounterRow
|
||||
label="Details fetched"
|
||||
value={result.details_fetched}
|
||||
total={result.total}
|
||||
/>
|
||||
<CounterRow label="Images downloaded" value={result.images_downloaded} />
|
||||
<CounterRow label="OCR completed" value={result.ocr_completed} />
|
||||
{(result.failed ?? 0) > 0 && (
|
||||
<div className="flex justify-between text-sm">
|
||||
<span className="text-red-500">Failed</span>
|
||||
<span className="font-mono tabular-nums text-red-500">{result.failed}</span>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
function LogViewer({ logs }: { logs: string[] }) {
|
||||
const scrollRef = useRef<HTMLDivElement>(null);
|
||||
const isAutoScrolling = useRef(true);
|
||||
|
||||
const handleScroll = () => {
|
||||
const el = scrollRef.current;
|
||||
if (!el) return;
|
||||
const atBottom = el.scrollHeight - el.scrollTop - el.clientHeight < 30;
|
||||
isAutoScrolling.current = atBottom;
|
||||
};
|
||||
|
||||
useEffect(() => {
|
||||
if (isAutoScrolling.current && scrollRef.current) {
|
||||
scrollRef.current.scrollTop = scrollRef.current.scrollHeight;
|
||||
}
|
||||
}, [logs]);
|
||||
|
||||
return (
|
||||
<div
|
||||
ref={scrollRef}
|
||||
onScroll={handleScroll}
|
||||
className="rounded-md bg-zinc-950 p-3 overflow-y-auto font-mono text-[11px] leading-4 text-zinc-300 min-h-[100px] h-full"
|
||||
>
|
||||
{logs.length === 0 ? (
|
||||
<span className="text-zinc-500 italic">Waiting for logs...</span>
|
||||
) : (
|
||||
logs.map((line, i) => (
|
||||
<div key={i} className="whitespace-pre-wrap break-all">
|
||||
{line}
|
||||
</div>
|
||||
))
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
export function TaskProgressDrawer({
|
||||
open,
|
||||
onOpenChange,
|
||||
taskResult,
|
||||
taskStatus,
|
||||
taskID,
|
||||
onCancel,
|
||||
isCancelling,
|
||||
}: TaskProgressDrawerProps) {
|
||||
const isInProgress =
|
||||
taskStatus !== null &&
|
||||
taskStatus !== TaskStatus.SUCCESS &&
|
||||
taskStatus !== TaskStatus.FAILURE &&
|
||||
taskStatus !== TaskStatus.REVOKED;
|
||||
|
||||
const progressPercent = taskResult
|
||||
? Math.min((taskResult.progress ?? 0) * 100, 100)
|
||||
: 0;
|
||||
|
||||
return (
|
||||
<Sheet open={open} onOpenChange={onOpenChange}>
|
||||
<SheetContent side="right" className="flex flex-col w-full sm:!max-w-lg">
|
||||
<SheetHeader>
|
||||
<div className="flex items-center justify-between pr-6">
|
||||
<SheetTitle>Crawl Job Progress</SheetTitle>
|
||||
<StatusBadge status={taskStatus} />
|
||||
</div>
|
||||
{taskID && (
|
||||
<SheetDescription>
|
||||
Task ID: {taskID.slice(0, 8)}...
|
||||
</SheetDescription>
|
||||
)}
|
||||
</SheetHeader>
|
||||
|
||||
{/* Fixed top section: timeline + counters + progress */}
|
||||
<div className="space-y-3 px-4 shrink-0">
|
||||
<PhaseTimeline
|
||||
currentPhase={taskResult?.phase}
|
||||
taskStatus={taskStatus}
|
||||
/>
|
||||
|
||||
{taskResult && <PhaseDetails result={taskResult} />}
|
||||
|
||||
{taskResult && taskResult.phase === 'processing' && (
|
||||
<div className="space-y-1">
|
||||
<div className="w-full h-2 bg-primary/20 rounded-full overflow-hidden">
|
||||
<div
|
||||
className="h-full bg-primary transition-all duration-300 ease-out rounded-full"
|
||||
style={{ width: `${progressPercent}%` }}
|
||||
/>
|
||||
</div>
|
||||
<div className="flex justify-between text-xs text-muted-foreground">
|
||||
<span>
|
||||
{taskResult.processed ?? 0} / {taskResult.total ?? '?'}
|
||||
</span>
|
||||
<span>{formatEta(taskResult.eta_seconds)}</span>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{taskResult?.message && (
|
||||
<p className="text-sm text-muted-foreground">{taskResult.message}</p>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{/* Log viewer fills remaining space */}
|
||||
<div className="flex-1 min-h-0 flex flex-col gap-1 px-4 pb-2">
|
||||
<p className="text-xs font-medium text-muted-foreground uppercase tracking-wide shrink-0">
|
||||
Worker Logs
|
||||
</p>
|
||||
<div className="flex-1 min-h-0">
|
||||
<LogViewer logs={taskResult?.logs ?? []} />
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{isInProgress && (
|
||||
<SheetFooter>
|
||||
<Button
|
||||
variant="destructive"
|
||||
onClick={onCancel}
|
||||
disabled={isCancelling}
|
||||
className="w-full"
|
||||
>
|
||||
{isCancelling ? (
|
||||
<>
|
||||
<Loader2 className="mr-2 h-4 w-4 animate-spin" />
|
||||
Cancelling...
|
||||
</>
|
||||
) : (
|
||||
'Cancel Job'
|
||||
)}
|
||||
</Button>
|
||||
</SheetFooter>
|
||||
)}
|
||||
</SheetContent>
|
||||
</Sheet>
|
||||
);
|
||||
}
|
||||
|
|
@ -48,13 +48,41 @@ export enum TaskStatus {
|
|||
|
||||
export interface TaskStatusResponse {
|
||||
status: TaskStatus;
|
||||
result: string; // JSON string containing { progress: number }
|
||||
result: string; // JSON string containing TaskResult
|
||||
message?: string;
|
||||
}
|
||||
|
||||
export type TaskPhase = 'splitting' | 'splitting_complete' | 'fetching' | 'filtering' | 'processing' | 'completed';
|
||||
|
||||
export interface TaskResult {
|
||||
progress: number;
|
||||
processed?: number;
|
||||
total?: number;
|
||||
phase?: TaskPhase;
|
||||
message?: string;
|
||||
// Splitting phase
|
||||
subqueries_probed?: number;
|
||||
subqueries_initial?: number;
|
||||
estimated_results?: number;
|
||||
subqueries_total?: number;
|
||||
// Fetching phase
|
||||
subqueries_completed?: number;
|
||||
ids_collected?: number;
|
||||
pages_fetched?: number;
|
||||
// Filtering phase
|
||||
total_found?: number;
|
||||
existing_in_db?: number;
|
||||
new_listings?: number;
|
||||
// Processing phase
|
||||
details_fetched?: number;
|
||||
images_downloaded?: number;
|
||||
ocr_completed?: number;
|
||||
failed?: number;
|
||||
elapsed_seconds?: number;
|
||||
rate_per_second?: number;
|
||||
eta_seconds?: number;
|
||||
// Live logs
|
||||
logs?: string[];
|
||||
}
|
||||
|
||||
export interface RefreshListingsResponse {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue