examples: async PRAW wrapper → RawPost
All checks were successful
ci/woodpecker/push/woodpecker Pipeline was successful
All checks were successful
ci/woodpecker/push/woodpecker Pipeline was successful
This commit is contained in:
parent
a378c7256e
commit
8fc0fd7646
2 changed files with 99 additions and 0 deletions
41
fire_planner/examples/praw_source.py
Normal file
41
fire_planner/examples/praw_source.py
Normal file
|
|
@ -0,0 +1,41 @@
|
||||||
|
"""Async PRAW wrapper — yields `RawPost` from a subreddit's top listing.
|
||||||
|
|
||||||
|
We use asyncpraw because the rest of the pipeline is asyncio-native and
|
||||||
|
we want to fan out across 12 subs concurrently via `asyncio.gather`.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import logging
|
||||||
|
from collections.abc import AsyncIterator
|
||||||
|
from datetime import date
|
||||||
|
from typing import Any, Literal
|
||||||
|
|
||||||
|
from fire_planner.examples.models import RawPost
|
||||||
|
|
||||||
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
TopWhen = Literal["all", "year", "month", "week", "day"]
|
||||||
|
REDDIT_BASE = "https://www.reddit.com"
|
||||||
|
|
||||||
|
|
||||||
|
async def fetch_top(
|
||||||
|
reddit: Any, # asyncpraw.Reddit
|
||||||
|
subreddit: str,
|
||||||
|
when: TopWhen,
|
||||||
|
limit: int = 1000,
|
||||||
|
) -> AsyncIterator[RawPost]:
|
||||||
|
"""Yield `RawPost`s from `r/{subreddit}/top/?t={when}` (PRAW 1000 cap)."""
|
||||||
|
sub = await reddit.subreddit(subreddit)
|
||||||
|
async for submission in sub.top(time_filter=when, limit=limit):
|
||||||
|
yield _to_raw_post(submission, subreddit)
|
||||||
|
|
||||||
|
|
||||||
|
def _to_raw_post(submission: Any, source_sub: str) -> RawPost:
|
||||||
|
return RawPost(
|
||||||
|
reddit_id=submission.id,
|
||||||
|
source_sub=source_sub,
|
||||||
|
url=f"{REDDIT_BASE}{submission.permalink}",
|
||||||
|
title=submission.title or "",
|
||||||
|
body=submission.selftext or "",
|
||||||
|
created_at=date.fromtimestamp(submission.created_utc),
|
||||||
|
)
|
||||||
58
tests/test_examples_praw_source.py
Normal file
58
tests/test_examples_praw_source.py
Normal file
|
|
@ -0,0 +1,58 @@
|
||||||
|
"""Tests for the asyncpraw wrapper — uses an in-test fake Submission iterator."""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from collections.abc import AsyncIterator
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from datetime import datetime
|
||||||
|
from unittest.mock import AsyncMock, MagicMock
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from fire_planner.examples.praw_source import fetch_top
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class _FakeSub:
|
||||||
|
id: str
|
||||||
|
title: str
|
||||||
|
selftext: str
|
||||||
|
permalink: str
|
||||||
|
created_utc: float
|
||||||
|
|
||||||
|
|
||||||
|
def _async_iter(items: list[_FakeSub]) -> AsyncIterator[_FakeSub]:
|
||||||
|
async def _gen() -> AsyncIterator[_FakeSub]:
|
||||||
|
for it in items:
|
||||||
|
yield it
|
||||||
|
return _gen()
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_fetch_top_normalises_submissions() -> None:
|
||||||
|
fakes = [
|
||||||
|
_FakeSub(
|
||||||
|
id="abc1",
|
||||||
|
title="t1",
|
||||||
|
selftext="b1",
|
||||||
|
permalink="/r/financialindependence/comments/abc1/",
|
||||||
|
created_utc=datetime(2026, 1, 1).timestamp(),
|
||||||
|
),
|
||||||
|
_FakeSub(
|
||||||
|
id="abc2",
|
||||||
|
title="t2",
|
||||||
|
selftext="b2",
|
||||||
|
permalink="/r/financialindependence/comments/abc2/",
|
||||||
|
created_utc=datetime(2026, 2, 1).timestamp(),
|
||||||
|
),
|
||||||
|
]
|
||||||
|
mock_subreddit = MagicMock()
|
||||||
|
mock_subreddit.top = MagicMock(return_value=_async_iter(fakes))
|
||||||
|
|
||||||
|
mock_reddit = MagicMock()
|
||||||
|
mock_reddit.subreddit = AsyncMock(return_value=mock_subreddit)
|
||||||
|
|
||||||
|
posts = [p async for p in fetch_top(mock_reddit, "financialindependence", "all", limit=1000)]
|
||||||
|
assert len(posts) == 2
|
||||||
|
assert posts[0].reddit_id == "abc1"
|
||||||
|
assert posts[0].url.endswith("/r/financialindependence/comments/abc1/")
|
||||||
|
assert posts[1].title == "t2"
|
||||||
Loading…
Add table
Add a link
Reference in a new issue