[ci skip] Move Terraform modules into stack directories
Move all 88 service modules (66 individual + 22 platform) from modules/kubernetes/<service>/ into their corresponding stack directories: - Service stacks: stacks/<service>/module/ - Platform stack: stacks/platform/modules/<service>/ This collocates module source code with its Terragrunt definition. Only shared utility modules remain in modules/kubernetes/: ingress_factory, setup_tls_secret, dockerhub_secret, oauth-proxy. All cross-references to shared modules updated to use correct relative paths. Verified with terragrunt run --all -- plan: 0 adds, 0 destroys across all 68 stacks.
This commit is contained in:
parent
73cb696f12
commit
e225e81ebf
614 changed files with 12075 additions and 352 deletions
359
stacks/f1-stream/module/files/internal/auth/auth.go
Normal file
359
stacks/f1-stream/module/files/internal/auth/auth.go
Normal file
|
|
@ -0,0 +1,359 @@
|
|||
package auth
|
||||
|
||||
import (
|
||||
"crypto/rand"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"f1-stream/internal/models"
|
||||
"f1-stream/internal/store"
|
||||
|
||||
"github.com/go-webauthn/webauthn/webauthn"
|
||||
)
|
||||
|
||||
var usernameRe = regexp.MustCompile(`^[a-zA-Z0-9_]{3,30}$`)
|
||||
|
||||
type Auth struct {
|
||||
store *store.Store
|
||||
webauthn *webauthn.WebAuthn
|
||||
adminUsername string
|
||||
sessionTTL time.Duration
|
||||
|
||||
// In-memory storage for WebAuthn ceremony session data (short-lived)
|
||||
regSessions map[string]*webauthn.SessionData
|
||||
loginSessions map[string]*webauthn.SessionData
|
||||
mu sync.Mutex
|
||||
}
|
||||
|
||||
func New(s *store.Store, rpDisplayName, rpID string, rpOrigins []string, adminUsername string, sessionTTL time.Duration) (*Auth, error) {
|
||||
wconfig := &webauthn.Config{
|
||||
RPDisplayName: rpDisplayName,
|
||||
RPID: rpID,
|
||||
RPOrigins: rpOrigins,
|
||||
}
|
||||
w, err := webauthn.New(wconfig)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("webauthn init: %w", err)
|
||||
}
|
||||
return &Auth{
|
||||
store: s,
|
||||
webauthn: w,
|
||||
adminUsername: adminUsername,
|
||||
sessionTTL: sessionTTL,
|
||||
regSessions: make(map[string]*webauthn.SessionData),
|
||||
loginSessions: make(map[string]*webauthn.SessionData),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// BeginRegistration starts the WebAuthn registration ceremony.
|
||||
func (a *Auth) BeginRegistration(w http.ResponseWriter, r *http.Request) {
|
||||
var req struct {
|
||||
Username string `json:"username"`
|
||||
}
|
||||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
||||
http.Error(w, `{"error":"invalid request"}`, http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
if !usernameRe.MatchString(req.Username) {
|
||||
http.Error(w, `{"error":"username must be 3-30 chars, alphanumeric or underscore"}`, http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
existing, err := a.store.GetUserByName(req.Username)
|
||||
if err != nil {
|
||||
http.Error(w, `{"error":"internal error"}`, http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
if existing != nil {
|
||||
http.Error(w, `{"error":"username already taken"}`, http.StatusConflict)
|
||||
return
|
||||
}
|
||||
|
||||
id, err := randomID()
|
||||
if err != nil {
|
||||
http.Error(w, `{"error":"internal error"}`, http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
isAdmin := false
|
||||
if a.adminUsername != "" && req.Username == a.adminUsername {
|
||||
isAdmin = true
|
||||
} else if a.adminUsername == "" {
|
||||
count, err := a.store.UserCount()
|
||||
if err == nil && count == 0 {
|
||||
isAdmin = true
|
||||
}
|
||||
}
|
||||
|
||||
user := &models.User{
|
||||
ID: id,
|
||||
Username: req.Username,
|
||||
IsAdmin: isAdmin,
|
||||
CreatedAt: time.Now(),
|
||||
}
|
||||
|
||||
options, session, err := a.webauthn.BeginRegistration(user)
|
||||
if err != nil {
|
||||
log.Printf("BeginRegistration error: %v", err)
|
||||
http.Error(w, `{"error":"failed to begin registration"}`, http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
a.mu.Lock()
|
||||
a.regSessions[req.Username] = session
|
||||
a.mu.Unlock()
|
||||
|
||||
// Clean up session after 5 minutes
|
||||
go func() {
|
||||
time.Sleep(5 * time.Minute)
|
||||
a.mu.Lock()
|
||||
delete(a.regSessions, req.Username)
|
||||
a.mu.Unlock()
|
||||
}()
|
||||
|
||||
// Store user temporarily - will be committed on finish
|
||||
// We create the user now so FinishRegistration can look it up
|
||||
if err := a.store.CreateUser(*user); err != nil {
|
||||
http.Error(w, `{"error":"failed to create user"}`, http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
json.NewEncoder(w).Encode(options)
|
||||
}
|
||||
|
||||
// FinishRegistration completes the WebAuthn registration ceremony.
|
||||
func (a *Auth) FinishRegistration(w http.ResponseWriter, r *http.Request) {
|
||||
var req struct {
|
||||
Username string `json:"username"`
|
||||
}
|
||||
// Username is passed as query param since body is the attestation response
|
||||
username := r.URL.Query().Get("username")
|
||||
if username == "" {
|
||||
// Try to decode from a wrapper
|
||||
http.Error(w, `{"error":"username required"}`, http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
req.Username = username
|
||||
|
||||
a.mu.Lock()
|
||||
session, ok := a.regSessions[req.Username]
|
||||
if ok {
|
||||
delete(a.regSessions, req.Username)
|
||||
}
|
||||
a.mu.Unlock()
|
||||
|
||||
if !ok {
|
||||
http.Error(w, `{"error":"no registration in progress"}`, http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
user, err := a.store.GetUserByName(req.Username)
|
||||
if err != nil || user == nil {
|
||||
http.Error(w, `{"error":"user not found"}`, http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
credential, err := a.webauthn.FinishRegistration(user, *session, r)
|
||||
if err != nil {
|
||||
log.Printf("FinishRegistration error: %v", err)
|
||||
http.Error(w, `{"error":"registration failed"}`, http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
user.Credentials = append(user.Credentials, *credential)
|
||||
if err := a.store.UpdateUserCredentials(user.ID, user.Credentials); err != nil {
|
||||
http.Error(w, `{"error":"failed to save credential"}`, http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
// Create session
|
||||
token, err := a.store.CreateSession(user.ID, a.sessionTTL)
|
||||
if err != nil {
|
||||
http.Error(w, `{"error":"failed to create session"}`, http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
http.SetCookie(w, &http.Cookie{
|
||||
Name: "session",
|
||||
Value: token,
|
||||
Path: "/",
|
||||
HttpOnly: true,
|
||||
SameSite: http.SameSiteStrictMode,
|
||||
Secure: r.TLS != nil,
|
||||
MaxAge: int(a.sessionTTL.Seconds()),
|
||||
})
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
json.NewEncoder(w).Encode(map[string]interface{}{
|
||||
"id": user.ID,
|
||||
"username": user.Username,
|
||||
"is_admin": user.IsAdmin,
|
||||
})
|
||||
}
|
||||
|
||||
// BeginLogin starts the WebAuthn login ceremony.
|
||||
func (a *Auth) BeginLogin(w http.ResponseWriter, r *http.Request) {
|
||||
var req struct {
|
||||
Username string `json:"username"`
|
||||
}
|
||||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
||||
http.Error(w, `{"error":"invalid request"}`, http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
user, err := a.store.GetUserByName(req.Username)
|
||||
if err != nil {
|
||||
http.Error(w, `{"error":"internal error"}`, http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
if user == nil {
|
||||
http.Error(w, `{"error":"user not found"}`, http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
if len(user.Credentials) == 0 {
|
||||
http.Error(w, `{"error":"no credentials registered"}`, http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
options, session, err := a.webauthn.BeginLogin(user)
|
||||
if err != nil {
|
||||
log.Printf("BeginLogin error: %v", err)
|
||||
http.Error(w, `{"error":"failed to begin login"}`, http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
a.mu.Lock()
|
||||
a.loginSessions[req.Username] = session
|
||||
a.mu.Unlock()
|
||||
|
||||
go func() {
|
||||
time.Sleep(5 * time.Minute)
|
||||
a.mu.Lock()
|
||||
delete(a.loginSessions, req.Username)
|
||||
a.mu.Unlock()
|
||||
}()
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
json.NewEncoder(w).Encode(options)
|
||||
}
|
||||
|
||||
// FinishLogin completes the WebAuthn login ceremony.
|
||||
func (a *Auth) FinishLogin(w http.ResponseWriter, r *http.Request) {
|
||||
username := r.URL.Query().Get("username")
|
||||
if username == "" {
|
||||
http.Error(w, `{"error":"username required"}`, http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
a.mu.Lock()
|
||||
session, ok := a.loginSessions[username]
|
||||
if ok {
|
||||
delete(a.loginSessions, username)
|
||||
}
|
||||
a.mu.Unlock()
|
||||
|
||||
if !ok {
|
||||
http.Error(w, `{"error":"no login in progress"}`, http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
user, err := a.store.GetUserByName(username)
|
||||
if err != nil || user == nil {
|
||||
http.Error(w, `{"error":"user not found"}`, http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
credential, err := a.webauthn.FinishLogin(user, *session, r)
|
||||
if err != nil {
|
||||
log.Printf("FinishLogin error: %v", err)
|
||||
http.Error(w, `{"error":"login failed"}`, http.StatusUnauthorized)
|
||||
return
|
||||
}
|
||||
|
||||
// Update credential sign count
|
||||
for i, c := range user.Credentials {
|
||||
if string(c.ID) == string(credential.ID) {
|
||||
user.Credentials[i].Authenticator.SignCount = credential.Authenticator.SignCount
|
||||
break
|
||||
}
|
||||
}
|
||||
if err := a.store.UpdateUserCredentials(user.ID, user.Credentials); err != nil {
|
||||
log.Printf("Failed to update credential sign count: %v", err)
|
||||
}
|
||||
|
||||
token, err := a.store.CreateSession(user.ID, a.sessionTTL)
|
||||
if err != nil {
|
||||
http.Error(w, `{"error":"failed to create session"}`, http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
http.SetCookie(w, &http.Cookie{
|
||||
Name: "session",
|
||||
Value: token,
|
||||
Path: "/",
|
||||
HttpOnly: true,
|
||||
SameSite: http.SameSiteStrictMode,
|
||||
Secure: r.TLS != nil,
|
||||
MaxAge: int(a.sessionTTL.Seconds()),
|
||||
})
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
json.NewEncoder(w).Encode(map[string]interface{}{
|
||||
"id": user.ID,
|
||||
"username": user.Username,
|
||||
"is_admin": user.IsAdmin,
|
||||
})
|
||||
}
|
||||
|
||||
// Logout clears the session.
|
||||
func (a *Auth) Logout(w http.ResponseWriter, r *http.Request) {
|
||||
cookie, err := r.Cookie("session")
|
||||
if err == nil {
|
||||
a.store.DeleteSession(cookie.Value)
|
||||
}
|
||||
http.SetCookie(w, &http.Cookie{
|
||||
Name: "session",
|
||||
Value: "",
|
||||
Path: "/",
|
||||
HttpOnly: true,
|
||||
MaxAge: -1,
|
||||
})
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.Write([]byte(`{"ok":true}`))
|
||||
}
|
||||
|
||||
// Me returns the current user info.
|
||||
func (a *Auth) Me(w http.ResponseWriter, r *http.Request) {
|
||||
user := UserFromContext(r.Context())
|
||||
if user == nil {
|
||||
http.Error(w, `{"error":"not authenticated"}`, http.StatusUnauthorized)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
json.NewEncoder(w).Encode(map[string]interface{}{
|
||||
"id": user.ID,
|
||||
"username": user.Username,
|
||||
"is_admin": user.IsAdmin,
|
||||
})
|
||||
}
|
||||
|
||||
// GetSessionUser returns the user for a session token.
|
||||
func (a *Auth) GetSessionUser(token string) (*models.User, error) {
|
||||
sess, err := a.store.GetSession(token)
|
||||
if err != nil || sess == nil {
|
||||
return nil, err
|
||||
}
|
||||
return a.store.GetUserByID(sess.UserID)
|
||||
}
|
||||
|
||||
func randomID() (string, error) {
|
||||
b := make([]byte, 16)
|
||||
if _, err := rand.Read(b); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return fmt.Sprintf("%x", b), nil
|
||||
}
|
||||
20
stacks/f1-stream/module/files/internal/auth/context.go
Normal file
20
stacks/f1-stream/module/files/internal/auth/context.go
Normal file
|
|
@ -0,0 +1,20 @@
|
|||
package auth
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"f1-stream/internal/models"
|
||||
)
|
||||
|
||||
type contextKey string
|
||||
|
||||
const userKey contextKey = "user"
|
||||
|
||||
func ContextWithUser(ctx context.Context, user *models.User) context.Context {
|
||||
return context.WithValue(ctx, userKey, user)
|
||||
}
|
||||
|
||||
func UserFromContext(ctx context.Context) *models.User {
|
||||
user, _ := ctx.Value(userKey).(*models.User)
|
||||
return user
|
||||
}
|
||||
38
stacks/f1-stream/module/files/internal/extractor/browser.go
Normal file
38
stacks/f1-stream/module/files/internal/extractor/browser.go
Normal file
|
|
@ -0,0 +1,38 @@
|
|||
package extractor
|
||||
|
||||
import (
|
||||
"log"
|
||||
"os/exec"
|
||||
)
|
||||
|
||||
const maxConcurrentSessions = 10
|
||||
|
||||
var sessionSem chan struct{}
|
||||
|
||||
// Init starts dbus, PulseAudio, and prepares the session semaphore.
|
||||
func Init() {
|
||||
// Start dbus (Chrome needs it for accessibility/service queries)
|
||||
if err := exec.Command("mkdir", "-p", "/var/run/dbus").Run(); err == nil {
|
||||
if err := exec.Command("dbus-daemon", "--system", "--nofork").Start(); err != nil {
|
||||
log.Printf("extractor: warning: failed to start dbus: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
if err := exec.Command("pulseaudio", "--start", "--exit-idle-time=-1").Run(); err != nil {
|
||||
log.Printf("extractor: warning: failed to start PulseAudio: %v", err)
|
||||
}
|
||||
// Create a null-sink as the default audio target for all sessions
|
||||
exec.Command("pactl", "load-module", "module-null-sink",
|
||||
"sink_name=virtual_sink",
|
||||
"sink_properties=device.description=VirtualSink").Run()
|
||||
exec.Command("pactl", "set-default-sink", "virtual_sink").Run()
|
||||
|
||||
sessionSem = make(chan struct{}, maxConcurrentSessions)
|
||||
log.Println("extractor: initialized")
|
||||
}
|
||||
|
||||
// Stop kills PulseAudio.
|
||||
func Stop() {
|
||||
exec.Command("pulseaudio", "--kill").Run()
|
||||
log.Println("extractor: stopped")
|
||||
}
|
||||
167
stacks/f1-stream/module/files/internal/extractor/capture.go
Normal file
167
stacks/f1-stream/module/files/internal/extractor/capture.go
Normal file
|
|
@ -0,0 +1,167 @@
|
|||
package extractor
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"os/exec"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
)
|
||||
|
||||
var displayCounter int64 = 99
|
||||
|
||||
func nextDisplay() int {
|
||||
return int(atomic.AddInt64(&displayCounter, 1))
|
||||
}
|
||||
|
||||
// Capture manages an Xvfb display and separate ffmpeg pipelines for video and audio.
|
||||
// Audio capture is best-effort — if PulseAudio is unavailable, video still works.
|
||||
type Capture struct {
|
||||
display int
|
||||
xvfbCmd *exec.Cmd
|
||||
videoCmd *exec.Cmd
|
||||
audioCmd *exec.Cmd
|
||||
videoR *os.File // IVF pipe reader (VP8 frames)
|
||||
audioR *os.File // OGG pipe reader (Opus frames)
|
||||
}
|
||||
|
||||
// NewCapture starts Xvfb on the given display and two ffmpeg processes:
|
||||
// one for video (x11grab → VP8/IVF) and one for audio (pulse → Opus/OGG).
|
||||
// Audio is best-effort — if it fails to start, video still works and audioR
|
||||
// is set to a pipe that will return EOF immediately.
|
||||
func NewCapture(display, width, height int) (*Capture, error) {
|
||||
c := &Capture{display: display}
|
||||
|
||||
// Start Xvfb
|
||||
screen := fmt.Sprintf("%dx%dx24", width, height)
|
||||
c.xvfbCmd = exec.Command("Xvfb", fmt.Sprintf(":%d", display),
|
||||
"-screen", "0", screen, "-ac", "-nolisten", "tcp")
|
||||
if err := c.xvfbCmd.Start(); err != nil {
|
||||
return nil, fmt.Errorf("capture: failed to start Xvfb: %w", err)
|
||||
}
|
||||
|
||||
// Wait for Xvfb to be ready (X11 socket must exist)
|
||||
ready := false
|
||||
for i := 0; i < 50; i++ {
|
||||
socketPath := fmt.Sprintf("/tmp/.X11-unix/X%d", display)
|
||||
if _, err := os.Stat(socketPath); err == nil {
|
||||
ready = true
|
||||
break
|
||||
}
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
}
|
||||
if !ready {
|
||||
c.xvfbCmd.Process.Kill()
|
||||
c.xvfbCmd.Wait()
|
||||
return nil, fmt.Errorf("capture: Xvfb did not start in time for display :%d", display)
|
||||
}
|
||||
|
||||
// --- Video pipeline (required) ---
|
||||
videoR, videoW, err := os.Pipe()
|
||||
if err != nil {
|
||||
c.cleanup()
|
||||
return nil, fmt.Errorf("capture: video pipe: %w", err)
|
||||
}
|
||||
|
||||
c.videoCmd = exec.Command("ffmpeg",
|
||||
"-loglevel", "warning",
|
||||
"-f", "x11grab", "-framerate", "30",
|
||||
"-video_size", fmt.Sprintf("%dx%d", width, height),
|
||||
"-i", fmt.Sprintf(":%d", display),
|
||||
"-c:v", "libvpx",
|
||||
"-quality", "realtime", "-cpu-used", "8",
|
||||
"-deadline", "realtime", "-b:v", "2M", "-g", "30",
|
||||
"-f", "ivf", "pipe:3",
|
||||
)
|
||||
c.videoCmd.ExtraFiles = []*os.File{videoW}
|
||||
c.videoCmd.Stdout = os.Stderr
|
||||
c.videoCmd.Stderr = os.Stderr
|
||||
|
||||
if err := c.videoCmd.Start(); err != nil {
|
||||
videoR.Close()
|
||||
videoW.Close()
|
||||
c.cleanup()
|
||||
return nil, fmt.Errorf("capture: failed to start video ffmpeg: %w", err)
|
||||
}
|
||||
videoW.Close()
|
||||
c.videoR = videoR
|
||||
|
||||
go func() {
|
||||
if err := c.videoCmd.Wait(); err != nil {
|
||||
log.Printf("capture: video ffmpeg exited on display :%d: %v", display, err)
|
||||
}
|
||||
}()
|
||||
|
||||
// --- Audio pipeline (best-effort) ---
|
||||
audioR, audioW, err := os.Pipe()
|
||||
if err != nil {
|
||||
log.Printf("capture: audio pipe failed on display :%d: %v (continuing without audio)", display, err)
|
||||
// Provide a closed pipe so StreamAudio gets EOF immediately
|
||||
r, w, _ := os.Pipe()
|
||||
w.Close()
|
||||
c.audioR = r
|
||||
log.Printf("capture: started display :%d (%dx%d) (video only)", display, width, height)
|
||||
return c, nil
|
||||
}
|
||||
|
||||
c.audioCmd = exec.Command("ffmpeg",
|
||||
"-loglevel", "warning",
|
||||
"-f", "pulse", "-i", "virtual_sink.monitor",
|
||||
"-c:a", "libopus",
|
||||
"-b:a", "128k", "-application", "lowdelay",
|
||||
"-f", "ogg", "pipe:3",
|
||||
)
|
||||
c.audioCmd.ExtraFiles = []*os.File{audioW}
|
||||
c.audioCmd.Stdout = os.Stderr
|
||||
c.audioCmd.Stderr = os.Stderr
|
||||
|
||||
if err := c.audioCmd.Start(); err != nil {
|
||||
log.Printf("capture: audio ffmpeg failed to start on display :%d: %v (continuing without audio)", display, err)
|
||||
audioR.Close()
|
||||
audioW.Close()
|
||||
// Provide a closed pipe so StreamAudio gets EOF immediately
|
||||
r, w, _ := os.Pipe()
|
||||
w.Close()
|
||||
c.audioR = r
|
||||
c.audioCmd = nil
|
||||
log.Printf("capture: started display :%d (%dx%d) (video only)", display, width, height)
|
||||
return c, nil
|
||||
}
|
||||
audioW.Close()
|
||||
c.audioR = audioR
|
||||
|
||||
go func() {
|
||||
if err := c.audioCmd.Wait(); err != nil {
|
||||
log.Printf("capture: audio ffmpeg exited on display :%d: %v", display, err)
|
||||
}
|
||||
}()
|
||||
|
||||
log.Printf("capture: started display :%d (%dx%d) (video + audio)", display, width, height)
|
||||
return c, nil
|
||||
}
|
||||
|
||||
func (c *Capture) cleanup() {
|
||||
if c.xvfbCmd != nil && c.xvfbCmd.Process != nil {
|
||||
c.xvfbCmd.Process.Kill()
|
||||
c.xvfbCmd.Wait()
|
||||
}
|
||||
}
|
||||
|
||||
// Close stops ffmpeg processes, Xvfb, and releases pipe resources.
|
||||
func (c *Capture) Close() {
|
||||
if c.videoCmd != nil && c.videoCmd.Process != nil {
|
||||
c.videoCmd.Process.Kill()
|
||||
}
|
||||
if c.audioCmd != nil && c.audioCmd.Process != nil {
|
||||
c.audioCmd.Process.Kill()
|
||||
}
|
||||
if c.videoR != nil {
|
||||
c.videoR.Close()
|
||||
}
|
||||
if c.audioR != nil {
|
||||
c.audioR.Close()
|
||||
}
|
||||
c.cleanup()
|
||||
log.Printf("capture: stopped display :%d", c.display)
|
||||
}
|
||||
383
stacks/f1-stream/module/files/internal/extractor/session.go
Normal file
383
stacks/f1-stream/module/files/internal/extractor/session.go
Normal file
|
|
@ -0,0 +1,383 @@
|
|||
package extractor
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log"
|
||||
"net"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/exec"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/chromedp/cdproto/fetch"
|
||||
"github.com/chromedp/cdproto/input"
|
||||
"github.com/chromedp/cdproto/network"
|
||||
"github.com/chromedp/cdproto/page"
|
||||
"github.com/chromedp/chromedp"
|
||||
"github.com/gobwas/ws"
|
||||
"github.com/gobwas/ws/wsutil"
|
||||
"github.com/pion/webrtc/v4"
|
||||
)
|
||||
|
||||
const (
|
||||
sessionTimeout = 5 * time.Minute
|
||||
defaultViewportW = 1280
|
||||
defaultViewportH = 720
|
||||
turnCredentialTTL = 24 * time.Hour
|
||||
)
|
||||
|
||||
var (
|
||||
turnURL string
|
||||
turnSharedSecret string
|
||||
turnInternalURL string
|
||||
)
|
||||
|
||||
// SetTURNConfig sets the TURN server URL, shared secret, and optional internal URL.
|
||||
// The internal URL is used by pion (server-side) to avoid hairpin NAT issues.
|
||||
// The public URL is sent to the browser client.
|
||||
func SetTURNConfig(url, secret, internalURL string) {
|
||||
turnURL = url
|
||||
turnSharedSecret = secret
|
||||
turnInternalURL = internalURL
|
||||
if turnInternalURL == "" {
|
||||
turnInternalURL = "turn:coturn.coturn.svc.cluster.local:3478"
|
||||
}
|
||||
log.Printf("extractor: TURN configured: public=%s internal=%s", url, turnInternalURL)
|
||||
}
|
||||
|
||||
var adDomains = []string{
|
||||
"doubleclick.net", "googlesyndication.com", "googleadservices.com",
|
||||
"google-analytics.com", "adnxs.com", "criteo.com", "outbrain.com",
|
||||
"taboola.com", "amazon-adsystem.com", "popads.net", "popcash.net",
|
||||
"juicyads.com", "exoclick.com", "trafficjunky.com", "propellerads.com",
|
||||
"adsterra.com", "hilltopads.net", "revcontent.com", "mgid.com",
|
||||
}
|
||||
|
||||
type inputMsg struct {
|
||||
Type string `json:"type"`
|
||||
X float64 `json:"x"`
|
||||
Y float64 `json:"y"`
|
||||
Button int `json:"button"`
|
||||
DeltaX float64 `json:"deltaX"`
|
||||
DeltaY float64 `json:"deltaY"`
|
||||
Key string `json:"key"`
|
||||
Code string `json:"code"`
|
||||
Mods int `json:"modifiers"`
|
||||
Width int `json:"width"`
|
||||
Height int `json:"height"`
|
||||
SDP string `json:"sdp"`
|
||||
Candidate *webrtc.ICECandidateInit `json:"candidate"`
|
||||
}
|
||||
|
||||
// HandleBrowserSession upgrades to WebSocket and runs a remote browser session
|
||||
// with WebRTC video/audio streaming and CDP input relay.
|
||||
func HandleBrowserSession(w http.ResponseWriter, r *http.Request, pageURL string) {
|
||||
// Check session capacity
|
||||
select {
|
||||
case sessionSem <- struct{}{}:
|
||||
defer func() { <-sessionSem }()
|
||||
default:
|
||||
http.Error(w, `{"error":"too many active browser sessions"}`, http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
|
||||
conn, _, _, err := ws.UpgradeHTTP(r, w)
|
||||
if err != nil {
|
||||
log.Printf("extractor: session: ws upgrade failed: %v", err)
|
||||
return
|
||||
}
|
||||
defer conn.Close()
|
||||
|
||||
ctx, cancel := context.WithCancel(r.Context())
|
||||
defer cancel()
|
||||
|
||||
// Allocate display and start capture pipeline
|
||||
display := nextDisplay()
|
||||
viewW, viewH := defaultViewportW, defaultViewportH
|
||||
|
||||
cap, err := NewCapture(display, viewW, viewH)
|
||||
if err != nil {
|
||||
sendWSError(conn, "failed to start capture: "+err.Error())
|
||||
log.Printf("extractor: session: capture error: %v", err)
|
||||
return
|
||||
}
|
||||
defer cap.Close()
|
||||
|
||||
// Start Chrome on the virtual display
|
||||
opts := append(chromedp.DefaultExecAllocatorOptions[:],
|
||||
chromedp.Flag("headless", false),
|
||||
chromedp.Flag("no-sandbox", true),
|
||||
chromedp.Flag("disable-gpu", true),
|
||||
chromedp.Flag("disable-software-rasterizer", true),
|
||||
chromedp.Flag("disable-dev-shm-usage", true),
|
||||
chromedp.Flag("disable-extensions", true),
|
||||
chromedp.Flag("disable-background-networking", true),
|
||||
chromedp.ModifyCmdFunc(func(cmd *exec.Cmd) {
|
||||
cmd.Env = append(os.Environ(), fmt.Sprintf("DISPLAY=:%d", display))
|
||||
}),
|
||||
chromedp.Flag("autoplay-policy", "no-user-gesture-required"),
|
||||
chromedp.Flag("window-size", fmt.Sprintf("%d,%d", viewW, viewH)),
|
||||
chromedp.WSURLReadTimeout(30 * time.Second),
|
||||
)
|
||||
allocCtx, allocCancel := chromedp.NewExecAllocator(ctx, opts...)
|
||||
defer allocCancel()
|
||||
|
||||
tabCtx, tabCancel := chromedp.NewContext(allocCtx)
|
||||
defer tabCancel()
|
||||
|
||||
var wsMu sync.Mutex
|
||||
|
||||
// Build ICE servers for pion (server-side) — uses internal TURN URL to avoid hairpin NAT
|
||||
iceServers := []webrtc.ICEServer{
|
||||
{URLs: []string{"stun:stun.l.google.com:19302"}},
|
||||
}
|
||||
var turnCreds *TURNCredentials
|
||||
if turnURL != "" && turnSharedSecret != "" {
|
||||
// Server-side: use internal k8s DNS for TURN to bypass NAT
|
||||
internalCreds := GenerateTURNCredentials(turnInternalURL, turnSharedSecret, turnCredentialTTL)
|
||||
turnCreds = &internalCreds
|
||||
iceServers = append(iceServers, webrtc.ICEServer{
|
||||
URLs: internalCreds.URLs,
|
||||
Username: internalCreds.Username,
|
||||
Credential: internalCreds.Credential,
|
||||
CredentialType: webrtc.ICECredentialTypePassword,
|
||||
})
|
||||
}
|
||||
|
||||
// Build ad-blocking fetch patterns
|
||||
adPatterns := make([]*fetch.RequestPattern, 0, len(adDomains))
|
||||
for _, domain := range adDomains {
|
||||
adPatterns = append(adPatterns, &fetch.RequestPattern{
|
||||
URLPattern: fmt.Sprintf("*://*.%s/*", domain),
|
||||
})
|
||||
}
|
||||
|
||||
// Set up event listeners before navigation
|
||||
chromedp.ListenTarget(tabCtx, func(ev interface{}) {
|
||||
switch e := ev.(type) {
|
||||
case *fetch.EventRequestPaused:
|
||||
go chromedp.Run(tabCtx, fetch.FailRequest(e.RequestID, network.ErrorReasonBlockedByClient))
|
||||
case *page.EventFrameNavigated:
|
||||
if e.Frame.ParentID == "" {
|
||||
go sendURLUpdate(tabCtx, conn, &wsMu, e.Frame.URL)
|
||||
}
|
||||
case *page.EventNavigatedWithinDocument:
|
||||
go sendURLUpdate(tabCtx, conn, &wsMu, e.URL)
|
||||
}
|
||||
})
|
||||
|
||||
// Enable fetch interception (ad blocking) and navigate
|
||||
if err := chromedp.Run(tabCtx,
|
||||
fetch.Enable().WithPatterns(adPatterns),
|
||||
chromedp.Navigate(pageURL),
|
||||
chromedp.WaitReady("body"),
|
||||
); err != nil {
|
||||
sendWSError(conn, "navigation failed")
|
||||
log.Printf("extractor: session: navigate error for %s: %v", pageURL, err)
|
||||
return
|
||||
}
|
||||
|
||||
// Create WebRTC media stream
|
||||
mediaStream, err := NewMediaStream(iceServers, func(c *webrtc.ICECandidate) {
|
||||
data, _ := json.Marshal(map[string]interface{}{
|
||||
"type": "ice",
|
||||
"candidate": c.ToJSON(),
|
||||
})
|
||||
wsMu.Lock()
|
||||
wsutil.WriteServerMessage(conn, ws.OpText, data)
|
||||
wsMu.Unlock()
|
||||
}, cancel)
|
||||
if err != nil {
|
||||
sendWSError(conn, "WebRTC setup failed")
|
||||
log.Printf("extractor: session: webrtc error: %v", err)
|
||||
return
|
||||
}
|
||||
defer mediaStream.Close()
|
||||
|
||||
// Create and send SDP offer
|
||||
sdp, err := mediaStream.Offer()
|
||||
if err != nil {
|
||||
sendWSError(conn, "WebRTC offer failed")
|
||||
log.Printf("extractor: session: offer error: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
// Send ICE config to client — uses PUBLIC TURN URL (for browser to reach from internet)
|
||||
clientICE := []map[string]interface{}{
|
||||
{"urls": []string{"stun:stun.l.google.com:19302"}},
|
||||
}
|
||||
if turnCreds != nil {
|
||||
// Client-side: use public IP for TURN (browser connects from internet)
|
||||
publicCreds := GenerateTURNCredentials(turnURL, turnSharedSecret, turnCredentialTTL)
|
||||
clientICE = append(clientICE, map[string]interface{}{
|
||||
"urls": publicCreds.URLs,
|
||||
"username": publicCreds.Username,
|
||||
"credential": publicCreds.Credential,
|
||||
})
|
||||
}
|
||||
iceMsg, _ := json.Marshal(map[string]interface{}{
|
||||
"type": "iceServers",
|
||||
"iceServers": clientICE,
|
||||
})
|
||||
wsMu.Lock()
|
||||
wsutil.WriteServerMessage(conn, ws.OpText, iceMsg)
|
||||
wsMu.Unlock()
|
||||
|
||||
offerMsg, _ := json.Marshal(map[string]interface{}{
|
||||
"type": "offer",
|
||||
"sdp": sdp,
|
||||
})
|
||||
wsMu.Lock()
|
||||
wsutil.WriteServerMessage(conn, ws.OpText, offerMsg)
|
||||
wsMu.Unlock()
|
||||
|
||||
// Send ready message with viewport dimensions
|
||||
readyMsg, _ := json.Marshal(map[string]interface{}{
|
||||
"type": "ready",
|
||||
"width": viewW,
|
||||
"height": viewH,
|
||||
})
|
||||
wsMu.Lock()
|
||||
wsutil.WriteServerMessage(conn, ws.OpText, readyMsg)
|
||||
wsMu.Unlock()
|
||||
|
||||
// Start streaming video and audio from capture pipes
|
||||
go mediaStream.StreamVideo(cap.videoR, ctx)
|
||||
go mediaStream.StreamAudio(cap.audioR, ctx)
|
||||
|
||||
log.Printf("extractor: session: started for %s (display :%d)", pageURL, display)
|
||||
|
||||
// Inactivity timer — cancels session after no client input
|
||||
inactivity := time.NewTimer(sessionTimeout)
|
||||
defer inactivity.Stop()
|
||||
go func() {
|
||||
select {
|
||||
case <-inactivity.C:
|
||||
log.Printf("extractor: session: inactivity timeout for %s", pageURL)
|
||||
cancel()
|
||||
case <-ctx.Done():
|
||||
}
|
||||
}()
|
||||
|
||||
// Read loop — process signaling and input messages
|
||||
for {
|
||||
msgs, err := wsutil.ReadClientMessage(conn, nil)
|
||||
if err != nil {
|
||||
break
|
||||
}
|
||||
for _, m := range msgs {
|
||||
if m.OpCode != ws.OpText {
|
||||
continue
|
||||
}
|
||||
|
||||
// Reset inactivity timer
|
||||
if !inactivity.Stop() {
|
||||
select {
|
||||
case <-inactivity.C:
|
||||
default:
|
||||
}
|
||||
}
|
||||
inactivity.Reset(sessionTimeout)
|
||||
|
||||
var msg inputMsg
|
||||
if err := json.Unmarshal(m.Payload, &msg); err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
switch msg.Type {
|
||||
case "answer":
|
||||
if err := mediaStream.SetAnswer(msg.SDP); err != nil {
|
||||
log.Printf("extractor: session: set answer error: %v", err)
|
||||
}
|
||||
case "ice":
|
||||
if msg.Candidate != nil {
|
||||
if err := mediaStream.AddICECandidate(*msg.Candidate); err != nil {
|
||||
log.Printf("extractor: session: add ICE error: %v", err)
|
||||
}
|
||||
}
|
||||
case "back":
|
||||
chromedp.Run(tabCtx, chromedp.NavigateBack())
|
||||
case "forward":
|
||||
chromedp.Run(tabCtx, chromedp.NavigateForward())
|
||||
default:
|
||||
handleInput(tabCtx, &msg)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
log.Printf("extractor: session: ended for %s", pageURL)
|
||||
}
|
||||
|
||||
func handleInput(ctx context.Context, msg *inputMsg) {
|
||||
switch msg.Type {
|
||||
case "mousemove":
|
||||
chromedp.Run(ctx,
|
||||
input.DispatchMouseEvent(input.MouseMoved, msg.X, msg.Y))
|
||||
case "mousedown":
|
||||
chromedp.Run(ctx,
|
||||
input.DispatchMouseEvent(input.MousePressed, msg.X, msg.Y).
|
||||
WithButton(mapButton(msg.Button)).WithClickCount(1))
|
||||
case "mouseup":
|
||||
chromedp.Run(ctx,
|
||||
input.DispatchMouseEvent(input.MouseReleased, msg.X, msg.Y).
|
||||
WithButton(mapButton(msg.Button)))
|
||||
case "scroll":
|
||||
chromedp.Run(ctx,
|
||||
input.DispatchMouseEvent(input.MouseWheel, msg.X, msg.Y).
|
||||
WithDeltaX(msg.DeltaX).WithDeltaY(msg.DeltaY))
|
||||
case "keydown":
|
||||
chromedp.Run(ctx,
|
||||
input.DispatchKeyEvent(input.KeyDown).
|
||||
WithKey(msg.Key).WithCode(msg.Code).
|
||||
WithModifiers(input.Modifier(msg.Mods)))
|
||||
case "keyup":
|
||||
chromedp.Run(ctx,
|
||||
input.DispatchKeyEvent(input.KeyUp).
|
||||
WithKey(msg.Key).WithCode(msg.Code).
|
||||
WithModifiers(input.Modifier(msg.Mods)))
|
||||
}
|
||||
}
|
||||
|
||||
func mapButton(jsButton int) input.MouseButton {
|
||||
switch jsButton {
|
||||
case 1:
|
||||
return input.Middle
|
||||
case 2:
|
||||
return input.Right
|
||||
default:
|
||||
return input.Left
|
||||
}
|
||||
}
|
||||
|
||||
func sendURLUpdate(tabCtx context.Context, conn net.Conn, mu *sync.Mutex, currentURL string) {
|
||||
var canBack, canForward bool
|
||||
var entries []*page.NavigationEntry
|
||||
var currentIndex int64
|
||||
|
||||
if err := chromedp.Run(tabCtx, chromedp.ActionFunc(func(ctx context.Context) error {
|
||||
var err error
|
||||
currentIndex, entries, err = page.GetNavigationHistory().Do(ctx)
|
||||
return err
|
||||
})); err == nil {
|
||||
canBack = currentIndex > 0
|
||||
canForward = int(currentIndex) < len(entries)-1
|
||||
}
|
||||
|
||||
data, _ := json.Marshal(map[string]interface{}{
|
||||
"type": "url",
|
||||
"url": currentURL,
|
||||
"canBack": canBack,
|
||||
"canForward": canForward,
|
||||
})
|
||||
mu.Lock()
|
||||
wsutil.WriteServerMessage(conn, ws.OpText, data)
|
||||
mu.Unlock()
|
||||
}
|
||||
|
||||
func sendWSError(conn net.Conn, msg string) {
|
||||
data, _ := json.Marshal(map[string]string{"type": "error", "message": msg})
|
||||
wsutil.WriteServerMessage(conn, ws.OpText, data)
|
||||
}
|
||||
248
stacks/f1-stream/module/files/internal/extractor/webrtc.go
Normal file
248
stacks/f1-stream/module/files/internal/extractor/webrtc.go
Normal file
|
|
@ -0,0 +1,248 @@
|
|||
package extractor
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/hmac"
|
||||
"crypto/sha1"
|
||||
"encoding/base64"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"time"
|
||||
|
||||
"github.com/pion/webrtc/v4"
|
||||
"github.com/pion/webrtc/v4/pkg/media"
|
||||
"github.com/pion/webrtc/v4/pkg/media/ivfreader"
|
||||
"github.com/pion/webrtc/v4/pkg/media/oggreader"
|
||||
)
|
||||
|
||||
// TURNCredentials holds ephemeral TURN credentials generated from a shared secret.
|
||||
type TURNCredentials struct {
|
||||
URLs []string `json:"urls"`
|
||||
Username string `json:"username"`
|
||||
Credential string `json:"credential"`
|
||||
}
|
||||
|
||||
// GenerateTURNCredentials creates time-limited TURN credentials using the
|
||||
// shared secret (TURN REST API / coturn --use-auth-secret).
|
||||
func GenerateTURNCredentials(turnURL, sharedSecret string, ttl time.Duration) TURNCredentials {
|
||||
expiry := time.Now().Add(ttl).Unix()
|
||||
username := fmt.Sprintf("%d", expiry)
|
||||
|
||||
mac := hmac.New(sha1.New, []byte(sharedSecret))
|
||||
mac.Write([]byte(username))
|
||||
credential := base64.StdEncoding.EncodeToString(mac.Sum(nil))
|
||||
|
||||
return TURNCredentials{
|
||||
URLs: []string{turnURL},
|
||||
Username: username,
|
||||
Credential: credential,
|
||||
}
|
||||
}
|
||||
|
||||
// MediaStream wraps a pion WebRTC PeerConnection with VP8 video and Opus audio tracks.
|
||||
type MediaStream struct {
|
||||
pc *webrtc.PeerConnection
|
||||
videoTrack *webrtc.TrackLocalStaticSample
|
||||
audioTrack *webrtc.TrackLocalStaticSample
|
||||
}
|
||||
|
||||
// NewMediaStream creates a PeerConnection with VP8 + Opus tracks and an ICE callback.
|
||||
// The cancel function is called when ICE fails to trigger session cleanup.
|
||||
func NewMediaStream(iceServers []webrtc.ICEServer, onICE func(*webrtc.ICECandidate), cancel context.CancelFunc) (*MediaStream, error) {
|
||||
config := webrtc.Configuration{
|
||||
ICEServers: iceServers,
|
||||
}
|
||||
|
||||
pc, err := webrtc.NewPeerConnection(config)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
videoTrack, err := webrtc.NewTrackLocalStaticSample(
|
||||
webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeVP8},
|
||||
"video", "stream",
|
||||
)
|
||||
if err != nil {
|
||||
pc.Close()
|
||||
return nil, err
|
||||
}
|
||||
|
||||
audioTrack, err := webrtc.NewTrackLocalStaticSample(
|
||||
webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeOpus},
|
||||
"audio", "stream",
|
||||
)
|
||||
if err != nil {
|
||||
pc.Close()
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if _, err = pc.AddTrack(videoTrack); err != nil {
|
||||
pc.Close()
|
||||
return nil, err
|
||||
}
|
||||
if _, err = pc.AddTrack(audioTrack); err != nil {
|
||||
pc.Close()
|
||||
return nil, err
|
||||
}
|
||||
|
||||
pc.OnICEConnectionStateChange(func(state webrtc.ICEConnectionState) {
|
||||
log.Printf("webrtc: ICE connection state: %s", state.String())
|
||||
if state == webrtc.ICEConnectionStateFailed {
|
||||
log.Printf("webrtc: ICE failed, cancelling session")
|
||||
cancel()
|
||||
return
|
||||
}
|
||||
if state == webrtc.ICEConnectionStateConnected {
|
||||
// Log selected candidate pair
|
||||
if stats := pc.GetStats(); stats != nil {
|
||||
for _, s := range stats {
|
||||
if cp, ok := s.(webrtc.ICECandidatePairStats); ok && cp.Nominated {
|
||||
log.Printf("webrtc: selected candidate pair: local=%s remote=%s",
|
||||
cp.LocalCandidateID, cp.RemoteCandidateID)
|
||||
}
|
||||
}
|
||||
}
|
||||
// Start periodic stats logging
|
||||
go func() {
|
||||
ticker := time.NewTicker(10 * time.Second)
|
||||
defer ticker.Stop()
|
||||
for range ticker.C {
|
||||
if pc.ICEConnectionState() != webrtc.ICEConnectionStateConnected &&
|
||||
pc.ICEConnectionState() != webrtc.ICEConnectionStateCompleted {
|
||||
return
|
||||
}
|
||||
stats := pc.GetStats()
|
||||
for _, s := range stats {
|
||||
if out, ok := s.(webrtc.OutboundRTPStreamStats); ok {
|
||||
log.Printf("webrtc: outbound-rtp kind=%s bytes=%d packets=%d",
|
||||
out.Kind, out.BytesSent, out.PacketsSent)
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
})
|
||||
|
||||
pc.OnConnectionStateChange(func(state webrtc.PeerConnectionState) {
|
||||
log.Printf("webrtc: peer connection state: %s", state.String())
|
||||
})
|
||||
|
||||
pc.OnICECandidate(func(c *webrtc.ICECandidate) {
|
||||
if c != nil {
|
||||
log.Printf("webrtc: gathered ICE candidate: type=%s addr=%s:%d",
|
||||
c.Typ.String(), c.Address, c.Port)
|
||||
if onICE != nil {
|
||||
onICE(c)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
return &MediaStream{
|
||||
pc: pc,
|
||||
videoTrack: videoTrack,
|
||||
audioTrack: audioTrack,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Offer creates an SDP offer, sets it as local description, and returns the SDP string.
|
||||
func (m *MediaStream) Offer() (string, error) {
|
||||
offer, err := m.pc.CreateOffer(nil)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if err := m.pc.SetLocalDescription(offer); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return offer.SDP, nil
|
||||
}
|
||||
|
||||
// SetAnswer sets the remote SDP answer.
|
||||
func (m *MediaStream) SetAnswer(sdp string) error {
|
||||
return m.pc.SetRemoteDescription(webrtc.SessionDescription{
|
||||
Type: webrtc.SDPTypeAnswer,
|
||||
SDP: sdp,
|
||||
})
|
||||
}
|
||||
|
||||
// AddICECandidate adds a remote ICE candidate.
|
||||
func (m *MediaStream) AddICECandidate(init webrtc.ICECandidateInit) error {
|
||||
return m.pc.AddICECandidate(init)
|
||||
}
|
||||
|
||||
// StreamVideo reads VP8 frames from an IVF stream and writes them to the video track.
|
||||
// Blocks until the reader returns an error or the context is cancelled.
|
||||
func (m *MediaStream) StreamVideo(r io.Reader, ctx context.Context) {
|
||||
ivf, _, err := ivfreader.NewWith(r)
|
||||
if err != nil {
|
||||
log.Printf("webrtc: ivf reader error: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
duration := time.Second / 30
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
default:
|
||||
}
|
||||
|
||||
frame, _, err := ivf.ParseNextFrame()
|
||||
if err != nil {
|
||||
if err != io.EOF {
|
||||
log.Printf("webrtc: video frame error: %v", err)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
if err := m.videoTrack.WriteSample(media.Sample{
|
||||
Data: frame,
|
||||
Duration: duration,
|
||||
}); err != nil {
|
||||
log.Printf("webrtc: video write error: %v", err)
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// StreamAudio reads Opus pages from an OGG stream and writes them to the audio track.
|
||||
// Blocks until the reader returns an error or the context is cancelled.
|
||||
func (m *MediaStream) StreamAudio(r io.Reader, ctx context.Context) {
|
||||
ogg, _, err := oggreader.NewWith(r)
|
||||
if err != nil {
|
||||
log.Printf("webrtc: ogg reader error: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
default:
|
||||
}
|
||||
|
||||
page, _, err := ogg.ParseNextPage()
|
||||
if err != nil {
|
||||
if err != io.EOF {
|
||||
log.Printf("webrtc: audio page error: %v", err)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
if err := m.audioTrack.WriteSample(media.Sample{
|
||||
Data: page,
|
||||
Duration: 20 * time.Millisecond,
|
||||
}); err != nil {
|
||||
log.Printf("webrtc: audio write error: %v", err)
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Close closes the underlying PeerConnection.
|
||||
func (m *MediaStream) Close() {
|
||||
if m.pc != nil {
|
||||
m.pc.Close()
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,188 @@
|
|||
package healthcheck
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log"
|
||||
"net/http"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"f1-stream/internal/models"
|
||||
"f1-stream/internal/store"
|
||||
)
|
||||
|
||||
const unhealthyThreshold = 5
|
||||
|
||||
const userAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"
|
||||
|
||||
// isReachable sends a GET request and returns true if the server responds with
|
||||
// an HTTP 2xx or 3xx status code.
|
||||
func isReachable(client *http.Client, rawURL string) bool {
|
||||
req, err := http.NewRequest("GET", rawURL, nil)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
req.Header.Set("User-Agent", userAgent)
|
||||
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
return resp.StatusCode >= 200 && resp.StatusCode < 400
|
||||
}
|
||||
|
||||
type HealthChecker struct {
|
||||
store *store.Store
|
||||
interval time.Duration
|
||||
timeout time.Duration
|
||||
client *http.Client
|
||||
mu sync.Mutex
|
||||
}
|
||||
|
||||
func New(s *store.Store, interval, timeout time.Duration) *HealthChecker {
|
||||
return &HealthChecker{
|
||||
store: s,
|
||||
interval: interval,
|
||||
timeout: timeout,
|
||||
client: &http.Client{
|
||||
Timeout: timeout,
|
||||
CheckRedirect: func(req *http.Request, via []*http.Request) error {
|
||||
if len(via) >= 3 {
|
||||
return http.ErrUseLastResponse
|
||||
}
|
||||
return nil
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func (hc *HealthChecker) Run(ctx context.Context) {
|
||||
log.Printf("healthcheck: starting with interval=%v timeout=%v", hc.interval, hc.timeout)
|
||||
hc.checkAll()
|
||||
|
||||
ticker := time.NewTicker(hc.interval)
|
||||
defer ticker.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
log.Println("healthcheck: shutting down")
|
||||
return
|
||||
case <-ticker.C:
|
||||
hc.checkAll()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (hc *HealthChecker) checkAll() {
|
||||
hc.mu.Lock()
|
||||
defer hc.mu.Unlock()
|
||||
|
||||
start := time.Now()
|
||||
urls := hc.collectURLs()
|
||||
log.Printf("healthcheck: checking %d URLs", len(urls))
|
||||
|
||||
existing, err := hc.store.LoadHealthStates()
|
||||
if err != nil {
|
||||
log.Printf("healthcheck: failed to load health states: %v", err)
|
||||
existing = nil
|
||||
}
|
||||
|
||||
stateMap := make(map[string]*models.HealthState, len(existing))
|
||||
for i := range existing {
|
||||
stateMap[existing[i].URL] = &existing[i]
|
||||
}
|
||||
|
||||
now := time.Now()
|
||||
var recovered, newlyUnhealthy int
|
||||
|
||||
for _, url := range urls {
|
||||
st, exists := stateMap[url]
|
||||
if !exists {
|
||||
st = &models.HealthState{
|
||||
URL: url,
|
||||
Healthy: true,
|
||||
}
|
||||
stateMap[url] = st
|
||||
}
|
||||
|
||||
ok := isReachable(hc.client, url)
|
||||
|
||||
if ok {
|
||||
if !st.Healthy {
|
||||
log.Printf("healthcheck: recovered %s", truncate(url, 80))
|
||||
recovered++
|
||||
}
|
||||
st.ConsecutiveFailures = 0
|
||||
st.Healthy = true
|
||||
} else {
|
||||
st.ConsecutiveFailures++
|
||||
if st.ConsecutiveFailures >= unhealthyThreshold && st.Healthy {
|
||||
st.Healthy = false
|
||||
log.Printf("healthcheck: marking unhealthy after %d failures: %s", st.ConsecutiveFailures, truncate(url, 80))
|
||||
newlyUnhealthy++
|
||||
}
|
||||
}
|
||||
st.LastCheckTime = now
|
||||
}
|
||||
|
||||
// Prune orphaned entries: only keep states whose URL is in the current set
|
||||
urlSet := make(map[string]bool, len(urls))
|
||||
for _, u := range urls {
|
||||
urlSet[u] = true
|
||||
}
|
||||
var finalStates []models.HealthState
|
||||
healthyCount := 0
|
||||
for _, st := range stateMap {
|
||||
if urlSet[st.URL] {
|
||||
finalStates = append(finalStates, *st)
|
||||
if st.Healthy {
|
||||
healthyCount++
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if err := hc.store.SaveHealthStates(finalStates); err != nil {
|
||||
log.Printf("healthcheck: failed to save health states: %v", err)
|
||||
}
|
||||
|
||||
log.Printf("healthcheck: done in %v, checked=%d healthy=%d recovered=%d newly_unhealthy=%d",
|
||||
time.Since(start).Round(time.Millisecond), len(urls), healthyCount, recovered, newlyUnhealthy)
|
||||
}
|
||||
|
||||
func (hc *HealthChecker) collectURLs() []string {
|
||||
seen := make(map[string]bool)
|
||||
|
||||
streams, err := hc.store.LoadStreams()
|
||||
if err != nil {
|
||||
log.Printf("healthcheck: failed to load streams: %v", err)
|
||||
} else {
|
||||
for _, s := range streams {
|
||||
seen[s.URL] = true
|
||||
}
|
||||
}
|
||||
|
||||
scraped, err := hc.store.LoadScrapedLinks()
|
||||
if err != nil {
|
||||
log.Printf("healthcheck: failed to load scraped links: %v", err)
|
||||
} else {
|
||||
for _, l := range scraped {
|
||||
seen[l.URL] = true
|
||||
}
|
||||
}
|
||||
|
||||
urls := make([]string, 0, len(seen))
|
||||
for u := range seen {
|
||||
urls = append(urls, u)
|
||||
}
|
||||
return urls
|
||||
}
|
||||
|
||||
func truncate(s string, maxLen int) string {
|
||||
if len(s) <= maxLen {
|
||||
return s
|
||||
}
|
||||
return s[:maxLen] + "..."
|
||||
}
|
||||
209
stacks/f1-stream/module/files/internal/hlsproxy/hlsproxy.go
Normal file
209
stacks/f1-stream/module/files/internal/hlsproxy/hlsproxy.go
Normal file
|
|
@ -0,0 +1,209 @@
|
|||
package hlsproxy
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"encoding/base64"
|
||||
"io"
|
||||
"log"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// NewHandler returns an http.Handler for /hls/{base64url_encoded_full_url}.
|
||||
// It proxies HLS playlists and segments, rewriting m3u8 URLs to route
|
||||
// through the proxy and forwarding X-Hls-Forward-* headers upstream.
|
||||
func NewHandler() http.Handler {
|
||||
client := &http.Client{
|
||||
Timeout: 30_000_000_000, // 30s
|
||||
CheckRedirect: func(req *http.Request, via []*http.Request) error {
|
||||
if len(via) >= 5 {
|
||||
return http.ErrUseLastResponse
|
||||
}
|
||||
return nil
|
||||
},
|
||||
}
|
||||
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method == http.MethodOptions {
|
||||
setCORS(w)
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
return
|
||||
}
|
||||
|
||||
// Parse: /hls/{base64url_encoded_full_url}
|
||||
trimmed := strings.TrimPrefix(r.URL.Path, "/hls/")
|
||||
if trimmed == "" || trimmed == r.URL.Path {
|
||||
http.Error(w, "bad hls proxy URL", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
// Decode the full upstream URL from base64url
|
||||
upstreamURL, err := base64.RawURLEncoding.DecodeString(trimmed)
|
||||
if err != nil {
|
||||
http.Error(w, "invalid base64url", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
target := string(upstreamURL)
|
||||
|
||||
parsed, err := url.Parse(target)
|
||||
if err != nil || (parsed.Scheme != "http" && parsed.Scheme != "https") {
|
||||
http.Error(w, "invalid upstream URL", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
log.Printf("hlsproxy: %s -> %s", r.URL.Path, target)
|
||||
|
||||
upReq, err := http.NewRequestWithContext(r.Context(), http.MethodGet, target, nil)
|
||||
if err != nil {
|
||||
http.Error(w, "failed to create request", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
// Set Referer and Origin. If the URL has a ?domain= param (CDN segments),
|
||||
// use that domain as the origin so the CDN accepts the request.
|
||||
refererOrigin := parsed.Scheme + "://" + parsed.Host
|
||||
if domainParam := parsed.Query().Get("domain"); domainParam != "" {
|
||||
refererOrigin = "https://" + domainParam
|
||||
}
|
||||
upReq.Header.Set("Referer", refererOrigin+"/")
|
||||
upReq.Header.Set("Origin", refererOrigin)
|
||||
upReq.Header.Set("User-Agent", r.Header.Get("User-Agent"))
|
||||
|
||||
// Forward X-Hls-Forward-* headers (strip prefix)
|
||||
for key, vals := range r.Header {
|
||||
if strings.HasPrefix(key, "X-Hls-Forward-") {
|
||||
realKey := strings.TrimPrefix(key, "X-Hls-Forward-")
|
||||
for _, v := range vals {
|
||||
upReq.Header.Set(realKey, v)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
resp, err := client.Do(upReq)
|
||||
if err != nil {
|
||||
log.Printf("hlsproxy: upstream fetch failed: %v", err)
|
||||
http.Error(w, "upstream fetch failed", http.StatusBadGateway)
|
||||
return
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
log.Printf("hlsproxy: %s <- %d (%s)", truncPath(r.URL.Path, 60), resp.StatusCode, resp.Header.Get("Content-Type"))
|
||||
|
||||
setCORS(w)
|
||||
|
||||
ct := resp.Header.Get("Content-Type")
|
||||
isM3U8 := strings.Contains(ct, "mpegurl") ||
|
||||
strings.Contains(ct, "x-mpegURL") ||
|
||||
strings.HasSuffix(parsed.Path, ".m3u8")
|
||||
|
||||
if isM3U8 {
|
||||
w.Header().Set("Content-Type", "application/vnd.apple.mpegurl")
|
||||
w.WriteHeader(resp.StatusCode)
|
||||
rewriteM3U8(w, resp.Body, target)
|
||||
return
|
||||
}
|
||||
|
||||
// Stream segment or other content directly
|
||||
for key, vals := range resp.Header {
|
||||
lk := strings.ToLower(key)
|
||||
if lk == "content-type" || lk == "content-length" || lk == "cache-control" || lk == "accept-ranges" {
|
||||
for _, v := range vals {
|
||||
w.Header().Add(key, v)
|
||||
}
|
||||
}
|
||||
}
|
||||
w.WriteHeader(resp.StatusCode)
|
||||
io.Copy(w, resp.Body)
|
||||
})
|
||||
}
|
||||
|
||||
// rewriteM3U8 reads an m3u8 playlist from r, rewrites segment/playlist URLs
|
||||
// to route through /hls/{b64}, and writes the result to w.
|
||||
func rewriteM3U8(w io.Writer, r io.Reader, playlistURL string) {
|
||||
base, err := url.Parse(playlistURL)
|
||||
if err != nil {
|
||||
io.Copy(w, r)
|
||||
return
|
||||
}
|
||||
|
||||
scanner := bufio.NewScanner(r)
|
||||
scanner.Buffer(make([]byte, 0, 64*1024), 1024*1024)
|
||||
|
||||
for scanner.Scan() {
|
||||
line := scanner.Text()
|
||||
|
||||
if strings.HasPrefix(line, "#") {
|
||||
// Rewrite URI="..." in directives like #EXT-X-KEY, #EXT-X-MAP
|
||||
rewritten := rewriteURIAttribute(line, base)
|
||||
w.Write([]byte(rewritten))
|
||||
w.Write([]byte("\n"))
|
||||
continue
|
||||
}
|
||||
|
||||
// Non-comment, non-empty lines are URLs
|
||||
trimmed := strings.TrimSpace(line)
|
||||
if trimmed == "" {
|
||||
w.Write([]byte("\n"))
|
||||
continue
|
||||
}
|
||||
|
||||
resolved := resolveURL(base, trimmed)
|
||||
encoded := encodeHLSURL(resolved)
|
||||
w.Write([]byte(encoded))
|
||||
w.Write([]byte("\n"))
|
||||
}
|
||||
}
|
||||
|
||||
// rewriteURIAttribute rewrites URI="..." attributes in HLS directives.
|
||||
func rewriteURIAttribute(line string, base *url.URL) string {
|
||||
// Look for URI="..." (case insensitive)
|
||||
uriIdx := strings.Index(strings.ToUpper(line), "URI=\"")
|
||||
if uriIdx == -1 {
|
||||
return line
|
||||
}
|
||||
|
||||
// Find the actual position (preserving original case)
|
||||
prefix := line[:uriIdx+5] // everything up to and including URI="
|
||||
rest := line[uriIdx+5:]
|
||||
endQuote := strings.Index(rest, "\"")
|
||||
if endQuote == -1 {
|
||||
return line
|
||||
}
|
||||
|
||||
uri := rest[:endQuote]
|
||||
suffix := rest[endQuote:] // closing quote and anything after
|
||||
|
||||
resolved := resolveURL(base, uri)
|
||||
encoded := encodeHLSURL(resolved)
|
||||
|
||||
return prefix + encoded + suffix
|
||||
}
|
||||
|
||||
// resolveURL resolves a potentially relative URL against a base URL.
|
||||
func resolveURL(base *url.URL, ref string) string {
|
||||
refURL, err := url.Parse(ref)
|
||||
if err != nil {
|
||||
return ref
|
||||
}
|
||||
return base.ResolveReference(refURL).String()
|
||||
}
|
||||
|
||||
// encodeHLSURL encodes a full URL into /hls/{base64url} format.
|
||||
func encodeHLSURL(fullURL string) string {
|
||||
encoded := base64.RawURLEncoding.EncodeToString([]byte(fullURL))
|
||||
return "/hls/" + encoded
|
||||
}
|
||||
|
||||
func setCORS(w http.ResponseWriter) {
|
||||
w.Header().Set("Access-Control-Allow-Origin", "*")
|
||||
w.Header().Set("Access-Control-Allow-Methods", "GET, OPTIONS")
|
||||
w.Header().Set("Access-Control-Allow-Headers", "*")
|
||||
}
|
||||
|
||||
func truncPath(s string, n int) string {
|
||||
if len(s) <= n {
|
||||
return s
|
||||
}
|
||||
return s[:n] + "..."
|
||||
}
|
||||
53
stacks/f1-stream/module/files/internal/models/models.go
Normal file
53
stacks/f1-stream/module/files/internal/models/models.go
Normal file
|
|
@ -0,0 +1,53 @@
|
|||
package models
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"github.com/go-webauthn/webauthn/webauthn"
|
||||
)
|
||||
|
||||
type User struct {
|
||||
ID string `json:"id"`
|
||||
Username string `json:"username"`
|
||||
IsAdmin bool `json:"is_admin"`
|
||||
Credentials []webauthn.Credential `json:"credentials"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
}
|
||||
|
||||
// WebAuthn interface implementation
|
||||
func (u *User) WebAuthnID() []byte { return []byte(u.ID) }
|
||||
func (u *User) WebAuthnName() string { return u.Username }
|
||||
func (u *User) WebAuthnDisplayName() string { return u.Username }
|
||||
func (u *User) WebAuthnCredentials() []webauthn.Credential { return u.Credentials }
|
||||
|
||||
type Stream struct {
|
||||
ID string `json:"id"`
|
||||
URL string `json:"url"`
|
||||
Title string `json:"title"`
|
||||
SubmittedBy string `json:"submitted_by"`
|
||||
Published bool `json:"published"`
|
||||
Source string `json:"source"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
}
|
||||
|
||||
type ScrapedLink struct {
|
||||
ID string `json:"id"`
|
||||
URL string `json:"url"`
|
||||
Title string `json:"title"`
|
||||
Source string `json:"source"`
|
||||
ScrapedAt time.Time `json:"scraped_at"`
|
||||
Stale bool `json:"stale"`
|
||||
}
|
||||
|
||||
type Session struct {
|
||||
Token string `json:"token"`
|
||||
UserID string `json:"user_id"`
|
||||
ExpiresAt time.Time `json:"expires_at"`
|
||||
}
|
||||
|
||||
type HealthState struct {
|
||||
URL string `json:"url"`
|
||||
ConsecutiveFailures int `json:"consecutive_failures"`
|
||||
LastCheckTime time.Time `json:"last_check_time"`
|
||||
Healthy bool `json:"healthy"`
|
||||
}
|
||||
|
|
@ -0,0 +1,512 @@
|
|||
package playerconfig
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// PlayerConfig is returned by the /api/streams/{id}/player-config endpoint.
|
||||
type PlayerConfig struct {
|
||||
Type string `json:"type"`
|
||||
HLSURL string `json:"hls_url,omitempty"`
|
||||
AuthToken string `json:"auth_token,omitempty"`
|
||||
ChannelKey string `json:"channel_key,omitempty"`
|
||||
ChannelSalt string `json:"channel_salt,omitempty"`
|
||||
Timestamp string `json:"timestamp,omitempty"`
|
||||
AuthModURL string `json:"auth_mod_url,omitempty"`
|
||||
ServerKey string `json:"server_key,omitempty"`
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
type cacheEntry struct {
|
||||
config *PlayerConfig
|
||||
expiresAt time.Time
|
||||
}
|
||||
|
||||
// Service handles stream type detection and DaddyLive config extraction.
|
||||
type Service struct {
|
||||
client *http.Client
|
||||
mu sync.RWMutex
|
||||
cache map[string]*cacheEntry
|
||||
}
|
||||
|
||||
// New creates a new playerconfig Service.
|
||||
func New() *Service {
|
||||
return &Service{
|
||||
client: &http.Client{
|
||||
Timeout: 15 * time.Second,
|
||||
},
|
||||
cache: make(map[string]*cacheEntry),
|
||||
}
|
||||
}
|
||||
|
||||
// DetectStreamType returns "hls", "daddylive", "vipleague", or "proxy" based on the URL.
|
||||
func DetectStreamType(rawURL string) string {
|
||||
lower := strings.ToLower(rawURL)
|
||||
|
||||
if strings.HasSuffix(strings.SplitN(lower, "?", 2)[0], ".m3u8") {
|
||||
return "hls"
|
||||
}
|
||||
|
||||
daddyPatterns := []string{"dlhd.link", "dlhd.sx", "dlhd.dad", "daddylive", "ksohls.ru"}
|
||||
for _, p := range daddyPatterns {
|
||||
if strings.Contains(lower, p) {
|
||||
return "daddylive"
|
||||
}
|
||||
}
|
||||
|
||||
vipPatterns := []string{"vipleague.io", "vipleague.im", "vipleague.cc", "casthill.net"}
|
||||
for _, p := range vipPatterns {
|
||||
if strings.Contains(lower, p) {
|
||||
return "vipleague"
|
||||
}
|
||||
}
|
||||
|
||||
return "proxy"
|
||||
}
|
||||
|
||||
// GetConfig returns a PlayerConfig for the given stream URL.
|
||||
func (s *Service) GetConfig(ctx context.Context, rawURL string) *PlayerConfig {
|
||||
streamType := DetectStreamType(rawURL)
|
||||
|
||||
switch streamType {
|
||||
case "hls":
|
||||
encoded := base64.RawURLEncoding.EncodeToString([]byte(rawURL))
|
||||
return &PlayerConfig{
|
||||
Type: "hls",
|
||||
HLSURL: "/hls/" + encoded,
|
||||
}
|
||||
|
||||
case "daddylive":
|
||||
return s.getDaddyLiveConfig(ctx, rawURL)
|
||||
|
||||
case "vipleague":
|
||||
return s.getVIPLeagueConfig(ctx, rawURL)
|
||||
|
||||
default:
|
||||
return &PlayerConfig{Type: "proxy"}
|
||||
}
|
||||
}
|
||||
|
||||
// Channel ID extraction patterns
|
||||
var channelIDPatterns = []*regexp.Regexp{
|
||||
regexp.MustCompile(`stream-(\d+)\.php`),
|
||||
regexp.MustCompile(`[?&]id=(\d+)`),
|
||||
regexp.MustCompile(`/(\d+)\.php`),
|
||||
}
|
||||
|
||||
// Page content extraction patterns
|
||||
var (
|
||||
iframeRe = regexp.MustCompile(`<iframe[^>]*src=["'](https?://[^"']*ksohls\.ru[^"']*)["']`)
|
||||
authTokenRe = regexp.MustCompile(`authToken\s*[:=]\s*['"]([^'"]+)['"]`)
|
||||
channelKeyRe = regexp.MustCompile(`channelKey\s*[:=]\s*['"]([^'"]+)['"]`)
|
||||
channelSaltRe = regexp.MustCompile(`channelSalt\s*[:=]\s*['"]([^'"]+)['"]`)
|
||||
timestampRe = regexp.MustCompile(`timestamp\s*[:=]\s*['"]?(\d+)['"]?`)
|
||||
authModRe = regexp.MustCompile(`<script[^>]*src=["'](https?://[^"']*aiaged\.fun[^"']*obfuscated[^"']*)["']`)
|
||||
)
|
||||
|
||||
func (s *Service) getDaddyLiveConfig(ctx context.Context, rawURL string) *PlayerConfig {
|
||||
// Check cache
|
||||
s.mu.RLock()
|
||||
if entry, ok := s.cache[rawURL]; ok && time.Now().Before(entry.expiresAt) {
|
||||
s.mu.RUnlock()
|
||||
return entry.config
|
||||
}
|
||||
s.mu.RUnlock()
|
||||
|
||||
config := s.fetchDaddyLiveConfig(ctx, rawURL)
|
||||
|
||||
// Cache the result (even errors, to avoid hammering)
|
||||
s.mu.Lock()
|
||||
s.cache[rawURL] = &cacheEntry{
|
||||
config: config,
|
||||
expiresAt: time.Now().Add(1 * time.Hour),
|
||||
}
|
||||
s.mu.Unlock()
|
||||
|
||||
return config
|
||||
}
|
||||
|
||||
func (s *Service) fetchDaddyLiveConfig(ctx context.Context, rawURL string) *PlayerConfig {
|
||||
// Step 1: Extract channel ID from URL
|
||||
channelID := ""
|
||||
for _, re := range channelIDPatterns {
|
||||
if m := re.FindStringSubmatch(rawURL); len(m) > 1 {
|
||||
channelID = m[1]
|
||||
break
|
||||
}
|
||||
}
|
||||
if channelID == "" {
|
||||
return &PlayerConfig{Type: "proxy", Error: "could not extract channel ID"}
|
||||
}
|
||||
|
||||
log.Printf("playerconfig: DaddyLive channel=%s from %s", channelID, rawURL)
|
||||
return s.fetchDaddyLiveConfigByID(ctx, channelID)
|
||||
}
|
||||
|
||||
func (s *Service) fetchDaddyLiveConfigByID(ctx context.Context, channelID string) *PlayerConfig {
|
||||
// Step 2: Fetch the cast page to find the ksohls iframe
|
||||
castURL := fmt.Sprintf("https://dlhd.link/cast/stream-%s.php", channelID)
|
||||
castBody, err := s.fetchPage(ctx, castURL, "https://dlhd.link/")
|
||||
if err != nil {
|
||||
log.Printf("playerconfig: failed to fetch cast page: %v", err)
|
||||
return &PlayerConfig{Type: "proxy", Error: "failed to fetch cast page"}
|
||||
}
|
||||
|
||||
// Step 3: Extract ksohls iframe URL
|
||||
iframeMatch := iframeRe.FindStringSubmatch(castBody)
|
||||
if iframeMatch == nil {
|
||||
log.Printf("playerconfig: no ksohls iframe found in cast page")
|
||||
return &PlayerConfig{Type: "proxy", Error: "no ksohls iframe found"}
|
||||
}
|
||||
ksohURL := iframeMatch[1]
|
||||
|
||||
// Step 4: Fetch the ksohls page
|
||||
referer := fmt.Sprintf("https://dlhd.link/stream/stream-%s.php", channelID)
|
||||
ksohBody, err := s.fetchPage(ctx, ksohURL, referer)
|
||||
if err != nil {
|
||||
log.Printf("playerconfig: failed to fetch ksohls page: %v", err)
|
||||
return &PlayerConfig{Type: "proxy", Error: "failed to fetch ksohls page"}
|
||||
}
|
||||
|
||||
// Step 5: Extract auth params from ksohls page
|
||||
config := &PlayerConfig{Type: "daddylive"}
|
||||
|
||||
if m := authTokenRe.FindStringSubmatch(ksohBody); len(m) > 1 {
|
||||
config.AuthToken = m[1]
|
||||
}
|
||||
if m := channelKeyRe.FindStringSubmatch(ksohBody); len(m) > 1 {
|
||||
config.ChannelKey = m[1]
|
||||
}
|
||||
if m := channelSaltRe.FindStringSubmatch(ksohBody); len(m) > 1 {
|
||||
config.ChannelSalt = m[1]
|
||||
}
|
||||
if m := timestampRe.FindStringSubmatch(ksohBody); len(m) > 1 {
|
||||
config.Timestamp = m[1]
|
||||
}
|
||||
if m := authModRe.FindStringSubmatch(ksohBody); len(m) > 1 {
|
||||
config.AuthModURL = m[1]
|
||||
}
|
||||
|
||||
if config.ChannelKey == "" {
|
||||
log.Printf("playerconfig: no channelKey found in ksohls page")
|
||||
return &PlayerConfig{Type: "proxy", Error: "no channelKey found"}
|
||||
}
|
||||
|
||||
// Step 6: Server lookup
|
||||
lookupURL := fmt.Sprintf("https://chevy.soyspace.cyou/server_lookup?channel_id=%s", config.ChannelKey)
|
||||
lookupBody, err := s.fetchPage(ctx, lookupURL, "")
|
||||
if err != nil {
|
||||
log.Printf("playerconfig: server lookup failed: %v", err)
|
||||
return &PlayerConfig{Type: "proxy", Error: "server lookup failed"}
|
||||
}
|
||||
|
||||
var lookupResp struct {
|
||||
ServerKey string `json:"server_key"`
|
||||
}
|
||||
if err := json.Unmarshal([]byte(lookupBody), &lookupResp); err != nil || lookupResp.ServerKey == "" {
|
||||
log.Printf("playerconfig: failed to parse server lookup: %v body=%s", err, lookupBody)
|
||||
return &PlayerConfig{Type: "proxy", Error: "server lookup parse failed"}
|
||||
}
|
||||
config.ServerKey = lookupResp.ServerKey
|
||||
|
||||
// Step 7: Build m3u8 URL
|
||||
m3u8URL := fmt.Sprintf("https://chevy.soyspace.cyou/proxy/%s/%s/mono.m3u8",
|
||||
config.ServerKey, config.ChannelKey)
|
||||
encoded := base64.RawURLEncoding.EncodeToString([]byte(m3u8URL))
|
||||
config.HLSURL = "/hls/" + encoded
|
||||
|
||||
log.Printf("playerconfig: DaddyLive config ready channel=%s server=%s", config.ChannelKey, config.ServerKey)
|
||||
return config
|
||||
}
|
||||
|
||||
// VIPLeague/casthill resolution
|
||||
|
||||
var zmidRe = regexp.MustCompile(`(?:const|var|let)\s+zmid\s*=\s*["']([^"']+)["']`)
|
||||
var casthillVRe = regexp.MustCompile(`[?&]v=([^&]+)`)
|
||||
|
||||
func (s *Service) getVIPLeagueConfig(ctx context.Context, rawURL string) *PlayerConfig {
|
||||
// Check cache using normalized URL
|
||||
s.mu.RLock()
|
||||
if entry, ok := s.cache[rawURL]; ok && time.Now().Before(entry.expiresAt) {
|
||||
s.mu.RUnlock()
|
||||
return entry.config
|
||||
}
|
||||
s.mu.RUnlock()
|
||||
|
||||
config := s.fetchVIPLeagueConfig(ctx, rawURL)
|
||||
|
||||
s.mu.Lock()
|
||||
s.cache[rawURL] = &cacheEntry{
|
||||
config: config,
|
||||
expiresAt: time.Now().Add(1 * time.Hour),
|
||||
}
|
||||
s.mu.Unlock()
|
||||
|
||||
return config
|
||||
}
|
||||
|
||||
func (s *Service) fetchVIPLeagueConfig(ctx context.Context, rawURL string) *PlayerConfig {
|
||||
lower := strings.ToLower(rawURL)
|
||||
|
||||
var zmid string
|
||||
|
||||
if strings.Contains(lower, "casthill.net") {
|
||||
// Extract zmid from casthill URL query param ?v=...
|
||||
if m := casthillVRe.FindStringSubmatch(rawURL); len(m) > 1 {
|
||||
zmid = m[1]
|
||||
}
|
||||
}
|
||||
|
||||
if zmid == "" {
|
||||
// Try to fetch VIPLeague page and extract zmid from JavaScript
|
||||
body, err := s.fetchPage(ctx, rawURL, "")
|
||||
if err != nil {
|
||||
log.Printf("playerconfig: failed to fetch VIPLeague page: %v, trying URL-based extraction", err)
|
||||
} else {
|
||||
if m := zmidRe.FindStringSubmatch(body); len(m) > 1 {
|
||||
zmid = m[1]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if zmid == "" {
|
||||
// Fallback: extract slug from URL path and use it directly for channel matching
|
||||
// e.g. /f-1/sky-sports-f1-streaming → "sky sports f1"
|
||||
zmid = extractSlugFromURL(rawURL)
|
||||
if zmid != "" {
|
||||
log.Printf("playerconfig: extracted slug %q from URL path", zmid)
|
||||
}
|
||||
}
|
||||
|
||||
if zmid == "" {
|
||||
log.Printf("playerconfig: no zmid found for VIPLeague URL %s", rawURL)
|
||||
return &PlayerConfig{Type: "proxy", Error: "no zmid found in VIPLeague page"}
|
||||
}
|
||||
|
||||
log.Printf("playerconfig: VIPLeague zmid=%q from %s", zmid, rawURL)
|
||||
|
||||
channelID, err := s.resolveChannelID(ctx, zmid)
|
||||
if err != nil {
|
||||
log.Printf("playerconfig: failed to resolve zmid %q: %v", zmid, err)
|
||||
return &PlayerConfig{Type: "proxy", Error: fmt.Sprintf("failed to resolve zmid: %v", err)}
|
||||
}
|
||||
|
||||
log.Printf("playerconfig: resolved zmid=%q to DaddyLive channel=%s", zmid, channelID)
|
||||
return s.fetchDaddyLiveConfigByID(ctx, channelID)
|
||||
}
|
||||
|
||||
// extractSlugFromURL extracts a channel-matching slug from a VIPLeague URL path.
|
||||
// e.g. "https://vipleague.io/f-1/sky-sports-f1-streaming" → "sky sports f1"
|
||||
// Strips common suffixes like "-streaming", "-live-stream", "-live", etc.
|
||||
func extractSlugFromURL(rawURL string) string {
|
||||
// Get the last path segment
|
||||
path := rawURL
|
||||
if idx := strings.Index(path, "?"); idx != -1 {
|
||||
path = path[:idx]
|
||||
}
|
||||
path = strings.TrimRight(path, "/")
|
||||
lastSlash := strings.LastIndex(path, "/")
|
||||
if lastSlash == -1 {
|
||||
return ""
|
||||
}
|
||||
slug := path[lastSlash+1:]
|
||||
|
||||
// Strip common suffixes
|
||||
for _, suffix := range []string{"-streaming", "-live-stream", "-stream", "-live", "-online", "-free"} {
|
||||
slug = strings.TrimSuffix(slug, suffix)
|
||||
}
|
||||
|
||||
// Replace hyphens with spaces for matching against channel names
|
||||
slug = strings.ReplaceAll(slug, "-", " ")
|
||||
slug = strings.TrimSpace(slug)
|
||||
|
||||
if slug == "" || len(slug) < 3 {
|
||||
return ""
|
||||
}
|
||||
return slug
|
||||
}
|
||||
|
||||
var channelLinkRe = regexp.MustCompile(`<a[^>]*href=["'][^"']*watch\.php\?id=(\d+)["'][^>]*data-title=["']([^"']+)["']`)
|
||||
var channelLinkRe2 = regexp.MustCompile(`<a[^>]*data-title=["']([^"']+)["'][^>]*href=["'][^"']*watch\.php\?id=(\d+)["']`)
|
||||
|
||||
func (s *Service) resolveChannelID(ctx context.Context, zmid string) (string, error) {
|
||||
channels, err := s.getChannelIndex(ctx)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
zmidLower := strings.ToLower(zmid)
|
||||
|
||||
// Build tokens: if zmid contains spaces, split on spaces; otherwise use tokenizer
|
||||
var tokens []string
|
||||
if strings.Contains(zmidLower, " ") {
|
||||
for _, word := range strings.Fields(zmidLower) {
|
||||
if len(word) >= 2 {
|
||||
tokens = append(tokens, word)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
tokens = tokenize(zmidLower)
|
||||
}
|
||||
|
||||
bestID := ""
|
||||
bestScore := 0
|
||||
bestNameLen := 0
|
||||
|
||||
for id, name := range channels {
|
||||
score := 0
|
||||
for _, tok := range tokens {
|
||||
if strings.Contains(name, tok) {
|
||||
score++
|
||||
}
|
||||
}
|
||||
// Tiebreaker: prefer shorter names (more specific match) and
|
||||
// English/UK channels which tend to have shorter names
|
||||
if score > bestScore || (score == bestScore && score > 0 && len(name) < bestNameLen) {
|
||||
bestScore = score
|
||||
bestID = id
|
||||
bestNameLen = len(name)
|
||||
}
|
||||
}
|
||||
|
||||
if bestID == "" || bestScore == 0 {
|
||||
return "", fmt.Errorf("no channel matched zmid %q (tried %d channels)", zmid, len(channels))
|
||||
}
|
||||
|
||||
log.Printf("playerconfig: zmid=%q matched channel %s (%s) with score %d/%d",
|
||||
zmid, bestID, channels[bestID], bestScore, len(tokens))
|
||||
return bestID, nil
|
||||
}
|
||||
|
||||
func (s *Service) getChannelIndex(ctx context.Context) (map[string]string, error) {
|
||||
const cacheKey = "__channel_index__"
|
||||
|
||||
s.mu.RLock()
|
||||
if entry, ok := s.cache[cacheKey]; ok && time.Now().Before(entry.expiresAt) {
|
||||
s.mu.RUnlock()
|
||||
// Decode from the Error field (ab)used as storage
|
||||
var idx map[string]string
|
||||
if err := json.Unmarshal([]byte(entry.config.Error), &idx); err == nil {
|
||||
return idx, nil
|
||||
}
|
||||
}
|
||||
s.mu.RUnlock()
|
||||
|
||||
body, err := s.fetchPage(ctx, "https://dlhd.link/24-7-channels.php", "https://dlhd.link/")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to fetch channel index: %w", err)
|
||||
}
|
||||
|
||||
channels := make(map[string]string)
|
||||
|
||||
// Try both attribute orderings
|
||||
for _, m := range channelLinkRe.FindAllStringSubmatch(body, -1) {
|
||||
channels[m[1]] = strings.ToLower(strings.TrimSpace(m[2]))
|
||||
}
|
||||
for _, m := range channelLinkRe2.FindAllStringSubmatch(body, -1) {
|
||||
channels[m[2]] = strings.ToLower(strings.TrimSpace(m[1]))
|
||||
}
|
||||
|
||||
if len(channels) == 0 {
|
||||
return nil, fmt.Errorf("no channels found in 24/7 page (%d bytes)", len(body))
|
||||
}
|
||||
|
||||
log.Printf("playerconfig: loaded %d channels from DaddyLive 24/7 page", len(channels))
|
||||
|
||||
// Cache as JSON in a fake PlayerConfig entry
|
||||
encoded, _ := json.Marshal(channels)
|
||||
s.mu.Lock()
|
||||
s.cache[cacheKey] = &cacheEntry{
|
||||
config: &PlayerConfig{Error: string(encoded)},
|
||||
expiresAt: time.Now().Add(6 * time.Hour),
|
||||
}
|
||||
s.mu.Unlock()
|
||||
|
||||
return channels, nil
|
||||
}
|
||||
|
||||
// tokenize splits a zmid slug into meaningful tokens.
|
||||
// e.g. "skyf1" -> ["sky", "f1"], "daznf1" -> ["dazn", "f1"]
|
||||
func tokenize(zmid string) []string {
|
||||
// Common known prefixes/suffixes in sports streaming slugs
|
||||
knownTokens := []string{
|
||||
"sky", "sports", "f1", "dazn", "espn", "fox", "bein", "bt",
|
||||
"star", "nbc", "cbs", "tnt", "abc", "tsn", "supersport",
|
||||
"canal", "rtl", "viaplay", "premier", "main", "event",
|
||||
"arena", "action", "cricket", "football", "tennis", "golf",
|
||||
"racing", "news", "extra", "max", "hd", "uhd",
|
||||
}
|
||||
|
||||
var tokens []string
|
||||
remaining := zmid
|
||||
|
||||
for len(remaining) > 0 {
|
||||
matched := false
|
||||
for _, tok := range knownTokens {
|
||||
if strings.HasPrefix(remaining, tok) {
|
||||
tokens = append(tokens, tok)
|
||||
remaining = remaining[len(tok):]
|
||||
matched = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !matched {
|
||||
// Try numeric suffix (like channel numbers)
|
||||
i := 0
|
||||
for i < len(remaining) && remaining[i] >= '0' && remaining[i] <= '9' {
|
||||
i++
|
||||
}
|
||||
if i > 0 {
|
||||
tokens = append(tokens, remaining[:i])
|
||||
remaining = remaining[i:]
|
||||
} else {
|
||||
// Skip single character and try again
|
||||
remaining = remaining[1:]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// If tokenization produced nothing useful, use the whole zmid as a single token
|
||||
if len(tokens) == 0 {
|
||||
tokens = []string{zmid}
|
||||
}
|
||||
|
||||
return tokens
|
||||
}
|
||||
|
||||
func (s *Service) fetchPage(ctx context.Context, pageURL, referer string) (string, error) {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, pageURL, nil)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36")
|
||||
if referer != "" {
|
||||
req.Header.Set("Referer", referer)
|
||||
}
|
||||
|
||||
resp, err := s.client.Do(req)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return "", fmt.Errorf("status %d from %s", resp.StatusCode, pageURL)
|
||||
}
|
||||
|
||||
body, err := io.ReadAll(io.LimitReader(resp.Body, 2*1024*1024)) // 2MB max
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return string(body), nil
|
||||
}
|
||||
473
stacks/f1-stream/module/files/internal/proxy/proxy.go
Normal file
473
stacks/f1-stream/module/files/internal/proxy/proxy.go
Normal file
|
|
@ -0,0 +1,473 @@
|
|||
package proxy
|
||||
|
||||
import (
|
||||
"encoding/base64"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"regexp"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// hopHeaders are headers that should not be forwarded by proxies.
|
||||
var hopHeaders = map[string]bool{
|
||||
"Connection": true,
|
||||
"Keep-Alive": true,
|
||||
"Proxy-Authenticate": true,
|
||||
"Proxy-Authorization": true,
|
||||
"Te": true,
|
||||
"Trailers": true,
|
||||
"Transfer-Encoding": true,
|
||||
"Upgrade": true,
|
||||
}
|
||||
|
||||
// antiFrameHeaders are headers we strip to allow iframe embedding.
|
||||
var antiFrameHeaders = []string{
|
||||
"X-Frame-Options",
|
||||
"Content-Security-Policy",
|
||||
"Content-Security-Policy-Report-Only",
|
||||
"X-Content-Type-Options",
|
||||
}
|
||||
|
||||
// forwardHeaders are request headers we copy from the client to the upstream.
|
||||
// NOTE: Accept-Encoding is intentionally omitted so Go's Transport handles
|
||||
// compression transparently (adds gzip, auto-decompresses response body).
|
||||
// This ensures we can do text replacements on HTML/CSS bodies.
|
||||
var forwardHeaders = []string{
|
||||
"User-Agent",
|
||||
"Accept",
|
||||
"Accept-Language",
|
||||
"Cookie",
|
||||
"Referer",
|
||||
"Range",
|
||||
"If-None-Match",
|
||||
"If-Modified-Since",
|
||||
"Cache-Control",
|
||||
}
|
||||
|
||||
// jsShimTemplate is injected into HTML responses to intercept JS-initiated requests.
|
||||
// It patches fetch, XMLHttpRequest, WebSocket, and EventSource to route through the proxy.
|
||||
const jsShimTemplate = `<script data-proxy-shim="1">(function(){
|
||||
var P='/proxy/%s';
|
||||
var O='%s';
|
||||
var H=location.origin;
|
||||
function b64(s){return btoa(s).replace(/\+/g,'-').replace(/\//g,'_').replace(/=+$/g,'');}
|
||||
function rw(u){
|
||||
if(!u||typeof u!=='string')return u;
|
||||
if(u.startsWith('/proxy/'))return u;
|
||||
if(u.startsWith(H+'/proxy/'))return u.slice(H.length);
|
||||
if(u.startsWith(H+'/')){var hp=u.slice(H.length);return P+hp;}
|
||||
if(u.startsWith(H))return P+'/';
|
||||
if(u.startsWith('/'))return P+u;
|
||||
if(u.startsWith(O))return P+u.slice(O.length);
|
||||
try{var p=new URL(u);if(p.protocol==='http:'||p.protocol==='https:'){return'/proxy/'+b64(p.origin)+p.pathname+p.search+p.hash;}}catch(e){}
|
||||
return u;
|
||||
}
|
||||
var _f=window.fetch;
|
||||
window.fetch=function(i,o){
|
||||
if(typeof i==='string')i=rw(i);
|
||||
else if(i&&i.url)i=new Request(rw(i.url),i);
|
||||
return _f.call(this,i,o);
|
||||
};
|
||||
var _xo=XMLHttpRequest.prototype.open;
|
||||
XMLHttpRequest.prototype.open=function(m,u){var a=[].slice.call(arguments);a[1]=rw(u);return _xo.apply(this,a);};
|
||||
var _ws=window.WebSocket;
|
||||
window.WebSocket=function(u,p){return new _ws(rw(u),p);};
|
||||
window.WebSocket.prototype=_ws.prototype;
|
||||
window.WebSocket.CONNECTING=_ws.CONNECTING;
|
||||
window.WebSocket.OPEN=_ws.OPEN;
|
||||
window.WebSocket.CLOSING=_ws.CLOSING;
|
||||
window.WebSocket.CLOSED=_ws.CLOSED;
|
||||
if(window.EventSource){var _es=window.EventSource;window.EventSource=function(u,o){return new _es(rw(u),o);};window.EventSource.prototype=_es.prototype;}
|
||||
var _ce=document.createElement.bind(document);
|
||||
document.createElement=function(t){
|
||||
var el=_ce(t);
|
||||
var tag=t.toLowerCase();
|
||||
if(tag==='script'||tag==='img'||tag==='link'||tag==='source'||tag==='video'||tag==='audio'){
|
||||
var _ss=Object.getOwnPropertyDescriptor(HTMLElement.prototype,'src')||Object.getOwnPropertyDescriptor(el.__proto__,'src');
|
||||
if(_ss&&_ss.set){Object.defineProperty(el,'src',{get:function(){return _ss.get?_ss.get.call(this):'';},set:function(v){_ss.set.call(this,rw(v));},configurable:true});}
|
||||
}
|
||||
return el;
|
||||
};
|
||||
/* Neutralize anti-debug: override setInterval/setTimeout to skip debugger-based detection */
|
||||
var _si=window.setInterval;
|
||||
window.setInterval=function(fn,ms){
|
||||
if(typeof fn==='function'){var s=fn.toString();if(s.indexOf('debugger')!==-1||s.indexOf('devtool')!==-1)return 0;}
|
||||
if(typeof fn==='string'&&(fn.indexOf('debugger')!==-1||fn.indexOf('devtool')!==-1))return 0;
|
||||
return _si.apply(this,arguments);
|
||||
};
|
||||
var _st=window.setTimeout;
|
||||
window.setTimeout=function(fn,ms){
|
||||
if(typeof fn==='function'){var s=fn.toString();if(s.indexOf('debugger')!==-1||s.indexOf('devtool')!==-1)return 0;}
|
||||
if(typeof fn==='string'&&(fn.indexOf('debugger')!==-1||fn.indexOf('devtool')!==-1))return 0;
|
||||
return _st.apply(this,arguments);
|
||||
};
|
||||
/* Override eval and Function to strip debugger statements */
|
||||
var _eval=window.eval;
|
||||
window.eval=function(s){if(typeof s==='string')s=s.replace(/\bdebugger\b\s*;?/g,'');return _eval.call(this,s);};
|
||||
var _Fn=Function;
|
||||
window.Function=function(){var a=[].slice.call(arguments);if(a.length>0){var last=a.length-1;if(typeof a[last]==='string')a[last]=a[last].replace(/\bdebugger\b\s*;?/g,'');}return _Fn.apply(this,a);};
|
||||
window.Function.prototype=_Fn.prototype;
|
||||
/* Block loading of known anti-debug scripts */
|
||||
var _ael=HTMLScriptElement.prototype.setAttribute;
|
||||
HTMLScriptElement.prototype.setAttribute=function(n,v){
|
||||
if(n==='src'&&typeof v==='string'&&(v.indexOf('disable-devtool')!==-1||v.indexOf('devtools-detect')!==-1)){return;}
|
||||
return _ael.apply(this,arguments);
|
||||
};
|
||||
})();</script>`
|
||||
|
||||
// NewHandler returns an http.Handler that serves the reverse proxy at /proxy/.
|
||||
// URL structure: /proxy/{base64_origin}/{path...}
|
||||
func NewHandler() http.Handler {
|
||||
client := &http.Client{
|
||||
Timeout: 30 * 1000000000, // 30s
|
||||
CheckRedirect: func(req *http.Request, via []*http.Request) error {
|
||||
return http.ErrUseLastResponse // don't follow redirects
|
||||
},
|
||||
}
|
||||
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
// Parse: /proxy/{base64_origin}/{path...}
|
||||
trimmed := strings.TrimPrefix(r.URL.Path, "/proxy/")
|
||||
if trimmed == "" || trimmed == r.URL.Path {
|
||||
http.Error(w, "bad proxy URL", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
// Split into base64 segment and remaining path
|
||||
slashIdx := strings.Index(trimmed, "/")
|
||||
var b64Origin, pathAndQuery string
|
||||
if slashIdx == -1 {
|
||||
b64Origin = trimmed
|
||||
pathAndQuery = "/"
|
||||
} else {
|
||||
b64Origin = trimmed[:slashIdx]
|
||||
pathAndQuery = trimmed[slashIdx:]
|
||||
}
|
||||
|
||||
originBytes, err := base64.RawURLEncoding.DecodeString(b64Origin)
|
||||
if err != nil {
|
||||
// Try standard encoding with padding
|
||||
originBytes, err = base64.StdEncoding.DecodeString(b64Origin)
|
||||
if err != nil {
|
||||
http.Error(w, "invalid base64 origin", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
}
|
||||
origin := string(originBytes)
|
||||
|
||||
// Validate origin is a valid URL
|
||||
originURL, err := url.Parse(origin)
|
||||
if err != nil || (originURL.Scheme != "http" && originURL.Scheme != "https") {
|
||||
http.Error(w, "invalid origin URL", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
// Build upstream URL
|
||||
targetURL := origin + pathAndQuery
|
||||
if r.URL.RawQuery != "" {
|
||||
targetURL += "?" + r.URL.RawQuery
|
||||
}
|
||||
|
||||
log.Printf("proxy: %s %s -> %s", r.Method, r.URL.Path, targetURL)
|
||||
|
||||
// Create upstream request
|
||||
upReq, err := http.NewRequestWithContext(r.Context(), r.Method, targetURL, r.Body)
|
||||
if err != nil {
|
||||
http.Error(w, "failed to create request", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
// Copy selected headers
|
||||
for _, h := range forwardHeaders {
|
||||
if v := r.Header.Get(h); v != "" {
|
||||
upReq.Header.Set(h, v)
|
||||
}
|
||||
}
|
||||
// Reconstruct the original Referer from the client's proxy-rewritten Referer.
|
||||
// The client sends e.g. "https://f1.viktorbarzin.me/proxy/{b64origin}/path"
|
||||
// and we need to decode that back to "https://original.com/path".
|
||||
upReq.Header.Set("Referer", decodeProxyReferer(r.Header.Get("Referer"), origin))
|
||||
|
||||
// Fetch upstream
|
||||
resp, err := client.Do(upReq)
|
||||
if err != nil {
|
||||
log.Printf("proxy: upstream fetch failed: %v", err)
|
||||
http.Error(w, "upstream fetch failed", http.StatusBadGateway)
|
||||
return
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
log.Printf("proxy: %s %s <- %d (%s)", r.Method, r.URL.Path, resp.StatusCode, resp.Header.Get("Content-Type"))
|
||||
|
||||
// Handle redirects: rewrite Location header through proxy
|
||||
if resp.StatusCode >= 300 && resp.StatusCode < 400 {
|
||||
loc := resp.Header.Get("Location")
|
||||
if loc != "" {
|
||||
rewritten := rewriteRedirect(loc, origin, b64Origin)
|
||||
w.Header().Set("Location", rewritten)
|
||||
log.Printf("proxy: redirect %s -> %s", loc, rewritten)
|
||||
}
|
||||
w.WriteHeader(resp.StatusCode)
|
||||
return
|
||||
}
|
||||
|
||||
// Copy response headers, stripping anti-frame, hop-by-hop, and encoding headers.
|
||||
// Content-Encoding is stripped because Go's Transport already decompressed the body.
|
||||
// Content-Length is stripped because we may rewrite the body (changing its length).
|
||||
for key, vals := range resp.Header {
|
||||
if hopHeaders[key] {
|
||||
continue
|
||||
}
|
||||
if strings.EqualFold(key, "Content-Encoding") || strings.EqualFold(key, "Content-Length") {
|
||||
continue
|
||||
}
|
||||
skip := false
|
||||
for _, ah := range antiFrameHeaders {
|
||||
if strings.EqualFold(key, ah) {
|
||||
skip = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if skip {
|
||||
continue
|
||||
}
|
||||
for _, v := range vals {
|
||||
w.Header().Add(key, v)
|
||||
}
|
||||
}
|
||||
|
||||
// Add permissive CORS headers so cross-origin XHR/fetch from the iframe works
|
||||
w.Header().Set("Access-Control-Allow-Origin", "*")
|
||||
w.Header().Set("Access-Control-Allow-Methods", "GET, POST, PUT, DELETE, OPTIONS")
|
||||
w.Header().Set("Access-Control-Allow-Headers", "*")
|
||||
|
||||
w.WriteHeader(resp.StatusCode)
|
||||
|
||||
// For HTML responses, rewrite URLs and inject JS shim
|
||||
ct := resp.Header.Get("Content-Type")
|
||||
if strings.Contains(ct, "text/html") {
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
log.Printf("proxy: failed to read HTML body: %v", err)
|
||||
return
|
||||
}
|
||||
rewritten := rewriteHTML(string(body), origin, b64Origin)
|
||||
w.Write([]byte(rewritten))
|
||||
return
|
||||
}
|
||||
|
||||
// For CSS responses, rewrite url() references
|
||||
if strings.Contains(ct, "text/css") {
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
log.Printf("proxy: failed to read CSS body: %v", err)
|
||||
return
|
||||
}
|
||||
rewritten := rewriteCSS(string(body), origin, b64Origin)
|
||||
w.Write([]byte(rewritten))
|
||||
return
|
||||
}
|
||||
|
||||
// For JavaScript responses, strip debugger statements
|
||||
if strings.Contains(ct, "javascript") || strings.Contains(ct, "ecmascript") {
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
log.Printf("proxy: failed to read JS body: %v", err)
|
||||
return
|
||||
}
|
||||
cleaned := debuggerStmtRe.ReplaceAllString(string(body), "/* */")
|
||||
w.Write([]byte(cleaned))
|
||||
return
|
||||
}
|
||||
|
||||
// Stream other responses directly
|
||||
io.Copy(w, resp.Body)
|
||||
})
|
||||
}
|
||||
|
||||
// rewriteRedirect rewrites a Location header value to route through the proxy.
|
||||
func rewriteRedirect(loc, origin, b64Origin string) string {
|
||||
// Absolute URL on the same origin
|
||||
if strings.HasPrefix(loc, origin) {
|
||||
path := strings.TrimPrefix(loc, origin)
|
||||
return "/proxy/" + b64Origin + path
|
||||
}
|
||||
// Absolute URL on a different origin — proxy it too
|
||||
parsed, err := url.Parse(loc)
|
||||
if err != nil {
|
||||
return loc
|
||||
}
|
||||
if parsed.IsAbs() {
|
||||
newOrigin := parsed.Scheme + "://" + parsed.Host
|
||||
newB64 := base64.RawURLEncoding.EncodeToString([]byte(newOrigin))
|
||||
return "/proxy/" + newB64 + parsed.RequestURI()
|
||||
}
|
||||
// Relative URL — it will resolve naturally
|
||||
return loc
|
||||
}
|
||||
|
||||
// Precompiled regexes for root-relative URL rewriting in HTML attributes.
|
||||
// Matches src="/...", href="/...", action="/...", poster="/..." but NOT "//..." (protocol-relative).
|
||||
var rootRelativeAttrRe = regexp.MustCompile(`((?:src|href|action|poster|data)\s*=\s*["'])/([^/"'][^"']*)`)
|
||||
|
||||
// Matches url("/...") or url('/...') or url(/...) in inline styles — but NOT url("//...")
|
||||
var rootRelativeCSSRe = regexp.MustCompile(`(url\(\s*["']?)/([^/"')[^"')]*)(["']?\s*\))`)
|
||||
|
||||
// disableDevtoolRe matches <script> tags that load disable-devtool or similar anti-debug libraries.
|
||||
var disableDevtoolRe = regexp.MustCompile(`(?i)<script[^>]*(?:disable-devtool|devtools-detect)[^>]*>(?:</script>)?`)
|
||||
|
||||
// adScriptRe matches <script> tags that load common ad/popup libraries.
|
||||
var adScriptRe = regexp.MustCompile(`(?i)<script[^>]*(?:acscdn\.com|popunder|popads|juicyads)[^>]*>\s*(?:</script>)?`)
|
||||
|
||||
// adInlineRe matches inline <script> blocks that call ad popup functions.
|
||||
var adInlineRe = regexp.MustCompile(`(?i)<script[^>]*>\s*(?:aclib\.run|popunder|pop_)\w*\([^)]*\);\s*</script>`)
|
||||
|
||||
// contextMenuBlockRe matches inline scripts that block right-click and dev tools shortcuts.
|
||||
var contextMenuBlockRe = regexp.MustCompile(`(?i)<script[^>]*>\s*document\.addEventListener\(\s*'contextmenu'[\s\S]{0,500}?</script>`)
|
||||
|
||||
// debuggerStmtRe matches debugger statements in JavaScript.
|
||||
var debuggerStmtRe = regexp.MustCompile(`\bdebugger\b\s*;?`)
|
||||
|
||||
// rewriteHTML replaces URLs and injects the JS shim to intercept runtime requests.
|
||||
func rewriteHTML(body, origin, b64Origin string) string {
|
||||
proxyPrefix := "/proxy/" + b64Origin
|
||||
|
||||
// 1. Rewrite absolute URLs matching the target origin
|
||||
escaped := regexp.QuoteMeta(origin)
|
||||
absRe := regexp.MustCompile(escaped + `(/[^"'\s>)]*)?`)
|
||||
body = absRe.ReplaceAllStringFunc(body, func(match string) string {
|
||||
path := strings.TrimPrefix(match, origin)
|
||||
if path == "" {
|
||||
path = "/"
|
||||
}
|
||||
return proxyPrefix + path
|
||||
})
|
||||
|
||||
// 2. Rewrite root-relative URLs in HTML attributes (src="/...", href="/...", etc.)
|
||||
// Skip URLs already rewritten by step 1 (starting with /proxy/)
|
||||
body = rootRelativeAttrRe.ReplaceAllStringFunc(body, func(match string) string {
|
||||
m := rootRelativeAttrRe.FindStringSubmatch(match)
|
||||
if len(m) < 3 {
|
||||
return match
|
||||
}
|
||||
// m[2] is the path after the leading "/", skip if already proxied
|
||||
if strings.HasPrefix(m[2], "proxy/") {
|
||||
return match
|
||||
}
|
||||
return m[1] + proxyPrefix + "/" + m[2]
|
||||
})
|
||||
|
||||
// 3. Rewrite root-relative URLs in inline CSS url() references
|
||||
// Skip URLs already rewritten by step 1 (starting with /proxy/)
|
||||
body = rootRelativeCSSRe.ReplaceAllStringFunc(body, func(match string) string {
|
||||
m := rootRelativeCSSRe.FindStringSubmatch(match)
|
||||
if len(m) < 4 {
|
||||
return match
|
||||
}
|
||||
if strings.HasPrefix(m[2], "proxy/") {
|
||||
return match
|
||||
}
|
||||
return m[1] + proxyPrefix + "/" + m[2] + m[3]
|
||||
})
|
||||
|
||||
// 4. Strip anti-debugging scripts (disable-devtool, devtools-detect)
|
||||
body = disableDevtoolRe.ReplaceAllString(body, "")
|
||||
|
||||
// 4b. Strip ad/popup scripts and context menu blockers
|
||||
body = adScriptRe.ReplaceAllString(body, "")
|
||||
body = adInlineRe.ReplaceAllString(body, "")
|
||||
body = contextMenuBlockRe.ReplaceAllString(body, "")
|
||||
|
||||
// 4c. Strip debugger statements from inline scripts
|
||||
body = debuggerStmtRe.ReplaceAllString(body, "/* */")
|
||||
|
||||
// 5. Inject JS shim right after <head> to intercept fetch/XHR/WebSocket
|
||||
shim := fmt.Sprintf(jsShimTemplate, b64Origin, origin)
|
||||
headIdx := strings.Index(strings.ToLower(body), "<head>")
|
||||
if headIdx != -1 {
|
||||
insertPos := headIdx + len("<head>")
|
||||
body = body[:insertPos] + shim + body[insertPos:]
|
||||
} else {
|
||||
// No <head> tag — prepend to body
|
||||
body = shim + body
|
||||
}
|
||||
|
||||
return body
|
||||
}
|
||||
|
||||
// rewriteCSS replaces root-relative url() references in CSS to route through the proxy.
|
||||
func rewriteCSS(body, origin, b64Origin string) string {
|
||||
proxyPrefix := "/proxy/" + b64Origin
|
||||
|
||||
// Rewrite absolute URLs matching origin
|
||||
escaped := regexp.QuoteMeta(origin)
|
||||
absRe := regexp.MustCompile(escaped + `(/[^"'\s)]*)?`)
|
||||
body = absRe.ReplaceAllStringFunc(body, func(match string) string {
|
||||
path := strings.TrimPrefix(match, origin)
|
||||
if path == "" {
|
||||
path = "/"
|
||||
}
|
||||
return proxyPrefix + path
|
||||
})
|
||||
|
||||
// Rewrite root-relative url() references, skip already-proxied
|
||||
body = rootRelativeCSSRe.ReplaceAllStringFunc(body, func(match string) string {
|
||||
m := rootRelativeCSSRe.FindStringSubmatch(match)
|
||||
if len(m) < 4 {
|
||||
return match
|
||||
}
|
||||
if strings.HasPrefix(m[2], "proxy/") {
|
||||
return match
|
||||
}
|
||||
return m[1] + proxyPrefix + "/" + m[2] + m[3]
|
||||
})
|
||||
|
||||
return body
|
||||
}
|
||||
|
||||
// decodeProxyReferer takes the client's Referer (which points to a proxy URL)
|
||||
// and decodes it back to the original upstream URL. This is critical for
|
||||
// cross-origin requests where the upstream checks the Referer (e.g. HLS servers).
|
||||
// Falls back to origin+"/" if decoding fails.
|
||||
func decodeProxyReferer(clientReferer, fallbackOrigin string) string {
|
||||
if clientReferer == "" {
|
||||
return fallbackOrigin + "/"
|
||||
}
|
||||
|
||||
// Find /proxy/ in the Referer URL path
|
||||
idx := strings.Index(clientReferer, "/proxy/")
|
||||
if idx == -1 {
|
||||
return fallbackOrigin + "/"
|
||||
}
|
||||
|
||||
// Extract everything after /proxy/
|
||||
rest := clientReferer[idx+len("/proxy/"):]
|
||||
if rest == "" {
|
||||
return fallbackOrigin + "/"
|
||||
}
|
||||
|
||||
// Split into base64 segment and remaining path
|
||||
slashIdx := strings.Index(rest, "/")
|
||||
var b64Seg, pathPart string
|
||||
if slashIdx == -1 {
|
||||
b64Seg = rest
|
||||
pathPart = "/"
|
||||
} else {
|
||||
b64Seg = rest[:slashIdx]
|
||||
pathPart = rest[slashIdx:]
|
||||
}
|
||||
|
||||
// Decode the base64 origin
|
||||
originBytes, err := base64.RawURLEncoding.DecodeString(b64Seg)
|
||||
if err != nil {
|
||||
originBytes, err = base64.StdEncoding.DecodeString(b64Seg)
|
||||
if err != nil {
|
||||
return fallbackOrigin + "/"
|
||||
}
|
||||
}
|
||||
|
||||
return string(originBytes) + pathPart
|
||||
}
|
||||
327
stacks/f1-stream/module/files/internal/scraper/reddit.go
Normal file
327
stacks/f1-stream/module/files/internal/scraper/reddit.go
Normal file
|
|
@ -0,0 +1,327 @@
|
|||
package scraper
|
||||
|
||||
import (
|
||||
"crypto/rand"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"math"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"f1-stream/internal/models"
|
||||
)
|
||||
|
||||
const (
|
||||
subredditURL = "https://www.reddit.com/r/motorsportsstreams2/new.json?limit=25"
|
||||
userAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"
|
||||
requestDelay = 1 * time.Second
|
||||
)
|
||||
|
||||
var (
|
||||
urlRe = regexp.MustCompile(`https?://[^\s\)\]\>"]+`)
|
||||
|
||||
// Keywords in post title that indicate F1 content (matched case-insensitively)
|
||||
f1Keywords = []string{
|
||||
"f1",
|
||||
"formula 1",
|
||||
"formula one",
|
||||
"formula1",
|
||||
"grand prix",
|
||||
"gp qualifying",
|
||||
"gp race",
|
||||
"gp sprint",
|
||||
"gp practice",
|
||||
}
|
||||
|
||||
f1NegativeKeywords = []string{
|
||||
"f1 key",
|
||||
"function 1",
|
||||
"help f1",
|
||||
}
|
||||
|
||||
// URLs to filter out (not stream sources)
|
||||
filteredDomains = map[string]bool{
|
||||
"reddit.com": true,
|
||||
"www.reddit.com": true,
|
||||
"imgur.com": true,
|
||||
"i.imgur.com": true,
|
||||
"redd.it": true,
|
||||
"i.redd.it": true,
|
||||
"v.redd.it": true,
|
||||
"youtu.be": true,
|
||||
"youtube.com": true,
|
||||
"twitter.com": true,
|
||||
"x.com": true,
|
||||
}
|
||||
)
|
||||
|
||||
type redditListing struct {
|
||||
Data struct {
|
||||
Children []struct {
|
||||
Data struct {
|
||||
Title string `json:"title"`
|
||||
SelfText string `json:"selftext"`
|
||||
Permalink string `json:"permalink"`
|
||||
CreatedUTC float64 `json:"created_utc"`
|
||||
} `json:"data"`
|
||||
} `json:"children"`
|
||||
} `json:"data"`
|
||||
}
|
||||
|
||||
type redditComments []struct {
|
||||
Data struct {
|
||||
Children []struct {
|
||||
Data struct {
|
||||
Body string `json:"body"`
|
||||
Replies json.RawMessage `json:"replies"`
|
||||
} `json:"data"`
|
||||
} `json:"children"`
|
||||
} `json:"data"`
|
||||
}
|
||||
|
||||
func scrapeReddit() ([]models.ScrapedLink, error) {
|
||||
client := &http.Client{Timeout: 15 * time.Second}
|
||||
var allLinks []models.ScrapedLink
|
||||
seen := make(map[string]bool)
|
||||
|
||||
log.Printf("scraper: fetching listing from %s", subredditURL)
|
||||
listing, err := fetchJSON[redditListing](client, subredditURL)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("fetch listing: %w", err)
|
||||
}
|
||||
|
||||
totalPosts := len(listing.Data.Children)
|
||||
matchedPosts := 0
|
||||
log.Printf("scraper: got %d posts from listing", totalPosts)
|
||||
|
||||
for _, child := range listing.Data.Children {
|
||||
post := child.Data
|
||||
|
||||
if !isF1Post(post.Title) {
|
||||
log.Printf("scraper: skipped post: %s", truncate(post.Title, 60))
|
||||
continue
|
||||
}
|
||||
|
||||
matchedPosts++
|
||||
log.Printf("scraper: matched post: %s", truncate(post.Title, 60))
|
||||
|
||||
selftextLinks := extractURLs(post.SelfText, post.Title)
|
||||
log.Printf("scraper: extracted %d URLs from selftext of %q", len(selftextLinks), truncate(post.Title, 40))
|
||||
for _, link := range selftextLinks {
|
||||
norm := normalizeURL(link.URL)
|
||||
if !seen[norm] {
|
||||
seen[norm] = true
|
||||
allLinks = append(allLinks, link)
|
||||
}
|
||||
}
|
||||
|
||||
time.Sleep(requestDelay)
|
||||
commentsURL := fmt.Sprintf("https://www.reddit.com%s.json", post.Permalink)
|
||||
comments, err := fetchJSONWithRetry[redditComments](client, commentsURL, 3)
|
||||
if err != nil {
|
||||
log.Printf("scraper: failed to fetch comments for %s: %v", post.Permalink, err)
|
||||
continue
|
||||
}
|
||||
|
||||
commentURLCount := 0
|
||||
walkComments(*comments, func(body string) {
|
||||
links := extractURLs(body, post.Title)
|
||||
commentURLCount += len(links)
|
||||
for _, link := range links {
|
||||
norm := normalizeURL(link.URL)
|
||||
if !seen[norm] {
|
||||
seen[norm] = true
|
||||
allLinks = append(allLinks, link)
|
||||
}
|
||||
}
|
||||
})
|
||||
log.Printf("scraper: extracted %d URLs from comments of %q", commentURLCount, truncate(post.Title, 40))
|
||||
|
||||
time.Sleep(requestDelay)
|
||||
}
|
||||
|
||||
log.Printf("scraper: summary — matched %d/%d posts, extracted %d unique URLs", matchedPosts, totalPosts, len(allLinks))
|
||||
return allLinks, nil
|
||||
}
|
||||
|
||||
func fetchJSON[T any](client *http.Client, rawURL string) (*T, error) {
|
||||
req, err := http.NewRequest("GET", rawURL, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
req.Header.Set("User-Agent", userAgent)
|
||||
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
log.Printf("scraper: GET %s -> %d", truncate(rawURL, 80), resp.StatusCode)
|
||||
|
||||
if resp.StatusCode != 200 {
|
||||
return nil, fmt.Errorf("status %d", resp.StatusCode)
|
||||
}
|
||||
|
||||
body, err := io.ReadAll(io.LimitReader(resp.Body, 5*1024*1024))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var result T
|
||||
if err := json.Unmarshal(body, &result); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &result, nil
|
||||
}
|
||||
|
||||
func fetchJSONWithRetry[T any](client *http.Client, rawURL string, maxRetries int) (*T, error) {
|
||||
var lastErr error
|
||||
for attempt := 0; attempt <= maxRetries; attempt++ {
|
||||
result, err := fetchJSON[T](client, rawURL)
|
||||
if err == nil {
|
||||
return result, nil
|
||||
}
|
||||
lastErr = err
|
||||
|
||||
errMsg := err.Error()
|
||||
if strings.Contains(errMsg, "status 429") {
|
||||
log.Printf("scraper: rate limited on %s, backing off 30s", truncate(rawURL, 60))
|
||||
time.Sleep(30 * time.Second)
|
||||
continue
|
||||
}
|
||||
if strings.Contains(errMsg, "status 502") || strings.Contains(errMsg, "status 503") {
|
||||
backoff := time.Duration(math.Pow(2, float64(attempt))) * time.Second
|
||||
log.Printf("scraper: server error on %s, retry %d/%d in %v", truncate(rawURL, 60), attempt+1, maxRetries, backoff)
|
||||
time.Sleep(backoff)
|
||||
continue
|
||||
}
|
||||
|
||||
return nil, err
|
||||
}
|
||||
return nil, fmt.Errorf("after %d retries: %w", maxRetries, lastErr)
|
||||
}
|
||||
|
||||
// deobfuscateText normalises obfuscated URLs commonly posted on Reddit to
|
||||
// evade auto-moderation. Examples:
|
||||
// - "pitsport . xyz/watch/f1" → "https://pitsport.xyz/watch/f1"
|
||||
// - "dlhd dot link" → "https://dlhd.link"
|
||||
func deobfuscateText(text string) string {
|
||||
// Common TLDs used in streaming links.
|
||||
tlds := `(?:com|net|org|xyz|link|info|live|tv|me|cc|to|io|co|stream|site|fun|top|club|watch|racing)`
|
||||
|
||||
// 1. Replace " dot " (case-insensitive) between word-like parts that
|
||||
// look like domain components: "dlhd dot link" → "dlhd.link"
|
||||
dotWord := regexp.MustCompile(`(?i)(\b\w[\w-]*)\s+dot\s+(` + tlds + `\b)`)
|
||||
text = dotWord.ReplaceAllString(text, "${1}.${2}")
|
||||
|
||||
// 2. Collapse spaces around dots in domain-like strings:
|
||||
// "pitsport . xyz" → "pitsport.xyz"
|
||||
spaceDot := regexp.MustCompile(`(\b\w[\w-]*)\s*\.\s*(` + tlds + `\b)`)
|
||||
text = spaceDot.ReplaceAllString(text, "${1}.${2}")
|
||||
|
||||
// 3. Prepend https:// to bare domain-like strings that the URL regex
|
||||
// would otherwise miss (no scheme present).
|
||||
bareDomain := regexp.MustCompile(`(?:^|[\s(>\[])(\w[\w-]*\.` + tlds + `(?:/[^\s)\]<"]*)?)`)
|
||||
text = bareDomain.ReplaceAllStringFunc(text, func(m string) string {
|
||||
// Preserve the leading whitespace/punctuation character.
|
||||
trimmed := strings.TrimLeft(m, " \t\n(>[")
|
||||
prefix := m[:len(m)-len(trimmed)]
|
||||
if strings.HasPrefix(trimmed, "http://") || strings.HasPrefix(trimmed, "https://") {
|
||||
return m
|
||||
}
|
||||
return prefix + "https://" + trimmed
|
||||
})
|
||||
|
||||
return text
|
||||
}
|
||||
|
||||
func extractURLs(text, postTitle string) []models.ScrapedLink {
|
||||
text = deobfuscateText(text)
|
||||
matches := urlRe.FindAllString(text, -1)
|
||||
var links []models.ScrapedLink
|
||||
filtered := 0
|
||||
for _, u := range matches {
|
||||
u = strings.TrimRight(u, ".,;:!?)")
|
||||
|
||||
parsed, err := url.Parse(u)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if filteredDomains[parsed.Hostname()] {
|
||||
filtered++
|
||||
continue
|
||||
}
|
||||
|
||||
id := make([]byte, 16)
|
||||
if _, err := rand.Read(id); err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
links = append(links, models.ScrapedLink{
|
||||
ID: fmt.Sprintf("%x", id),
|
||||
URL: u,
|
||||
Title: postTitle,
|
||||
Source: "r/motorsportsstreams2",
|
||||
ScrapedAt: time.Now(),
|
||||
})
|
||||
}
|
||||
if filtered > 0 {
|
||||
log.Printf("scraper: filtered %d URLs from known domains in %q", filtered, truncate(postTitle, 40))
|
||||
}
|
||||
return links
|
||||
}
|
||||
|
||||
func walkComments(comments redditComments, fn func(string)) {
|
||||
for _, listing := range comments {
|
||||
for _, child := range listing.Data.Children {
|
||||
if child.Data.Body != "" {
|
||||
fn(child.Data.Body)
|
||||
}
|
||||
// Recurse into replies
|
||||
if len(child.Data.Replies) > 0 && child.Data.Replies[0] == '{' {
|
||||
var nested redditComments
|
||||
if err := json.Unmarshal([]byte("["+string(child.Data.Replies)+"]"), &nested); err == nil {
|
||||
walkComments(nested, fn)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func normalizeURL(u string) string {
|
||||
parsed, err := url.Parse(u)
|
||||
if err != nil {
|
||||
return strings.ToLower(u)
|
||||
}
|
||||
parsed.Host = strings.ToLower(parsed.Host)
|
||||
path := strings.TrimRight(parsed.Path, "/")
|
||||
return fmt.Sprintf("%s://%s%s", parsed.Scheme, parsed.Host, path)
|
||||
}
|
||||
|
||||
func isF1Post(title string) bool {
|
||||
lower := strings.ToLower(title)
|
||||
for _, neg := range f1NegativeKeywords {
|
||||
if strings.Contains(lower, neg) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
for _, kw := range f1Keywords {
|
||||
if strings.Contains(lower, kw) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func truncate(s string, maxLen int) string {
|
||||
if len(s) <= maxLen {
|
||||
return s
|
||||
}
|
||||
return s[:maxLen] + "..."
|
||||
}
|
||||
105
stacks/f1-stream/module/files/internal/scraper/scraper.go
Normal file
105
stacks/f1-stream/module/files/internal/scraper/scraper.go
Normal file
|
|
@ -0,0 +1,105 @@
|
|||
package scraper
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"f1-stream/internal/models"
|
||||
"f1-stream/internal/store"
|
||||
)
|
||||
|
||||
type Scraper struct {
|
||||
store *store.Store
|
||||
interval time.Duration
|
||||
validateTimeout time.Duration
|
||||
mu sync.Mutex
|
||||
}
|
||||
|
||||
func New(s *store.Store, interval time.Duration, validateTimeout time.Duration) *Scraper {
|
||||
return &Scraper{store: s, interval: interval, validateTimeout: validateTimeout}
|
||||
}
|
||||
|
||||
func (s *Scraper) Run(ctx context.Context) {
|
||||
log.Printf("scraper: starting with interval %v", s.interval)
|
||||
// Run immediately on start
|
||||
s.scrape()
|
||||
|
||||
ticker := time.NewTicker(s.interval)
|
||||
defer ticker.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
log.Println("scraper: shutting down")
|
||||
return
|
||||
case <-ticker.C:
|
||||
s.scrape()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Scraper) TriggerScrape() {
|
||||
go s.scrape()
|
||||
}
|
||||
|
||||
func (s *Scraper) scrape() {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
|
||||
start := time.Now()
|
||||
log.Println("scraper: starting scrape")
|
||||
links, err := scrapeReddit()
|
||||
if err != nil {
|
||||
log.Printf("scraper: error after %v: %v", time.Since(start).Round(time.Millisecond), err)
|
||||
return
|
||||
}
|
||||
log.Printf("scraper: reddit scrape completed in %v, got %d links", time.Since(start).Round(time.Millisecond), len(links))
|
||||
|
||||
// Merge with existing links, filtering out non-F1 entries
|
||||
existing, err := s.store.LoadScrapedLinks()
|
||||
if err != nil {
|
||||
log.Printf("scraper: failed to load existing links: %v", err)
|
||||
existing = nil
|
||||
}
|
||||
seen := make(map[string]bool)
|
||||
var filtered []models.ScrapedLink
|
||||
for _, l := range existing {
|
||||
if !isF1Post(l.Title) {
|
||||
continue
|
||||
}
|
||||
norm := normalizeURL(l.URL)
|
||||
seen[norm] = true
|
||||
filtered = append(filtered, l)
|
||||
}
|
||||
existing = filtered
|
||||
|
||||
added := 0
|
||||
for _, l := range links {
|
||||
norm := normalizeURL(l.URL)
|
||||
if !seen[norm] {
|
||||
existing = append(existing, l)
|
||||
seen[norm] = true
|
||||
added++
|
||||
}
|
||||
}
|
||||
|
||||
if err := s.store.SaveScrapedLinks(existing); err != nil {
|
||||
log.Printf("scraper: failed to save: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
// Auto-publish newly validated links as streams
|
||||
for _, l := range links {
|
||||
if err := s.store.PublishScrapedStream(l.URL, l.Title); err != nil {
|
||||
u := l.URL
|
||||
if len(u) > 80 {
|
||||
u = u[:80] + "..."
|
||||
}
|
||||
log.Printf("scraper: failed to auto-publish %s: %v", u, err)
|
||||
}
|
||||
}
|
||||
|
||||
log.Printf("scraper: done in %v, added %d new links (total: %d)", time.Since(start).Round(time.Millisecond), added, len(existing))
|
||||
}
|
||||
142
stacks/f1-stream/module/files/internal/scraper/validate.go
Normal file
142
stacks/f1-stream/module/files/internal/scraper/validate.go
Normal file
|
|
@ -0,0 +1,142 @@
|
|||
package scraper
|
||||
|
||||
import (
|
||||
"io"
|
||||
"log"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"f1-stream/internal/models"
|
||||
)
|
||||
|
||||
// videoMarkers are substrings checked (case-insensitively) against the HTML
|
||||
// body to detect the presence of a video player or streaming manifest.
|
||||
var videoMarkers = []string{
|
||||
// HTML5 video element
|
||||
"<video",
|
||||
// HLS manifests
|
||||
".m3u8",
|
||||
"application/x-mpegurl",
|
||||
"application/vnd.apple.mpegurl",
|
||||
// DASH manifests
|
||||
".mpd",
|
||||
"application/dash+xml",
|
||||
// Player libraries
|
||||
"hls.js",
|
||||
"hls.min.js",
|
||||
"dash.js",
|
||||
"dash.all.min.js",
|
||||
"video.js",
|
||||
"video.min.js",
|
||||
"videojs",
|
||||
"jwplayer",
|
||||
"clappr",
|
||||
"flowplayer",
|
||||
"plyr",
|
||||
"shaka-player",
|
||||
"mediaelement",
|
||||
"fluidplayer",
|
||||
}
|
||||
|
||||
// videoContentTypes are Content-Type prefixes/substrings that indicate a
|
||||
// direct video response (no HTML inspection needed).
|
||||
var videoContentTypes = []string{
|
||||
"video/",
|
||||
"application/x-mpegurl",
|
||||
"application/vnd.apple.mpegurl",
|
||||
"application/dash+xml",
|
||||
}
|
||||
|
||||
// validateBodyLimit caps how much HTML we read when looking for markers.
|
||||
const validateBodyLimit = 2 * 1024 * 1024 // 2 MB
|
||||
|
||||
// validateLinks fetches each link and keeps only those whose response
|
||||
// contains video/player content markers.
|
||||
func validateLinks(links []models.ScrapedLink, timeout time.Duration) []models.ScrapedLink {
|
||||
client := &http.Client{
|
||||
Timeout: timeout,
|
||||
CheckRedirect: func(req *http.Request, via []*http.Request) error {
|
||||
if len(via) >= 3 {
|
||||
return http.ErrUseLastResponse
|
||||
}
|
||||
return nil
|
||||
},
|
||||
}
|
||||
|
||||
var kept []models.ScrapedLink
|
||||
for _, link := range links {
|
||||
if HasVideoContent(client, link.URL) {
|
||||
kept = append(kept, link)
|
||||
} else {
|
||||
log.Printf("scraper: discarded %s (no video markers)", truncate(link.URL, 60))
|
||||
}
|
||||
}
|
||||
return kept
|
||||
}
|
||||
|
||||
// HasVideoContent performs a GET request for rawURL and returns true if the
|
||||
// response is a direct video file (by Content-Type) or an HTML page that
|
||||
// contains at least one video marker substring.
|
||||
func HasVideoContent(client *http.Client, rawURL string) bool {
|
||||
req, err := http.NewRequest("GET", rawURL, nil)
|
||||
if err != nil {
|
||||
log.Printf("scraper: validate request error for %s: %v", truncate(rawURL, 60), err)
|
||||
return false
|
||||
}
|
||||
req.Header.Set("User-Agent", userAgent)
|
||||
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
log.Printf("scraper: validate fetch error for %s: %v", truncate(rawURL, 60), err)
|
||||
return false
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode < 200 || resp.StatusCode >= 400 {
|
||||
return false
|
||||
}
|
||||
|
||||
ct := strings.ToLower(resp.Header.Get("Content-Type"))
|
||||
|
||||
// Direct video content type — no need to inspect body.
|
||||
if isDirectVideoContentType(ct) {
|
||||
return true
|
||||
}
|
||||
|
||||
// Only inspect HTML pages for markers.
|
||||
if !strings.Contains(ct, "text/html") && !strings.Contains(ct, "application/xhtml") {
|
||||
return false
|
||||
}
|
||||
|
||||
body, err := io.ReadAll(io.LimitReader(resp.Body, validateBodyLimit))
|
||||
if err != nil {
|
||||
log.Printf("scraper: validate read error for %s: %v", truncate(rawURL, 60), err)
|
||||
return false
|
||||
}
|
||||
|
||||
return containsVideoMarkers(strings.ToLower(string(body)))
|
||||
}
|
||||
|
||||
// containsVideoMarkers returns true if loweredBody contains any known video
|
||||
// player or streaming marker substring.
|
||||
func containsVideoMarkers(loweredBody string) bool {
|
||||
for _, marker := range videoMarkers {
|
||||
if strings.Contains(loweredBody, marker) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// isDirectVideoContentType returns true if ct (already lowercased) matches a
|
||||
// known video content type.
|
||||
func isDirectVideoContentType(ct string) bool {
|
||||
ct = strings.ToLower(ct)
|
||||
for _, vct := range videoContentTypes {
|
||||
if strings.Contains(ct, vct) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
124
stacks/f1-stream/module/files/internal/scraper/validate_test.go
Normal file
124
stacks/f1-stream/module/files/internal/scraper/validate_test.go
Normal file
|
|
@ -0,0 +1,124 @@
|
|||
package scraper
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestContainsVideoMarkers(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
body string
|
||||
want bool
|
||||
}{
|
||||
// Positive cases
|
||||
{
|
||||
name: "video tag",
|
||||
body: `<div><video src="stream.mp4"></video></div>`,
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "HLS manifest reference",
|
||||
body: `var url = "https://cdn.example.com/live.m3u8";`,
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "DASH manifest reference",
|
||||
body: `<source src="stream.mpd" type="application/dash+xml">`,
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "HLS.js library",
|
||||
body: `<script src="/js/hls.min.js"></script>`,
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "Video.js library",
|
||||
body: `<script src="https://cdn.example.com/video.js"></script>`,
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "JW Player",
|
||||
body: `<div id="jwplayer-container"></div><script>jwplayer("jwplayer-container")</script>`,
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "Clappr player",
|
||||
body: `<script src="clappr.min.js"></script>`,
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "Flowplayer",
|
||||
body: `<script>flowplayer("#player")</script>`,
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "Plyr player",
|
||||
body: `<link rel="stylesheet" href="plyr.css"><script src="plyr.js"></script>`,
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "Shaka Player",
|
||||
body: `<script src="shaka-player.compiled.js"></script>`,
|
||||
want: true,
|
||||
},
|
||||
// Negative cases
|
||||
{
|
||||
name: "plain HTML",
|
||||
body: `<html><body><p>Hello world</p></body></html>`,
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "reddit link page",
|
||||
body: `<html><body><a href="https://example.com">Click here</a></body></html>`,
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "blog post",
|
||||
body: `<html><body><article>F1 race results and analysis...</article></body></html>`,
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "empty string",
|
||||
body: "",
|
||||
want: false,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := containsVideoMarkers(tt.body)
|
||||
if got != tt.want {
|
||||
t.Errorf("containsVideoMarkers(%q) = %v, want %v", truncate(tt.body, 60), got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsDirectVideoContentType(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
ct string
|
||||
want bool
|
||||
}{
|
||||
// Positive cases
|
||||
{name: "video/mp4", ct: "video/mp4", want: true},
|
||||
{name: "video/webm", ct: "video/webm", want: true},
|
||||
{name: "HLS content type", ct: "application/x-mpegurl", want: true},
|
||||
{name: "Apple HLS content type", ct: "application/vnd.apple.mpegurl", want: true},
|
||||
{name: "DASH content type", ct: "application/dash+xml", want: true},
|
||||
{name: "video with params", ct: "video/mp4; charset=utf-8", want: true},
|
||||
// Negative cases
|
||||
{name: "text/html", ct: "text/html", want: false},
|
||||
{name: "application/json", ct: "application/json", want: false},
|
||||
{name: "image/png", ct: "image/png", want: false},
|
||||
{name: "text/plain", ct: "text/plain", want: false},
|
||||
{name: "empty string", ct: "", want: false},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := isDirectVideoContentType(tt.ct)
|
||||
if got != tt.want {
|
||||
t.Errorf("isDirectVideoContentType(%q) = %v, want %v", tt.ct, got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
93
stacks/f1-stream/module/files/internal/server/middleware.go
Normal file
93
stacks/f1-stream/module/files/internal/server/middleware.go
Normal file
|
|
@ -0,0 +1,93 @@
|
|||
package server
|
||||
|
||||
import (
|
||||
"log"
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"f1-stream/internal/auth"
|
||||
)
|
||||
|
||||
func LoggingMiddleware(next http.Handler) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
log.Printf("%s %s %s", r.Method, r.URL.Path, r.RemoteAddr)
|
||||
next.ServeHTTP(w, r)
|
||||
})
|
||||
}
|
||||
|
||||
func RecoveryMiddleware(next http.Handler) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
defer func() {
|
||||
if err := recover(); err != nil {
|
||||
log.Printf("panic: %v", err)
|
||||
http.Error(w, "internal server error", http.StatusInternalServerError)
|
||||
}
|
||||
}()
|
||||
next.ServeHTTP(w, r)
|
||||
})
|
||||
}
|
||||
|
||||
// AuthMiddleware injects user into context if session cookie is present.
|
||||
func AuthMiddleware(a *auth.Auth) func(http.Handler) http.Handler {
|
||||
return func(next http.Handler) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
cookie, err := r.Cookie("session")
|
||||
if err == nil && cookie.Value != "" {
|
||||
user, err := a.GetSessionUser(cookie.Value)
|
||||
if err == nil && user != nil {
|
||||
r = r.WithContext(auth.ContextWithUser(r.Context(), user))
|
||||
}
|
||||
}
|
||||
next.ServeHTTP(w, r)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// RequireAuth rejects unauthenticated requests.
|
||||
func RequireAuth(next http.HandlerFunc) http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
user := auth.UserFromContext(r.Context())
|
||||
if user == nil {
|
||||
http.Error(w, `{"error":"authentication required"}`, http.StatusUnauthorized)
|
||||
return
|
||||
}
|
||||
next(w, r)
|
||||
}
|
||||
}
|
||||
|
||||
// RequireAdmin rejects non-admin requests.
|
||||
func RequireAdmin(next http.HandlerFunc) http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
user := auth.UserFromContext(r.Context())
|
||||
if user == nil || !user.IsAdmin {
|
||||
http.Error(w, `{"error":"admin access required"}`, http.StatusForbidden)
|
||||
return
|
||||
}
|
||||
next(w, r)
|
||||
}
|
||||
}
|
||||
|
||||
// OriginCheck validates Origin header on mutation requests (CSRF protection).
|
||||
func OriginCheck(allowedOrigins []string) func(http.Handler) http.Handler {
|
||||
return func(next http.Handler) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != "GET" && r.Method != "HEAD" && r.Method != "OPTIONS" {
|
||||
origin := r.Header.Get("Origin")
|
||||
if origin != "" {
|
||||
allowed := false
|
||||
for _, o := range allowedOrigins {
|
||||
if strings.EqualFold(origin, o) {
|
||||
allowed = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !allowed {
|
||||
http.Error(w, `{"error":"origin not allowed"}`, http.StatusForbidden)
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
next.ServeHTTP(w, r)
|
||||
})
|
||||
}
|
||||
}
|
||||
338
stacks/f1-stream/module/files/internal/server/server.go
Normal file
338
stacks/f1-stream/module/files/internal/server/server.go
Normal file
|
|
@ -0,0 +1,338 @@
|
|||
package server
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"html"
|
||||
"log"
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"f1-stream/internal/auth"
|
||||
"f1-stream/internal/extractor"
|
||||
"f1-stream/internal/hlsproxy"
|
||||
"f1-stream/internal/playerconfig"
|
||||
"f1-stream/internal/proxy"
|
||||
"f1-stream/internal/scraper"
|
||||
"f1-stream/internal/store"
|
||||
)
|
||||
|
||||
type Server struct {
|
||||
store *store.Store
|
||||
auth *auth.Auth
|
||||
scraper *scraper.Scraper
|
||||
playerConfig *playerconfig.Service
|
||||
mux *http.ServeMux
|
||||
headlessEnabled bool
|
||||
}
|
||||
|
||||
func New(s *store.Store, a *auth.Auth, sc *scraper.Scraper, pc *playerconfig.Service, origins []string, headlessEnabled bool) *Server {
|
||||
srv := &Server{
|
||||
store: s,
|
||||
auth: a,
|
||||
scraper: sc,
|
||||
playerConfig: pc,
|
||||
mux: http.NewServeMux(),
|
||||
headlessEnabled: headlessEnabled,
|
||||
}
|
||||
srv.registerRoutes(origins)
|
||||
return srv
|
||||
}
|
||||
|
||||
func (s *Server) Handler() http.Handler {
|
||||
return s.mux
|
||||
}
|
||||
|
||||
func (s *Server) registerRoutes(origins []string) {
|
||||
// Apply middleware chain
|
||||
authMw := AuthMiddleware(s.auth)
|
||||
originMw := OriginCheck(origins)
|
||||
|
||||
// Static files
|
||||
fs := http.FileServer(http.Dir("static"))
|
||||
s.mux.Handle("GET /static/", http.StripPrefix("/static/", fs))
|
||||
s.mux.HandleFunc("GET /", func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path != "/" {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
http.ServeFile(w, r, "static/index.html")
|
||||
})
|
||||
|
||||
// Health
|
||||
s.mux.HandleFunc("GET /api/health", func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.Write([]byte(`{"status":"ok"}`))
|
||||
})
|
||||
|
||||
// Reverse proxy for iframe embedding (strips anti-framing headers)
|
||||
proxyHandler := proxy.NewHandler()
|
||||
s.mux.Handle("GET /proxy/", proxyHandler)
|
||||
s.mux.Handle("POST /proxy/", proxyHandler)
|
||||
s.mux.Handle("HEAD /proxy/", proxyHandler)
|
||||
s.mux.Handle("OPTIONS /proxy/", proxyHandler)
|
||||
|
||||
// HLS proxy for native video playback
|
||||
hlsHandler := hlsproxy.NewHandler()
|
||||
s.mux.Handle("GET /hls/", hlsHandler)
|
||||
s.mux.Handle("OPTIONS /hls/", hlsHandler)
|
||||
|
||||
// Public API - wrap with middleware
|
||||
wrapAll := func(h http.HandlerFunc) http.Handler {
|
||||
return RecoveryMiddleware(LoggingMiddleware(originMw(authMw(h))))
|
||||
}
|
||||
|
||||
// Auth endpoints
|
||||
s.mux.Handle("POST /api/auth/register/begin", wrapAll(s.auth.BeginRegistration))
|
||||
s.mux.Handle("POST /api/auth/register/finish", wrapAll(s.auth.FinishRegistration))
|
||||
s.mux.Handle("POST /api/auth/login/begin", wrapAll(s.auth.BeginLogin))
|
||||
s.mux.Handle("POST /api/auth/login/finish", wrapAll(s.auth.FinishLogin))
|
||||
s.mux.Handle("POST /api/auth/logout", wrapAll(s.auth.Logout))
|
||||
s.mux.Handle("GET /api/auth/me", wrapAll(s.auth.Me))
|
||||
|
||||
// Public streams
|
||||
s.mux.Handle("GET /api/streams/public", wrapAll(s.handlePublicStreams))
|
||||
s.mux.Handle("GET /api/streams/{id}/browse", wrapAll(s.handleBrowseStream))
|
||||
s.mux.Handle("GET /api/streams/{id}/player-config", wrapAll(s.handlePlayerConfig))
|
||||
|
||||
// Scraped links
|
||||
s.mux.Handle("GET /api/scraped", wrapAll(s.handleScrapedLinks))
|
||||
s.mux.Handle("POST /api/scraped/refresh", wrapAll(s.handleTriggerScrape))
|
||||
s.mux.Handle("POST /api/scraped/{id}/import", wrapAll(s.handleImportScraped))
|
||||
|
||||
// Authenticated endpoints
|
||||
s.mux.Handle("GET /api/streams/mine", wrapAll(RequireAuth(s.handleMyStreams)))
|
||||
s.mux.Handle("POST /api/streams", wrapAll(s.handleSubmitStream))
|
||||
s.mux.Handle("DELETE /api/streams/{id}", wrapAll(s.handleDeleteStream))
|
||||
|
||||
// Admin endpoints
|
||||
s.mux.Handle("PUT /api/streams/{id}/publish", wrapAll(RequireAdmin(s.handleTogglePublish)))
|
||||
s.mux.Handle("GET /api/admin/streams", wrapAll(RequireAdmin(s.handleAllStreams)))
|
||||
s.mux.Handle("POST /api/admin/scrape", wrapAll(RequireAdmin(s.handleTriggerScrape)))
|
||||
}
|
||||
|
||||
func (s *Server) handlePublicStreams(w http.ResponseWriter, r *http.Request) {
|
||||
streams, err := s.store.PublicStreams()
|
||||
if err != nil {
|
||||
http.Error(w, `{"error":"internal error"}`, http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
json.NewEncoder(w).Encode(streams)
|
||||
}
|
||||
|
||||
func (s *Server) handleScrapedLinks(w http.ResponseWriter, r *http.Request) {
|
||||
links, err := s.store.GetActiveScrapedLinks()
|
||||
if err != nil {
|
||||
http.Error(w, `{"error":"internal error"}`, http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
if links == nil {
|
||||
w.Write([]byte("[]"))
|
||||
return
|
||||
}
|
||||
json.NewEncoder(w).Encode(links)
|
||||
}
|
||||
|
||||
func (s *Server) handleImportScraped(w http.ResponseWriter, r *http.Request) {
|
||||
id := r.PathValue("id")
|
||||
link, err := s.store.GetScrapedLinkByID(id)
|
||||
if err != nil {
|
||||
http.Error(w, `{"error":"not found"}`, http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
if err := s.store.PublishScrapedStream(link.URL, link.Title); err != nil {
|
||||
http.Error(w, `{"error":"internal error"}`, http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.Write([]byte(`{"ok":true}`))
|
||||
}
|
||||
|
||||
func (s *Server) handleMyStreams(w http.ResponseWriter, r *http.Request) {
|
||||
user := auth.UserFromContext(r.Context())
|
||||
streams, err := s.store.UserStreams(user.ID)
|
||||
if err != nil {
|
||||
http.Error(w, `{"error":"internal error"}`, http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
if streams == nil {
|
||||
w.Write([]byte("[]"))
|
||||
return
|
||||
}
|
||||
json.NewEncoder(w).Encode(streams)
|
||||
}
|
||||
|
||||
func (s *Server) handleSubmitStream(w http.ResponseWriter, r *http.Request) {
|
||||
user := auth.UserFromContext(r.Context())
|
||||
|
||||
var req struct {
|
||||
URL string `json:"url"`
|
||||
Title string `json:"title"`
|
||||
}
|
||||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
||||
http.Error(w, `{"error":"invalid request"}`, http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
req.URL = strings.TrimSpace(req.URL)
|
||||
req.Title = strings.TrimSpace(req.Title)
|
||||
|
||||
if req.URL == "" {
|
||||
http.Error(w, `{"error":"url required"}`, http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
if len(req.URL) > 2048 {
|
||||
http.Error(w, `{"error":"url too long"}`, http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
if !strings.HasPrefix(req.URL, "https://") && !strings.HasPrefix(req.URL, "http://") {
|
||||
http.Error(w, `{"error":"url must start with http:// or https://"}`, http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
if req.Title == "" {
|
||||
req.Title = req.URL
|
||||
}
|
||||
req.Title = html.EscapeString(req.Title)
|
||||
|
||||
submittedBy := "anonymous"
|
||||
published := true
|
||||
if user != nil {
|
||||
submittedBy = user.ID
|
||||
published = false
|
||||
}
|
||||
|
||||
stream, err := s.store.AddStream(req.URL, req.Title, submittedBy, published, "user")
|
||||
if err != nil {
|
||||
http.Error(w, `{"error":"internal error"}`, http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(http.StatusCreated)
|
||||
json.NewEncoder(w).Encode(stream)
|
||||
}
|
||||
|
||||
func (s *Server) handleDeleteStream(w http.ResponseWriter, r *http.Request) {
|
||||
user := auth.UserFromContext(r.Context())
|
||||
id := r.PathValue("id")
|
||||
|
||||
var userID string
|
||||
var isAdmin bool
|
||||
if user != nil {
|
||||
userID = user.ID
|
||||
isAdmin = user.IsAdmin
|
||||
}
|
||||
|
||||
if err := s.store.DeleteStream(id, userID, isAdmin); err != nil {
|
||||
if strings.Contains(err.Error(), "not authorized") {
|
||||
http.Error(w, `{"error":"not authorized"}`, http.StatusForbidden)
|
||||
return
|
||||
}
|
||||
if strings.Contains(err.Error(), "not found") {
|
||||
http.Error(w, `{"error":"not found"}`, http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
http.Error(w, `{"error":"internal error"}`, http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.Write([]byte(`{"ok":true}`))
|
||||
}
|
||||
|
||||
func (s *Server) handleTogglePublish(w http.ResponseWriter, r *http.Request) {
|
||||
id := r.PathValue("id")
|
||||
if err := s.store.TogglePublish(id); err != nil {
|
||||
http.Error(w, `{"error":"stream not found"}`, http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.Write([]byte(`{"ok":true}`))
|
||||
}
|
||||
|
||||
func (s *Server) handleAllStreams(w http.ResponseWriter, r *http.Request) {
|
||||
streams, err := s.store.LoadStreams()
|
||||
if err != nil {
|
||||
http.Error(w, `{"error":"internal error"}`, http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
json.NewEncoder(w).Encode(streams)
|
||||
}
|
||||
|
||||
func (s *Server) handleTriggerScrape(w http.ResponseWriter, r *http.Request) {
|
||||
s.scraper.TriggerScrape()
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.Write([]byte(`{"ok":true,"message":"scrape triggered"}`))
|
||||
}
|
||||
|
||||
func (s *Server) handleBrowseStream(w http.ResponseWriter, r *http.Request) {
|
||||
if !s.headlessEnabled {
|
||||
http.Error(w, `{"error":"browser sessions not available"}`, http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
|
||||
id := r.PathValue("id")
|
||||
streams, err := s.store.LoadStreams()
|
||||
if err != nil {
|
||||
log.Printf("server: browse: failed to load streams: %v", err)
|
||||
http.Error(w, `{"error":"internal error"}`, http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
var streamURL string
|
||||
var found bool
|
||||
for _, st := range streams {
|
||||
if st.ID == id {
|
||||
if !st.Published {
|
||||
http.Error(w, `{"error":"stream not found"}`, http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
streamURL = st.URL
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
http.Error(w, `{"error":"stream not found"}`, http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
|
||||
extractor.HandleBrowserSession(w, r, streamURL)
|
||||
}
|
||||
|
||||
func (s *Server) handlePlayerConfig(w http.ResponseWriter, r *http.Request) {
|
||||
id := r.PathValue("id")
|
||||
streams, err := s.store.LoadStreams()
|
||||
if err != nil {
|
||||
log.Printf("server: player-config: failed to load streams: %v", err)
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
json.NewEncoder(w).Encode(playerconfig.PlayerConfig{Type: "proxy"})
|
||||
return
|
||||
}
|
||||
|
||||
var streamURL string
|
||||
var found bool
|
||||
for _, st := range streams {
|
||||
if st.ID == id {
|
||||
if !st.Published {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
json.NewEncoder(w).Encode(playerconfig.PlayerConfig{Type: "proxy"})
|
||||
return
|
||||
}
|
||||
streamURL = st.URL
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
json.NewEncoder(w).Encode(playerconfig.PlayerConfig{Type: "proxy"})
|
||||
return
|
||||
}
|
||||
|
||||
config := s.playerConfig.GetConfig(r.Context(), streamURL)
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
json.NewEncoder(w).Encode(config)
|
||||
}
|
||||
37
stacks/f1-stream/module/files/internal/store/health.go
Normal file
37
stacks/f1-stream/module/files/internal/store/health.go
Normal file
|
|
@ -0,0 +1,37 @@
|
|||
package store
|
||||
|
||||
import (
|
||||
"f1-stream/internal/models"
|
||||
)
|
||||
|
||||
func (s *Store) LoadHealthStates() ([]models.HealthState, error) {
|
||||
s.healthMu.RLock()
|
||||
defer s.healthMu.RUnlock()
|
||||
var states []models.HealthState
|
||||
if err := readJSON(s.filePath("health_state.json"), &states); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return states, nil
|
||||
}
|
||||
|
||||
func (s *Store) SaveHealthStates(states []models.HealthState) error {
|
||||
s.healthMu.Lock()
|
||||
defer s.healthMu.Unlock()
|
||||
return writeJSON(s.filePath("health_state.json"), states)
|
||||
}
|
||||
|
||||
// HealthMap returns a map of URL -> Healthy status. It reads the health state
|
||||
// file directly without acquiring healthMu to avoid deadlock when called from
|
||||
// methods that already hold other locks (e.g., PublicStreams, GetActiveScrapedLinks).
|
||||
// URLs not present in the map are implicitly healthy.
|
||||
func (s *Store) HealthMap() map[string]bool {
|
||||
var states []models.HealthState
|
||||
if err := readJSON(s.filePath("health_state.json"), &states); err != nil {
|
||||
return make(map[string]bool)
|
||||
}
|
||||
m := make(map[string]bool, len(states))
|
||||
for _, st := range states {
|
||||
m[st.URL] = st.Healthy
|
||||
}
|
||||
return m
|
||||
}
|
||||
63
stacks/f1-stream/module/files/internal/store/scraped.go
Normal file
63
stacks/f1-stream/module/files/internal/store/scraped.go
Normal file
|
|
@ -0,0 +1,63 @@
|
|||
package store
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"f1-stream/internal/models"
|
||||
)
|
||||
|
||||
func (s *Store) LoadScrapedLinks() ([]models.ScrapedLink, error) {
|
||||
s.scrapedMu.RLock()
|
||||
defer s.scrapedMu.RUnlock()
|
||||
var links []models.ScrapedLink
|
||||
if err := readJSON(s.filePath("scraped_links.json"), &links); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return links, nil
|
||||
}
|
||||
|
||||
func (s *Store) SaveScrapedLinks(links []models.ScrapedLink) error {
|
||||
s.scrapedMu.Lock()
|
||||
defer s.scrapedMu.Unlock()
|
||||
return writeJSON(s.filePath("scraped_links.json"), links)
|
||||
}
|
||||
|
||||
func (s *Store) GetScrapedLinkByID(id string) (models.ScrapedLink, error) {
|
||||
s.scrapedMu.RLock()
|
||||
defer s.scrapedMu.RUnlock()
|
||||
var links []models.ScrapedLink
|
||||
if err := readJSON(s.filePath("scraped_links.json"), &links); err != nil {
|
||||
return models.ScrapedLink{}, err
|
||||
}
|
||||
for _, l := range links {
|
||||
if l.ID == id {
|
||||
return l, nil
|
||||
}
|
||||
}
|
||||
return models.ScrapedLink{}, fmt.Errorf("not found")
|
||||
}
|
||||
|
||||
func (s *Store) GetActiveScrapedLinks() ([]models.ScrapedLink, error) {
|
||||
s.scrapedMu.RLock()
|
||||
defer s.scrapedMu.RUnlock()
|
||||
var links []models.ScrapedLink
|
||||
if err := readJSON(s.filePath("scraped_links.json"), &links); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
healthMap := s.HealthMap()
|
||||
now := time.Now()
|
||||
var active []models.ScrapedLink
|
||||
for _, l := range links {
|
||||
l.Stale = now.Sub(l.ScrapedAt) > 7*24*time.Hour
|
||||
if l.Stale {
|
||||
continue
|
||||
}
|
||||
// Filter unhealthy scraped links. URLs not in healthMap are assumed healthy.
|
||||
if healthy, exists := healthMap[l.URL]; exists && !healthy {
|
||||
continue
|
||||
}
|
||||
active = append(active, l)
|
||||
}
|
||||
return active, nil
|
||||
}
|
||||
98
stacks/f1-stream/module/files/internal/store/sessions.go
Normal file
98
stacks/f1-stream/module/files/internal/store/sessions.go
Normal file
|
|
@ -0,0 +1,98 @@
|
|||
package store
|
||||
|
||||
import (
|
||||
"crypto/rand"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"f1-stream/internal/models"
|
||||
)
|
||||
|
||||
func (s *Store) LoadSessions() ([]models.Session, error) {
|
||||
s.sessionsMu.RLock()
|
||||
defer s.sessionsMu.RUnlock()
|
||||
var sessions []models.Session
|
||||
if err := readJSON(s.filePath("sessions.json"), &sessions); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return sessions, nil
|
||||
}
|
||||
|
||||
func (s *Store) CreateSession(userID string, ttl time.Duration) (string, error) {
|
||||
s.sessionsMu.Lock()
|
||||
defer s.sessionsMu.Unlock()
|
||||
var sessions []models.Session
|
||||
if err := readJSON(s.filePath("sessions.json"), &sessions); err != nil {
|
||||
return "", err
|
||||
}
|
||||
b := make([]byte, 32)
|
||||
if _, err := rand.Read(b); err != nil {
|
||||
return "", err
|
||||
}
|
||||
token := hex.EncodeToString(b)
|
||||
sess := models.Session{
|
||||
Token: token,
|
||||
UserID: userID,
|
||||
ExpiresAt: time.Now().Add(ttl),
|
||||
}
|
||||
sessions = append(sessions, sess)
|
||||
if err := writeJSON(s.filePath("sessions.json"), sessions); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return token, nil
|
||||
}
|
||||
|
||||
func (s *Store) GetSession(token string) (*models.Session, error) {
|
||||
s.sessionsMu.RLock()
|
||||
defer s.sessionsMu.RUnlock()
|
||||
var sessions []models.Session
|
||||
if err := readJSON(s.filePath("sessions.json"), &sessions); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, sess := range sessions {
|
||||
if sess.Token == token && time.Now().Before(sess.ExpiresAt) {
|
||||
return &sess, nil
|
||||
}
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (s *Store) DeleteSession(token string) error {
|
||||
s.sessionsMu.Lock()
|
||||
defer s.sessionsMu.Unlock()
|
||||
var sessions []models.Session
|
||||
if err := readJSON(s.filePath("sessions.json"), &sessions); err != nil {
|
||||
return err
|
||||
}
|
||||
var updated []models.Session
|
||||
found := false
|
||||
for _, sess := range sessions {
|
||||
if sess.Token == token {
|
||||
found = true
|
||||
continue
|
||||
}
|
||||
updated = append(updated, sess)
|
||||
}
|
||||
if !found {
|
||||
return fmt.Errorf("session not found")
|
||||
}
|
||||
return writeJSON(s.filePath("sessions.json"), updated)
|
||||
}
|
||||
|
||||
func (s *Store) CleanExpiredSessions() error {
|
||||
s.sessionsMu.Lock()
|
||||
defer s.sessionsMu.Unlock()
|
||||
var sessions []models.Session
|
||||
if err := readJSON(s.filePath("sessions.json"), &sessions); err != nil {
|
||||
return err
|
||||
}
|
||||
now := time.Now()
|
||||
var valid []models.Session
|
||||
for _, sess := range sessions {
|
||||
if now.Before(sess.ExpiresAt) {
|
||||
valid = append(valid, sess)
|
||||
}
|
||||
}
|
||||
return writeJSON(s.filePath("sessions.json"), valid)
|
||||
}
|
||||
53
stacks/f1-stream/module/files/internal/store/store.go
Normal file
53
stacks/f1-stream/module/files/internal/store/store.go
Normal file
|
|
@ -0,0 +1,53 @@
|
|||
package store
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
)
|
||||
|
||||
type Store struct {
|
||||
dir string
|
||||
streamsMu sync.RWMutex
|
||||
usersMu sync.RWMutex
|
||||
scrapedMu sync.RWMutex
|
||||
sessionsMu sync.RWMutex
|
||||
healthMu sync.RWMutex
|
||||
}
|
||||
|
||||
func New(dir string) (*Store, error) {
|
||||
if err := os.MkdirAll(dir, 0755); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &Store{dir: dir}, nil
|
||||
}
|
||||
|
||||
func (s *Store) filePath(name string) string {
|
||||
return filepath.Join(s.dir, name)
|
||||
}
|
||||
|
||||
// readJSON reads a JSON file into the target. Returns nil if file doesn't exist.
|
||||
func readJSON(path string, target interface{}) error {
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
return json.Unmarshal(data, target)
|
||||
}
|
||||
|
||||
// writeJSON atomically writes target as JSON to path using temp-file-then-rename.
|
||||
func writeJSON(path string, data interface{}) error {
|
||||
b, err := json.MarshalIndent(data, "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
tmp := path + ".tmp"
|
||||
if err := os.WriteFile(tmp, b, 0644); err != nil {
|
||||
return err
|
||||
}
|
||||
return os.Rename(tmp, path)
|
||||
}
|
||||
176
stacks/f1-stream/module/files/internal/store/streams.go
Normal file
176
stacks/f1-stream/module/files/internal/store/streams.go
Normal file
|
|
@ -0,0 +1,176 @@
|
|||
package store
|
||||
|
||||
import (
|
||||
"crypto/rand"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"f1-stream/internal/models"
|
||||
)
|
||||
|
||||
func (s *Store) LoadStreams() ([]models.Stream, error) {
|
||||
s.streamsMu.RLock()
|
||||
defer s.streamsMu.RUnlock()
|
||||
var streams []models.Stream
|
||||
if err := readJSON(s.filePath("streams.json"), &streams); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return streams, nil
|
||||
}
|
||||
|
||||
func (s *Store) PublicStreams() ([]models.Stream, error) {
|
||||
s.streamsMu.RLock()
|
||||
defer s.streamsMu.RUnlock()
|
||||
var streams []models.Stream
|
||||
if err := readJSON(s.filePath("streams.json"), &streams); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
healthMap := s.HealthMap()
|
||||
var pub []models.Stream
|
||||
for _, st := range streams {
|
||||
if !st.Published {
|
||||
continue
|
||||
}
|
||||
// Filter unhealthy streams. URLs not in healthMap are assumed healthy (new/unchecked).
|
||||
if healthy, exists := healthMap[st.URL]; exists && !healthy {
|
||||
continue
|
||||
}
|
||||
pub = append(pub, st)
|
||||
}
|
||||
return pub, nil
|
||||
}
|
||||
|
||||
func (s *Store) UserStreams(userID string) ([]models.Stream, error) {
|
||||
s.streamsMu.RLock()
|
||||
defer s.streamsMu.RUnlock()
|
||||
var streams []models.Stream
|
||||
if err := readJSON(s.filePath("streams.json"), &streams); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var result []models.Stream
|
||||
for _, st := range streams {
|
||||
if st.SubmittedBy == userID {
|
||||
result = append(result, st)
|
||||
}
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (s *Store) AddStream(url, title, submittedBy string, published bool, source string) (models.Stream, error) {
|
||||
s.streamsMu.Lock()
|
||||
defer s.streamsMu.Unlock()
|
||||
var streams []models.Stream
|
||||
if err := readJSON(s.filePath("streams.json"), &streams); err != nil {
|
||||
return models.Stream{}, err
|
||||
}
|
||||
id, err := randomID()
|
||||
if err != nil {
|
||||
return models.Stream{}, err
|
||||
}
|
||||
st := models.Stream{
|
||||
ID: id,
|
||||
URL: url,
|
||||
Title: title,
|
||||
SubmittedBy: submittedBy,
|
||||
Published: published,
|
||||
Source: source,
|
||||
CreatedAt: time.Now(),
|
||||
}
|
||||
streams = append(streams, st)
|
||||
if err := writeJSON(s.filePath("streams.json"), streams); err != nil {
|
||||
return models.Stream{}, err
|
||||
}
|
||||
return st, nil
|
||||
}
|
||||
|
||||
func (s *Store) PublishScrapedStream(url, title string) error {
|
||||
s.streamsMu.Lock()
|
||||
defer s.streamsMu.Unlock()
|
||||
var streams []models.Stream
|
||||
if err := readJSON(s.filePath("streams.json"), &streams); err != nil {
|
||||
return err
|
||||
}
|
||||
// Deduplicate: skip if URL already exists in streams
|
||||
for _, st := range streams {
|
||||
if st.URL == url {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
id, err := randomID()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
streams = append(streams, models.Stream{
|
||||
ID: id,
|
||||
URL: url,
|
||||
Title: title,
|
||||
SubmittedBy: "scraper",
|
||||
Published: true,
|
||||
Source: "scraped",
|
||||
CreatedAt: time.Now(),
|
||||
})
|
||||
return writeJSON(s.filePath("streams.json"), streams)
|
||||
}
|
||||
|
||||
func (s *Store) DeleteStream(id, userID string, isAdmin bool) error {
|
||||
s.streamsMu.Lock()
|
||||
defer s.streamsMu.Unlock()
|
||||
var streams []models.Stream
|
||||
if err := readJSON(s.filePath("streams.json"), &streams); err != nil {
|
||||
return err
|
||||
}
|
||||
var updated []models.Stream
|
||||
found := false
|
||||
for _, st := range streams {
|
||||
if st.ID == id {
|
||||
if userID != "" && !isAdmin && st.SubmittedBy != userID {
|
||||
return fmt.Errorf("not authorized")
|
||||
}
|
||||
found = true
|
||||
continue
|
||||
}
|
||||
updated = append(updated, st)
|
||||
}
|
||||
if !found {
|
||||
return fmt.Errorf("stream not found")
|
||||
}
|
||||
return writeJSON(s.filePath("streams.json"), updated)
|
||||
}
|
||||
|
||||
func (s *Store) TogglePublish(id string) error {
|
||||
s.streamsMu.Lock()
|
||||
defer s.streamsMu.Unlock()
|
||||
var streams []models.Stream
|
||||
if err := readJSON(s.filePath("streams.json"), &streams); err != nil {
|
||||
return err
|
||||
}
|
||||
for i, st := range streams {
|
||||
if st.ID == id {
|
||||
streams[i].Published = !st.Published
|
||||
return writeJSON(s.filePath("streams.json"), streams)
|
||||
}
|
||||
}
|
||||
return fmt.Errorf("stream not found")
|
||||
}
|
||||
|
||||
func (s *Store) SeedStreams(defaults []models.Stream) error {
|
||||
s.streamsMu.Lock()
|
||||
defer s.streamsMu.Unlock()
|
||||
var existing []models.Stream
|
||||
if err := readJSON(s.filePath("streams.json"), &existing); err != nil {
|
||||
return err
|
||||
}
|
||||
if len(existing) > 0 {
|
||||
return nil
|
||||
}
|
||||
return writeJSON(s.filePath("streams.json"), defaults)
|
||||
}
|
||||
|
||||
func randomID() (string, error) {
|
||||
b := make([]byte, 16)
|
||||
if _, err := rand.Read(b); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return hex.EncodeToString(b), nil
|
||||
}
|
||||
91
stacks/f1-stream/module/files/internal/store/users.go
Normal file
91
stacks/f1-stream/module/files/internal/store/users.go
Normal file
|
|
@ -0,0 +1,91 @@
|
|||
package store
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"f1-stream/internal/models"
|
||||
|
||||
"github.com/go-webauthn/webauthn/webauthn"
|
||||
)
|
||||
|
||||
func (s *Store) LoadUsers() ([]models.User, error) {
|
||||
s.usersMu.RLock()
|
||||
defer s.usersMu.RUnlock()
|
||||
var users []models.User
|
||||
if err := readJSON(s.filePath("users.json"), &users); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return users, nil
|
||||
}
|
||||
|
||||
func (s *Store) GetUserByName(username string) (*models.User, error) {
|
||||
s.usersMu.RLock()
|
||||
defer s.usersMu.RUnlock()
|
||||
var users []models.User
|
||||
if err := readJSON(s.filePath("users.json"), &users); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, u := range users {
|
||||
if u.Username == username {
|
||||
return &u, nil
|
||||
}
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (s *Store) GetUserByID(id string) (*models.User, error) {
|
||||
s.usersMu.RLock()
|
||||
defer s.usersMu.RUnlock()
|
||||
var users []models.User
|
||||
if err := readJSON(s.filePath("users.json"), &users); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, u := range users {
|
||||
if u.ID == id {
|
||||
return &u, nil
|
||||
}
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (s *Store) CreateUser(user models.User) error {
|
||||
s.usersMu.Lock()
|
||||
defer s.usersMu.Unlock()
|
||||
var users []models.User
|
||||
if err := readJSON(s.filePath("users.json"), &users); err != nil {
|
||||
return err
|
||||
}
|
||||
for _, u := range users {
|
||||
if u.Username == user.Username {
|
||||
return fmt.Errorf("username already exists")
|
||||
}
|
||||
}
|
||||
users = append(users, user)
|
||||
return writeJSON(s.filePath("users.json"), users)
|
||||
}
|
||||
|
||||
func (s *Store) UpdateUserCredentials(userID string, creds []webauthn.Credential) error {
|
||||
s.usersMu.Lock()
|
||||
defer s.usersMu.Unlock()
|
||||
var users []models.User
|
||||
if err := readJSON(s.filePath("users.json"), &users); err != nil {
|
||||
return err
|
||||
}
|
||||
for i, u := range users {
|
||||
if u.ID == userID {
|
||||
users[i].Credentials = creds
|
||||
return writeJSON(s.filePath("users.json"), users)
|
||||
}
|
||||
}
|
||||
return fmt.Errorf("user not found")
|
||||
}
|
||||
|
||||
func (s *Store) UserCount() (int, error) {
|
||||
s.usersMu.RLock()
|
||||
defer s.usersMu.RUnlock()
|
||||
var users []models.User
|
||||
if err := readJSON(s.filePath("users.json"), &users); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return len(users), nil
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue