fix(web): add comic delete UI and fix container Cloudflare bypass for #6
- Add delete button (SVG X, hover-reveal) and confirmation modal to comic cards - Add DELETE /api/comics/delete endpoint with path traversal protection - Fix container downloads: delegate Cloudflare-blocked requests to FlareSolverr (headless Chrome sidecar) instead of retrying with Go HTTP client, whose Linux TCP fingerprint is flagged by Cloudflare even with network_mode: host - Add FlareSolverr service to docker-compose; inject FLARESOLVERR_URL env var - Add diagnostic logging to BatcaveBizMarkup request flow - Trim URL whitespace before storing in download job - Guard Archive() against empty filelist; fix runJob error-check ordering
This commit is contained in:
@@ -1,10 +1,15 @@
|
||||
package comic
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"net/http"
|
||||
"net/http/cookiejar"
|
||||
"net/url"
|
||||
"os"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
@@ -51,6 +56,63 @@ func Markup(url string, c chan *goquery.Document) *goquery.Document {
|
||||
return markup
|
||||
}
|
||||
|
||||
// fetchViaFlareSolverr fetches a URL through FlareSolverr (headless Chrome),
|
||||
// returning the final page HTML as a Document. Cookies from the browser session
|
||||
// are written into jar for use in subsequent requests (e.g. image downloads).
|
||||
func fetchViaFlareSolverr(targetURL string, jar *cookiejar.Jar) (*goquery.Document, error) {
|
||||
fsURL := os.Getenv("FLARESOLVERR_URL")
|
||||
if fsURL == "" {
|
||||
return nil, fmt.Errorf("FLARESOLVERR_URL not set")
|
||||
}
|
||||
|
||||
payload, _ := json.Marshal(map[string]interface{}{
|
||||
"cmd": "request.get",
|
||||
"url": targetURL,
|
||||
"maxTimeout": 60000,
|
||||
})
|
||||
|
||||
resp, err := http.Post(fsURL+"/v1", "application/json", bytes.NewReader(payload))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
var result struct {
|
||||
Status string `json:"status"`
|
||||
Solution struct {
|
||||
Response string `json:"response"`
|
||||
Cookies []struct {
|
||||
Name string `json:"name"`
|
||||
Value string `json:"value"`
|
||||
Domain string `json:"domain"`
|
||||
Path string `json:"path"`
|
||||
Secure bool `json:"secure"`
|
||||
} `json:"cookies"`
|
||||
} `json:"solution"`
|
||||
}
|
||||
if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if result.Status != "ok" {
|
||||
return nil, fmt.Errorf("flaresolverr: %s", result.Status)
|
||||
}
|
||||
|
||||
parsed, _ := url.Parse(targetURL)
|
||||
var cookies []*http.Cookie
|
||||
for _, c := range result.Solution.Cookies {
|
||||
cookies = append(cookies, &http.Cookie{
|
||||
Name: c.Name,
|
||||
Value: c.Value,
|
||||
Domain: c.Domain,
|
||||
Path: c.Path,
|
||||
Secure: c.Secure,
|
||||
})
|
||||
}
|
||||
jar.SetCookies(parsed, cookies)
|
||||
|
||||
return goquery.NewDocumentFromReader(strings.NewReader(result.Solution.Response))
|
||||
}
|
||||
|
||||
func BatcaveBizMarkup(referer string, c chan *goquery.Document, clientChan chan *http.Client) *goquery.Document {
|
||||
sendErr := func() *goquery.Document {
|
||||
if c != nil {
|
||||
@@ -72,12 +134,12 @@ func BatcaveBizMarkup(referer string, c chan *goquery.Document, clientChan chan
|
||||
}
|
||||
|
||||
headers := map[string]string{
|
||||
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36",
|
||||
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
|
||||
"Accept-Language": "en-US,en;q=0.9",
|
||||
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8",
|
||||
}
|
||||
|
||||
// GET the challange page to obtain cookies and any necessary tokens
|
||||
// GET the challenge page to obtain cookies and any necessary tokens
|
||||
req, err := http.NewRequest("GET", referer, nil)
|
||||
if err != nil {
|
||||
return sendErr()
|
||||
@@ -88,8 +150,30 @@ func BatcaveBizMarkup(referer string, c chan *goquery.Document, clientChan chan
|
||||
|
||||
res, err := client.Do(req)
|
||||
if err != nil {
|
||||
log.Printf("[batcave] initial GET failed: %v", err)
|
||||
return sendErr()
|
||||
}
|
||||
log.Printf("[batcave] initial GET status: %d", res.StatusCode)
|
||||
|
||||
// Cloudflare challenge — use FlareSolverr (headless Chrome) to fetch the
|
||||
// full page and solve any JS challenges. cf_clearance is stored in jar for
|
||||
// subsequent image downloads.
|
||||
if res.StatusCode == 403 || res.StatusCode == 503 {
|
||||
res.Body.Close()
|
||||
log.Printf("[batcave] Cloudflare challenge detected, fetching via FlareSolverr")
|
||||
doc, err := fetchViaFlareSolverr(referer, jar)
|
||||
if err != nil {
|
||||
log.Printf("[batcave] FlareSolverr failed: %v", err)
|
||||
return sendErr()
|
||||
}
|
||||
if c != nil {
|
||||
c <- doc
|
||||
}
|
||||
if clientChan != nil {
|
||||
clientChan <- client
|
||||
}
|
||||
return doc
|
||||
}
|
||||
defer res.Body.Close()
|
||||
|
||||
body, err := io.ReadAll(res.Body)
|
||||
@@ -121,7 +205,7 @@ func BatcaveBizMarkup(referer string, c chan *goquery.Document, clientChan chan
|
||||
token = encodedToken
|
||||
}
|
||||
|
||||
// Step 3: POST to /_v with fake browser metrics
|
||||
// POST to /_v with fake browser metrics
|
||||
params := url.Values{}
|
||||
params.Set("token", token)
|
||||
params.Set("mode", "modern")
|
||||
@@ -145,9 +229,11 @@ func BatcaveBizMarkup(referer string, c chan *goquery.Document, clientChan chan
|
||||
|
||||
postRes, err := client.Do(postReq)
|
||||
if err != nil {
|
||||
log.Printf("[batcave] POST to /_v failed: %v", err)
|
||||
return sendErr()
|
||||
}
|
||||
defer postRes.Body.Close()
|
||||
log.Printf("[batcave] POST to /_v status: %d", postRes.StatusCode)
|
||||
io.ReadAll(postRes.Body)
|
||||
|
||||
// GET the real page with the set cookie
|
||||
@@ -161,8 +247,10 @@ func BatcaveBizMarkup(referer string, c chan *goquery.Document, clientChan chan
|
||||
|
||||
realRes, err := client.Do(realReq)
|
||||
if err != nil {
|
||||
log.Printf("[batcave] final GET failed: %v", err)
|
||||
return sendErr()
|
||||
}
|
||||
log.Printf("[batcave] final GET status: %d", realRes.StatusCode)
|
||||
defer realRes.Body.Close()
|
||||
|
||||
doc, err := goquery.NewDocumentFromReader(realRes.Body)
|
||||
|
||||
Reference in New Issue
Block a user