Compare commits
14 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 9ec1301317 | |||
| dcb41deea9 | |||
| a7c3b632a5 | |||
| d53af6b84f | |||
| 9cd4af9bb6 | |||
| 96f9301b32 | |||
| 16b7545757 | |||
| 412438fa22 | |||
| 551a5b2b2a | |||
| 1a567a19fe | |||
| 9d1ca16704 | |||
| 25eee6f76a | |||
| 5d7c324fb7 | |||
| a0ee2db58f |
5
.dockerignore
Normal file
5
.dockerignore
Normal file
@@ -0,0 +1,5 @@
|
|||||||
|
.git
|
||||||
|
.github
|
||||||
|
*.md
|
||||||
|
library/
|
||||||
|
*_test.go
|
||||||
185
.github/workflows/release.yaml
vendored
185
.github/workflows/release.yaml
vendored
@@ -1,4 +1,4 @@
|
|||||||
name: Build and Release
|
name: Release
|
||||||
|
|
||||||
on:
|
on:
|
||||||
push:
|
push:
|
||||||
@@ -6,185 +6,32 @@ on:
|
|||||||
- 'v*'
|
- 'v*'
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
build-and-release-linux:
|
release:
|
||||||
name: Build and Release for Linux
|
name: Build and Release
|
||||||
runs-on: [self-hosted, linux, arm64]
|
runs-on: [self-hosted, linux, arm64]
|
||||||
|
|
||||||
steps:
|
steps:
|
||||||
- name: Checkout code
|
- name: Checkout
|
||||||
uses: actions/checkout@v3
|
uses: actions/checkout@v4
|
||||||
|
|
||||||
- name: Set up Go
|
- name: Set up Go
|
||||||
uses: actions/setup-go@v3
|
uses: actions/setup-go@v5
|
||||||
with:
|
with:
|
||||||
go-version: '1.22.3'
|
go-version: '1.22.3'
|
||||||
|
|
||||||
- name: Build for Linux
|
- name: Build
|
||||||
run: go build -o myapp
|
run: make all
|
||||||
|
|
||||||
- name: Create Release
|
- name: Release
|
||||||
id: create_release
|
uses: softprops/action-gh-release@v2
|
||||||
uses: actions/create-release@v1
|
|
||||||
env:
|
|
||||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
||||||
with:
|
with:
|
||||||
tag_name: ${{ github.ref_name }}
|
tag_name: ${{ github.ref_name }}
|
||||||
release_name: Release ${{ github.ref_name }}
|
name: ${{ github.ref_name }}
|
||||||
body: Release notes for ${{ github.ref_name }}
|
files: build/*
|
||||||
draft: false
|
generate_release_notes: true
|
||||||
prerelease: false
|
|
||||||
|
|
||||||
- name: Save upload_url
|
|
||||||
run: echo "UPLOAD_URL=${{ steps.create_release.outputs.upload_url }}" > upload_url.txt
|
|
||||||
shell: bash
|
|
||||||
|
|
||||||
- name: Upload upload_url artifact
|
|
||||||
uses: actions/upload-artifact@v3
|
|
||||||
with:
|
|
||||||
name: upload_url
|
|
||||||
path: upload_url.txt
|
|
||||||
|
|
||||||
build-and-release-windows:
|
|
||||||
name: Build and Release for Windows
|
|
||||||
runs-on: windows-latest
|
|
||||||
needs: build-and-release-linux
|
|
||||||
|
|
||||||
steps:
|
|
||||||
- name: Checkout code
|
|
||||||
uses: actions/checkout@v3
|
|
||||||
|
|
||||||
- name: Set up Go
|
|
||||||
uses: actions/setup-go@v3
|
|
||||||
with:
|
|
||||||
go-version: '1.22.3'
|
|
||||||
|
|
||||||
- name: Build for Windows
|
|
||||||
run: go build -o myapp.exe
|
|
||||||
|
|
||||||
- name: Download upload_url artifact
|
|
||||||
uses: actions/download-artifact@v3
|
|
||||||
with:
|
|
||||||
name: upload_url
|
|
||||||
path: .
|
|
||||||
|
|
||||||
- name: Read upload_url
|
|
||||||
id: read_upload_url
|
|
||||||
run: |
|
|
||||||
UPLOAD_URL=$(cat upload_url.txt)
|
|
||||||
echo "::set-output name=upload_url::$UPLOAD_URL"
|
|
||||||
shell: bash
|
|
||||||
|
|
||||||
- name: Debug upload_url
|
|
||||||
run: |
|
|
||||||
echo "Upload URL: ${{ steps.read_upload_url.outputs.upload_url }}"
|
|
||||||
|
|
||||||
- name: Upload Windows Release Asset
|
|
||||||
if: startsWith(github.ref, 'refs/tags/')
|
|
||||||
uses: actions/upload-release-asset@v1
|
|
||||||
env:
|
env:
|
||||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||||
with:
|
|
||||||
upload_url: ${{ steps.read_upload_url.outputs.upload_url }}
|
|
||||||
asset_path: ./myapp.exe
|
|
||||||
asset_name: myapp-windows.exe
|
|
||||||
asset_content_type: application/octet-stream
|
|
||||||
|
|
||||||
|
- name: Clean up
|
||||||
# name: Yoink Release
|
if: always()
|
||||||
|
run: make clean
|
||||||
# on:
|
|
||||||
# push:
|
|
||||||
# tags:
|
|
||||||
# - 'v*'
|
|
||||||
|
|
||||||
# jobs:
|
|
||||||
# yoink-release:
|
|
||||||
# name: Yoink Build and Release
|
|
||||||
# runs-on: [self-hosted, linux, arm64]
|
|
||||||
|
|
||||||
# steps:
|
|
||||||
# - name: Checkout
|
|
||||||
# uses: actions/checkout@v3
|
|
||||||
|
|
||||||
# - name: Setup Go
|
|
||||||
# uses: actions/setup-go@v4
|
|
||||||
# with:
|
|
||||||
# go-version: '1.22.3'
|
|
||||||
|
|
||||||
# - name: Linux Build
|
|
||||||
# run: go build -o yoink
|
|
||||||
|
|
||||||
# - name: Create Release
|
|
||||||
# id: create_release
|
|
||||||
# uses: actions/create-release@v1
|
|
||||||
# env:
|
|
||||||
# GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
||||||
# with:
|
|
||||||
# tag_name: ${{ github.ref_name }}
|
|
||||||
# release_name: Release ${{ github.ref_name }}
|
|
||||||
# body: Release notes for ${{ github.ref_name }}
|
|
||||||
# draft: false
|
|
||||||
# prerelease: false
|
|
||||||
|
|
||||||
# - name: Save upload_url
|
|
||||||
# run: echo "UPLOAD_URL=${{ steps.create_release.outputs.upload_url }}" > upload_url.txt
|
|
||||||
# shell: bash
|
|
||||||
|
|
||||||
# - name: Upload upload_url artifact
|
|
||||||
# uses: actions/upload-artifact@v3
|
|
||||||
# with:
|
|
||||||
# name: upload_url.txt
|
|
||||||
# path: ./upload_url.txt
|
|
||||||
|
|
||||||
# - name: Upload Linux Release Asset
|
|
||||||
# uses: actions/upload-release-asset@v1
|
|
||||||
# env:
|
|
||||||
# GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
||||||
# with:
|
|
||||||
# upload_url: ${{ steps.create_release.outputs.upload_url }}
|
|
||||||
# asset_path: ./yoink
|
|
||||||
# asset_name: yoink
|
|
||||||
# asset_content_type: application/x-executable
|
|
||||||
|
|
||||||
# build-and-release-win:
|
|
||||||
# runs-on: windows-latest
|
|
||||||
# needs: yoink-release
|
|
||||||
|
|
||||||
# steps:
|
|
||||||
# - name: Checkout
|
|
||||||
# uses: actions/checkout@v3
|
|
||||||
|
|
||||||
# - name: Setup Go
|
|
||||||
# uses: actions/setup-go@v4
|
|
||||||
# with:
|
|
||||||
# go-version: '1.22.3'
|
|
||||||
|
|
||||||
# - name: Windows Build
|
|
||||||
# run: go build -o yoink.exe
|
|
||||||
|
|
||||||
# - name: Download upload_url artifact
|
|
||||||
# uses: actions/download-artifact@v3
|
|
||||||
# with:
|
|
||||||
# name: upload_url.txt
|
|
||||||
# path: .
|
|
||||||
|
|
||||||
# - name: Read upload_url artifact
|
|
||||||
# id: read_upload_url
|
|
||||||
# run: |
|
|
||||||
# UPLOAD_URL=$(cat upload_url.txt)
|
|
||||||
# echo "::set-output name=upload_url::$UPLOAD_URL"
|
|
||||||
# shell: bash
|
|
||||||
|
|
||||||
# - name: Debug upload_url
|
|
||||||
# run: echo ${{ steps.read_upload_url.outputs.upload_url }}
|
|
||||||
# shell: bash
|
|
||||||
|
|
||||||
# - name: Upload Windows Release Asset
|
|
||||||
# uses: actions/upload-release-asset@v1
|
|
||||||
# env:
|
|
||||||
# GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
||||||
# with:
|
|
||||||
# upload_url: ${{ steps.read_upload_url.outputs.upload_url }}
|
|
||||||
# asset_path: ./yoink.exe
|
|
||||||
# asset_name: yoink.exe
|
|
||||||
# asset_content_type: application/x-executable
|
|
||||||
|
|||||||
13
.gitignore
vendored
13
.gitignore
vendored
@@ -19,4 +19,15 @@ go.work
|
|||||||
go.work.sum
|
go.work.sum
|
||||||
|
|
||||||
# env file
|
# env file
|
||||||
.env
|
.env
|
||||||
|
|
||||||
|
# Built binary
|
||||||
|
yoink
|
||||||
|
yoink.exe
|
||||||
|
|
||||||
|
# Comic library (downloaded content)
|
||||||
|
library/
|
||||||
|
|
||||||
|
# IDE
|
||||||
|
.vscode/
|
||||||
|
.idea/
|
||||||
37
Dockerfile
Normal file
37
Dockerfile
Normal file
@@ -0,0 +1,37 @@
|
|||||||
|
# ── Build stage ────────────────────────────────────────────────────────────
|
||||||
|
FROM mcr.microsoft.com/oss/go/microsoft/golang:1.22-bullseye AS builder
|
||||||
|
|
||||||
|
WORKDIR /app
|
||||||
|
|
||||||
|
# Restore modules in a separate layer so it's cached until go.mod/go.sum change
|
||||||
|
COPY go.mod go.sum ./
|
||||||
|
RUN go mod download && go mod verify
|
||||||
|
|
||||||
|
# Copy source and build a fully static binary
|
||||||
|
COPY . .
|
||||||
|
RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 \
|
||||||
|
go build -ldflags="-s -w" -trimpath -o yoink .
|
||||||
|
|
||||||
|
# ── Runtime stage ──────────────────────────────────────────────────────────
|
||||||
|
# distroless/base-debian12:nonroot — minimal attack surface, non-root by default
|
||||||
|
FROM gcr.io/distroless/base-debian12:nonroot
|
||||||
|
|
||||||
|
LABEL org.opencontainers.image.title="yoink" \
|
||||||
|
org.opencontainers.image.description="Comic downloader web UI" \
|
||||||
|
org.opencontainers.image.source="https://git.brizzle.dev/bryan/yoink-go"
|
||||||
|
|
||||||
|
WORKDIR /app
|
||||||
|
|
||||||
|
COPY --from=builder --chown=nonroot:nonroot /app/yoink .
|
||||||
|
|
||||||
|
ENV YOINK_LIBRARY=/library
|
||||||
|
|
||||||
|
VOLUME ["/library"]
|
||||||
|
EXPOSE 8080
|
||||||
|
|
||||||
|
HEALTHCHECK --interval=30s --timeout=5s --start-period=5s --retries=3 \
|
||||||
|
CMD ["/app/yoink", "healthcheck"]
|
||||||
|
|
||||||
|
USER nonroot
|
||||||
|
|
||||||
|
CMD ["/app/yoink", "serve"]
|
||||||
38
Makefile
38
Makefile
@@ -1,7 +1,10 @@
|
|||||||
BIN := yoink
|
BIN := yoink
|
||||||
BUILD_DIR := build
|
BUILD_DIR := build
|
||||||
|
REGISTRY := git.brizzle.dev/bryan/yoink-go
|
||||||
|
VERSION ?= $(shell git describe --tags --always --dirty)
|
||||||
|
NOTES ?= ""
|
||||||
|
|
||||||
.PHONY: all windows linux darwin clean
|
.PHONY: all windows linux darwin clean docker-build docker-push tag gitea-release release
|
||||||
|
|
||||||
all: windows linux darwin
|
all: windows linux darwin
|
||||||
|
|
||||||
@@ -16,5 +19,38 @@ darwin:
|
|||||||
GOOS=darwin GOARCH=amd64 go build -o $(BUILD_DIR)/$(BIN)-darwin-amd64
|
GOOS=darwin GOARCH=amd64 go build -o $(BUILD_DIR)/$(BIN)-darwin-amd64
|
||||||
GOOS=darwin GOARCH=arm64 go build -o $(BUILD_DIR)/$(BIN)-darwin-arm64
|
GOOS=darwin GOARCH=arm64 go build -o $(BUILD_DIR)/$(BIN)-darwin-arm64
|
||||||
|
|
||||||
|
docker-build:
|
||||||
|
podman build --format docker \
|
||||||
|
-t $(REGISTRY):$(VERSION) \
|
||||||
|
-t $(REGISTRY):latest \
|
||||||
|
.
|
||||||
|
|
||||||
|
docker-push: docker-build
|
||||||
|
podman push $(REGISTRY):$(VERSION)
|
||||||
|
podman push $(REGISTRY):latest
|
||||||
|
|
||||||
|
tag:
|
||||||
|
@if [ -z "$(VERSION)" ]; then echo "Usage: make tag VERSION=1.2.0"; exit 1; fi
|
||||||
|
git tag $(VERSION)
|
||||||
|
git push origin $(VERSION)
|
||||||
|
|
||||||
|
gitea-release:
|
||||||
|
tea release create \
|
||||||
|
--tag $(VERSION) \
|
||||||
|
--title "$(VERSION)" \
|
||||||
|
--note $(NOTES) \
|
||||||
|
--asset $(BUILD_DIR)/$(BIN)-windows-amd64.exe \
|
||||||
|
--asset $(BUILD_DIR)/$(BIN)-linux-amd64 \
|
||||||
|
--asset $(BUILD_DIR)/$(BIN)-linux-arm64 \
|
||||||
|
--asset $(BUILD_DIR)/$(BIN)-darwin-amd64 \
|
||||||
|
--asset $(BUILD_DIR)/$(BIN)-darwin-arm64
|
||||||
|
|
||||||
|
release:
|
||||||
|
@if [ -z "$(VERSION)" ]; then echo "Usage: make release VERSION=1.3.0 NOTES='...'"; exit 1; fi
|
||||||
|
$(MAKE) tag VERSION=$(VERSION)
|
||||||
|
$(MAKE) clean all
|
||||||
|
$(MAKE) gitea-release VERSION=$(VERSION) NOTES=$(NOTES)
|
||||||
|
$(MAKE) docker-push VERSION=$(VERSION)
|
||||||
|
|
||||||
clean:
|
clean:
|
||||||
rm -rf $(BUILD_DIR)
|
rm -rf $(BUILD_DIR)
|
||||||
|
|||||||
108
README.md
108
README.md
@@ -1,6 +1,6 @@
|
|||||||
# yoink
|
# yoink
|
||||||
|
|
||||||
A CLI tool for downloading comics from readallcomics.com and packaging them as `.cbz` archives.
|
A tool for downloading comics from readallcomics.com and packaging them as `.cbz` archives. Available as a CLI command or a self-hosted web application. The web UI also lets you package local image folders into `.cbz` archives directly from your browser.
|
||||||
|
|
||||||
## How it works
|
## How it works
|
||||||
|
|
||||||
@@ -9,17 +9,33 @@ A CLI tool for downloading comics from readallcomics.com and packaging them as `
|
|||||||
3. Packages the images into a `.cbz` (Comic Book Zip) archive
|
3. Packages the images into a `.cbz` (Comic Book Zip) archive
|
||||||
4. Cleans up downloaded images, keeping only the cover (`001`)
|
4. Cleans up downloaded images, keeping only the cover (`001`)
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
## Installation
|
## Installation
|
||||||
|
|
||||||
Build from source (requires Go 1.22.3+):
|
### From source
|
||||||
|
|
||||||
|
Requires Go 1.22.3+:
|
||||||
|
|
||||||
```shell
|
```shell
|
||||||
go build -o yoink
|
go build -o yoink
|
||||||
```
|
```
|
||||||
|
|
||||||
Pre-built binaries for Linux (arm64) and Windows are available on the [releases page](../../releases).
|
### Pre-built binaries
|
||||||
|
|
||||||
## Usage
|
Pre-built binaries for Linux (arm64) and Windows are available on the [releases page](https://git.brizzle.dev/bryan/yoink-go/releases).
|
||||||
|
|
||||||
|
### Docker
|
||||||
|
|
||||||
|
```shell
|
||||||
|
docker pull git.brizzle.dev/bryan/yoink-go:latest
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## CLI
|
||||||
|
|
||||||
|
Download a single comic issue:
|
||||||
|
|
||||||
```shell
|
```shell
|
||||||
yoink <url>
|
yoink <url>
|
||||||
@@ -37,16 +53,94 @@ The comic title is extracted from the page and used to name the archive. Output
|
|||||||
<library>/<Title>/<Title>.cbz
|
<library>/<Title>/<Title>.cbz
|
||||||
```
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Web UI
|
||||||
|
|
||||||
|
Yoink includes a self-hosted web interface for browsing and downloading comics from your browser.
|
||||||
|
|
||||||
|

|
||||||
|
|
||||||
|
### Running directly
|
||||||
|
|
||||||
|
```shell
|
||||||
|
yoink serve
|
||||||
|
```
|
||||||
|
|
||||||
|
By default the server listens on port `8080`. Use the `-p` flag to change it:
|
||||||
|
|
||||||
|
```shell
|
||||||
|
yoink serve -p 3000
|
||||||
|
```
|
||||||
|
|
||||||
|
### Running with Docker
|
||||||
|
|
||||||
|
A `docker-compose.yml` is included for quick deployment:
|
||||||
|
|
||||||
|
```shell
|
||||||
|
docker compose up -d
|
||||||
|
```
|
||||||
|
|
||||||
|
Or with Podman:
|
||||||
|
|
||||||
|
```shell
|
||||||
|
podman compose up -d
|
||||||
|
```
|
||||||
|
|
||||||
|
The web UI is then available at `http://localhost:8080`.
|
||||||
|
|
||||||
|
### Features
|
||||||
|
|
||||||
|
- **Download queue** — paste a comic URL into the input bar and track download progress in real time
|
||||||
|
- **Local packaging** — drag and drop a folder of images (or use the file picker) to package them as a `.cbz` archive and add it to your library without downloading anything
|
||||||
|
- **Library grid** — browse your comics as a 150×300 cover grid with title-initial placeholders for missing covers
|
||||||
|
- **Filter & sort** — filter by title and sort by newest, oldest, A–Z, or Z–A
|
||||||
|
- **One-click download** — click any cover to download the `.cbz` archive directly
|
||||||
|
|
||||||
|
#### Packaging local images
|
||||||
|
|
||||||
|

|
||||||
|
|
||||||
|
Click the upload icon (↑) in the header to open the packaging panel. Enter a title, then either:
|
||||||
|
|
||||||
|
- **Drag and drop** a folder or image files onto the drop zone
|
||||||
|
- **Select folder** to pick an entire directory at once
|
||||||
|
- **Select files** to pick individual images
|
||||||
|
|
||||||
|
Images are sorted by filename, the first image is used as the cover, and the result is saved to your library as `<Title>/<Title>.cbz`.
|
||||||
|
|
||||||
|
### Library volume
|
||||||
|
|
||||||
|
Downloaded comics are stored at the path set by `YOINK_LIBRARY`. When using Docker, mount this as a volume to persist your library across container restarts:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
# docker-compose.yml
|
||||||
|
services:
|
||||||
|
yoink:
|
||||||
|
image: git.brizzle.dev/bryan/yoink-go:latest
|
||||||
|
ports:
|
||||||
|
- "8080:8080"
|
||||||
|
volumes:
|
||||||
|
- ./library:/library
|
||||||
|
environment:
|
||||||
|
- YOINK_LIBRARY=/library
|
||||||
|
restart: unless-stopped
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
## Configuration
|
## Configuration
|
||||||
|
|
||||||
| Variable | Default | Description |
|
| Variable | Default | Description |
|
||||||
|-----------------|--------------|--------------------------------------|
|
|-----------------|------------|-----------------------------------|
|
||||||
| `YOINK_LIBRARY` | `~/.yoink` | Directory where comics are stored |
|
| `YOINK_LIBRARY` | `~/.yoink` | Directory where comics are stored |
|
||||||
|
|
||||||
```shell
|
```shell
|
||||||
YOINK_LIBRARY=/mnt/media/comics yoink https://readallcomics.com/some-comic-001/
|
YOINK_LIBRARY=/mnt/media/comics yoink https://readallcomics.com/some-comic-001/
|
||||||
```
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
## Dependencies
|
## Dependencies
|
||||||
|
|
||||||
- [goquery](https://github.com/PuerkitoBio/goquery) — HTML parsing
|
- [goquery](https://github.com/PuerkitoBio/goquery) — HTML parsing
|
||||||
|
|||||||
BIN
Screenshot_01.png
Normal file
BIN
Screenshot_01.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 560 KiB |
BIN
Screenshot_02.png
Normal file
BIN
Screenshot_02.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 270 KiB |
28
cli/healthcheck.go
Normal file
28
cli/healthcheck.go
Normal file
@@ -0,0 +1,28 @@
|
|||||||
|
package cli
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"net/http"
|
||||||
|
"os"
|
||||||
|
|
||||||
|
"github.com/spf13/cobra"
|
||||||
|
)
|
||||||
|
|
||||||
|
var healthcheckCmd = &cobra.Command{
|
||||||
|
Use: "healthcheck",
|
||||||
|
Short: "Check if the web server is running (used by Docker HEALTHCHECK)",
|
||||||
|
Args: cobra.NoArgs,
|
||||||
|
Hidden: true,
|
||||||
|
Run: func(cmd *cobra.Command, args []string) {
|
||||||
|
port, _ := cmd.Flags().GetString("port")
|
||||||
|
resp, err := http.Get(fmt.Sprintf("http://localhost:%s/health", port))
|
||||||
|
if err != nil || resp.StatusCode != http.StatusOK {
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
func init() {
|
||||||
|
healthcheckCmd.Flags().StringP("port", "p", "8080", "Port the server is listening on")
|
||||||
|
cli.AddCommand(healthcheckCmd)
|
||||||
|
}
|
||||||
36
cli/serve.go
Normal file
36
cli/serve.go
Normal file
@@ -0,0 +1,36 @@
|
|||||||
|
package cli
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"log"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
|
||||||
|
"github.com/spf13/cobra"
|
||||||
|
"yoink/web"
|
||||||
|
)
|
||||||
|
|
||||||
|
var serveCmd = &cobra.Command{
|
||||||
|
Use: "serve",
|
||||||
|
Short: "Start the Yoink web UI",
|
||||||
|
Args: cobra.NoArgs,
|
||||||
|
Run: func(cmd *cobra.Command, args []string) {
|
||||||
|
library, ok := os.LookupEnv("YOINK_LIBRARY")
|
||||||
|
if !ok {
|
||||||
|
userHome, _ := os.UserHomeDir()
|
||||||
|
library = filepath.Join(userHome, ".yoink")
|
||||||
|
}
|
||||||
|
|
||||||
|
port, _ := cmd.Flags().GetString("port")
|
||||||
|
addr := fmt.Sprintf(":%s", port)
|
||||||
|
|
||||||
|
if err := web.Listen(addr, library); err != nil {
|
||||||
|
log.Fatal(err)
|
||||||
|
}
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
func init() {
|
||||||
|
serveCmd.Flags().StringP("port", "p", "8080", "Port to listen on")
|
||||||
|
cli.AddCommand(serveCmd)
|
||||||
|
}
|
||||||
110
comic/archive_test.go
Normal file
110
comic/archive_test.go
Normal file
@@ -0,0 +1,110 @@
|
|||||||
|
package comic
|
||||||
|
|
||||||
|
import (
|
||||||
|
"archive/zip"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestArchiveError(t *testing.T) {
|
||||||
|
err := ArchiveError{Message: "archive failed", Code: 1}
|
||||||
|
if err.Error() != "archive failed" {
|
||||||
|
t.Errorf("Error() = %q, want %q", err.Error(), "archive failed")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestArchive(t *testing.T) {
|
||||||
|
t.Run("creates cbz with image files", func(t *testing.T) {
|
||||||
|
tmpDir := t.TempDir()
|
||||||
|
title := "TestComic"
|
||||||
|
comicDir := filepath.Join(tmpDir, title)
|
||||||
|
os.MkdirAll(comicDir, os.ModePerm)
|
||||||
|
|
||||||
|
// Create fake image files
|
||||||
|
for _, name := range []string{"TestComic 001.jpg", "TestComic 002.jpg", "TestComic 003.png"} {
|
||||||
|
os.WriteFile(filepath.Join(comicDir, name), []byte("fake image"), 0644)
|
||||||
|
}
|
||||||
|
|
||||||
|
c := &Comic{
|
||||||
|
Title: title,
|
||||||
|
LibraryPath: tmpDir,
|
||||||
|
}
|
||||||
|
|
||||||
|
err := c.Archive()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Archive() unexpected error: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
archivePath := filepath.Join(comicDir, title+".cbz")
|
||||||
|
if _, err := os.Stat(archivePath); os.IsNotExist(err) {
|
||||||
|
t.Fatalf("expected archive %s to exist", archivePath)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Verify the zip contains the image files
|
||||||
|
reader, err := zip.OpenReader(archivePath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("failed to open archive: %v", err)
|
||||||
|
}
|
||||||
|
defer reader.Close()
|
||||||
|
|
||||||
|
if len(reader.File) != 3 {
|
||||||
|
t.Errorf("archive contains %d files, want 3", len(reader.File))
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("excludes non-image files from archive", func(t *testing.T) {
|
||||||
|
tmpDir := t.TempDir()
|
||||||
|
title := "TestComic"
|
||||||
|
comicDir := filepath.Join(tmpDir, title)
|
||||||
|
os.MkdirAll(comicDir, os.ModePerm)
|
||||||
|
|
||||||
|
// Create mixed files
|
||||||
|
os.WriteFile(filepath.Join(comicDir, "page-001.jpg"), []byte("image"), 0644)
|
||||||
|
os.WriteFile(filepath.Join(comicDir, "readme.txt"), []byte("text"), 0644)
|
||||||
|
os.WriteFile(filepath.Join(comicDir, "data.json"), []byte("json"), 0644)
|
||||||
|
|
||||||
|
c := &Comic{
|
||||||
|
Title: title,
|
||||||
|
LibraryPath: tmpDir,
|
||||||
|
}
|
||||||
|
|
||||||
|
err := c.Archive()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Archive() unexpected error: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
archivePath := filepath.Join(comicDir, title+".cbz")
|
||||||
|
reader, err := zip.OpenReader(archivePath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("failed to open archive: %v", err)
|
||||||
|
}
|
||||||
|
defer reader.Close()
|
||||||
|
|
||||||
|
if len(reader.File) != 1 {
|
||||||
|
t.Errorf("archive contains %d files, want 1 (only .jpg)", len(reader.File))
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("handles empty directory", func(t *testing.T) {
|
||||||
|
tmpDir := t.TempDir()
|
||||||
|
title := "EmptyComic"
|
||||||
|
comicDir := filepath.Join(tmpDir, title)
|
||||||
|
os.MkdirAll(comicDir, os.ModePerm)
|
||||||
|
|
||||||
|
c := &Comic{
|
||||||
|
Title: title,
|
||||||
|
LibraryPath: tmpDir,
|
||||||
|
}
|
||||||
|
|
||||||
|
err := c.Archive()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Archive() unexpected error: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
archivePath := filepath.Join(comicDir, title+".cbz")
|
||||||
|
if _, err := os.Stat(archivePath); os.IsNotExist(err) {
|
||||||
|
t.Fatalf("expected archive %s to exist even if empty", archivePath)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
93
comic/cleanup_test.go
Normal file
93
comic/cleanup_test.go
Normal file
@@ -0,0 +1,93 @@
|
|||||||
|
package comic
|
||||||
|
|
||||||
|
import (
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestCleanup(t *testing.T) {
|
||||||
|
t.Run("keeps cover image 001 and removes others", func(t *testing.T) {
|
||||||
|
tmpDir := t.TempDir()
|
||||||
|
title := "TestComic"
|
||||||
|
comicDir := filepath.Join(tmpDir, title)
|
||||||
|
os.MkdirAll(comicDir, os.ModePerm)
|
||||||
|
|
||||||
|
files := map[string]bool{
|
||||||
|
"TestComic 001.jpg": true, // should be kept
|
||||||
|
"TestComic 002.jpg": false, // should be removed
|
||||||
|
"TestComic 003.jpg": false, // should be removed
|
||||||
|
}
|
||||||
|
|
||||||
|
for name := range files {
|
||||||
|
os.WriteFile(filepath.Join(comicDir, name), []byte("fake"), 0644)
|
||||||
|
}
|
||||||
|
|
||||||
|
c := &Comic{
|
||||||
|
Title: title,
|
||||||
|
LibraryPath: tmpDir,
|
||||||
|
}
|
||||||
|
|
||||||
|
err := c.Cleanup()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Cleanup() unexpected error: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
for name, shouldExist := range files {
|
||||||
|
path := filepath.Join(comicDir, name)
|
||||||
|
_, err := os.Stat(path)
|
||||||
|
exists := !os.IsNotExist(err)
|
||||||
|
|
||||||
|
if shouldExist && !exists {
|
||||||
|
t.Errorf("expected %s to be kept, but it was removed", name)
|
||||||
|
}
|
||||||
|
if !shouldExist && exists {
|
||||||
|
t.Errorf("expected %s to be removed, but it still exists", name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("keeps non-image files", func(t *testing.T) {
|
||||||
|
tmpDir := t.TempDir()
|
||||||
|
title := "TestComic"
|
||||||
|
comicDir := filepath.Join(tmpDir, title)
|
||||||
|
os.MkdirAll(comicDir, os.ModePerm)
|
||||||
|
|
||||||
|
os.WriteFile(filepath.Join(comicDir, "TestComic.cbz"), []byte("archive"), 0644)
|
||||||
|
os.WriteFile(filepath.Join(comicDir, "metadata.json"), []byte("data"), 0644)
|
||||||
|
|
||||||
|
c := &Comic{
|
||||||
|
Title: title,
|
||||||
|
LibraryPath: tmpDir,
|
||||||
|
}
|
||||||
|
|
||||||
|
err := c.Cleanup()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Cleanup() unexpected error: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, name := range []string{"TestComic.cbz", "metadata.json"} {
|
||||||
|
path := filepath.Join(comicDir, name)
|
||||||
|
if _, err := os.Stat(path); os.IsNotExist(err) {
|
||||||
|
t.Errorf("expected non-image file %s to be kept", name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("handles empty directory", func(t *testing.T) {
|
||||||
|
tmpDir := t.TempDir()
|
||||||
|
title := "EmptyComic"
|
||||||
|
comicDir := filepath.Join(tmpDir, title)
|
||||||
|
os.MkdirAll(comicDir, os.ModePerm)
|
||||||
|
|
||||||
|
c := &Comic{
|
||||||
|
Title: title,
|
||||||
|
LibraryPath: tmpDir,
|
||||||
|
}
|
||||||
|
|
||||||
|
err := c.Cleanup()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Cleanup() unexpected error for empty dir: %v", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
@@ -26,21 +26,52 @@ type Comic struct {
|
|||||||
// Returns the extracted title as a string.
|
// Returns the extracted title as a string.
|
||||||
func extractTitleFromMarkup(c Comic) string {
|
func extractTitleFromMarkup(c Comic) string {
|
||||||
yearFormat := `^(.*?)\s+\(\d{4}(?:\s+.+)?\)`
|
yearFormat := `^(.*?)\s+\(\d{4}(?:\s+.+)?\)`
|
||||||
selection := c.Markup.Find("title")
|
|
||||||
|
|
||||||
if selection.Length() == 0 {
|
|
||||||
return "Untitled"
|
|
||||||
}
|
|
||||||
|
|
||||||
content := selection.First().Text()
|
|
||||||
regex := regexp.MustCompile(yearFormat)
|
regex := regexp.MustCompile(yearFormat)
|
||||||
matches := regex.FindStringSubmatch(content)
|
|
||||||
|
|
||||||
if len(matches) != 2 {
|
extractFrom := func(text string) string {
|
||||||
return "Untitled"
|
matches := regex.FindStringSubmatch(text)
|
||||||
|
if len(matches) != 2 {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return strings.ReplaceAll(matches[1], ":", "")
|
||||||
}
|
}
|
||||||
|
|
||||||
return strings.ReplaceAll(matches[1], ":", "")
|
title := extractFrom(c.Markup.Find("title").First().Text())
|
||||||
|
|
||||||
|
if strings.HasPrefix(title, "#") {
|
||||||
|
if h1 := extractFrom(c.Markup.Find("h1").First().Text()); h1 != "" && !strings.HasPrefix(h1, "#") {
|
||||||
|
return h1
|
||||||
|
}
|
||||||
|
if slug := titleFromSlug(c.URL); slug != "" {
|
||||||
|
return slug
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if title != "" {
|
||||||
|
return title
|
||||||
|
}
|
||||||
|
|
||||||
|
return "Untitled"
|
||||||
|
}
|
||||||
|
|
||||||
|
// titleFromSlug derives a comic title from the last path segment of a URL.
|
||||||
|
// It strips a trailing year (-YYYY), replaces hyphens with spaces, and title-cases the result.
|
||||||
|
func titleFromSlug(url string) string {
|
||||||
|
slug := strings.TrimRight(url, "/")
|
||||||
|
if i := strings.LastIndex(slug, "/"); i >= 0 {
|
||||||
|
slug = slug[i+1:]
|
||||||
|
}
|
||||||
|
slug = regexp.MustCompile(`-\d{4}$`).ReplaceAllString(slug, "")
|
||||||
|
if slug == "" {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
words := strings.Split(slug, "-")
|
||||||
|
for i, w := range words {
|
||||||
|
if len(w) > 0 {
|
||||||
|
words[i] = strings.ToUpper(w[:1]) + w[1:]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return strings.Join(words, " ")
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewComic creates a new Comic instance from the provided URL and library path.
|
// NewComic creates a new Comic instance from the provided URL and library path.
|
||||||
@@ -61,13 +92,21 @@ func NewComic(
|
|||||||
LibraryPath: libraryPath,
|
LibraryPath: libraryPath,
|
||||||
}
|
}
|
||||||
|
|
||||||
go Markup(c.URL, markupChannel)
|
if strings.Contains(url, "batcave.biz") {
|
||||||
|
go BatcaveBizMarkup(url, markupChannel)
|
||||||
|
} else {
|
||||||
|
go Markup(url, markupChannel)
|
||||||
|
}
|
||||||
|
|
||||||
markup := <-markupChannel
|
markup := <-markupChannel
|
||||||
c.Markup = markup
|
c.Markup = markup
|
||||||
c.Title = extractTitleFromMarkup(*c)
|
c.Title = extractTitleFromMarkup(*c)
|
||||||
|
|
||||||
go ParseImageLinks(markup, imageChannel)
|
if strings.Contains(url, "batcave.biz") {
|
||||||
|
go ParseBatcaveBizImageLinks(markup, imageChannel)
|
||||||
|
} else {
|
||||||
|
go ParseImageLinks(markup, imageChannel)
|
||||||
|
}
|
||||||
links := <-imageChannel
|
links := <-imageChannel
|
||||||
|
|
||||||
c.Filelist = links
|
c.Filelist = links
|
||||||
|
|||||||
170
comic/comic_test.go
Normal file
170
comic/comic_test.go
Normal file
@@ -0,0 +1,170 @@
|
|||||||
|
package comic
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/PuerkitoBio/goquery"
|
||||||
|
)
|
||||||
|
|
||||||
|
func newDocFromHTML(html string) *goquery.Document {
|
||||||
|
doc, _ := goquery.NewDocumentFromReader(strings.NewReader(html))
|
||||||
|
return doc
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExtractTitleFromMarkup(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
html string
|
||||||
|
url string
|
||||||
|
expected string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "standard title with year",
|
||||||
|
html: `<html><head><title>Ultraman X Avengers 001 (2024)</title></head></html>`,
|
||||||
|
expected: "Ultraman X Avengers 001",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "title with year and extra text",
|
||||||
|
html: `<html><head><title>Batman 042 (2023 Digital)</title></head></html>`,
|
||||||
|
expected: "Batman 042",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "title with colon removed",
|
||||||
|
html: `<html><head><title>Spider-Man: No Way Home 001 (2022)</title></head></html>`,
|
||||||
|
expected: "Spider-Man No Way Home 001",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "no title tag",
|
||||||
|
html: `<html><head></head></html>`,
|
||||||
|
expected: "Untitled",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "title without year pattern",
|
||||||
|
html: `<html><head><title>Some Random Page</title></head></html>`,
|
||||||
|
expected: "Untitled",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "empty title",
|
||||||
|
html: `<html><head><title></title></head></html>`,
|
||||||
|
expected: "Untitled",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "title starts with # falls back to h1",
|
||||||
|
html: `<html><head><title>#018 (2026)</title></head><body><h1>Absolute Batman #018 (2026)</h1></body></html>`,
|
||||||
|
expected: "Absolute Batman #018",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "title starts with # but h1 also starts with #, falls back to slug",
|
||||||
|
html: `<html><head><title>#018 (2026)</title></head><body><h1>#018 (2026)</h1></body></html>`,
|
||||||
|
url: "https://readallcomics.com/absolute-batman-018-2026/",
|
||||||
|
expected: "Absolute Batman 018",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "title starts with # falls back to slug when no h1",
|
||||||
|
html: `<html><head><title>#018 (2026)</title></head></html>`,
|
||||||
|
url: "https://readallcomics.com/absolute-batman-018-2026/",
|
||||||
|
expected: "Absolute Batman 018",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "title starts with # no h1 no url",
|
||||||
|
html: `<html><head><title>#018 (2026)</title></head></html>`,
|
||||||
|
expected: "#018",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
doc := newDocFromHTML(tt.html)
|
||||||
|
c := Comic{Markup: doc, URL: tt.url}
|
||||||
|
result := extractTitleFromMarkup(c)
|
||||||
|
if result != tt.expected {
|
||||||
|
t.Errorf("extractTitleFromMarkup() = %q, want %q", result, tt.expected)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTitleFromSlug(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
url string
|
||||||
|
expected string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "standard comic URL",
|
||||||
|
url: "https://readallcomics.com/absolute-batman-018-2026/",
|
||||||
|
expected: "Absolute Batman 018",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "no trailing slash",
|
||||||
|
url: "https://readallcomics.com/absolute-batman-018-2026",
|
||||||
|
expected: "Absolute Batman 018",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "no year in slug",
|
||||||
|
url: "https://readallcomics.com/absolute-batman-018/",
|
||||||
|
expected: "Absolute Batman 018",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "single word slug",
|
||||||
|
url: "https://readallcomics.com/batman/",
|
||||||
|
expected: "Batman",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
result := titleFromSlug(tt.url)
|
||||||
|
if result != tt.expected {
|
||||||
|
t.Errorf("titleFromSlug() = %q, want %q", result, tt.expected)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCover(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
filelist []string
|
||||||
|
wantSuffix string
|
||||||
|
expectErr bool
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "finds cover ending in 001.jpg",
|
||||||
|
filelist: []string{"https://example.com/image-002.jpg", "https://example.com/image-001.jpg", "https://example.com/image-003.jpg"},
|
||||||
|
wantSuffix: "image-001.jpg",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "finds cover ending in 000.jpg",
|
||||||
|
filelist: []string{"https://example.com/image-000.jpg", "https://example.com/image-001.jpg"},
|
||||||
|
wantSuffix: "image-000.jpg",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "returns error when no cover found",
|
||||||
|
filelist: []string{"https://example.com/image-002.jpg", "https://example.com/image-003.jpg"},
|
||||||
|
expectErr: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "returns error for empty filelist",
|
||||||
|
filelist: []string{},
|
||||||
|
expectErr: true,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
c := &Comic{Filelist: tt.filelist}
|
||||||
|
cover, err := c.Cover()
|
||||||
|
if tt.expectErr && err == nil {
|
||||||
|
t.Error("Cover() expected error, got nil")
|
||||||
|
}
|
||||||
|
if !tt.expectErr && err != nil {
|
||||||
|
t.Errorf("Cover() unexpected error: %v", err)
|
||||||
|
}
|
||||||
|
if tt.wantSuffix != "" && !strings.HasSuffix(cover, tt.wantSuffix) {
|
||||||
|
t.Errorf("Cover() = %q, want path ending in %q", cover, tt.wantSuffix)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
145
comic/download_test.go
Normal file
145
comic/download_test.go
Normal file
@@ -0,0 +1,145 @@
|
|||||||
|
package comic
|
||||||
|
|
||||||
|
import (
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestComicDownloadError(t *testing.T) {
|
||||||
|
err := ComicDownloadError{Message: "download failed", Code: 1}
|
||||||
|
if err.Error() != "download failed" {
|
||||||
|
t.Errorf("Error() = %q, want %q", err.Error(), "download failed")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHandleRequest(t *testing.T) {
|
||||||
|
t.Run("successful request", func(t *testing.T) {
|
||||||
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
if r.Header.Get("User-Agent") == "" {
|
||||||
|
t.Error("expected User-Agent header to be set")
|
||||||
|
}
|
||||||
|
w.WriteHeader(http.StatusOK)
|
||||||
|
w.Write([]byte("image data"))
|
||||||
|
}))
|
||||||
|
defer server.Close()
|
||||||
|
|
||||||
|
resp, err := handleRequest(server.URL)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("handleRequest() unexpected error: %v", err)
|
||||||
|
}
|
||||||
|
defer resp.Body.Close()
|
||||||
|
|
||||||
|
if resp.StatusCode != http.StatusOK {
|
||||||
|
t.Errorf("handleRequest() status = %d, want %d", resp.StatusCode, http.StatusOK)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("non-200 response", func(t *testing.T) {
|
||||||
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
w.WriteHeader(http.StatusNotFound)
|
||||||
|
}))
|
||||||
|
defer server.Close()
|
||||||
|
|
||||||
|
_, err := handleRequest(server.URL)
|
||||||
|
if err == nil {
|
||||||
|
t.Error("handleRequest() expected error for 404 response, got nil")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("invalid URL", func(t *testing.T) {
|
||||||
|
_, err := handleRequest("http://invalid.localhost:0/bad")
|
||||||
|
if err == nil {
|
||||||
|
t.Error("handleRequest() expected error for invalid URL, got nil")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDownloadFile(t *testing.T) {
|
||||||
|
t.Run("successful download", func(t *testing.T) {
|
||||||
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
w.WriteHeader(http.StatusOK)
|
||||||
|
w.Write([]byte("fake image content"))
|
||||||
|
}))
|
||||||
|
defer server.Close()
|
||||||
|
|
||||||
|
tmpDir := t.TempDir()
|
||||||
|
c := &Comic{
|
||||||
|
Title: "TestComic",
|
||||||
|
LibraryPath: tmpDir,
|
||||||
|
}
|
||||||
|
|
||||||
|
err := downloadFile(server.URL+"/image.jpg", 1, c)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("downloadFile() unexpected error: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
expectedPath := filepath.Join(tmpDir, "TestComic", "TestComic 001.jpg")
|
||||||
|
if _, err := os.Stat(expectedPath); os.IsNotExist(err) {
|
||||||
|
t.Errorf("expected file %s to exist", expectedPath)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("formats page number with leading zeros", func(t *testing.T) {
|
||||||
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
w.WriteHeader(http.StatusOK)
|
||||||
|
w.Write([]byte("fake image content"))
|
||||||
|
}))
|
||||||
|
defer server.Close()
|
||||||
|
|
||||||
|
tmpDir := t.TempDir()
|
||||||
|
c := &Comic{
|
||||||
|
Title: "TestComic",
|
||||||
|
LibraryPath: tmpDir,
|
||||||
|
}
|
||||||
|
|
||||||
|
err := downloadFile(server.URL+"/image.jpg", 42, c)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("downloadFile() unexpected error: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
expectedPath := filepath.Join(tmpDir, "TestComic", "TestComic 042.jpg")
|
||||||
|
if _, err := os.Stat(expectedPath); os.IsNotExist(err) {
|
||||||
|
t.Errorf("expected file %s to exist", expectedPath)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("server error returns error", func(t *testing.T) {
|
||||||
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
w.WriteHeader(http.StatusInternalServerError)
|
||||||
|
}))
|
||||||
|
defer server.Close()
|
||||||
|
|
||||||
|
tmpDir := t.TempDir()
|
||||||
|
c := &Comic{
|
||||||
|
Title: "TestComic",
|
||||||
|
LibraryPath: tmpDir,
|
||||||
|
}
|
||||||
|
|
||||||
|
err := downloadFile(server.URL+"/image.jpg", 1, c)
|
||||||
|
if err == nil {
|
||||||
|
t.Error("downloadFile() expected error for server error, got nil")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("empty response body returns error", func(t *testing.T) {
|
||||||
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
w.WriteHeader(http.StatusOK)
|
||||||
|
// write nothing
|
||||||
|
}))
|
||||||
|
defer server.Close()
|
||||||
|
|
||||||
|
tmpDir := t.TempDir()
|
||||||
|
c := &Comic{
|
||||||
|
Title: "TestComic",
|
||||||
|
LibraryPath: tmpDir,
|
||||||
|
}
|
||||||
|
|
||||||
|
err := downloadFile(server.URL+"/image.jpg", 1, c)
|
||||||
|
if err == nil {
|
||||||
|
t.Error("downloadFile() expected error for empty body, got nil")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
192
comic/parser.go
192
comic/parser.go
@@ -3,6 +3,9 @@ package comic
|
|||||||
import (
|
import (
|
||||||
"io"
|
"io"
|
||||||
"net/http"
|
"net/http"
|
||||||
|
"net/http/cookiejar"
|
||||||
|
"net/url"
|
||||||
|
"regexp"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"github.com/PuerkitoBio/goquery"
|
"github.com/PuerkitoBio/goquery"
|
||||||
@@ -47,6 +50,143 @@ func Markup(url string, c chan *goquery.Document) *goquery.Document {
|
|||||||
return markup
|
return markup
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func BatcaveBizMarkup(referer string, c chan *goquery.Document) *goquery.Document {
|
||||||
|
jar, _ := cookiejar.New(nil)
|
||||||
|
client := &http.Client{
|
||||||
|
Jar: jar,
|
||||||
|
CheckRedirect: func(req *http.Request, via []*http.Request) error {
|
||||||
|
return nil
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
headers := map[string]string{
|
||||||
|
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36",
|
||||||
|
"Accept-Language": "en-US,en;q=0.9",
|
||||||
|
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8",
|
||||||
|
}
|
||||||
|
|
||||||
|
// GET the challange page to obtain cookies and any necessary tokens
|
||||||
|
req, err := http.NewRequest("GET", referer, nil)
|
||||||
|
if err != nil {
|
||||||
|
if c != nil {
|
||||||
|
c <- &goquery.Document{}
|
||||||
|
}
|
||||||
|
return &goquery.Document{}
|
||||||
|
}
|
||||||
|
for k, v := range headers {
|
||||||
|
req.Header.Set(k, v)
|
||||||
|
}
|
||||||
|
|
||||||
|
res, err := client.Do(req)
|
||||||
|
if err != nil {
|
||||||
|
if c != nil {
|
||||||
|
c <- &goquery.Document{}
|
||||||
|
}
|
||||||
|
return &goquery.Document{}
|
||||||
|
}
|
||||||
|
defer res.Body.Close()
|
||||||
|
|
||||||
|
body, err := io.ReadAll(res.Body)
|
||||||
|
if err != nil {
|
||||||
|
if c != nil {
|
||||||
|
c <- &goquery.Document{}
|
||||||
|
}
|
||||||
|
return &goquery.Document{}
|
||||||
|
}
|
||||||
|
|
||||||
|
tokenRegex := regexp.MustCompile(`token:\s*"([^"]+)"`)
|
||||||
|
matches := tokenRegex.FindSubmatch(body)
|
||||||
|
|
||||||
|
if matches == nil {
|
||||||
|
// no challenge, parse directly
|
||||||
|
doc, err := goquery.NewDocumentFromReader(strings.NewReader(string(body)))
|
||||||
|
if err != nil {
|
||||||
|
if c != nil {
|
||||||
|
c <- &goquery.Document{}
|
||||||
|
}
|
||||||
|
return &goquery.Document{}
|
||||||
|
}
|
||||||
|
if c != nil {
|
||||||
|
c <- doc
|
||||||
|
}
|
||||||
|
return doc
|
||||||
|
}
|
||||||
|
|
||||||
|
encodedToken := string(matches[1])
|
||||||
|
token, err := url.QueryUnescape(encodedToken)
|
||||||
|
if err != nil {
|
||||||
|
token = encodedToken
|
||||||
|
}
|
||||||
|
|
||||||
|
// Step 3: POST to /_v with fake browser metrics
|
||||||
|
params := url.Values{}
|
||||||
|
params.Set("token", token)
|
||||||
|
params.Set("mode", "modern")
|
||||||
|
params.Set("workTime", "462")
|
||||||
|
params.Set("iterations", "183")
|
||||||
|
params.Set("webdriver", "0")
|
||||||
|
params.Set("touch", "0")
|
||||||
|
params.Set("screen_w", "1920")
|
||||||
|
params.Set("screen_h", "1080")
|
||||||
|
params.Set("screen_cd", "24")
|
||||||
|
|
||||||
|
postReq, err := http.NewRequest("POST", "https://batcave.biz/_v", strings.NewReader(params.Encode()))
|
||||||
|
if err != nil {
|
||||||
|
if c != nil {
|
||||||
|
c <- &goquery.Document{}
|
||||||
|
}
|
||||||
|
return &goquery.Document{}
|
||||||
|
}
|
||||||
|
for k, v := range headers {
|
||||||
|
postReq.Header.Set(k, v)
|
||||||
|
}
|
||||||
|
postReq.Header.Set("Content-Type", "application/x-www-form-urlencoded")
|
||||||
|
postReq.Header.Set("Referer", referer)
|
||||||
|
|
||||||
|
postRes, err := client.Do(postReq)
|
||||||
|
if err != nil {
|
||||||
|
if c != nil {
|
||||||
|
c <- &goquery.Document{}
|
||||||
|
}
|
||||||
|
return &goquery.Document{}
|
||||||
|
}
|
||||||
|
defer postRes.Body.Close()
|
||||||
|
io.ReadAll(postRes.Body)
|
||||||
|
|
||||||
|
// GET the real page with the set cookie
|
||||||
|
realReq, err := http.NewRequest("GET", referer, nil)
|
||||||
|
if err != nil {
|
||||||
|
if c != nil {
|
||||||
|
c <- &goquery.Document{}
|
||||||
|
}
|
||||||
|
return &goquery.Document{}
|
||||||
|
}
|
||||||
|
for k, v := range headers {
|
||||||
|
realReq.Header.Set(k, v)
|
||||||
|
}
|
||||||
|
|
||||||
|
realRes, err := client.Do(realReq)
|
||||||
|
if err != nil {
|
||||||
|
if c != nil {
|
||||||
|
c <- &goquery.Document{}
|
||||||
|
}
|
||||||
|
return &goquery.Document{}
|
||||||
|
}
|
||||||
|
defer realRes.Body.Close()
|
||||||
|
|
||||||
|
doc, err := goquery.NewDocumentFromReader(realRes.Body)
|
||||||
|
if err != nil {
|
||||||
|
if c != nil {
|
||||||
|
c <- &goquery.Document{}
|
||||||
|
}
|
||||||
|
return &goquery.Document{}
|
||||||
|
}
|
||||||
|
if c != nil {
|
||||||
|
c <- doc
|
||||||
|
}
|
||||||
|
return doc
|
||||||
|
}
|
||||||
|
|
||||||
// ParseImageLinks parses a goquery document to extract image links.
|
// ParseImageLinks parses a goquery document to extract image links.
|
||||||
//
|
//
|
||||||
// markup is the goquery document to parse for image links.
|
// markup is the goquery document to parse for image links.
|
||||||
@@ -69,3 +209,55 @@ func ParseImageLinks(markup *goquery.Document, c chan []string) ([]string, error
|
|||||||
|
|
||||||
return links, ImageParseError{Message: "No images found", Code: 1}
|
return links, ImageParseError{Message: "No images found", Code: 1}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func ParseReadAllComicsLinks(markup *goquery.Document, c chan []string) ([]string, error) {
|
||||||
|
var links []string
|
||||||
|
markup.Find("img").Each(func(_ int, image *goquery.Selection) {
|
||||||
|
link, _ := image.Attr("src")
|
||||||
|
if !strings.Contains(link, "logo") && (strings.Contains(link, "bp.blogspot.com") || strings.Contains(link, "blogger.googleusercontent") || strings.Contains(link, "covers")) {
|
||||||
|
links = append(links, link)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
c <- links
|
||||||
|
|
||||||
|
if len(links) > 0 {
|
||||||
|
return links, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
return links, ImageParseError{Message: "No images found", Code: 1}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ParseBatcaveBizImageLinks extracts image URLs from the __DATA__.images JavaScript
|
||||||
|
// variable embedded in a batcave.biz page.
|
||||||
|
func ParseBatcaveBizImageLinks(markup *goquery.Document, c chan []string) ([]string, error) {
|
||||||
|
var links []string
|
||||||
|
|
||||||
|
markup.Find("script").Each(func(_ int, s *goquery.Selection) {
|
||||||
|
text := s.Text()
|
||||||
|
if !strings.Contains(text, "__DATA__") {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
arrayRegex := regexp.MustCompile(`"images"\s*:\s*\[([^\]]+)\]`)
|
||||||
|
arrayMatch := arrayRegex.FindStringSubmatch(text)
|
||||||
|
if len(arrayMatch) < 2 {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
urlRegex := regexp.MustCompile(`"([^"]+)"`)
|
||||||
|
for _, m := range urlRegex.FindAllStringSubmatch(arrayMatch[1], -1) {
|
||||||
|
if len(m) >= 2 {
|
||||||
|
links = append(links, m[1])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
c <- links
|
||||||
|
|
||||||
|
if len(links) > 0 {
|
||||||
|
return links, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
return links, ImageParseError{Message: "No images found", Code: 1}
|
||||||
|
}
|
||||||
|
|||||||
183
comic/parser_test.go
Normal file
183
comic/parser_test.go
Normal file
@@ -0,0 +1,183 @@
|
|||||||
|
package comic
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/PuerkitoBio/goquery"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestParseBatcaveBizImageLinks(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
html string
|
||||||
|
expectCount int
|
||||||
|
expectErr bool
|
||||||
|
expectURLs []string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "extracts images from __DATA__",
|
||||||
|
html: `<html><body><script>
|
||||||
|
var __DATA__ = {"images":["https://cdn.batcave.biz/img/001.jpg","https://cdn.batcave.biz/img/002.jpg"]};
|
||||||
|
</script></body></html>`,
|
||||||
|
expectCount: 2,
|
||||||
|
expectErr: false,
|
||||||
|
expectURLs: []string{"https://cdn.batcave.biz/img/001.jpg", "https://cdn.batcave.biz/img/002.jpg"},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "extracts images with spaces around colon and bracket",
|
||||||
|
html: `<html><body><script>
|
||||||
|
var __DATA__ = {"images" : [ "https://cdn.batcave.biz/img/001.jpg" ]};
|
||||||
|
</script></body></html>`,
|
||||||
|
expectCount: 1,
|
||||||
|
expectErr: false,
|
||||||
|
expectURLs: []string{"https://cdn.batcave.biz/img/001.jpg"},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "no __DATA__ script",
|
||||||
|
html: `<html><body><script>
|
||||||
|
var foo = "bar";
|
||||||
|
</script></body></html>`,
|
||||||
|
expectCount: 0,
|
||||||
|
expectErr: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "__DATA__ present but no images key",
|
||||||
|
html: `<html><body><script>
|
||||||
|
var __DATA__ = {"title":"Nightwing"};
|
||||||
|
</script></body></html>`,
|
||||||
|
expectCount: 0,
|
||||||
|
expectErr: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "no script tags",
|
||||||
|
html: `<html><body><p>nothing here</p></body></html>`,
|
||||||
|
expectCount: 0,
|
||||||
|
expectErr: true,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
doc, _ := goquery.NewDocumentFromReader(strings.NewReader(tt.html))
|
||||||
|
ch := make(chan []string, 1)
|
||||||
|
|
||||||
|
links, err := ParseBatcaveBizImageLinks(doc, ch)
|
||||||
|
|
||||||
|
if tt.expectErr && err == nil {
|
||||||
|
t.Error("ParseBatcaveBizImageLinks() expected error, got nil")
|
||||||
|
}
|
||||||
|
if !tt.expectErr && err != nil {
|
||||||
|
t.Errorf("ParseBatcaveBizImageLinks() unexpected error: %v", err)
|
||||||
|
}
|
||||||
|
if len(links) != tt.expectCount {
|
||||||
|
t.Errorf("ParseBatcaveBizImageLinks() returned %d links, want %d", len(links), tt.expectCount)
|
||||||
|
}
|
||||||
|
for i, expected := range tt.expectURLs {
|
||||||
|
if i >= len(links) {
|
||||||
|
t.Errorf("missing link at index %d: want %q", i, expected)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if links[i] != expected {
|
||||||
|
t.Errorf("links[%d] = %q, want %q", i, links[i], expected)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
channelLinks := <-ch
|
||||||
|
if len(channelLinks) != tt.expectCount {
|
||||||
|
t.Errorf("channel received %d links, want %d", len(channelLinks), tt.expectCount)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestImageParseError(t *testing.T) {
|
||||||
|
err := ImageParseError{Message: "test error", Code: 1}
|
||||||
|
if err.Error() != "test error" {
|
||||||
|
t.Errorf("Error() = %q, want %q", err.Error(), "test error")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseImageLinks(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
html string
|
||||||
|
expectCount int
|
||||||
|
expectErr bool
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "extracts blogspot images",
|
||||||
|
html: `<html><body>
|
||||||
|
<img src="https://bp.blogspot.com/page-001.jpg" />
|
||||||
|
<img src="https://bp.blogspot.com/page-002.jpg" />
|
||||||
|
</body></html>`,
|
||||||
|
expectCount: 2,
|
||||||
|
expectErr: false,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "extracts blogger googleusercontent images",
|
||||||
|
html: `<html><body>
|
||||||
|
<img src="https://blogger.googleusercontent.com/page-001.jpg" />
|
||||||
|
</body></html>`,
|
||||||
|
expectCount: 1,
|
||||||
|
expectErr: false,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "extracts covers images",
|
||||||
|
html: `<html><body>
|
||||||
|
<img src="https://example.com/covers/cover-001.jpg" />
|
||||||
|
</body></html>`,
|
||||||
|
expectCount: 1,
|
||||||
|
expectErr: false,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "excludes logo images",
|
||||||
|
html: `<html><body>
|
||||||
|
<img src="https://bp.blogspot.com/logo-site.jpg" />
|
||||||
|
<img src="https://bp.blogspot.com/page-001.jpg" />
|
||||||
|
</body></html>`,
|
||||||
|
expectCount: 1,
|
||||||
|
expectErr: false,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "excludes non-matching images",
|
||||||
|
html: `<html><body>
|
||||||
|
<img src="https://other-site.com/image.jpg" />
|
||||||
|
<img src="https://cdn.example.com/banner.png" />
|
||||||
|
</body></html>`,
|
||||||
|
expectCount: 0,
|
||||||
|
expectErr: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "no images at all",
|
||||||
|
html: `<html><body><p>No images here</p></body></html>`,
|
||||||
|
expectCount: 0,
|
||||||
|
expectErr: true,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
doc, _ := goquery.NewDocumentFromReader(strings.NewReader(tt.html))
|
||||||
|
ch := make(chan []string, 1)
|
||||||
|
|
||||||
|
links, err := ParseImageLinks(doc, ch)
|
||||||
|
|
||||||
|
if tt.expectErr && err == nil {
|
||||||
|
t.Error("ParseImageLinks() expected error, got nil")
|
||||||
|
}
|
||||||
|
if !tt.expectErr && err != nil {
|
||||||
|
t.Errorf("ParseImageLinks() unexpected error: %v", err)
|
||||||
|
}
|
||||||
|
if len(links) != tt.expectCount {
|
||||||
|
t.Errorf("ParseImageLinks() returned %d links, want %d", len(links), tt.expectCount)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Verify the channel also received the links
|
||||||
|
channelLinks := <-ch
|
||||||
|
if len(channelLinks) != tt.expectCount {
|
||||||
|
t.Errorf("channel received %d links, want %d", len(channelLinks), tt.expectCount)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
10
docker-compose.yml
Normal file
10
docker-compose.yml
Normal file
@@ -0,0 +1,10 @@
|
|||||||
|
services:
|
||||||
|
yoink:
|
||||||
|
build: .
|
||||||
|
ports:
|
||||||
|
- "8080:8080"
|
||||||
|
volumes:
|
||||||
|
- ./library:/library
|
||||||
|
environment:
|
||||||
|
- YOINK_LIBRARY=/library
|
||||||
|
restart: unless-stopped
|
||||||
1
go.mod
1
go.mod
@@ -5,6 +5,7 @@ go 1.22.3
|
|||||||
require (
|
require (
|
||||||
github.com/DaRealFreak/cloudflare-bp-go v1.0.4
|
github.com/DaRealFreak/cloudflare-bp-go v1.0.4
|
||||||
github.com/PuerkitoBio/goquery v1.9.2
|
github.com/PuerkitoBio/goquery v1.9.2
|
||||||
|
github.com/andybalholm/brotli v1.2.0
|
||||||
github.com/spf13/cobra v1.8.1
|
github.com/spf13/cobra v1.8.1
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|||||||
4
go.sum
4
go.sum
@@ -4,6 +4,8 @@ github.com/EDDYCJY/fake-useragent v0.2.0 h1:Jcnkk2bgXmDpX0z+ELlUErTkoLb/mxFBNd2Y
|
|||||||
github.com/EDDYCJY/fake-useragent v0.2.0/go.mod h1:5wn3zzlDxhKW6NYknushqinPcAqZcAPHy8lLczCdJdc=
|
github.com/EDDYCJY/fake-useragent v0.2.0/go.mod h1:5wn3zzlDxhKW6NYknushqinPcAqZcAPHy8lLczCdJdc=
|
||||||
github.com/PuerkitoBio/goquery v1.9.2 h1:4/wZksC3KgkQw7SQgkKotmKljk0M6V8TUvA8Wb4yPeE=
|
github.com/PuerkitoBio/goquery v1.9.2 h1:4/wZksC3KgkQw7SQgkKotmKljk0M6V8TUvA8Wb4yPeE=
|
||||||
github.com/PuerkitoBio/goquery v1.9.2/go.mod h1:GHPCaP0ODyyxqcNoFGYlAprUFH81NuRPd0GX3Zu2Mvk=
|
github.com/PuerkitoBio/goquery v1.9.2/go.mod h1:GHPCaP0ODyyxqcNoFGYlAprUFH81NuRPd0GX3Zu2Mvk=
|
||||||
|
github.com/andybalholm/brotli v1.2.0 h1:ukwgCxwYrmACq68yiUqwIWnGY0cTPox/M94sVwToPjQ=
|
||||||
|
github.com/andybalholm/brotli v1.2.0/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY=
|
||||||
github.com/andybalholm/cascadia v1.3.2 h1:3Xi6Dw5lHF15JtdcmAHD3i1+T8plmv7BQ/nsViSLyss=
|
github.com/andybalholm/cascadia v1.3.2 h1:3Xi6Dw5lHF15JtdcmAHD3i1+T8plmv7BQ/nsViSLyss=
|
||||||
github.com/andybalholm/cascadia v1.3.2/go.mod h1:7gtRlve5FxPPgIgX36uWBX58OdBsSS6lUvCFb+h7KvU=
|
github.com/andybalholm/cascadia v1.3.2/go.mod h1:7gtRlve5FxPPgIgX36uWBX58OdBsSS6lUvCFb+h7KvU=
|
||||||
github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
|
github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
|
||||||
@@ -20,6 +22,8 @@ github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
|
|||||||
github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
|
github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
|
||||||
github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
|
github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
|
||||||
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
|
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
|
||||||
|
github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU=
|
||||||
|
github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E=
|
||||||
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
|
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
|
||||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||||
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
|
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
|
||||||
|
|||||||
358
web/server.go
Normal file
358
web/server.go
Normal file
@@ -0,0 +1,358 @@
|
|||||||
|
package web
|
||||||
|
|
||||||
|
import (
|
||||||
|
"archive/zip"
|
||||||
|
"embed"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"io/fs"
|
||||||
|
"net/http"
|
||||||
|
"net/url"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/PuerkitoBio/goquery"
|
||||||
|
"yoink/comic"
|
||||||
|
)
|
||||||
|
|
||||||
|
//go:embed static
|
||||||
|
var staticFiles embed.FS
|
||||||
|
|
||||||
|
type JobStatus string
|
||||||
|
|
||||||
|
const (
|
||||||
|
StatusPending JobStatus = "pending"
|
||||||
|
StatusRunning JobStatus = "running"
|
||||||
|
StatusComplete JobStatus = "complete"
|
||||||
|
StatusError JobStatus = "error"
|
||||||
|
)
|
||||||
|
|
||||||
|
type Job struct {
|
||||||
|
ID string `json:"id"`
|
||||||
|
URL string `json:"url"`
|
||||||
|
Title string `json:"title"`
|
||||||
|
Status JobStatus `json:"status"`
|
||||||
|
Error string `json:"error,omitempty"`
|
||||||
|
CreatedAt time.Time `json:"created_at"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type ComicEntry struct {
|
||||||
|
Title string `json:"title"`
|
||||||
|
CoverURL string `json:"cover_url"`
|
||||||
|
FileURL string `json:"file_url"`
|
||||||
|
DownloadedAt time.Time `json:"downloaded_at"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type Server struct {
|
||||||
|
libraryPath string
|
||||||
|
jobs map[string]*Job
|
||||||
|
mu sync.RWMutex
|
||||||
|
}
|
||||||
|
|
||||||
|
func NewServer(libraryPath string) *Server {
|
||||||
|
return &Server{
|
||||||
|
libraryPath: libraryPath,
|
||||||
|
jobs: make(map[string]*Job),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Server) Handler() http.Handler {
|
||||||
|
mux := http.NewServeMux()
|
||||||
|
|
||||||
|
// Embedded static assets
|
||||||
|
staticFS, _ := fs.Sub(staticFiles, "static")
|
||||||
|
mux.Handle("/static/", http.StripPrefix("/static/", http.FileServer(http.FS(staticFS))))
|
||||||
|
|
||||||
|
// Library files: covers (inline) and cbz downloads (attachment)
|
||||||
|
mux.Handle("/covers/", http.StripPrefix("/covers/", http.FileServer(http.Dir(s.libraryPath))))
|
||||||
|
mux.Handle("/files/", http.StripPrefix("/files/", s.downloadHandler()))
|
||||||
|
|
||||||
|
// API
|
||||||
|
mux.HandleFunc("/api/download", s.handleDownload)
|
||||||
|
mux.HandleFunc("/api/upload", s.handleUpload)
|
||||||
|
mux.HandleFunc("/api/comics", s.handleComics)
|
||||||
|
mux.HandleFunc("/api/jobs", s.handleJobs)
|
||||||
|
mux.HandleFunc("/health", func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
w.WriteHeader(http.StatusOK)
|
||||||
|
})
|
||||||
|
|
||||||
|
// SPA root
|
||||||
|
mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
if r.URL.Path != "/" {
|
||||||
|
http.NotFound(w, r)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
data, _ := staticFiles.ReadFile("static/index.html")
|
||||||
|
w.Header().Set("Content-Type", "text/html; charset=utf-8")
|
||||||
|
w.Write(data)
|
||||||
|
})
|
||||||
|
|
||||||
|
return mux
|
||||||
|
}
|
||||||
|
|
||||||
|
// downloadHandler wraps the library file server to force Content-Disposition: attachment.
|
||||||
|
func (s *Server) downloadHandler() http.Handler {
|
||||||
|
fs := http.FileServer(http.Dir(s.libraryPath))
|
||||||
|
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
w.Header().Set("Content-Disposition", "attachment")
|
||||||
|
fs.ServeHTTP(w, r)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Server) handleDownload(w http.ResponseWriter, r *http.Request) {
|
||||||
|
if r.Method != http.MethodPost {
|
||||||
|
http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
var req struct {
|
||||||
|
URL string `json:"url"`
|
||||||
|
}
|
||||||
|
if err := json.NewDecoder(r.Body).Decode(&req); err != nil || strings.TrimSpace(req.URL) == "" {
|
||||||
|
http.Error(w, "invalid request", http.StatusBadRequest)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
job := &Job{
|
||||||
|
ID: fmt.Sprintf("%d", time.Now().UnixNano()),
|
||||||
|
URL: req.URL,
|
||||||
|
Status: StatusPending,
|
||||||
|
CreatedAt: time.Now(),
|
||||||
|
}
|
||||||
|
|
||||||
|
s.mu.Lock()
|
||||||
|
s.jobs[job.ID] = job
|
||||||
|
s.mu.Unlock()
|
||||||
|
|
||||||
|
go s.runJob(job)
|
||||||
|
|
||||||
|
w.Header().Set("Content-Type", "application/json")
|
||||||
|
json.NewEncoder(w).Encode(job)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Server) runJob(job *Job) {
|
||||||
|
s.mu.Lock()
|
||||||
|
job.Status = StatusRunning
|
||||||
|
s.mu.Unlock()
|
||||||
|
|
||||||
|
markupCh := make(chan *goquery.Document)
|
||||||
|
imageCh := make(chan []string)
|
||||||
|
|
||||||
|
c := comic.NewComic(job.URL, s.libraryPath, imageCh, markupCh)
|
||||||
|
|
||||||
|
s.mu.Lock()
|
||||||
|
job.Title = c.Title
|
||||||
|
s.mu.Unlock()
|
||||||
|
|
||||||
|
errs := c.Download(len(c.Filelist))
|
||||||
|
if len(errs) > 0 {
|
||||||
|
s.mu.Lock()
|
||||||
|
job.Status = StatusError
|
||||||
|
job.Error = errs[0].Error()
|
||||||
|
s.mu.Unlock()
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := c.Archive(); err != nil {
|
||||||
|
s.mu.Lock()
|
||||||
|
job.Status = StatusError
|
||||||
|
job.Error = err.Error()
|
||||||
|
s.mu.Unlock()
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
c.Cleanup()
|
||||||
|
|
||||||
|
s.mu.Lock()
|
||||||
|
job.Status = StatusComplete
|
||||||
|
s.mu.Unlock()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Server) handleComics(w http.ResponseWriter, r *http.Request) {
|
||||||
|
w.Header().Set("Content-Type", "application/json")
|
||||||
|
|
||||||
|
entries := []ComicEntry{}
|
||||||
|
|
||||||
|
dirs, err := os.ReadDir(s.libraryPath)
|
||||||
|
if err != nil {
|
||||||
|
json.NewEncoder(w).Encode(entries)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, dir := range dirs {
|
||||||
|
if !dir.IsDir() {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
title := dir.Name()
|
||||||
|
dirPath := filepath.Join(s.libraryPath, title)
|
||||||
|
|
||||||
|
var coverURL, fileURL string
|
||||||
|
var downloadedAt time.Time
|
||||||
|
|
||||||
|
files, _ := os.ReadDir(dirPath)
|
||||||
|
for _, f := range files {
|
||||||
|
name := f.Name()
|
||||||
|
|
||||||
|
if strings.HasSuffix(name, ".cbz") {
|
||||||
|
fileURL = "/files/" + url.PathEscape(title) + "/" + url.PathEscape(name)
|
||||||
|
if info, err := f.Info(); err == nil {
|
||||||
|
downloadedAt = info.ModTime()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Cover kept by Cleanup: "<Title> 001.jpg"
|
||||||
|
stripped := strings.TrimSpace(strings.TrimPrefix(name, title))
|
||||||
|
if strings.HasPrefix(strings.ToLower(stripped), "001") {
|
||||||
|
coverURL = "/covers/" + url.PathEscape(title) + "/" + url.PathEscape(name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if fileURL != "" {
|
||||||
|
entries = append(entries, ComicEntry{
|
||||||
|
Title: title,
|
||||||
|
CoverURL: coverURL,
|
||||||
|
FileURL: fileURL,
|
||||||
|
DownloadedAt: downloadedAt,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Default: newest first
|
||||||
|
sort.Slice(entries, func(i, j int) bool {
|
||||||
|
return entries[i].DownloadedAt.After(entries[j].DownloadedAt)
|
||||||
|
})
|
||||||
|
|
||||||
|
json.NewEncoder(w).Encode(entries)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Server) handleJobs(w http.ResponseWriter, r *http.Request) {
|
||||||
|
s.mu.RLock()
|
||||||
|
jobs := make([]*Job, 0, len(s.jobs))
|
||||||
|
for _, j := range s.jobs {
|
||||||
|
jobs = append(jobs, j)
|
||||||
|
}
|
||||||
|
s.mu.RUnlock()
|
||||||
|
|
||||||
|
w.Header().Set("Content-Type", "application/json")
|
||||||
|
json.NewEncoder(w).Encode(jobs)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Server) handleUpload(w http.ResponseWriter, r *http.Request) {
|
||||||
|
if r.Method != http.MethodPost {
|
||||||
|
http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// 500 MB limit
|
||||||
|
if err := r.ParseMultipartForm(500 << 20); err != nil {
|
||||||
|
http.Error(w, "request too large", http.StatusRequestEntityTooLarge)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
title := strings.TrimSpace(r.FormValue("title"))
|
||||||
|
if title == "" {
|
||||||
|
http.Error(w, "title required", http.StatusBadRequest)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
// Sanitize: no path separators or shell-special characters
|
||||||
|
title = filepath.Base(title)
|
||||||
|
title = strings.Map(func(r rune) rune {
|
||||||
|
if strings.ContainsRune(`/\:*?"<>|`, r) {
|
||||||
|
return '_'
|
||||||
|
}
|
||||||
|
return r
|
||||||
|
}, title)
|
||||||
|
|
||||||
|
fileHeaders := r.MultipartForm.File["images"]
|
||||||
|
if len(fileHeaders) == 0 {
|
||||||
|
http.Error(w, "no images provided", http.StatusBadRequest)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// Sort by original filename so page order is preserved
|
||||||
|
sort.Slice(fileHeaders, func(i, j int) bool {
|
||||||
|
return fileHeaders[i].Filename < fileHeaders[j].Filename
|
||||||
|
})
|
||||||
|
|
||||||
|
dir := filepath.Join(s.libraryPath, title)
|
||||||
|
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||||||
|
http.Error(w, "failed to create directory", http.StatusInternalServerError)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
cbzPath := filepath.Join(dir, title+".cbz")
|
||||||
|
cbzFile, err := os.Create(cbzPath)
|
||||||
|
if err != nil {
|
||||||
|
http.Error(w, "failed to create archive", http.StatusInternalServerError)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
defer cbzFile.Close()
|
||||||
|
|
||||||
|
zw := zip.NewWriter(cbzFile)
|
||||||
|
defer zw.Close()
|
||||||
|
|
||||||
|
imageExts := map[string]bool{".jpg": true, ".jpeg": true, ".png": true, ".webp": true}
|
||||||
|
idx := 1
|
||||||
|
|
||||||
|
for _, fh := range fileHeaders {
|
||||||
|
ext := strings.ToLower(filepath.Ext(fh.Filename))
|
||||||
|
if !imageExts[ext] {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if ext == ".jpeg" {
|
||||||
|
ext = ".jpg"
|
||||||
|
}
|
||||||
|
|
||||||
|
entryName := fmt.Sprintf("%03d%s", idx, ext)
|
||||||
|
|
||||||
|
src, err := fh.Open()
|
||||||
|
if err != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
// Save first image as cover: "<Title> 001.jpg"
|
||||||
|
if idx == 1 {
|
||||||
|
coverPath := filepath.Join(dir, title+" "+entryName)
|
||||||
|
if cf, err := os.Create(coverPath); err == nil {
|
||||||
|
io.Copy(cf, src)
|
||||||
|
cf.Close()
|
||||||
|
src.Close()
|
||||||
|
src, err = fh.Open()
|
||||||
|
if err != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
ze, err := zw.Create(entryName)
|
||||||
|
if err != nil {
|
||||||
|
src.Close()
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
io.Copy(ze, src)
|
||||||
|
src.Close()
|
||||||
|
idx++
|
||||||
|
}
|
||||||
|
|
||||||
|
if idx == 1 {
|
||||||
|
// Nothing was written — no valid images
|
||||||
|
os.RemoveAll(dir)
|
||||||
|
http.Error(w, "no valid images in upload", http.StatusBadRequest)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
w.Header().Set("Content-Type", "application/json")
|
||||||
|
json.NewEncoder(w).Encode(map[string]string{"title": title, "status": "complete"})
|
||||||
|
}
|
||||||
|
|
||||||
|
func Listen(addr string, libraryPath string) error {
|
||||||
|
srv := NewServer(libraryPath)
|
||||||
|
fmt.Printf("Yoink web server listening on %s\n", addr)
|
||||||
|
return http.ListenAndServe(addr, srv.Handler())
|
||||||
|
}
|
||||||
1510
web/static/index.html
Normal file
1510
web/static/index.html
Normal file
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user