yoink-go/comic/download.go

package comic

import (
	"fmt"
	"io"
	"net/http"
	"os"
	"path/filepath"
	"time"

	cloudflarebp "github.com/DaRealFreak/cloudflare-bp-go"
)

// func _downloadFile(wg *sync.WaitGroup, url string, page int, c *Comic) error {
// 	defer wg.Done()

// 	pageNumber := fmt.Sprintf("%03d", page)
// 	formattedImagePath := fmt.Sprintf("%s %s.jpg", c.Title, pageNumber)
// 	imageFilepath, _ := filepath.Abs(filepath.Join(c.LibraryPath, c.Title, formattedImagePath))

// 	if err := os.MkdirAll(
// 		filepath.Dir(imageFilepath),
// 		os.ModePerm,
// 	); err != nil {
// 		return ComicDownloadError{
// 			Message: "error creating directory",
// 			Code:    1,
// 		}
// 	}

// 	// get image data
// 	res, err := handleRequest(url)
// 	if err != nil {
// 		return ComicDownloadError{
// 			Message: "invalid request",
// 			Code:    1,
// 		}
// 	}
// 	defer res.Body.Close()

// 	var fileChannel = make(chan *os.File)

// 	go func() error {
// 		imageFile, err := os.Create(imageFilepath)
// 		if err != nil {
// 			return ComicDownloadError{
// 				Message: "error creating image file",
// 				Code:    1,
// 			}
// 		}
// 		defer imageFile.Close()

// 		fileChannel <- imageFile

// 		return nil
// 	}()

// 	println("Downloading", imageFilepath)

// 	go func(
// 		fc chan *os.File,
// 		res *http.Response,
// 	) error {
// 		buffer := make([]byte, 64*1024)

// 		defer close(fileChannel)

// 		// write image data
// 		_, err := io.CopyBuffer(<-fc, res.Body, buffer)
// 		if err != nil {
// 			return ComicDownloadError{
// 				Message: "Unable to save file contents",
// 				Code:    1,
// 			}
// 		}

// 		return nil
// 	}(fileChannel, res)

// 	return nil
// }

func downloadFile(url string, page int, c *Comic) error {
	pageNumber := fmt.Sprintf("%03d", page)
	formattedImagePath := fmt.Sprintf("%s %s.jpg", c.Title, pageNumber)
	imageFilepath, _ := filepath.Abs(filepath.Join(c.LibraryPath, c.Title, formattedImagePath))

	if err := os.MkdirAll(
		filepath.Dir(imageFilepath),
		os.ModePerm,
	); err != nil {
		return ComicDownloadError{
			Message: "error creating directory",
			Code:    1,
		}
	}

	res, err := handleRequest(url)
	if err != nil {
		return ComicDownloadError{
			Message: "invalid request",
			Code:    1,
		}
	}
	defer res.Body.Close()

	imageFile, err := os.Create(imageFilepath)
	if err != nil {
		return ComicDownloadError{
			Message: "error creating image file",
			Code:    1,
		}
	}
	defer imageFile.Close()

	written, err := io.Copy(imageFile, res.Body)
	if err != nil {
		return ComicDownloadError{
			Message: "Unable to save file contents",
			Code:    1,
		}
	}

	if written == 0 {
		return ComicDownloadError{
			Message: "Unable to save file contents",
			Code:    1,
		}
	}

	return nil
}

func handleRequest(url string) (*http.Response, error) {
	// adjust timeout and keep-alive to avoid connection timeout
	transport := &http.Transport{
		DisableKeepAlives:   false,
		MaxIdleConnsPerHost: 32,
	}

	// add cloudflare bypass
	cfTransport := cloudflarebp.AddCloudFlareByPass(transport)

	// prevents cloudflarebp from occasionally returning the wrong type
	if converted, ok := cfTransport.(*http.Transport); ok {
		transport = converted
	}

	client := &http.Client{
		Timeout:   time.Second * 30,
		Transport: transport,
	}

	// mimic generic browser
	req, err := http.NewRequest("GET", url, nil)
	if err != nil {
		return nil, ComicDownloadError{
			Message: "invalid request",
			Code:    1,
		}
	}

	req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.114 Safari/537.36")

	res, err := client.Do(req)
	if err != nil {
		return nil, ComicDownloadError{
			Message: "invalid request",
			Code:    1,
		}
	}

	if res.StatusCode != http.StatusOK {
		return nil, ComicDownloadError{
			Message: "bad response",
			Code:    1,
		}
	}

	return res, nil
}

func (c *Comic) Download(concurrency int) []error {
	// var wg sync.WaitGroup
	// wg.Add(len(c.Filelist))

	// for i, link := range c.Filelist {
	// 	go downloadFile(link, i+1, c)
	// }

	// wg.Wait()
	// return nil
	jobs := make(chan Download)
	results := make(chan error)

	for worker := 1; worker <= concurrency; worker++ {
		go workerPool(jobs, results)
	}

	for i, url := range c.Filelist {
		jobs <- Download{
			URL:   url,
			Page:  i + 1,
			Comic: c,
		}
	}

	var errors []error
	for i := 0; i < len(c.Filelist); i++ {
		err := <-results
		if err != nil {
			errors = append(errors, err)
		}
	}
	return errors
}

type Download struct {
	URL   string
	Page  int
	Comic *Comic
}

func workerPool(jobs <-chan Download, results chan<- error) {
	for job := range jobs {
		results <- downloadFile(job.URL, job.Page, job.Comic)
	}
}

func DownloadComicImages(c *Comic, concurrency int) []error {
	jobs := make(chan Download)
	results := make(chan error)

	for worker := 1; worker <= concurrency; worker++ {
		go workerPool(jobs, results)
	}

	for i, url := range c.Filelist {
		jobs <- Download{
			URL:   url,
			Page:  i + 1,
			Comic: c,
		}
	}

	var errors []error
	for i := 0; i < len(c.Filelist); i++ {
		err := <-results
		if err != nil {
			errors = append(errors, err)
		}
	}
	return errors
}