yoink-go initial commit

comic/download.go

@@ -0,0 +1,254 @@
+package comic
+
+import (
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"path/filepath"
+	"time"
+
+	cloudflarebp "github.com/DaRealFreak/cloudflare-bp-go"
+)
+
+// func _downloadFile(wg *sync.WaitGroup, url string, page int, c *Comic) error {
+// 	defer wg.Done()
+
+// 	pageNumber := fmt.Sprintf("%03d", page)
+// 	formattedImagePath := fmt.Sprintf("%s %s.jpg", c.Title, pageNumber)
+// 	imageFilepath, _ := filepath.Abs(filepath.Join(c.LibraryPath, c.Title, formattedImagePath))
+
+// 	if err := os.MkdirAll(
+// 		filepath.Dir(imageFilepath),
+// 		os.ModePerm,
+// 	); err != nil {
+// 		return ComicDownloadError{
+// 			Message: "error creating directory",
+// 			Code:    1,
+// 		}
+// 	}
+
+// 	// get image data
+// 	res, err := handleRequest(url)
+// 	if err != nil {
+// 		return ComicDownloadError{
+// 			Message: "invalid request",
+// 			Code:    1,
+// 		}
+// 	}
+// 	defer res.Body.Close()
+
+// 	var fileChannel = make(chan *os.File)
+
+// 	go func() error {
+// 		imageFile, err := os.Create(imageFilepath)
+// 		if err != nil {
+// 			return ComicDownloadError{
+// 				Message: "error creating image file",
+// 				Code:    1,
+// 			}
+// 		}
+// 		defer imageFile.Close()
+
+// 		fileChannel <- imageFile
+
+// 		return nil
+// 	}()
+
+// 	println("Downloading", imageFilepath)
+
+// 	go func(
+// 		fc chan *os.File,
+// 		res *http.Response,
+// 	) error {
+// 		buffer := make([]byte, 64*1024)
+
+// 		defer close(fileChannel)
+
+// 		// write image data
+// 		_, err := io.CopyBuffer(<-fc, res.Body, buffer)
+// 		if err != nil {
+// 			return ComicDownloadError{
+// 				Message: "Unable to save file contents",
+// 				Code:    1,
+// 			}
+// 		}
+
+// 		return nil
+// 	}(fileChannel, res)
+
+// 	return nil
+// }
+
+func downloadFile(url string, page int, c *Comic) error {
+	pageNumber := fmt.Sprintf("%03d", page)
+	formattedImagePath := fmt.Sprintf("%s %s.jpg", c.Title, pageNumber)
+	imageFilepath, _ := filepath.Abs(filepath.Join(c.LibraryPath, c.Title, formattedImagePath))
+
+	if err := os.MkdirAll(
+		filepath.Dir(imageFilepath),
+		os.ModePerm,
+	); err != nil {
+		return ComicDownloadError{
+			Message: "error creating directory",
+			Code:    1,
+		}
+	}
+
+	res, err := handleRequest(url)
+	if err != nil {
+		return ComicDownloadError{
+			Message: "invalid request",
+			Code:    1,
+		}
+	}
+	defer res.Body.Close()
+
+	imageFile, err := os.Create(imageFilepath)
+	if err != nil {
+		return ComicDownloadError{
+			Message: "error creating image file",
+			Code:    1,
+		}
+	}
+	defer imageFile.Close()
+
+	written, err := io.Copy(imageFile, res.Body)
+	if err != nil {
+		return ComicDownloadError{
+			Message: "Unable to save file contents",
+			Code:    1,
+		}
+	}
+
+	if written == 0 {
+		return ComicDownloadError{
+			Message: "Unable to save file contents",
+			Code:    1,
+		}
+	}
+
+	return nil
+}
+
+func handleRequest(url string) (*http.Response, error) {
+	// adjust timeout and keep-alive to avoid connection timeout
+	transport := &http.Transport{
+		DisableKeepAlives:   false,
+		MaxIdleConnsPerHost: 32,
+	}
+
+	// add cloudflare bypass
+	cfTransport := cloudflarebp.AddCloudFlareByPass(transport)
+
+	// prevents cloudflarebp from occasionally returning the wrong type
+	if converted, ok := cfTransport.(*http.Transport); ok {
+		transport = converted
+	}
+
+	client := &http.Client{
+		Timeout:   time.Second * 30,
+		Transport: transport,
+	}
+
+	// mimic generic browser
+	req, err := http.NewRequest("GET", url, nil)
+	if err != nil {
+		return nil, ComicDownloadError{
+			Message: "invalid request",
+			Code:    1,
+		}
+	}
+
+	req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.114 Safari/537.36")
+
+	res, err := client.Do(req)
+	if err != nil {
+		return nil, ComicDownloadError{
+			Message: "invalid request",
+			Code:    1,
+		}
+	}
+
+	if res.StatusCode != http.StatusOK {
+		return nil, ComicDownloadError{
+			Message: "bad response",
+			Code:    1,
+		}
+	}
+
+	return res, nil
+}
+
+func (c *Comic) Download(concurrency int) []error {
+	// var wg sync.WaitGroup
+	// wg.Add(len(c.Filelist))
+
+	// for i, link := range c.Filelist {
+	// 	go downloadFile(link, i+1, c)
+	// }
+
+	// wg.Wait()
+	// return nil
+	jobs := make(chan Download)
+	results := make(chan error)
+
+	for worker := 1; worker <= concurrency; worker++ {
+		go workerPool(jobs, results)
+	}
+
+	for i, url := range c.Filelist {
+		jobs <- Download{
+			URL:   url,
+			Page:  i + 1,
+			Comic: c,
+		}
+	}
+
+	var errors []error
+	for i := 0; i < len(c.Filelist); i++ {
+		err := <-results
+		if err != nil {
+			errors = append(errors, err)
+		}
+	}
+	return errors
+}
+
+type Download struct {
+	URL   string
+	Page  int
+	Comic *Comic
+}
+
+func workerPool(jobs <-chan Download, results chan<- error) {
+	for job := range jobs {
+		results <- downloadFile(job.URL, job.Page, job.Comic)
+	}
+}
+
+func DownloadComicImages(c *Comic, concurrency int) []error {
+	jobs := make(chan Download)
+	results := make(chan error)
+
+	for worker := 1; worker <= concurrency; worker++ {
+		go workerPool(jobs, results)
+	}
+
+	for i, url := range c.Filelist {
+		jobs <- Download{
+			URL:   url,
+			Page:  i + 1,
+			Comic: c,
+		}
+	}
+
+	var errors []error
+	for i := 0; i < len(c.Filelist); i++ {
+		err := <-results
+		if err != nil {
+			errors = append(errors, err)
+		}
+	}
+	return errors
+}