From d421ac310a6c1eb41ca08a8330b5ae0572750cb0 Mon Sep 17 00:00:00 2001 From: Bryan Bailey Date: Thu, 10 Mar 2022 22:26:50 -0500 Subject: [PATCH] working comic module TODO fix torrent downloads --- requirements.txt | 15 ++++--------- yoink/cli.py | 44 +++++++++++++++++++++++++++------------ yoink/comic.py | 1 + yoink/common.py | 2 -- yoink/scraper.py | 30 +++++++++++++++++++------- yoink/tests/test_basic.py | 19 ++++++++++++++--- yoink/torrent.py | 24 ++++++++++++++++----- 7 files changed, 94 insertions(+), 41 deletions(-) diff --git a/requirements.txt b/requirements.txt index 46fa72c..9c99ee3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,17 +5,10 @@ charset-normalizer==2.0.12 click==8.0.4 coverage==6.3.2 idna==3.3 +lbry-libtorrent==1.2.4 +python-qbittorrent==0.4.2 +qbittorrent-api==2022.2.28 requests==2.27.1 -soupsieve==2.3.1 -urllib3==1.26.8 -beautifulsoup4==4.10.0 -bs4==0.0.1 -certifi==2021.10.8 -charset-normalizer==2.0.12 -click==8.0.4 -coverage==6.3.2 -idna==3.3 -qbittorrent==0.1.6 -requests==2.27.1 +six==1.16.0 soupsieve==2.3.1 urllib3==1.26.8 diff --git a/yoink/cli.py b/yoink/cli.py index 050327b..bc43b4e 100644 --- a/yoink/cli.py +++ b/yoink/cli.py @@ -1,11 +1,14 @@ -from pydoc import cli import click +import libtorrent as lib -from yoink.common import qb_client, app_root, library_path, config_path -from yoink.comic import Comic, ComicArchiver -from yoink.torrent import Torrent, TorrentDownloader +from yoink.common import app_root, library_path, config_path +from yoink.comic import Comic +from yoink.torrent import Torrent, downloader +session = lib.session() +session.listen_on(6881, 6891) +queue = [] @click.group() def yoink(): @@ -20,15 +23,30 @@ def download(url): click.echo('url cannot be blank') return 1 - comic = Comic(url) - archiver = ComicArchiver(comic) - click.echo(f'Downloading {comic.title}') - archiver.download() - click.echo('Building comic archive') - archiver.generate_archive() - click.echo('Cleaning up') - archiver.cleanup_worktree() - click.echo('Success') + # comic = Comic(url) + # click.echo(f'Downloading {comic.title}') + # comic.archiver.download() + # click.echo('Building comic archive') + # comic.archiver.generate_archive() + # click.echo('Cleaning up') + # comic.archiver.cleanup_worktree() + # click.echo('Success') + + torrent = Torrent(url) + print(torrent.magnet_link) + + # queue.append(lib.add_magnet_uri(session, torrent.magnet_link, {'save_path': library_path})) + + # while queue: + # next_shift = 0 + + # for index, download in enumerate(queue): + # if not download.is_seed(): + # status = download.status() + + + # downloader.add(torrent) + # downloader.download() if __name__=='__main__': yoink() \ No newline at end of file diff --git a/yoink/comic.py b/yoink/comic.py index 36a1313..96ef030 100644 --- a/yoink/comic.py +++ b/yoink/comic.py @@ -12,6 +12,7 @@ import urllib class Comic(Scrapable): def __init__(self, url) -> None: super().__init__(url) + self.archiver = ComicArchiver(self) def __get_image_src(self, comic): diff --git a/yoink/common.py b/yoink/common.py index 47906f0..7bd74f6 100644 --- a/yoink/common.py +++ b/yoink/common.py @@ -1,6 +1,5 @@ import pathlib -from qbittorrent import Client import pathlib # TODO replace os path with pathlib @@ -14,6 +13,5 @@ library_path = os.path.abspath(os.path.join(os.environ.get('HOME'), 'yoink/libra required_comic_files = ('.cbr', '.cbz', '000.jpg', '001.jpg') skippable_images = ('logo-1.png', 'logo.png', 'report.png', 'request.png', 'prev.png', 'Next.png', 'Donate.png', '11.png') torrent_concurrent_download_limit = 1 -qb_client = Client('127.0.0.1:8080').login('admin', 'adminadmin') supported_sites = ['readallcomics.com', 'tpb.party'] headers = {'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.75 Safari/537.36'} diff --git a/yoink/scraper.py b/yoink/scraper.py index bd98c7a..593b435 100644 --- a/yoink/scraper.py +++ b/yoink/scraper.py @@ -10,16 +10,32 @@ class Scrapable: def __init__(self, url) -> None: self.url = url - for link in supported_sites: - if link in self.url: - return - else: - raise ValueError('Unsupported site') - + + self.__check_site_support() + # for link in supported_sites: + # if link in self.url: + # return + # else: + # raise ValueError('Unsupported site') + # if not any(url in link for link in supported_sites): + # raise ValueError('Unsupported site') @property def markup(self) -> str: return requests.get(self.url).content @property - def soup(self) -> BeautifulSoup: return BeautifulSoup(self.markup, 'html.parser') \ No newline at end of file + def soup(self) -> BeautifulSoup: return BeautifulSoup(self.markup, 'html.parser') + + + def __check_site_support(self): + num_of_sites = len(supported_sites) + + while num_of_sites > 0: + for link in supported_sites: + if link in self.url: + return + + num_of_sites = num_of_sites - 1 + + raise ValueError('Unsupported site') \ No newline at end of file diff --git a/yoink/tests/test_basic.py b/yoink/tests/test_basic.py index ba2c6ac..fb41e5e 100644 --- a/yoink/tests/test_basic.py +++ b/yoink/tests/test_basic.py @@ -1,11 +1,11 @@ -import imp from bs4 import BeautifulSoup import os import unittest -from yoink.common import app_root, library_path, config_path, skippable_images, supported_sites, qb_client, required_comic_files, torrent_concurrent_download_limit, headers +from yoink.common import app_root, library_path, config_path, skippable_images, supported_sites, required_comic_files, torrent_concurrent_download_limit, headers from yoink.comic import Comic, ComicArchiver +from yoink.scraper import Scrapable @@ -38,4 +38,17 @@ class BasicTestCase(unittest.TestCase): def test_006_folder_cleaned_after_archive_generation(self): self.archiver.cleanup_worktree() - self.assertAlmostEqual(len(os.listdir(os.path.join(library_path, f'comics/{self.comic.title}'))), 3) \ No newline at end of file + self.assertAlmostEqual(len(os.listdir(os.path.join(library_path, f'comics/{self.comic.title}'))), 3) + + def test_007_comic_instance_has_archiver(self): + self.assertIsInstance(self.comic.archiver, ComicArchiver) + + def test_008_comic_is_subclass_scrapable(self): + self.assertTrue(issubclass(Comic, Scrapable)) + + def test_009_invalid_comic_link(self): + + with self.assertRaises(ValueError) as condition: + comic = Comic('https://viz.com') + + self.assertTrue('Unsupported' in str(condition.exception)) \ No newline at end of file diff --git a/yoink/torrent.py b/yoink/torrent.py index 2695d3d..abf8107 100644 --- a/yoink/torrent.py +++ b/yoink/torrent.py @@ -1,9 +1,10 @@ from bs4 import BeautifulSoup +from qbittorrent import Client import requests import os -from yoink.common import qb_client, library_path, config_path, app_root, headers +from yoink.common import library_path, config_path, app_root, headers from yoink.scraper import Scrapable @@ -12,6 +13,9 @@ stopped_state = ('pausedUP', 'stalledUP', 'uploading', 'seeding') + + + class TorrentDownloader: def __init__(self) -> None: self.limit = 1 @@ -24,7 +28,7 @@ class TorrentDownloader: @classmethod def get_torrent(cls, name): - return [torrent for torrent in qb_client.torrents() if name == torrent['name']][0] + return [torrent for torrent in new_downloader.torrents() if name == torrent['name']][0] @classmethod def quick_download(cls, url): @@ -36,7 +40,8 @@ class TorrentDownloader: soup = BeautifulSoup(markup, 'html.parser') magnet_link = soup.find('a', attrs={'title': 'Get this torrent'}.attrs['href']) - qb_client.download_from_link(url if url.startswith('magnet') else magnet_link) + # qb_client.download_from_link(url if url.startswith('magnet') else magnet_link) + new_downloader.torrents.add(urls=url if url.startswith('magnet') else magnet_link) def set_path(self, path): @@ -53,21 +58,30 @@ class TorrentDownloader: def add(self, torrent): if not isinstance(torrent, Torrent): raise TypeError('Not a valid torrent') - + self.queue.append(torrent) + # TODO separate download method into new thread def download(self): while len(self.queue) > 0: for torrent in self.queue: if not isinstance(torrent, Torrent): raise TypeError('Not a valid torrent') - qb_client.download_from_link(torrent.magnet_link) + # qb_client.download_from_link(torrent.magnet_link) + print(torrent.magnet_link) + # qb_client2.torrents.add(urls=torrent.magnet_link) + # print(new_downloader.torrents()) + + +downloader = TorrentDownloader() + class Torrent(Scrapable): def __init__(self, url) -> None: super().__init__(url) + @property def name(self) -> str: return self.soup.find('div', attrs={'id': 'title'})