bryan/yoink
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

working comic module TODO fix torrent downloads

Browse Source
This commit is contained in:
Bryan Bailey
2022-03-10 22:26:50 -05:00
parent b0e2ad0a3b
commit d421ac310a
7 changed files with 94 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
requirements.txt
View File

@@ -5,17 +5,10 @@ charset-normalizer==2.0.12
click==8.0.4
coverage==6.3.2
idna==3.3
lbry-libtorrent==1.2.4
python-qbittorrent==0.4.2
qbittorrent-api==2022.2.28
requests==2.27.1
soupsieve==2.3.1
urllib3==1.26.8
beautifulsoup4==4.10.0
bs4==0.0.1
certifi==2021.10.8
charset-normalizer==2.0.12
click==8.0.4
coverage==6.3.2
idna==3.3
qbittorrent==0.1.6
requests==2.27.1
six==1.16.0
soupsieve==2.3.1
urllib3==1.26.8

44
yoink/cli.py
View File

@@ -1,11 +1,14 @@
from pydoc import cli
import click
import libtorrent as lib
from yoink.common import qb_client, app_root, library_path, config_path
from yoink.comic import Comic, ComicArchiver
from yoink.torrent import Torrent, TorrentDownloader
from yoink.common import app_root, library_path, config_path
from yoink.comic import Comic
from yoink.torrent import Torrent, downloader
session = lib.session()
session.listen_on(6881, 6891)
queue = []
@click.group()
def yoink():
@@ -20,15 +23,30 @@ def download(url):
click.echo('url cannot be blank')
return 1
comic = Comic(url)
archiver = ComicArchiver(comic)
click.echo(f'Downloading {comic.title}')
archiver.download()
click.echo('Building comic archive')
archiver.generate_archive()
click.echo('Cleaning up')
archiver.cleanup_worktree()
click.echo('Success')
# comic = Comic(url)
# click.echo(f'Downloading {comic.title}')
# comic.archiver.download()
# click.echo('Building comic archive')
# comic.archiver.generate_archive()
# click.echo('Cleaning up')
# comic.archiver.cleanup_worktree()
# click.echo('Success')
torrent = Torrent(url)
print(torrent.magnet_link)
# queue.append(lib.add_magnet_uri(session, torrent.magnet_link, {'save_path': library_path}))
# while queue:
# next_shift = 0
# for index, download in enumerate(queue):
# if not download.is_seed():
# status = download.status()
# downloader.add(torrent)
# downloader.download()
if __name__=='__main__':
yoink()

1
yoink/comic.py
View File

@@ -12,6 +12,7 @@ import urllib
class Comic(Scrapable):
def __init__(self, url) -> None:
super().__init__(url)
self.archiver = ComicArchiver(self)
def __get_image_src(self, comic):

2
yoink/common.py
View File

@@ -1,6 +1,5 @@
import pathlib
from qbittorrent import Client
import pathlib
# TODO replace os path with pathlib
@@ -14,6 +13,5 @@ library_path = os.path.abspath(os.path.join(os.environ.get('HOME'), 'yoink/libra
required_comic_files = ('.cbr', '.cbz', '000.jpg', '001.jpg')
skippable_images = ('logo-1.png', 'logo.png', 'report.png', 'request.png', 'prev.png', 'Next.png', 'Donate.png', '11.png')
torrent_concurrent_download_limit = 1
qb_client = Client('127.0.0.1:8080').login('admin', 'adminadmin')
supported_sites = ['readallcomics.com', 'tpb.party']
headers = {'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.75 Safari/537.36'}

30
yoink/scraper.py
View File

@@ -10,16 +10,32 @@ class Scrapable:
def __init__(self, url) -> None:
self.url = url
for link in supported_sites:
if link in self.url:
return
else:
raise ValueError('Unsupported site')
self.__check_site_support()
# for link in supported_sites:
# if link in self.url:
# return
# else:
# raise ValueError('Unsupported site')
# if not any(url in link for link in supported_sites):
# raise ValueError('Unsupported site')
@property
def markup(self) -> str: return requests.get(self.url).content
@property
def soup(self) -> BeautifulSoup: return BeautifulSoup(self.markup, 'html.parser')
def soup(self) -> BeautifulSoup: return BeautifulSoup(self.markup, 'html.parser')
def __check_site_support(self):
num_of_sites = len(supported_sites)
while num_of_sites > 0:
for link in supported_sites:
if link in self.url:
return
num_of_sites = num_of_sites - 1
raise ValueError('Unsupported site')

19
yoink/tests/test_basic.py
View File

@@ -1,11 +1,11 @@
import imp
from bs4 import BeautifulSoup
import os
import unittest
from yoink.common import app_root, library_path, config_path, skippable_images, supported_sites, qb_client, required_comic_files, torrent_concurrent_download_limit, headers
from yoink.common import app_root, library_path, config_path, skippable_images, supported_sites, required_comic_files, torrent_concurrent_download_limit, headers
from yoink.comic import Comic, ComicArchiver
from yoink.scraper import Scrapable
@@ -38,4 +38,17 @@ class BasicTestCase(unittest.TestCase):
def test_006_folder_cleaned_after_archive_generation(self):
self.archiver.cleanup_worktree()
self.assertAlmostEqual(len(os.listdir(os.path.join(library_path, f'comics/{self.comic.title}'))), 3)
self.assertAlmostEqual(len(os.listdir(os.path.join(library_path, f'comics/{self.comic.title}'))), 3)
def test_007_comic_instance_has_archiver(self):
self.assertIsInstance(self.comic.archiver, ComicArchiver)
def test_008_comic_is_subclass_scrapable(self):
self.assertTrue(issubclass(Comic, Scrapable))
def test_009_invalid_comic_link(self):
with self.assertRaises(ValueError) as condition:
comic = Comic('https://viz.com')
self.assertTrue('Unsupported' in str(condition.exception))

24
yoink/torrent.py
View File

@@ -1,9 +1,10 @@
from bs4 import BeautifulSoup
from qbittorrent import Client
import requests
import os
from yoink.common import qb_client, library_path, config_path, app_root, headers
from yoink.common import library_path, config_path, app_root, headers
from yoink.scraper import Scrapable
@@ -12,6 +13,9 @@ stopped_state = ('pausedUP', 'stalledUP', 'uploading', 'seeding')
class TorrentDownloader:
def __init__(self) -> None:
self.limit = 1
@@ -24,7 +28,7 @@ class TorrentDownloader:
@classmethod
def get_torrent(cls, name):
return [torrent for torrent in qb_client.torrents() if name == torrent['name']][0]
return [torrent for torrent in new_downloader.torrents() if name == torrent['name']][0]
@classmethod
def quick_download(cls, url):
@@ -36,7 +40,8 @@ class TorrentDownloader:
soup = BeautifulSoup(markup, 'html.parser')
magnet_link = soup.find('a', attrs={'title': 'Get this torrent'}.attrs['href'])
qb_client.download_from_link(url if url.startswith('magnet') else magnet_link)
# qb_client.download_from_link(url if url.startswith('magnet') else magnet_link)
new_downloader.torrents.add(urls=url if url.startswith('magnet') else magnet_link)
def set_path(self, path):
@@ -53,21 +58,30 @@ class TorrentDownloader:
def add(self, torrent):
if not isinstance(torrent, Torrent):
raise TypeError('Not a valid torrent')
self.queue.append(torrent)
# TODO separate download method into new thread
def download(self):
while len(self.queue) > 0:
for torrent in self.queue:
if not isinstance(torrent, Torrent):
raise TypeError('Not a valid torrent')
qb_client.download_from_link(torrent.magnet_link)
# qb_client.download_from_link(torrent.magnet_link)
print(torrent.magnet_link)
# qb_client2.torrents.add(urls=torrent.magnet_link)
# print(new_downloader.torrents())
downloader = TorrentDownloader()
class Torrent(Scrapable):
def __init__(self, url) -> None:
super().__init__(url)
@property
def name(self) -> str: return self.soup.find('div', attrs={'id': 'title'})