diff --git a/.gitignore b/.gitignore index 0f4df66..06647ff 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,7 @@ env __pycache__ +.pytest_cache .coverage *.egg-info -sample \ No newline at end of file +sample +.tox \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..2b90214 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,3 @@ +[build-system] +requires = ["setuptools>=42.0", "wheel"] +build-backend = "setuptools.build_meta" \ No newline at end of file diff --git a/report.xml b/report.xml new file mode 100644 index 0000000..6300d72 --- /dev/null +++ b/report.xml @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/requirements_dev.txt b/requirements_dev.txt new file mode 100644 index 0000000..0094d29 --- /dev/null +++ b/requirements_dev.txt @@ -0,0 +1,5 @@ +flake8==3.9.2 +tox==3.24.3 +pytest==6.2.5 +pytest-cov==2.12.1 +mypy==0.910 \ No newline at end of file diff --git a/test.sh b/test.sh deleted file mode 100644 index d5c8a52..0000000 --- a/test.sh +++ /dev/null @@ -1,69 +0,0 @@ -# : ${DIALOG_OK=0} -# : ${DIALOG_CANCEL=1} -# : ${DIALOG_HELP=2} -# : ${DIALOG_EXTRA=3} -# : ${DIALOG_ITEM_HELP=4} -# : ${DIALOG_ESC=255} - -# lipsum="Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum." -# tmp_file=$(tempfile 2>/dev/null) || temp_file=/tmp/test$$ -# trap "rm -f $tmp_file" 0 1 2 5 15 - -# dialog --title "Testing" --clear --inputbox "${lipsum}" 16 51 2> $tmp_file - -# return_value=$? - -# case $return_value in -# $DIALOG_OK) -# echo "Result: $(cat $tmp_file)";; -# $DIALOG_CANCEL) -# echo "Cancel pressed.";; -# $DIALOG_HELP) -# echo "Help pressed.";; -# $DIALOG_EXTRA) -# echo "Extra button pressed.";; -# $DIALOG_ITEM_HELP) -# echo "Item-help button pressed.";; -# $DIALOG_ESC) -# if test -s $tmp_file ; then -# cat $tmp_file -# else -# echo "ESC pressed." -# fi -# ;; -# esac - -# dialog --begin 5 70 --backtitle "Shirak v0.1.0" --title "Info" --clear --msgbox 'Greetings, mortal...' 16 56 2> /dev/null - -output="/tmp/shit.txt" ->$output - -function hello() { - local name=${@-"Stranger"} - - dialog --backtitle "Shirak v0.1.0 | test" --title "Greetings" --clear --msgbox "Greetings, ${name}..." 10 41 -} - - -trap "rm $output; exit" SIGHUP SIGINT SIGTERM - - -dialog --title "Input your name" --backtitle "Shirak v0.1.0 | test" --inputbox "Enter your name " 8 60 2>$output - -response=$? - -name=$(<$output) - -case $response in - 0) - hello ${name} - ;; - 1) - echo "Cancel pressed." - ;; - 255) - echo "[esc] pressed." -esac - - -rm $output \ No newline at end of file diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..4ef2ab2 --- /dev/null +++ b/tox.ini @@ -0,0 +1,13 @@ +[tox] +minversion = 3.8.0 +envlist = py39 +isolated_build = true + + +[testenv] +setenv = + PYTHONPATH = {toxinidir} +deps = + -r{toxinidir}/requirements_dev.txt +commands = + pytest --junitxml report.xml yoink/tests/test_basic.py diff --git a/yoink/comic.py b/yoink/comic.py index f19a098..866f4e3 100644 --- a/yoink/comic.py +++ b/yoink/comic.py @@ -1,3 +1,4 @@ +from urllib.error import HTTPError from yoink.config import required_archive_files, skippable_images, library_path, config from yoink.scraper import Scrapable @@ -71,7 +72,7 @@ class Comic(Scrapable): @property def issue_number(self) -> int: # matches any year in parentheses (xxxx) - year_regex = re.search("(\([12]\d{3}\))", self.title) + year_regex = re.search("(\\([12]\\d{3}\\))", self.title) try: return int(self.title[:year_regex.start() - 1][-1]) @@ -123,22 +124,27 @@ class ComicArchiver: opener.addheaders = [('User-Agent', 'Mozilla/5.0')] urllib.request.install_opener(opener) - for index,url in enumerate(self.comic.filelist): + try: + for index,url in enumerate(self.comic.filelist): - if not url.endswith('.jpg'): - formatted_file = os.path.join(self.worktree, f'{self.comic.title} ' + ''.join([str(index).zfill(3), '.jpg'])) - print(formatted_file, end='\r') - urllib.request.urlretrieve(url, filename=formatted_file) - else: - page_number = str(index).zfill(3) - file_extension = url.split('/')[-1].split('.')[1] + if not url.endswith('.jpg'): + formatted_file = os.path.join(self.worktree, f'{self.comic.title} ' + ''.join([str(index).zfill(3), '.jpg'])) + print(formatted_file, end='\r') + urllib.request.urlretrieve(url, filename=formatted_file) + else: + page_number = str(index).zfill(3) + file_extension = url.split('/')[-1].split('.')[1] - if len(file_extension) > 3: - file_extension = 'jpg' + if len(file_extension) > 3: + file_extension = 'jpg' + + formatted_file = f'{self.comic.title} - {page_number}.{file_extension}' + print(formatted_file, end='\r',) + urllib.request.urlretrieve(url, filename=os.path.join(self.worktree, formatted_file)) + except HTTPError: + # the page itself loads but the images (stored on another server) 4040 + raise ReferenceError(f'Issue {self.comic.title} #{self.comic.issue_number} could not be found. The page may be down or the images might have errored: {self.comic.url}') - formatted_file = f'{self.comic.title} - {page_number}.{file_extension}' - print(formatted_file, end='\r',) - urllib.request.urlretrieve(url, filename=os.path.join(self.worktree, formatted_file)) print() def generate_archive(self, archive_format='.cbr'): diff --git a/yoink/tests/test_basic.py b/yoink/tests/test_basic.py index de77ee1..70ed0a6 100644 --- a/yoink/tests/test_basic.py +++ b/yoink/tests/test_basic.py @@ -12,18 +12,19 @@ from yoink.scraper import Scrapable class BasicTestCase(unittest.TestCase): def setUp(self): - self.test_comic = 'http://readallcomics.com/static-season-one-4-2021/' + self.test_comic = 'http://readallcomics.com/static-season-one-6-2022/' self.test_comic_b = 'http://readallcomics.com/captain-marvel-vs-rogue-2021-part-1/' self.comic = Comic(self.test_comic) self.archiver = ComicArchiver(self.comic) self.remove_queue = [] - self.expected_title = 'Static Season One 4 (2021)' + self.expected_title = 'Static Season One 6 (2022)' self.expected_title_b = 'Captain Marvel vs. Rogue (2021 – Part 1)' self.expected_category = 'Static: Season One' self.expected_category_b = 'Captain Marvel vs. Rogue' - self.expected_issue_num = 4 - self.expected_next_url = 'http://readallcomics.com/static-season-one-5-2022/' - self.expected_prev_url = 'http://readallcomics.com/static-season-one-003-2021/' + self.expected_issue_num = 6 + self.expected_next_url = None + self.expected_prev_url = 'http://readallcomics.com/static-season-one-5-2022/' + self.erroneous_comic = 'http://readallcomics.com/static-season-one-4-2021/' def tearDown(self) -> None: @@ -81,4 +82,11 @@ class BasicTestCase(unittest.TestCase): def test_012_has_prev_link(self): self.assertEqual(self.comic.prev, self.expected_prev_url) + + def test_013_broken_comic_images_raise_ref_error(self): + with self.assertRaises(ReferenceError) as condition: + Comic(self.erroneous_comic).archiver.download() + + self.assertTrue('images might have errored' in str(condition.exception)) +