commit fbcf6769ce6fe4956691a6ad824c62694c3b8428 Author: Bryan Bailey Date: Thu Nov 7 10:21:56 2019 -0500 proof of concept diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..7b1aa46 --- /dev/null +++ b/.gitignore @@ -0,0 +1,10 @@ +env +*.zip +*.cbr +*.cbz +*.cfg +*.jpg +settings.json +.flaskenv +*.db +__pycache__ \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..60ed3bb --- /dev/null +++ b/README.md @@ -0,0 +1 @@ +# saveallcomics diff --git a/app/__init__.py b/app/__init__.py new file mode 100644 index 0000000..bd840b8 --- /dev/null +++ b/app/__init__.py @@ -0,0 +1,44 @@ +from flask import Flask, render_template +from flask_sqlalchemy import SQLAlchemy +from flask_migrate import Migrate +from flask_mail import Mail +from flask_moment import Moment +from config import config +from flask_login import LoginManager +from flask_assets import Environment, Bundle + + +mail = Mail() +moment = Moment() +db = SQLAlchemy() +login_manager = LoginManager() +login_manager.login_view = 'auth.login' + + +def create_app(config_name): + app = Flask(__name__) + app.config.from_object(config[config_name]) + config[config_name].init_app(app) + + mail.init_app(app) + moment.init_app(app) + db.init_app(app) + login_manager.init_app(app) + assets = Environment(app) + assets.url = app.static_url_path + assets.debug = True + scss = Bundle('scss/_variables.scss', + filters='pyscss', + output='gen/ripper.min.css') + assets.register('scss_all', scss) + + + from .main import main as main_blueprint + from .admin import admin as admin_blueprint + from .auth import auth as auth_blueprint + + app.register_blueprint(main_blueprint) + app.register_blueprint(admin_blueprint) + app.register_blueprint(auth_blueprint) + + return app \ No newline at end of file diff --git a/app/admin/__init__.py b/app/admin/__init__.py new file mode 100644 index 0000000..4190ef9 --- /dev/null +++ b/app/admin/__init__.py @@ -0,0 +1,7 @@ +from flask import Blueprint + + +admin = Blueprint('admin', __name__) + + +from . import views, errors, forms, models \ No newline at end of file diff --git a/app/admin/errors.py b/app/admin/errors.py new file mode 100644 index 0000000..65a2dbe --- /dev/null +++ b/app/admin/errors.py @@ -0,0 +1 @@ +from flask import Flask \ No newline at end of file diff --git a/app/admin/forms.py b/app/admin/forms.py new file mode 100644 index 0000000..e69de29 diff --git a/app/admin/models.py b/app/admin/models.py new file mode 100644 index 0000000..e69de29 diff --git a/app/admin/views.py b/app/admin/views.py new file mode 100644 index 0000000..e69de29 diff --git a/app/auth/__init__.py b/app/auth/__init__.py new file mode 100644 index 0000000..bfa788e --- /dev/null +++ b/app/auth/__init__.py @@ -0,0 +1,7 @@ +from flask import Blueprint + + +auth = Blueprint('auth', __name__) + + +from . import views, errors, forms, models \ No newline at end of file diff --git a/app/auth/errors.py b/app/auth/errors.py new file mode 100644 index 0000000..e69de29 diff --git a/app/auth/forms.py b/app/auth/forms.py new file mode 100644 index 0000000..e69de29 diff --git a/app/auth/models.py b/app/auth/models.py new file mode 100644 index 0000000..e69de29 diff --git a/app/auth/views.py b/app/auth/views.py new file mode 100644 index 0000000..e69de29 diff --git a/app/main/__init__.py b/app/main/__init__.py new file mode 100644 index 0000000..82abc46 --- /dev/null +++ b/app/main/__init__.py @@ -0,0 +1,7 @@ +from flask import Blueprint + + +main = Blueprint('main', __name__) + + +from . import views, errors, forms, models \ No newline at end of file diff --git a/app/main/errors.py b/app/main/errors.py new file mode 100644 index 0000000..e69de29 diff --git a/app/main/forms.py b/app/main/forms.py new file mode 100644 index 0000000..e3b34a0 --- /dev/null +++ b/app/main/forms.py @@ -0,0 +1,9 @@ +from flask_wtf import FlaskForm +from wtforms import StringField, SubmitField +import wtforms.validators +from wtforms.validators import Length, MacAddress, IPAddress, DataRequired, EqualTo + + +class Ripper(FlaskForm): + search = StringField('Search') + submit = SubmitField('Download') \ No newline at end of file diff --git a/app/main/models.py b/app/main/models.py new file mode 100644 index 0000000..e69de29 diff --git a/app/main/views.py b/app/main/views.py new file mode 100644 index 0000000..daefcbe --- /dev/null +++ b/app/main/views.py @@ -0,0 +1,42 @@ +import os +from flask import render_template, session, redirect, url_for, flash, request, current_app +from . import main +from .forms import Ripper +from .. import db +from flask_login import current_user, login_required +from ..panelrip import ripper, zipper + + +accepted_methods = ['GET', 'POST'] +headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64)'} + + +def download_comic_files(soup): + # create list for comic file links + comic_files = [] + + # set the image_folder name to the title of the comic + image_folder = os.path.join(current_app.config['STASH_FOLDER'], ripper.get_title(soup)) + print(image_folder) + + # append each comic link found in the bs4 response to the comic file list + for link in ripper.get_img_links(soup): + comic_files.append(link) + + # download the completed comic list + ripper.download_img_links(comic_files, soup, current_app.config['STASH_FOLDER']) + + # create a comic archive from all images in image_folder + zipper.create_comic_archive(ripper.get_title(soup), image_folder) + + +@main.route('/', methods=accepted_methods) +def index(): + form = Ripper() + + if form.validate_on_submit(): + soup = ripper.get_soup_obj(form.search.data, headers) + download_comic_files(soup) + return render_template('success.html') + + return render_template('index.html', form=form) \ No newline at end of file diff --git a/app/panelrip/__init__.py b/app/panelrip/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/app/panelrip/ripper.py b/app/panelrip/ripper.py new file mode 100644 index 0000000..af2a3e5 --- /dev/null +++ b/app/panelrip/ripper.py @@ -0,0 +1,47 @@ +import os +import urllib.request +from bs4 import BeautifulSoup + + +def get_soup_obj(url, headers): + req = urllib.request.Request(url, headers=headers) + raw = urllib.request.urlopen(req) + + return BeautifulSoup(raw, 'html.parser') + + +def get_title(soup): + ''' Grab the title element string from a Beautiful Soup object + removing the last 45 garbage characters''' + comic_title = soup.title.string.replace('Read All Comics Online For Free', '').replace('…', '').replace('|', '') + return comic_title.strip() + + +def get_img_links(soup): + images = [] + for link in soup.findAll('a'): + if not link.has_attr('href'): + img = link.img['src'] + images.append(img) + + return images + +def download_img_links(links, soup, folder, title=None): + # test to see if the folder exists + if title is None: + print('No title specified\ncreating from reqest') + title = get_title(soup) + # if it doesn't, create it + if not os.path.exists(os.path.join(folder, title)): + os.makedirs(os.path.join(folder, title), mode=0o777) + print('Folder {} created...'.format(os.path.join(folder, title))) + # retrieve files from server + for link in links: + opener = urllib.request.build_opener() + opener.addheaders = [('User-agent', 'Mozilla/5.0')] + urllib.request.install_opener(opener) + print('Downloading file {} from {}'.format((link[-6:] if link[-11:-4] != '%2Bcopy' else (link[-13:-11] + link[-4:]) ), title)) + urllib.request.urlretrieve(link, filename=os.path.join(folder + '/' + title, title + ' 0' + (link[-6:] if link[-11:-4] != '%2Bcopy' else (link[-13:-11] + link[-4:]) ))) + + print("Download Complete!") + diff --git a/app/panelrip/zipper.py b/app/panelrip/zipper.py new file mode 100644 index 0000000..230c0c8 --- /dev/null +++ b/app/panelrip/zipper.py @@ -0,0 +1,15 @@ +import os +import shutil +import zipfile + +basedir = os.path.dirname(__file__) + +def create_comic_archive(title, folder): + print('Creating comic archive: {}.cbr'.format(title)) + + output = shutil.make_archive(title, 'zip', folder) + os.rename(output, os.path.join(folder, title +'.cbr')) + print('Performing folder cleanup...') + for img in os.listdir(folder): + if not img.endswith('.cbr'): + os.remove(os.path.join(folder, img)) \ No newline at end of file diff --git a/app/templates/index.html b/app/templates/index.html new file mode 100644 index 0000000..9465c18 --- /dev/null +++ b/app/templates/index.html @@ -0,0 +1,6 @@ +

Save All Comics!

+
+{{ form.hidden_tag() }} +{{ form.search(placeholder='Comic URL, eg: https://readallcomics.com/marauders-001-2019/') }} +{{ form.submit() }} +
\ No newline at end of file diff --git a/app/templates/success.html b/app/templates/success.html new file mode 100644 index 0000000..76fb975 --- /dev/null +++ b/app/templates/success.html @@ -0,0 +1,2 @@ +

Huzzah! It Worked!

+« Back to Home \ No newline at end of file diff --git a/config.py b/config.py new file mode 100644 index 0000000..b80437d --- /dev/null +++ b/config.py @@ -0,0 +1,53 @@ +import os +from dotenv import load_dotenv + + +basedir = os.path.abspath(os.path.dirname(__file__)) +load_dotenv(os.path.join(basedir, '.flaskenv')) + +class Config(object): + VERSION = '0.0.1' + SECRET_KEY = os.environ.get('SECRET_KEY') or 'snapekilledumbledore' + MAIL_SERVER = os.environ.get('MAIL_SERVER') + MAIL_PORT = 587 + MAIL_USE_TLS = True + MAIL_USERNAME = os.environ.get('MAIL_USERNAME') + MAIL_PASSWORD = os.environ.get('MAIL_PASSWORD') + MAIL_SUBJECT_PREFIX = os.environ.get('MAIL_SUBJECT_PREFIX') + ADMIN = os.environ.get('MAIL_SENDER') or 'bryan.bailey@brizzle.dev' + WTF_CSRF_ENABLED = True + SQLALCHEMY_TRACK_MODIFICATIONS = True + SQLALCHEMY_DATABASE_URI = os.environ.get('DATABASE_URI') or \ + 'sqlite:///' + os.path.join(basedir, 'app.db') + UPLOAD_FOLDER = os.path.join(basedir, 'app/static/uploads') + STASH_FOLDER = os.path.join(UPLOAD_FOLDER, 'stash') + CLOUD_PLATFORM = 'aws' + + @staticmethod + def init_app(app): + pass + + +class DevConfig(Config): + DEBUG = True + SQLALCHEMY_DATABASE_URI = os.environ.get('DATABASE_URI') or \ + 'sqlite:///' + os.path.join(basedir, 'dev.db') + + +class TestConfig(Config): + DEBUG = False + SQLALCHEMY_DATABASE_URI = os.environ.get('DATABASE_URI') or \ + 'sqlite:///' + os.path.join(basedir, 'test.db') + + +# class ProductionConfig(Config): +# SQLALCHET_BINDS = { +# 'main': SQLALCHEMY_DATABASE_URI +# } + +config = { + 'development': DevConfig, + 'testing': TestConfig, + 'production': Config, + 'default': DevConfig +} \ No newline at end of file diff --git a/main.py b/main.py new file mode 100644 index 0000000..9999b6e --- /dev/null +++ b/main.py @@ -0,0 +1,7 @@ +import os +from flask import render_template, session, redirect, url_for, flash, request +from app import create_app, db +from flask_migrate import Migrate + +app = create_app(os.getenv('FLASK_CONFIG') or 'default') +migrate = Migrate(app, db) \ No newline at end of file