changed current_app import from flask to app remix

.github/ISSUE_TEMPLATE/bug_report.md

@@ -1,38 +0,0 @@
----
-name: Bug report
-about: Create a report to help us improve
-title: ''
-labels: ''
-assignees: Rigil-Kent
-
----
-
-**Describe the bug**
-A clear and concise description of what the bug is.
-
-**To Reproduce**
-Steps to reproduce the behavior:
-1. Go to '...'
-2. Click on '....'
-3. Scroll down to '....'
-4. See error
-
-**Expected behavior**
-A clear and concise description of what you expected to happen.
-
-**Screenshots**
-If applicable, add screenshots to help explain your problem.
-
-**Desktop (please complete the following information):**
- - OS: [e.g. iOS]
- - Browser [e.g. chrome, safari]
- - Version [e.g. 22]
-
-**Smartphone (please complete the following information):**
- - Device: [e.g. iPhone6]
- - OS: [e.g. iOS8.1]
- - Browser [e.g. stock browser, safari]
- - Version [e.g. 22]
-
-**Additional context**
-Add any other context about the problem here.

.github/workflows/tests.yml

@@ -10,7 +10,7 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-latest, windows-latest]
-        python-version: ['3.7', '3.8', '3.9']
+        python-version: ['3.8', '3.9', '3.10']
 
     steps:
       - uses: actions/checkout@v2

.gitignore

@@ -1,8 +1,10 @@
 env
-venv
 __pycache__
 .pytest_cache
 .coverage
 *.egg-info
 sample
 .tox
+*.sqlite
+migrations
+.vscode

.gitlab-ci.yml

@@ -13,5 +13,7 @@ before_script:
   - pip install -r requirements.txt
 test:
   script:
-    - pytest --junitxml report.xml yoink/tests/test_basic.py
+    - pytest --junitxml report.xml tests/test_basic.py
+    - pytest --junitxml report.xml tests/test_web.py
+    - pytest --junitxml report.xml tests/test_model_user.py
 

.vscode/settings.json

@@ -1,16 +0,0 @@
-{
-    "cSpell.words": [
-        "Archiver",
-        "dbsuper",
-        "draggable",
-        "errored",
-        "excaliber",
-        "mangadex",
-        "readallcomics",
-        "Scrapable",
-        "skippable",
-        "Uncategorized",
-        "worktree",
-        "yoink"
-    ]
-}

app/__init__.py

@@ -0,0 +1,48 @@
+import atexit
+
+from apscheduler.schedulers.background import BackgroundScheduler
+import requests
+
+from flask import Flask, current_app, g
+from flask_login import LoginManager
+from flask_mail import Mail
+from flask_moment import Moment
+from flask_sqlalchemy import SQLAlchemy
+
+from config import configuration, DevelopmentConfig, TestingConfig, ProductionConfig
+
+
+
+mail =Mail()
+moment = Moment()
+db = SQLAlchemy()
+login_manager = LoginManager()
+login_manager.login_view = 'auth.login'
+
+def scheduled_task(func, seconds=60, args=None):
+
+    scheduler = BackgroundScheduler()
+    scheduler.add_job(func=func, trigger='interval', seconds=seconds, args=args)
+    scheduler.start()
+
+    atexit.register(lambda: scheduler.shutdown())
+
+
+def create_app(config_name):
+    app = Flask(__name__)
+
+    app.config.from_object(configuration[config_name])
+    configuration[config_name].init_app(app)
+
+    mail.init_app(app)
+    moment.init_app(app)
+    db.init_app(app)
+    login_manager.init_app(app)
+
+    from .main import main as main_blueprint
+    from app.auth import auth as auth_blueprint
+    
+    app.register_blueprint(main_blueprint)
+    app.register_blueprint(auth_blueprint, url_prefix='/auth')
+
+    return app

app/auth/__init__.py

@@ -0,0 +1,7 @@
+from flask import Blueprint
+
+
+auth = Blueprint('auth', __name__)
+
+
+from . import views

app/auth/forms.py

@@ -0,0 +1,29 @@
+from flask_wtf import FlaskForm
+from wtforms import StringField, PasswordField, BooleanField, SubmitField, ValidationError
+from wtforms.validators import DataRequired, Length, Email, EqualTo, Regexp
+
+from app.models import User
+
+
+class LoginForm(FlaskForm):
+    email = StringField('Email', validators=[DataRequired(), Length(1, 64), Email()])
+    password = PasswordField('Password', validators=[DataRequired()])
+    remember_me = BooleanField('Keep me logged in')
+    submit = SubmitField('Log In')
+
+
+class RegistrationForm(FlaskForm):
+    email = StringField('Email', validators=[DataRequired(), Length(1, 64), Email()])
+    username = StringField('Username', validators=[DataRequired(), Regexp('^[A-Za-z][A-Za-z0-9_.]*$', 0, 'Usernames must have only letters, numbers, dots or underscors')])
+    password = PasswordField('Password', validators=[DataRequired(), EqualTo('password2', message='Passwords must match.')])
+    password2 = PasswordField('Confirm Password', validators=[DataRequired()])
+    submit = SubmitField('Register')
+
+
+    def validate_email(self, field):
+        if User.query.filter_by(email=field.data).first():
+            raise ValidationError('Email already registered')
+
+    def validate_username(self, field):
+        if User.query.filter_by(username=field.data).first():
+            raise ValidationError('Username already in use')

app/auth/views.py

@@ -0,0 +1,101 @@
+from flask import render_template, redirect, request, url_for, flash
+from flask_login import login_user, logout_user, login_required, current_user
+
+from app import db
+from app.auth import auth
+from app.email import send_email
+from app.models import User
+from app.auth.forms import LoginForm, RegistrationForm
+
+
+
+@auth.before_app_request
+def before_request():
+    if current_user.is_authenticated and not current_user.confirmed and request.blueprint != 'auth' and request.endpoint != 'static':
+        return redirect(url_for('auth.unconfirmed'))
+
+
+@auth.route('/unconfirmed')
+def unconfirmed():
+    if current_user.is_anonymous or current_user.confirmed:
+        return redirect(url_for('main.index'))
+    return render_template('auth/unconfirmed.html')
+
+@auth.route('/login', methods=['get', 'post'])
+def login():
+    form = LoginForm()
+
+    if form.validate_on_submit():
+        user = User.query.filter_by(email=form.email.data).first()
+
+        if user is not None and user.verify_password(form.password.data):
+            login_user(user, form.remember_me.data)
+
+            next = request.args.get('next')
+
+            if next is None or next.startswith('/'):
+                next = url_for('main.index')
+            
+            return redirect(next)
+        flash('Invalid username or password', category='error')
+    return render_template('auth/login.html', form=form)
+
+
+@auth.route('/logout')
+@login_required
+def logout():
+    logout_user()
+    flash('You have been logged out.', category='info')
+    return redirect(url_for('main.index'))
+
+
+@auth.route('/register', methods=['get', 'post'])
+def register():
+    form = RegistrationForm()
+
+    if form.validate_on_submit():
+        user = User(email=form.email.data, username=form.username.data, password=form.password.data)
+        db.session.add(user)
+
+        db.session.commit()
+
+        token = user.generate_confirmation_token()
+
+        send_email(user.email, 'Confirm Your Yo!nk Account', 'auth/email/confirm', user=user, token=token)
+
+        flash(f'A confirmation email was sent to {user.email}', category='success')
+        return redirect(url_for('auth.login'))
+    
+    return render_template('auth/register.html', form=form)
+
+
+@auth.route('/confirm/<token>')
+@login_required
+def confirm(token):
+    if current_user.confirmed:
+        return redirect(url_for('main.index'))
+
+    if current_user.confirm(token):
+        db.session.commit()
+        flash('Your account has been confirmed. Thank you!')
+    else:
+        flash('Invalid or expired confirmation link')
+
+    return redirect(url_for('main.index'))
+
+
+@auth.route('/confirm')
+@login_required
+def resend_confirm():
+    token = current_user.generate_confirmation_token()
+
+    send_email(current_user.email, 'Confirm Your Yo!nk Account', 'auth/email/confirm', user=current_user, token=token)
+
+    flash(f'A new confirmation email was sent to {current_user.email}', category='success')
+
+    return redirect(url_for('main.index'))
+
+
+# TODO: password updates
+# TODO: password reset
+# TODO: email updates

app/email.py

@@ -0,0 +1,23 @@
+import threading
+
+from flask import current_app, render_template
+from flask_mail import Message
+
+from app import mail
+
+
+def send_async_email(app, msg):
+    with app.app_context():
+        mail.send(msg)
+
+def send_email(to, subject, template, **kwargs):
+    msg = Message(
+        
+        current_app.config['MAIL_PREFIX'] + subject,
+        sender=current_app.config['MAIL_SENDER'], recipients=to)
+    msg.body = render_template(template + '.txt', **kwargs)
+    msg.html = render_template(template + '.html', **kwargs)
+    
+    mail_thread = threading.Thread(target=send_async_email, args=[current_app._get_current_object(), msg])
+    mail_thread.start()
+    return mail_thread

app/main/__init__.py

@@ -0,0 +1,7 @@
+from flask import Blueprint
+
+
+main = Blueprint('main', __name__)
+
+
+from . import views, errors

app/main/errors.py

@@ -0,0 +1,12 @@
+from flask import render_template
+from . import main
+
+
+
+@main.app_errorhandler(404)
+def not_found(e):
+    return render_template('404.html'), 404
+
+@main.errorhandler(500)
+def server_error(e):
+    return render_template('500.html'), 500

app/main/forms.py

@@ -0,0 +1,9 @@
+from flask_wtf import FlaskForm
+from wtforms import StringField, BooleanField, SubmitField
+from wtforms.validators import DataRequired
+
+
+class DownloadForm(FlaskForm):
+    url = StringField('Comic URL', validators=[DataRequired()])
+    series = BooleanField('Series? ')
+    download = SubmitField('Download')

app/main/views.py

@@ -0,0 +1,104 @@
+import os
+from datetime import datetime
+
+from flask import render_template, session, send_from_directory, redirect, url_for, flash, g
+
+from app.main import main
+from app.main.forms import DownloadForm
+from app import db
+from app.email import send_email
+from app.models import User, Role, ComicMeta
+from yoink.comic import Comic
+from yoink.config import config
+
+
+@main.route('/uploads/<path:filename>')
+def download_file(filename):
+    return send_from_directory(os.path.join(config.library_path, 'comics'), filename, as_attachment=True)
+
+
+def get_cover_path(comic):
+    return [image for image in os.listdir(os.path.join(config.library_path, 'comics', comic.title)) if image.endswith('000.jpg')][0]
+
+
+def get_archive_path(comic):
+    return [image for image in os.listdir(os.path.join(config.library_path, 'comics', comic.title)) if image.endswith('.cbr')][0]
+    
+
+def get_comic_library_meta():
+    comic_meta = []
+
+    for comic in ComicMeta.query.order_by(ComicMeta.id.desc()).all():
+        comic_meta.append({
+            'cover': comic.cover_path,
+            'title': comic.title,
+            'archive': comic.archive_path
+        })
+
+    return comic_meta
+
+
+def new_user(username, db, app, form):
+    user = User.query.filter_by(username=username).first()
+
+    if user is None:
+        user = User(username=form.username.data, role_id=3)
+        db.session.add(user)
+        if app.config['YOINK_ADMIN']:
+            send_email(app.config['YOINK_ADMIN'], 'New User', 'mail/new_user', user=user)
+        db.session.commit()
+
+
+def check_setup():
+    if User.query.all() is None:
+        return redirect(url_for('setup'))
+
+
+@main.route('/', methods=['post', 'get'])
+def index():
+    url = None
+    series = False
+    form = DownloadForm()
+    latest = get_comic_library_meta()
+
+    if form.validate_on_submit():
+        url = form.url.data.strip()
+        series = form.series.data
+
+        comic = Comic(url)
+        comic_meta = ComicMeta.query.filter_by(title=comic.title).first()
+
+        if comic_meta is None:
+            comic.archiver.download()
+            comic.archiver.generate_archive()
+
+            comic_meta = ComicMeta()
+            comic_meta.title = comic.title
+            comic_meta.category = comic.category
+            comic_meta.issue = comic.issue_number
+            comic_meta.next_issue = comic.next
+            comic_meta.previous_issue = comic.prev
+            comic_meta.cover_path = os.path.join(comic.title, get_cover_path(comic))
+            comic_meta.archive_path = os.path.join(comic.title, get_archive_path(comic))
+
+            db.session.add(comic_meta)
+            db.session.commit()
+        else:
+            flash(f'Comic {comic.title} exists')
+
+            latest = get_comic_library_meta()
+            form.url.data = ''
+
+            return render_template('index.html', form=form, url=url, series=series, latest=latest, status=g.status_code), 200
+
+
+        if form.series.data:
+            print('Download the whole damn lot')
+
+        flash(f'{comic.title} downloaded to {os.path.join(config.library_path, "comics/" + comic.title)}')
+
+        latest = get_comic_library_meta()
+        comic.archiver.cleanup_worktree()
+        form.url.data = ''
+
+    return render_template('index.html', form=form, url=url, series=series, latest=latest), 200

app/models.py

@@ -0,0 +1,77 @@
+from flask import current_app
+from flask_login import UserMixin
+from itsdangerous import TimedJSONWebSignatureSerializer
+from werkzeug.security import generate_password_hash, check_password_hash
+
+from app import db, login_manager
+
+
+
+@login_manager.user_loader
+def load_user(user_id): return User.query.get(int(user_id))
+
+
+class Role(db.Model):
+    __tablename__ = 'roles'
+    id = db.Column(db.Integer, primary_key=True)
+    name = db.Column(db.String(64), unique=True)
+    users = db.relationship('User', backref='role', lazy='dynamic')
+
+    def __repr__(self):
+        return f'<Role {self.name!r}>'
+
+
+class User(UserMixin, db.Model):
+    __tablename__ = 'users'
+    id = db.Column(db.Integer, primary_key=True)
+    email = db.Column(db.String(64), unique=True, index=True)
+    username = db.Column(db.String(64), unique=True, index=True)
+    role_id = db.Column(db.Integer, db.ForeignKey('roles.id'))
+    password_hash = db.Column(db.String(128))
+    confirmed = db.Column(db.Boolean, default=False)
+
+    def __repr__(self): return f'<User {self.username!r}>'
+
+    @property
+    def password(self):
+        raise AttributeError('password is not a readable attribute')
+
+    @password.setter
+    def password(self, password):
+        self.password_hash = generate_password_hash(password, method='pbkdf2:sha256', salt_length=20)
+
+    def verify_password(self, password):
+        return check_password_hash(self.password_hash, password)
+
+    def generate_confirmation_token(self, expiration=3600):
+        s = TimedJSONWebSignatureSerializer(current_app, expiration)
+        return s.dumps({'confirm': self.id}).decode('utf-8')
+
+    def confirm(self, token):
+        s = TimedJSONWebSignatureSerializer(current_app.config['SECRET_KEY'])
+
+        try:
+            data = s.loads(token.encode('utf-8'))
+        except:
+            return False
+
+        if data.get('confirm') != self.id:
+            return False
+
+        self.confirmed = True
+        db.session.add(self)
+        db.session.commit()
+
+        return True
+
+
+class ComicMeta(db.Model):
+    __tablename__ = 'comicmeta'
+    id = db.Column(db.Integer, primary_key=True)
+    title = db.Column(db.String(256), unique=True, index=True)
+    issue = db.Column(db.Integer, nullable=True, index=True)
+    category = db.Column(db.String(128), index=True, nullable=True)
+    previous_issue = db.Column(db.String(256), nullable=True)
+    next_issue = db.Column(db.String(256), nullable=True)
+    cover_path = db.Column(db.String(256))
+    archive_path = db.Column(db.String(256))

app/static/css/yoink.css

@@ -0,0 +1,3 @@
+* {
+    box-sizing: border-box;
+}

app/templates/404.html

@@ -0,0 +1,14 @@
+{% extends "base.html" %}
+
+{% block meta %}
+{{ super() }}
+<!-- Add additional meta tags here -->
+{% endblock %}
+
+{% block title %}Testing{% endblock %}
+
+{% block page_content %}
+<div class="page-header">
+    <h1>Opps! File not found</h1>
+</div>
+{% endblock %}

app/templates/500.html

@@ -0,0 +1,14 @@
+{% extends "base.html" %}
+
+{% block meta %}
+{{ super() }}
+<!-- Add additional meta tags here -->
+{% endblock %}
+
+{% block title %}Testing{% endblock %}
+
+{% block page_content %}
+<div class="page-header">
+    <h1>Opps! Technical difficulties</h1>
+</div>
+{% endblock %}

app/templates/auth/email/confirm.html

@@ -0,0 +1,12 @@
+Welcome to Yo!nk {{ user.username }},
+
+To confirm your account please click on the following link:
+
+{{ url_for('auth.confirm', token=token, _external=True) }}
+
+Sincerely,
+
+The Yo!nk Team
+
+
+Note: replies to this email address are unmonitored.

app/templates/auth/login.html

@@ -0,0 +1,19 @@
+{% extends "base.html" %}
+
+{% block title %}Yo!nk - Login{% endblock %}
+
+{% block page_content %}
+<div class="page-header">
+    <h1>Login</h1>
+</div>
+<div class="col-md-4">
+    <form method="post">
+        {{ form.hidden_tag() }}
+        {{ form.email.label }} {{ form.email() }}
+        {{ form.password.label }} {{ form.password() }}
+        {{ form.remember_me.label }} {{ form.remember_me() }}
+        {{ form.submit() }}
+    </form>
+    <p>New User? Click <a href="{{ url_for('auth.register') }}">here</a> to register a free account.</p>
+</div>
+{% endblock %}

app/templates/auth/register.html

@@ -0,0 +1,14 @@
+{% extends "base.html" %}
+
+{% block meta %}
+{{ super() }}
+<!-- Add additional meta tags here -->
+{% endblock %}
+
+{% block title %}Testing{% endblock %}
+
+{% block page_content %}
+<div class="page-header">
+    <h1>Register</h1>
+</div>
+{% endblock %}

app/templates/base.html

@@ -0,0 +1,75 @@
+{% import 'macros/connection_status.html' as connection %}
+
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    {% block meta %}
+    <meta charset="UTF-8">
+    <meta http-equiv="X-UA-Compatible" content="IE=edge">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    {% endblock %}
+    <title>{% block title %}Y!oink Web App{% endblock %}</title>
+    {% block styles %}
+    <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/bootstrap@5.1.3/dist/css/bootstrap.min.css" integrity="sha384-1BmE4kWBq78iYhFldvKuhfTAU6auU8tT94WrHftjDbrCEXSU1oBoqyl2QvZ6jIW3" crossorigin="anonymous">
+    <link rel="stylesheet" href="{{ url_for('static', filename='css/yoink.css') }}">
+    {% endblock %}
+    {% block scripts %}
+    <script src="https://cdn.jsdelivr.net/npm/bootstrap@5.1.3/dist/js/bootstrap.min.js" integrity="sha384-QJHtvGhmr9XOIpI6YVutG+2QOK9T+ZnN4kzFN1RtK3zEFEIsxhlmWl5/YESvpZ13" crossorigin="anonymous"></script>
+    {{ moment.include_moment() }}
+    {% endblock %}
+    <link rel="shortcut icon" href="{{ connection.toggle_connection_status(status_code) }}" type="image/x-icon">
+</head>
+<body>
+    {% block navbar %}
+    <nav class="navbar navbar-inverse">
+        <div class="container">
+            <div class="navbar-header">
+                <button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-collapse">
+                    <span class="sr-only">Toggle Navigation</span>
+                    <span class="icon-bar"></span>
+                    <span class="icon-bar"></span>
+                    <span class="icon-bar"></span>
+                </button>
+
+                <a href="/" class="navbar-brand">Y!oink</a>
+
+                <ul class="nav navbar-nav navbar-right">
+                    {% if current_user.is_authenticated %}
+                    <li>
+                        <a href="{{ url_for('auth.logout') }}">Log Out</a>
+                    </li>
+                    {% else %}
+                    <li>
+                        <a href="{{ url_for('auth.login') }}">Log In</a>
+                    </li>
+                    {% endif %}
+                </ul>
+            </div>
+
+            <div class="navbar-collapse collapse">
+                <ul class="nav navbar-nav">
+                    <li>
+                        <a href="/">Y!oink</a>
+                    </li>
+                </ul>
+            </div>
+        </div>
+    </nav>
+    {% endblock %}
+
+    {% block content %}
+    <div class="container-fluid">
+        {% for message in get_flashed_messages() %}
+        <div class="alert alert-success alert-dismissible fade show" role="alert">
+            <button type="button" class="btn-close" data-bs-dismiss="alert" aria-label="Close"></button>
+            {{message}}
+        </div>
+        {% endfor %}
+        {% block page_content %}
+        {% endblock %}
+    </div>
+    {% endblock %}
+
+    <script src="{{ url_for('static', filename='js/yoink.js') }}"></script>
+</body>
+</html>

app/templates/index.html

@@ -0,0 +1,36 @@
+{% extends "base.html" %}
+{% import 'macros/library.html' as library %}
+{% import 'macros/connection_status.html' as connection %}
+
+
+{% block meta %}
+{{ super() }}
+<!-- Add additional meta tags here -->
+{% endblock %}
+
+{% block title %}Testing{% endblock %}
+
+{% block page_content %}
+<div class="page-header">
+    <h1>Y!oink Web App</h1>
+</div>
+<form method="post" onsubmit="download.disabled = true; url.readOnly = true; return true;">
+    {{ form.hidden_tag() }}
+    <div class="input-group mb-3">
+        {{ form.series.label(class_='input-group-text') }}
+        <div class="input-group-text">
+            {{ form.series() }}
+        </div>
+        
+        {{ form.url(class_='form-control') }}
+        {{ form.download(class_='btn btn-outline-primary') }}
+    </div>
+</form>
+{% if current_user.is_authenticated %}
+<h2>{{ current_user.username }}'s Library</h2>
+{% else %}
+<h2>Comic Library</h2>
+{% endif %}
+{{ library.latest_downloads(latest) }}
+
+{% endblock %}

app/templates/macros/comments.html

@@ -0,0 +1,3 @@
+{% macro render_comment(comment) %}
+    <li>{{ comment }}</li>
+{% endmacro %}

app/templates/macros/connection_status.html

@@ -0,0 +1,7 @@
+{% macro toggle_connection_status(status_code) %}
+{% if status_code == 200 %}
+{{ url_for('static', filename='up.png') }}
+{% else %}
+{{ url_for('static', filename='down.png') }}
+{% endif %}
+{% endmacro %}

app/templates/macros/library.html

@@ -0,0 +1,38 @@
+<!-- latest downloads - limit to a max of 6 -->
+{% macro latest_downloads(comics) %}
+    {% if comics %}
+    <div class="row">
+        <h2>Latest Downloads</h2>
+        {% for comic in comics %}
+            <div class="col col-md-3">
+                <div class="card">
+                    <img src="{{ url_for('main.download_file', filename=comic['cover']) }}" alt="{{ comic['title'] }}" class="card-image-top position-relative">
+                    <div class="card-body">
+                        <h4 class="card-title">{{ comic['title'] }}</h4>
+                        <p class="card-text">{{ comic['title'] }}</p>
+                    </div>
+                    <div class="card-footer text-center">
+                        <a href="{{url_for('main.download_file', filename=comic.archive) }}" class="btn btn-info">Download Archive</a>
+                    </div>
+                </div>
+            </div>
+        {% endfor %}
+    </div>
+    {% else %}
+        <p>Empty Comic Library</p>
+    {% endif %}
+{% endmacro %}
+
+<!-- all downloaded comics - TODO: pagination -->
+{% macro all_downloads(comics) %}
+    {% if comics %}
+        {% for comic in comics %}
+
+        <h3>{{ comic['title'] }}</h3>
+        <img src="{{ url_for('main.download_file', filename=comic['cover']) }}">
+        
+        {% endfor %}
+    {% else %}
+        <p>Empty Comic Library</p>
+    {% endif %}
+{% endmacro %}

config.py

@@ -0,0 +1,46 @@
+import os
+from yoink.config import config
+
+
+
+basedir = os.path.abspath(os.path.dirname(__file__))
+
+
+
+class Config:
+    SECRET_KEY = os.environ.get('SECRET_KEY', 'snapekilleddumpledork')
+    SQLALCHEMY_TRACK_MODIFICATIONS = False
+    MAIL_SERVER = os.environ.get('MAIL_SERVER', 'smtp.gmail.com')
+    MAIL_PORT = os.environ.get('MAIL_PORT', 465)
+    MAIL_USE_TLS = os.environ.get('MAIL_USE_TLS', False)
+    MAIL_USE_SSL = os.environ.get('MAIL_USE_SSL', True)
+    MAIL_USERNAME = os.environ.get('MAIL_USERNAME')
+    MAIL_PASSWORD = os.environ.get('MAIL_PASSWORD')
+    MAIL_PREFIX = os.environ.get('MAIL_PREFIX', '[YO!NK]')
+    MAIL_SENDER = os.environ.get('MAIL_SENDER', 'YO!NK Admin <yoinkmediaapp@gmail.com>')
+
+
+    @staticmethod
+    def init_app(app):
+        pass
+
+
+class DevelopmentConfig(Config):
+    DEBUG = True
+    SQLALCHEMY_DATABASE_URI = os.environ.get('DEV_DATABASE_URL') or f'sqlite:///{os.path.join(config.app_root, "data-dev.sqlite")}'
+
+
+class TestingConfig(Config):
+    TESTING = True
+    SQLALCHEMY_DATABASE_URI = os.environ.get('TEST_DATABASE_URL') or f'sqlite:///{os.path.join(config.app_root, "data-test.sqlite")}'
+
+class ProductionConfig(Config):
+    SQLALCHEMY_DATABASE_URI = os.environ.get('TEST_DATABASE_URL') or f'sqlite:///{os.path.join(config.app_root, "data.sqlite")}'
+
+
+configuration = {
+    'development': DevelopmentConfig,
+    'testing': TestingConfig,
+    'production': ProductionConfig,
+    'default': DevelopmentConfig
+}

requirements.txt

@@ -1,6 +1,9 @@
+alembic==1.7.7
+APScheduler==3.9.1
 asgiref==3.5.0
 attrs==21.4.0
 beautifulsoup4==4.10.0
+blinker==1.4
 bs4==0.0.1
 certifi==2021.10.8
 charset-normalizer==2.0.12
@@ -8,13 +11,23 @@ click==8.0.4
 click-default-group==1.2.2
 coverage==6.3.2
 distlib==0.3.4
+dnspython==2.2.1
+email-validator==1.1.3
 filelock==3.6.0
 Flask==2.1.1
+Flask-Login==0.6.0
+Flask-Mail==0.9.1
+Flask-Migrate==3.1.0
+Flask-Moment==1.0.2
+Flask-SQLAlchemy==2.5.1
+Flask-WTF==1.0.1
+greenlet==1.1.2
 idna==3.3
 importlib-metadata==4.11.3
 iniconfig==1.1.1
-itsdangerous==2.1.2
+itsdangerous==2.0.1
 Jinja2==3.1.1
+Mako==1.2.0
 MarkupSafe==2.1.1
 packaging==21.3
 platformdirs==2.5.1
@@ -22,13 +35,19 @@ pluggy==1.0.0
 py==1.11.0
 pyparsing==3.0.7
 pytest==7.1.0
+pytz==2022.1
+pytz-deprecation-shim==0.1.0.post0
 requests==2.27.1
 six==1.16.0
 soupsieve==2.3.1
+SQLAlchemy==1.4.34
 toml==0.10.2
 tomli==2.0.1
 tox==3.24.5
+tzdata==2022.1
+tzlocal==4.2
 urllib3==1.26.8
 virtualenv==20.14.0
 Werkzeug==2.1.0
+WTForms==3.0.1
 zipp==3.7.0

setup.py

@@ -10,7 +10,7 @@ with open('README.md', 'r') as readme:
 
 setuptools.setup(
     name='yoink',
-    version='0.2.0',
+    version='0.3.2',
     author='Bryan Bailey',
     author_email='brizzledev@gmail.com',
     description='',
@@ -28,5 +28,5 @@ setuptools.setup(
             'yoink = yoink.cli:yoink'
         ]
     },
-    install_requires=['click', 'bs4', 'requests', 'click-default-group']
+    install_requires=['click', 'bs4', 'requests']
 )

tests/test_basic.py

@@ -5,7 +5,7 @@ import unittest
 from shutil import rmtree
 
 from yoink.config import app_root, library_path, config_path, skippable_images, supported_sites, required_archive_files
-from yoink.comic import Comic, download_comic_files, generate_archive, clean_up
+from yoink.comic import Comic, ComicArchiver
 from yoink.scraper import Scrapable
 
 
@@ -21,7 +21,6 @@ class BasicTestCase(unittest.TestCase):
         self.expected_category = 'Static: Season One'
         self.expected_category_b = 'Captain Marvel vs. Rogue'
         self.expected_issue_num = 6
-        self.expected_vol_num = 1
         self.expected_next_url = None
         self.expected_prev_url = 'http://readallcomics.com/static-season-one-5-2022/'
         self.erroneous_comic = 'http://readallcomics.com/static-season-one-4-2021/'
@@ -39,28 +38,28 @@ class BasicTestCase(unittest.TestCase):
     def test_001_comic_has_valid_title(self):
         self.assertEqual(self.expected_title, self.comic.title)
 
-    def test_002_comic_has_valid_volume_numer(self):
+    def test_002_comic_has_valid_category(self):
-        self.assertEqual(self.expected_vol_num, self.comic.volume)
-
-    def test_003_comic_has_valid_category(self):
         self.assertEqual(self.expected_category, self.comic.category)
 
-    def test_004_empty_comic_folder(self):
+    def test_003_empty_comic_folder(self):
         self.assertEqual(len(os.listdir(os.path.join(library_path, 'comics'))), 0)
 
-    def test_005_comic_folder_created_and_populated(self):
+    def test_004_comic_folder_created_and_populated(self):
-        download_comic_files(self.comic)
+        self.comic.archiver.download()
         self.assertTrue(os.path.exists(os.path.join(library_path, f'comics/{self.comic.title}')))
         self.assertGreater(len(os.listdir(os.path.join(library_path, f'comics/{self.comic.title}'))), 0)
 
-    def test_006_comic_archive_generated(self):
+    def test_005_comic_archive_generated(self):
-        generate_archive(self.comic)
+        self.comic.archiver.generate_archive()
         self.assertTrue(os.path.exists(os.path.join(library_path, f'comics/{self.comic.title}/{self.comic.title}.cbr')))
 
-    def test_007_folder_cleaned_after_archive_generation(self):
+    def test_006_folder_cleaned_after_archive_generation(self):
-        clean_up(self.comic)
+        self.comic.archiver.cleanup_worktree()
         self.assertLessEqual(len(os.listdir(os.path.join(library_path, f'comics/{self.comic.title}'))), 3)
 
+    def test_007_comic_instance_has_archiver(self):
+        self.assertIsInstance(self.comic.archiver, ComicArchiver)
+
     def test_008_comic_is_subclass_scrapable(self):
         self.assertTrue(issubclass(Comic, Scrapable))
 

tests/test_model_user.py

@@ -0,0 +1,26 @@
+import unittest
+
+from app.models import User
+
+
+
+class UserModelTestCase(unittest.TestCase):
+
+    def test_password_setter(self):
+        u = User(password='abcdefg')
+        self.assertTrue(u.password_hash is not None)
+
+    def test_no_password_getter(self):
+        u = User(password='abcdefg')
+        with self.assertRaises(AttributeError):
+            u.password
+
+    def test_password_verification(self):
+        u = User(password='abcdefg')
+        self.assertTrue(u.verify_password('abcdefg'))
+        self.assertFalse(u.verify_password('password'))
+
+    def test_password_salts_are_random(self):
+        u = User(password='abcdefg')
+        u2 = User(password='abcdefg')
+        self.assertTrue(u.password_hash != u2.password_hash)

tests/test_web.py

@@ -0,0 +1,22 @@
+import unittest
+
+from app import create_app, db, current_app
+
+
+class BasicTestCase(unittest.TestCase):
+    def setUp(self) -> None:
+        self.app = create_app('testing')
+        self.app_context = self.app.app_context()
+        self.app_context.push()
+        db.create_all()
+
+    def tearDown(self) -> None:
+        db.session.remove()
+        db.drop_all()
+        self.app_context.pop()
+
+    def test_app_exists(self):
+        self.assertFalse(current_app is None)
+
+    def test_app_is_testing(self):
+        self.assertTrue(current_app.config['TESTING'])

tox.ini

@@ -1,13 +1,13 @@
 [tox]
 minversion = 3.8.0
-envlist = py37, py38, py39
+envlist = py38, py39, py310
 isolated_build = true
 
 [gh-actions]
 python =
-    3.7: py37
     3.8: py38
     3.9: py39
+    3.10: py310
 
 [testenv]
 setenv =
@@ -15,4 +15,6 @@ setenv =
 deps =
     -r{toxinidir}/requirements_dev.txt
 commands = 
-    pytest --junitxml report.xml yoink/tests/test_basic.py
+    pytest --junitxml report.xml tests/test_basic.py
+    pytest --junitxml report.xml tests/test_web.py
+    pytest --junitxml report.xml tests/test_model_user.py

web.py

@@ -0,0 +1,36 @@
+import atexit
+import click
+import os
+import time
+import requests
+
+from apscheduler.schedulers.background import BackgroundScheduler
+from flask_migrate import Migrate
+
+from app import create_app, db
+from app.models import User, Role, ComicMeta
+from yoink.config import config
+from yoink.comic import Comic
+
+
+def ping(url: str):
+    ''' check url for connection status'''
+    return requests.get(url).status_code
+
+
+app = create_app(os.environ.get('FLASK_CONFIG') or 'default')
+migrate = Migrate(app, db)
+
+@app.shell_context_processor
+def make_shell_ctx(): return dict(db=db, User=User, Role=Role, ComicMeta=ComicMeta)
+
+@app.context_processor
+def inject_status_code():
+    return dict(status_code=ping('http://readallcomics.com'))
+
+@app.cli.command()
+def test():
+    ''' run unit tests '''
+    import unittest
+    tests = unittest.TestLoader().discover('tests')
+    unittest.TextTestRunner(verbosity=2).run(tests)

yoink/cli.py

@@ -6,7 +6,7 @@ import click
 from click_default_group import DefaultGroup
 
 from yoink.config import  YoinkConfig, app_root, config_from_file, library_path, config_path
-from yoink.comic import Comic, download_comic_files, generate_archive, clean_up
+from yoink.comic import Comic
 
 
 
@@ -22,13 +22,13 @@ def download_comic(url, series):
         return 1
 
     click.echo(f'Downloading {comic.title}')
-    download_comic_files(comic)
+    comic.archiver.download()
 
     click.echo('Building comic archive')
-    generate_archive(comic)
+    comic.archiver.generate_archive()
 
     click.echo('Cleaning up')
-    clean_up(comic)
+    comic.archiver.cleanup_worktree()
 
     click.echo('Success')
 
@@ -50,6 +50,8 @@ def init():
 
 
 @yoink.command()
+# @click.option('-c', '--comic', is_flag=True, help='Download a Comic file')
+# @click.option('-t', '--torrent', is_flag=True, help='Download a Torrent')
 @click.option('-s', '--series', is_flag=True, help='Download the entire series')
 @click.argument('url')
 def download(url, series):

yoink/comic.py

@@ -1,5 +1,5 @@
 from urllib.error import HTTPError
-from yoink.config import required_archive_files, skippable_images, config
+from yoink.config import required_archive_files, skippable_images, library_path, config
 from yoink.scraper import Scrapable
 
 import os
@@ -12,6 +12,7 @@ import re
 class Comic(Scrapable):
     def __init__(self, url, path=None) -> None:
         super().__init__(url)
+        self.archiver = ComicArchiver(self, library=path)
 
     def __is_supported_image(self, image):
         return image.endswith('.jpg' or '.jpeg')
@@ -82,14 +83,7 @@ class Comic(Scrapable):
 
     @property
     def volume(self) -> int:
-        delimiter = ' v'
+        return
-        try:
-            if self.title.find(delimiter) and self.title[self.title.index(delimiter) + 2].isdigit():
-                return self.title[self.title.index(delimiter) + 2]
-            else:
-                return 1
-        except ValueError:
-            return 1
 
     @property
     def next(self) -> str:
@@ -112,22 +106,29 @@ class Comic(Scrapable):
         return not filename.endswith(config.skippable_images)
 
 
+class ComicArchiver:
+    def __init__(self, comic : Comic, library=None) -> None:
+        self.comic = comic
+        self.worktree = library if library else os.path.join(config.library_path, f'comics/{self.comic.title}')
+        self.queue = []
 
-def download_comic_files(comic: Comic, worktree = None):
+    def add(self, link : str) -> None:
-    if not worktree:
+        self.queue.append(link)
-        worktree = os.path.join(config.library_path, f'comics/{comic.title}')
     
-    if not os.path.exists(worktree):
+    def download(self) -> None:
-        os.makedirs(worktree, mode=0o777)
+
+        if not os.path.exists(self.worktree):
+            os.makedirs(self.worktree, mode=0o777)
 
         opener = urllib.request.build_opener()
-    opener.addheaders = [('User-Agent', "Mozilla/5.0")]
+        opener.addheaders = [('User-Agent', 'Mozilla/5.0')]
         urllib.request.install_opener(opener)
 
         try:
-        for index,url in enumerate(comic.filelist):
+            for index,url in enumerate(self.comic.filelist):
+
                 if not url.endswith('.jpg'):
-                formatted_file = os.path.join(worktree, f'{comic.title} ' + ''.join([str(index).zfill(3), '.jpg']))
+                    formatted_file = os.path.join(self.worktree, f'{self.comic.title} ' + ''.join([str(index).zfill(3), '.jpg']))
                     print(formatted_file, end='\r')
                     urllib.request.urlretrieve(url, filename=formatted_file)
                 else:
@@ -137,43 +138,38 @@ def download_comic_files(comic: Comic, worktree = None):
                     if len(file_extension) > 3:
                         file_extension = 'jpg'
 
-                formatted_file = f'{comic.title} - {page_number}.{file_extension}'
+                    formatted_file = f'{self.comic.title} - {page_number}.{file_extension}'
                     print(formatted_file, end='\r',)
-                urllib.request.urlretrieve(url, filename=os.path.join(worktree, formatted_file))
+                    urllib.request.urlretrieve(url, filename=os.path.join(self.worktree, formatted_file))
         except HTTPError:
             # the page itself loads but the images (stored on another server) 4040
-        raise ReferenceError(f'Issue {comic.title} #{comic.issue_number} could not be found. The page may be down or the images might have errored: {self.comic.url}')
+            raise ReferenceError(f'Issue {self.comic.title} #{self.comic.issue_number} could not be found. The page may be down or the images might have errored: {self.comic.url}')
 
-def generate_archive(comic: Comic, worktree = None, archive_format = '.cbr'):
+        print()
-    if not worktree:
-        worktree = os.path.join(config.library_path, f'comics/{comic.title}')
 
-    archive_from = os.path.basename(worktree)
+    def generate_archive(self, archive_format='.cbr'):
-    if os.path.exists(os.path.join(worktree, f'{comic.title}{archive_format}')):
+
+        archive_from = os.path.basename(self.worktree)
+        if os.path.exists(os.path.join(self.worktree, f'{self.comic.title}{archive_format}')):
             return
 
-    output = shutil.make_archive(comic.title, 'zip', worktree, worktree)
+        output = shutil.make_archive(self.comic.title, 'zip', self.worktree, self.worktree)
-    # os.rename causes OSError: [Errno 18] Invalid cross-device link and files build test
+        # os.rename casuses OSError: [Errno 18] Invalid cross-device link and files build test
         # os rename only works if src and dest are on the same file system
-    shutil.move(output, os.path.join(worktree, f'{comic.title}{archive_format}'))
+        shutil.move(output, os.path.join(self.worktree, f'{self.comic.title}{archive_format}'))
 
-def clean_up(comic: Comic):
-    worktree = os.path.join(config.library_path, f'comics/{comic.title}')
 
-    for image in os.listdir(worktree):
+    def cleanup_worktree(self):
+        for image in os.listdir(self.worktree):
             if not image.endswith(required_archive_files):
-                os.remove(os.path.join(worktree, image))
+                os.remove(os.path.join(self.worktree, image))
-                
-
 
 if __name__ == '__main__':
-    # comic = Comic('http://www.readallcomics.com/static-season-one-4-2021/') # all links
+    comic = Comic('http://www.readallcomics.com/static-season-one-4-2021/') # all links
-    comic = Comic('http://readallcomics.com/x-men-v6-12-2022/')
     # comic = Comic('http://readallcomics.com/static-season-one-001-2021/') # no prev link
     # comic = Comic('http://readallcomics.com/static-season-one-6-2022/') # no next link
     # comic = Comic('http://readallcomics.com/superman-vs-lobo-4-2022/')
-    # comic = Comic('http://readallcomics.com/captain-marvel-vs-rogue-2021-part-1/')
+    # test_comic_b = 'http://readallcomics.com/captain-marvel-vs-rogue-2021-part-1/'
-    print(comic.volume)
     # print(comic.next)
     # print(comic.prev)
     # print(comic.issue_number)

yoink/yoink.json

@@ -11,7 +11,7 @@
         {
             "name": "readallcomics",
             "url": "http://readallcomics.com",
-            "search": {
+            "filters": {
                 "default": {
                     "element": "div",
                     "class": "separator",
@@ -34,7 +34,7 @@
         {
             "name": "dragonballsupermanga",
             "url": "https://www.dragonballsupermanga.net",
-            "search": {
+            "filters": {
                 "dbsuper": {
                     "element": "meta",
                     "class": null,
@@ -47,7 +47,7 @@
         {
             "name": "mangadex",
             "url": "https://www.mangadex.tv",
-            "search": {
+            "filters": {
                 "mangadex": {
                     "element": "img",
                     "class": null,