diff --git a/.gitignore b/.gitignore index 1cd9956..0c8e2a4 100644 --- a/.gitignore +++ b/.gitignore @@ -16,3 +16,6 @@ worker_*.lock /.ipynb_checkpoints .idea/misc.xml .idea/moa.iml + +Pipfile +Pipfile.lock \ No newline at end of file diff --git a/README.md b/README.md index 3f67a9a..1b0322d 100644 --- a/README.md +++ b/README.md @@ -36,9 +36,9 @@ Moa is a flask app and can be run with `python` or proxied via WSGI. * Install pipenv `pip3 install pipenv` * `PIPENV_VENV_IN_PROJECT=1 pipenv install` * `cp config.py.sample config.py` and override the settings from `defaults.py` -* `MOA_CONFIG=config.DevelopmentConfig /usr/local/bin/pipenv run python -m moa.models` to create the DB tables -* `MOA_CONFIG=config.DevelopmentConfig /usr/local/bin/pipenv run python app.py` -* run the worker with `MOA_CONFIG=DevelopmentConfig /usr/local/bin/pipenv run python -m moa.worker` +* `MOA_CONFIG=config.DevelopmentConfig pipenv run python -m moa.models` to create the DB tables +* `MOA_CONFIG=config.DevelopmentConfig pipenv run python app.py` +* run the worker with `MOA_CONFIG=DevelopmentConfig pipenv run python -m moa.worker` ## Features * preserves image alt text diff --git a/app.py b/app.py index 967f747..72a88e6 100644 --- a/app.py +++ b/app.py @@ -25,6 +25,9 @@ from sentry_sdk.integrations.sqlalchemy import SqlalchemyIntegration from sqlalchemy import exc, func from twitter import TwitterError +# import python_gitlab as gitlab +from loginpass import create_gitlab_backend, register_to +import requests from moa.forms import MastodonIDForm, SettingsForm from moa.helpers import blacklisted, email_bridge_details, send_blacklisted_email, timespan, FORMAT @@ -468,7 +471,7 @@ def mastodon_oauthorized(): except (MastodonUnauthorizedError, MastodonAPIError) as e: flash(f"There was a problem connecting to the mastodon server. The error was {e}") return redirect(url_for('index')) - + print(creds) username = creds["username"] account_id = creds["id"] @@ -481,7 +484,7 @@ def mastodon_oauthorized(): bridge = get_or_create_bridge() bridge.mastodon_host = get_or_create_host(host) bridge.md.is_bot = creds['bot'] - + print(account_id) try: bridge.mastodon_account_id = int(account_id) except ValueError: @@ -510,6 +513,11 @@ def mastodon_oauthorized(): return redirect(url_for('index')) +# +# Instagram +# + + @app.route('/instagram_activate', methods=["GET"]) def instagram_activate(): client_id = app.config['INSTAGRAM_CLIENT_ID'] @@ -583,6 +591,99 @@ def instagram_oauthorized(): return redirect(url_for('index')) +# +# Gitlab +# + +@app.route('/gitlab_activate', methods=["GET"]) +def gitlab_activate(): + client_id = app.config['GITLAB_CLIENT_ID'] + client_secret = app.config['GITLAB_SECRET'] + gitlab_app_name = app.config['GITLAB_APP_NAME'] + gitlab_host = app.config['GITLAB_HOST'] + redirect_uri = url_for('gitlab_oauthorized', _external=True) + # app.logger.info(redirect_uri) + + scope = ["basic"] + gitlab_backend = create_gitlab_backend(gitlab_app_name, gitlab_host) + # api = InstagramAPI(client_id=client_id, client_secret=client_secret, redirect_uri=redirect_uri) + # print(gitlab_backend) + # try: + # # redirect_uri = api.get_authorize_login_url(scope=scope) + # # return gitlab_backend.authorize_redirect(request, redirect_uri) + # except ServerNotFoundError as e: + # flash(f"There was a problem connecting to Instagram. Please try again") + # return redirect(url_for('index')) + # else: + auth_url = '{}?client_id={}&redirect_uri={}&response_type=code'.format(gitlab_backend.OAUTH_CONFIG['authorize_url'], client_id, redirect_uri) + # print(gitlab_backend.OAUTH_CONFIG['authorize_url']) + # print(auth_url) + return redirect(auth_url) + +@app.route('/gitlab_oauthorized') +def gitlab_oauthorized(): + code = request.args.get('code', None) + gitlab_app_name = app.config['GITLAB_APP_NAME'] + gitlab_host = app.config['GITLAB_HOST'] + client_id = app.config['GITLAB_CLIENT_ID'] + client_secret = app.config['GITLAB_SECRET'] + + if code: + gitlab_backend = create_gitlab_backend(gitlab_app_name, gitlab_host) + # token_url = '{}'.format(gitlab_backend.OAUTH_CONFIG['token_url']) + # client_id = app.config['GITLAB_CLIENT_ID'] + # client_secret = app.config['GITLAB_SECRET'] + redirect_uri = url_for('gitlab_oauthorized', _external=True) + # api = InstagramAPI(client_id=client_id, client_secret=client_secret, redirect_uri=redirect_uri) + token_data = requests.post(gitlab_backend.OAUTH_CONFIG['access_token_url'], data={'client_id': client_id, 'client_secret': client_secret, 'code': code, 'grant_type': 'authorization_code', 'redirect_uri': redirect_uri}) + # print(dir(token_data)) + print(token_data.json()) + try: + access_token = token_data.json()['access_token'] + # access_token = api.exchange_code_for_access_token(code) + except OAuth2AuthExchangeError as e: + flash("Gitlan authorization failed") + return redirect(url_for('index')) + except ServerNotFoundError as e: + flash("Gitlab authorization failed") + return redirect(url_for('index')) + + if 'bridge_id' in session: + bridge = get_or_create_bridge(bridge_id=session['bridge_id']) + + if not bridge: + pass # this should be an error + else: + bridge = get_or_create_bridge() + + bridge.gitlab_access_code = access_token + + # get token info + token_info = requests.get('https://{}/api/v4/user?access_token={}'.format(gitlab_host, access_token)) + print(token_info.json()) + # data = access_token[1] + bridge.gitlab_account_id = token_info.json()['id'] + bridge.gitlab_handle = token_info.json()['username'] + + # user_api = InstagramAPI(access_token=bridge.instagram_access_code, client_secret=client_secret) + + # try: + # latest_media, _ = user_api.user_recent_media(user_id=bridge.instagram_account_id, count=1) + # except Exception: + # latest_media = [] + + # if len(latest_media) > 0: + # bridge.instagram_last_id = datetime_to_timestamp(latest_media[0].created_time) + # else: + # bridge.instagram_last_id = 0 + + db.session.commit() + + else: + flash("Gitlab authorization failed") + + return redirect(url_for('index')) + @app.route('/logout') def logout(): session.pop('bridge_id', None) diff --git a/defaults.py b/defaults.py index b7d854c..81513d6 100644 --- a/defaults.py +++ b/defaults.py @@ -10,6 +10,10 @@ class DefaultConfig(object): TWITTER_CONSUMER_SECRET = '' INSTAGRAM_CLIENT_ID = '' INSTAGRAM_SECRET = '' + GITLAB_CLIENT_ID = '' + GITLAB_SECRET = '' + GITLAB_APP_NAME = '' + GITLAB_HOST = '' SQLALCHEMY_DATABASE_URI = 'sqlite:///moa.db' # SQLALCHEMY_DATABASE_URI = 'mysql+pymysql://moa:moa@localhost/moa' SEND = True diff --git a/migrations/alembic.ini b/migrations/alembic.ini index f8ed480..160151d 100644 --- a/migrations/alembic.ini +++ b/migrations/alembic.ini @@ -7,7 +7,7 @@ # set to 'true' to run the environment during # the 'revision' command, regardless of autogenerate # revision_environment = false - +script_location = . # Logging configuration [loggers] diff --git a/migrations/versions/0b051136582c_update_bridge_table.py b/migrations/versions/0b051136582c_update_bridge_table.py new file mode 100644 index 0000000..5a61167 --- /dev/null +++ b/migrations/versions/0b051136582c_update_bridge_table.py @@ -0,0 +1,32 @@ +"""update bridge table + +Revision ID: 0b051136582c +Revises: 3ac471544742 +Create Date: 2021-02-14 22:51:03.352845 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = '0b051136582c' +down_revision = '3ac471544742' +branch_labels = None +depends_on = None + + +def upgrade(): + op.add_column('bridge', sa.Column('gitlab_access_code', sa.String(length=80), nullable=True)) + op.add_column('bridge', sa.Column('gitlab_account_id', sa.BigInteger(), nullable=True)) + op.add_column('bridge', sa.Column('gitlab_handle', sa.String(length=30), nullable=True)) + op.add_column('settings', sa.Column('gitlab_project', sa.String(length=100), nullable=False, default="")) + op.add_column('settings', sa.Column('post_to_gitlab', sa.Boolean, nullable=False, default=True)) + + +def downgrade(): + op.drop_column('bridge', 'gitlab_access_code') + op.drop_column('bridge', 'gitlab_account_id') + op.drop_column('bridge', 'gitlab_handle') + op.drop_column('settings', 'gitlab_project') + op.drop_column('settings', 'post_to_gitlab') \ No newline at end of file diff --git a/moa/forms.py b/moa/forms.py index bcc7f23..fe5a6f8 100644 --- a/moa/forms.py +++ b/moa/forms.py @@ -39,6 +39,10 @@ class SettingsForm(FlaskForm): instagram_post_to_mastodon = BooleanField('Post Instagrams to Mastodon?') instagram_include_link = BooleanField('Include link to instagram post') + post_to_gitlab = BooleanField('Post Public posts to Gitlab?') + # TODO add validator when I figure out how to do it conditionally + gitlab_project = StringField('') + def remove_masto_and_twitter_fields(self): del self.post_to_twitter del self.post_private_to_twitter diff --git a/moa/git_poster.py b/moa/git_poster.py new file mode 100644 index 0000000..1b1269c --- /dev/null +++ b/moa/git_poster.py @@ -0,0 +1,48 @@ +import logging +import requests +import re +import base64 +from datetime import datetime + +from moa.poster import Poster +from moa.message import Message + +logger = logging.getLogger('worker') + +class GitPoster(Poster): + def __init__(self, send, session, gitlab_host, bridge): + + super().__init__(send, session) + + self.bridge = bridge + self.gitlab_host = gitlab_host + + def post(self, post: Message) -> bool: + self.reset() + + + logger.info('Post body: {}'.format(post.clean_content)) + reg = re.compile('.*\[\[.*?\]\]') + m = reg.match(post.clean_content) + if m is None: + logger.info("no wikilink found in post") + return + + date = datetime.now().date().isoformat() + access_token = self.bridge.gitlab_access_code + url = 'https://{}/api/v4/projects/{}/repository/files/{}.md'.format(self.gitlab_host, self.bridge.t_settings.gitlab_project, date) + raw_url = '{}/raw?ref=master&access_token={}'.format(url, access_token) + raw = requests.get(raw_url) + # logger.info(raw.text) + if raw.status_code == 200: + content = '{}\n\n{}'.format(raw.text, post.clean_content) + file_info = requests.put(url, data={'branch': 'master', 'content': content, 'commit_message': 'update from moa', 'access_token': access_token}) + else: + content = '{}'.format(post.clean_content) + file_info = requests.post(url, data={'branch': 'master', 'content': content, 'commit_message': 'update from moa', 'access_token': access_token}) + # logger.info(content) + if file_info.status_code != 200: + logger.info(file_info.text) + return + logger.info(file_info.status_code) + diff --git a/moa/models.py b/moa/models.py index 972c006..c304c44 100644 --- a/moa/models.py +++ b/moa/models.py @@ -81,6 +81,10 @@ class TSettings(Base): instagram_post_to_mastodon = Column(Boolean, nullable=False, default=False) instagram_include_link = Column(Boolean, nullable=False, default=True) + # Gitlab + post_to_gitlab = Column(Boolean, nullable=False, default=True) + gitlab_project = Column(String(100), nullable=False, default="") + def __init__(self, **kwargs): kwargs.setdefault('post_to_twitter', True) kwargs.setdefault('post_private_to_twitter', False) @@ -101,6 +105,8 @@ def __init__(self, **kwargs): kwargs.setdefault('instagram_post_to_mastodon', False) kwargs.setdefault('instagram_include_link', True) + kwargs.setdefault('gitlab_project', "") + super(TSettings, self).__init__(**kwargs) @property @@ -138,6 +144,10 @@ class Bridge(Base): instagram_account_id = Column(BigInteger, default=0) instagram_handle = Column(String(30)) + gitlab_access_code = Column(String(80)) + gitlab_account_id = Column(BigInteger, default=0) + gitlab_handle = Column(String(30)) + t_settings_id = Column(Integer, ForeignKey('settings.id'), nullable=True) metadata_id = Column(Integer, ForeignKey('bridgemetadata.id'), nullable=True) @@ -252,6 +262,7 @@ def receive_time_set(target, value, oldvalue, initiator): from sqlalchemy import create_engine moa_config = os.environ.get('MOA_CONFIG', 'DevelopmentConfig') + print(moa_config) config = getattr(importlib.import_module('config'), moa_config) if "mysql" in config.SQLALCHEMY_DATABASE_URI: diff --git a/moa/worker.py b/moa/worker.py index 9891c0f..483f099 100644 --- a/moa/worker.py +++ b/moa/worker.py @@ -2,6 +2,7 @@ import importlib import logging import os +import pytz import smtplib import sys import time @@ -34,6 +35,8 @@ from moa.toot_poster import TootPoster from moa.tweet import Tweet from moa.tweet_poster import TweetPoster +from moa.git_poster import GitPoster + start_time = time.time() @@ -402,8 +405,8 @@ def check_worker_stop(): l.info(f"{bridge.id}: M - {bridge.mastodon_user}@{mastodonhost.hostname}") tweet_poster = TweetPoster(c.SEND, session, twitter_api, bridge) - - if settings.post_to_twitter_enabled and len(new_toots) > 0: + git_poster = GitPoster(c.SEND, session, c.GITLAB_HOST, bridge) + if len(new_toots) > 0: l.info(f"{len(new_toots)} new toots found") @@ -414,17 +417,26 @@ def check_worker_stop(): t = Toot(settings, toot, c) try: - result = tweet_poster.post(t) + if settings.post_to_twitter_enabled: + result = tweet_poster.post(t) + if settings.post_to_gitlab: + result = git_poster.post(t) except MoaMediaUploadException as e: continue + l.info('Result: {}'.format(result)) if result: worker_stat.add_toot() bridge_stat.add_toot() - bridge.md.last_toot = t.data['created_at'] + session.commit() + + + + + # # Post Tweets to Mastodon # @@ -435,7 +447,7 @@ def check_worker_stop(): if bridge.twitter_oauth_token: l.info(f"{bridge.id}: T - @{bridge.twitter_handle}") - if settings.post_to_mastodon_enabled and len(new_tweets) > 0: + if len(new_tweets) > 0: l.info(f"{len(new_tweets)} new tweets found") if not bridge_stat: @@ -446,11 +458,16 @@ def check_worker_stop(): tweet = Tweet(settings, status, twitter_api) try: - result = toot_poster.post(tweet) + if settings.post_to_mastodon_enabled: + result = toot_poster.post(tweet) + if settings.post_to_gitlab: + result = git_poster.post(tweet) except MoaMediaUploadException as e: continue + l.info('Result: {}'.format(result)) + if result: worker_stat.add_tweet() bridge_stat.add_tweet() diff --git a/requirements.in b/requirements.in index 8803d38..8cf8b94 100644 --- a/requirements.in +++ b/requirements.in @@ -1,9 +1,11 @@ certifi>=2019.3.9 +gitlab Flask==1.1.2 Flask-SQLAlchemy==2.4.1 Flask-Mail==0.9.1 Flask-Migrate==2.5.3 Flask-WTF==0.14.3 +loginpass==0.4 git+https://github.com/foozmeat/python-instagram.git#egg=instagram Mastodon.py==1.5.1 pandas @@ -12,6 +14,7 @@ pygal==2.4.0 python-twitter==3.5 PyMySQL==0.9.3 pip-check +requests sentry-sdk[flask] authlib==0.13 cairosvg diff --git a/requirements.txt b/requirements.txt index cb32791..28cad15 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,63 +4,173 @@ # # pip-compile # -alembic==1.4.2 # via flask-migrate -authlib==0.13 # via -r requirements.in -blinker==1.4 # via flask-mail, sentry-sdk -blurhash==1.1.4 # via mastodon.py -cairocffi==1.1.0 # via cairosvg -cairosvg==2.4.2 # via -r requirements.in -certifi==2020.4.5.1 # via -r requirements.in, requests, sentry-sdk -cffi==1.14.0 # via cairocffi, cryptography -chardet==3.0.4 # via requests -click==7.1.2 # via flask, pip-tools -colorclass==2.2.0 # via pip-check -cryptography==2.9.2 # via authlib -cssselect2==0.3.0 # via cairosvg -decorator==4.4.2 # via mastodon.py -defusedxml==0.6.0 # via cairosvg -flask-mail==0.9.1 # via -r requirements.in -flask-migrate==2.5.3 # via -r requirements.in -flask-sqlalchemy==2.4.1 # via -r requirements.in, flask-migrate -flask-wtf==0.14.3 # via -r requirements.in -flask==1.1.2 # via -r requirements.in, flask-mail, flask-migrate, flask-sqlalchemy, flask-wtf, sentry-sdk -future==0.18.2 # via python-twitter -httplib2==0.18.0 # via instagram -idna==2.9 # via requests -git+https://github.com/foozmeat/python-instagram.git#egg=instagram # via -r requirements.in -itsdangerous==1.1.0 # via flask, flask-wtf -jinja2==2.11.2 # via flask -mako==1.1.2 # via alembic -markupsafe==1.1.1 # via jinja2, mako, wtforms -mastodon.py==1.5.1 # via -r requirements.in -numpy==1.18.4 # via pandas -oauthlib==3.1.0 # via requests-oauthlib -pandas==1.0.3 # via -r requirements.in -pillow==7.1.2 # via cairosvg -pip-check==2.6 # via -r requirements.in -pip-tools==5.3.1 # via -r requirements.in -psutil==5.7.0 # via -r requirements.in -pycparser==2.20 # via cffi -pygal==2.4.0 # via -r requirements.in -pymysql==0.9.3 # via -r requirements.in -python-dateutil==2.8.1 # via alembic, mastodon.py, pandas -python-editor==1.0.4 # via alembic -python-magic==0.4.15 # via mastodon.py -python-twitter==3.5 # via -r requirements.in -pytz==2020.1 # via instagram, mastodon.py, pandas -requests-oauthlib==1.3.0 # via python-twitter -requests==2.23.0 # via mastodon.py, python-twitter, requests-oauthlib -sentry-sdk[flask]==0.17.3 # via -r requirements.in -simplejson==3.17.0 # via instagram -six==1.14.0 # via cryptography, instagram, mastodon.py, pip-tools, python-dateutil -sqlalchemy==1.3.16 # via alembic, flask-sqlalchemy -terminaltables==3.1.0 # via pip-check -tinycss2==1.0.2 # via cairosvg, cssselect2 -urllib3==1.25.9 # via requests, sentry-sdk -webencodings==0.5.1 # via cssselect2, tinycss2 -werkzeug==0.16.1 # via -r requirements.in, flask -wheel==0.34.2 # via -r requirements.in -wtforms==2.3.1 # via flask-wtf +alembic==1.4.2 + # via flask-migrate +authlib==0.13 + # via + # -r requirements.in + # loginpass +blinker==1.4 + # via + # flask-mail + # sentry-sdk +blurhash==1.1.4 + # via mastodon.py +cairocffi==1.1.0 + # via cairosvg +cairosvg==2.4.2 + # via -r requirements.in +certifi==2020.4.5.1 + # via + # -r requirements.in + # requests + # sentry-sdk +cffi==1.14.0 + # via + # cairocffi + # cryptography +chardet==3.0.4 + # via requests +click==7.1.2 + # via + # flask + # pip-tools +colorclass==2.2.0 + # via pip-check +cryptography==2.9.2 + # via authlib +cssselect2==0.3.0 + # via cairosvg +decorator==4.4.2 + # via mastodon.py +defusedxml==0.6.0 + # via cairosvg +flask-mail==0.9.1 + # via -r requirements.in +flask-migrate==2.5.3 + # via -r requirements.in +flask-sqlalchemy==2.4.1 + # via + # -r requirements.in + # flask-migrate +flask-wtf==0.14.3 + # via -r requirements.in +flask==1.1.2 + # via + # -r requirements.in + # flask-mail + # flask-migrate + # flask-sqlalchemy + # flask-wtf + # sentry-sdk +future==0.18.2 + # via python-twitter +gitlab==1.0.2 + # via -r requirements.in +httplib2==0.18.0 + # via instagram +idna==2.9 + # via requests +git+https://github.com/foozmeat/python-instagram.git#egg=instagram + # via -r requirements.in +itsdangerous==1.1.0 + # via + # flask + # flask-wtf +jinja2==2.11.2 + # via flask +loginpass==0.4 + # via -r requirements.in +mako==1.1.2 + # via alembic +markupsafe==1.1.1 + # via + # jinja2 + # mako + # wtforms +mastodon.py==1.5.1 + # via -r requirements.in +numpy==1.18.4 + # via pandas +oauthlib==3.1.0 + # via requests-oauthlib +pandas==1.0.3 + # via -r requirements.in +pillow==7.1.2 + # via cairosvg +pip-check==2.6 + # via -r requirements.in +pip-tools==5.3.1 + # via -r requirements.in +psutil==5.7.0 + # via -r requirements.in +pycparser==2.20 + # via cffi +pygal==2.4.0 + # via -r requirements.in +pymysql==0.9.3 + # via -r requirements.in +python-dateutil==2.8.1 + # via + # alembic + # mastodon.py + # pandas +python-editor==1.0.4 + # via alembic +python-magic==0.4.15 + # via mastodon.py +python-twitter==3.5 + # via -r requirements.in +pytz==2020.1 + # via + # instagram + # mastodon.py + # pandas +requests-oauthlib==1.3.0 + # via python-twitter +requests==2.23.0 + # via + # loginpass + # mastodon.py + # python-twitter + # requests-oauthlib +sentry-sdk[flask]==0.17.3 + # via -r requirements.in +simplejson==3.17.0 + # via instagram +six==1.14.0 + # via + # cryptography + # instagram + # mastodon.py + # pip-tools + # python-dateutil +sqlalchemy==1.3.16 + # via + # alembic + # flask-sqlalchemy +terminaltables==3.1.0 + # via pip-check +tinycss2==1.0.2 + # via + # cairosvg + # cssselect2 +urllib3==1.25.9 + # via + # requests + # sentry-sdk +webencodings==0.5.1 + # via + # cssselect2 + # tinycss2 +werkzeug==0.16.1 + # via + # -r requirements.in + # flask +wheel==0.34.2 + # via -r requirements.in +wtforms==2.3.1 + # via flask-wtf # The following packages are considered to be unsafe in a requirements file: # pip diff --git a/templates/index.html.j2 b/templates/index.html.j2 index 2ecf930..02bf636 100644 --- a/templates/index.html.j2 +++ b/templates/index.html.j2 @@ -30,6 +30,15 @@ {% endif %} + {# Gitlab #} + {% if g.bridge.gitlab_access_code %} +

Gitlab: {{ g.bridge.gitlab_handle }} [X]

+ + {% else %} +

[ ] Connect to Gitlab

+ + {% endif %} + {# Options #}
{{ form.csrf_token }} {% if g.bridge.twitter_oauth_token or g.bridge.instagram_access_code %}