Commit 3d3fd5ab authored by tsukumi's avatar tsukumi

Refactor: organize the composition, add docker-compose.yaml

parent 4112ddd8
# Show debug log messages (0 or 1)
DEBUG=1
ACCOUNT_FILE=./.htaccounts
COOKIE_DIR=./.htcookies
# JSON file with reference account credentials
ACCOUNT_FILE=./accounts.json
# Directory for session cookies
COOKIE_DIR=./cookies
# File to write logs to
LOG_FILE=./logs/results.log
# host/port which to listen on
HOST=0.0.0.0
PORT=4040
HOST=localhost
# Auth key for Twitter guest session
TWITTER_AUTH_KEY=AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA
# Value for Access-Control-Allow-Origin header
CORS_ALLOW=http://localhost:3000
# Number of Twitter guest sessions to use
GUEST_SESSIONS=3
# MongoDB Connection
MONGO_HOST=localhost
MONGO_PORT=27017
MONGO_DB=shadowban-testing
MONGO_DB=tester
MONGO_USERNAME=root
MONGO_PASSWORD=5X-rl[(EMdJKll1|qMDU}5xY<t?F.UEo
......@@ -54,7 +54,11 @@ node_modules
.env
.env.*
!.env.example
.ht*
# credential
accounts.json
cookies
mongo
# python misc
.venv
......
FROM python:3.7-slim-buster
RUN apt update
# Use Python 3.9 as a base image
FROM python:3.9-slim-buster
# Package installation
RUN apt-get update -y && apt-get upgrade -y
RUN apt install gcc python3-dev -y
RUN mkdir /app
ADD . /app
# Dependencies installation
COPY requirements.txt /tmp/
RUN pip install --no-cache-dir -r /tmp/requirements.txt && rm /tmp/requirements.txt
# Working directory
WORKDIR /app/
ENV NO_VENV=1
WORKDIR /app
RUN ./bin/install.sh
# entry point
ENTRYPOINT /bin/bash docker-entry.sh .env
[
{
"username": "example_account",
"password": "example_account_password"
}
]
\ No newline at end of file
......@@ -82,8 +82,8 @@ def ensure_dir(path):
os.mkdir(path)
parser = argparse.ArgumentParser(description='Twitter Shadowban Tester')
parser.add_argument('--account-file', type=str, default='.htaccounts', help='json file with reference account credentials')
parser.add_argument('--cookie-dir', type=str, default=None, help='directory for session cookies')
parser.add_argument('--account-file', type=str, default='./accounts.json', help='json file with reference account credentials')
parser.add_argument('--cookie-dir', type=str, default='./cookies', help='directory for session cookies')
parser.add_argument('--log', type=str, default='./logs/backend.log', help='file to write logs to (default: ./logs/backend.log)')
parser.add_argument('--daemon', action='store_true', help='run in background')
parser.add_argument('--debug', action='store_true', help='show debug log messages')
......@@ -137,15 +137,15 @@ async def clean_up(app):
shutdown_logging()
def run():
global db
db = None
global DB
DB = None
if args.mongo_host is not None:
db = Database(
DB = connect(
host=args.mongo_host,
port=args.mongo_port,
db=args.mongo_db,
username=args.mongo_username,
password=args.mongo_password,
db=args.mongo_db
)
loop = asyncio.get_event_loop()
loop.run_until_complete(login_accounts(TwitterSession.accounts, args.cookie_dir))
......
echo -n "Looking for python3: "
if ! hash python3 &> /dev/null; then
echo -e "\nFatal: Please install Python3 to use this program!"
exit 1
fi
echo "OK"
if [ "$NO_VENV" != "1" ]; then
if [ ! -f .venv/bin/activate ]; then
echo "Creating new venv in: ./.venv"
python3 -m venv ./.venv
fi
echo "Loading virtualenv: ./.venv"
source ./.venv/bin/activate
else
echo "Skipping venv setup"
fi
#!/usr/bin/env bash
source ./bin/_venv
if [ "$1" != "" ] && [ -f $1 ]; then
echo "Using provided .env file: $1"
export $(cat $1 | xargs)
shift
fi
CMD="python3 -u ./backend.py"
if [ "$1" == "mprof" ]; then
shift
CMD="mprof run $@ ./backend.py"
echo -e "\nRecording memory profile\n"
fi
SERVICE_ARGS="\
--account-file $ACCOUNT_FILE \
--cookie-dir $COOKIE_DIR \
--log $LOG_FILE \
--port "$PORT" \
--host "$HOST" \
--mongo-host $MONGO_HOST \
--mongo-port $MONGO_PORT \
--mongo-db $MONGO_DB \
--mongo-username $MONGO_USERNAME \
--mongo-password $MONGO_PASSWORD \
--twitter-auth-key $TWITTER_AUTH_KEY \
--cors-allow $CORS_ALLOW \
--guest-sessions $GUEST_SESSIONS \
"
if [ -n "$DEBUG" ]; then
SERVICE_ARGS="$SERVICE_ARGS --debug"
fi
CMD="$CMD $SERVICE_ARGS $@"
echo -n "Starting server: "
if [ -n "$DEBUG" ]; then
echo $CMD
else
echo ""
fi
$CMD
#!/usr/bin/env bash
source ./bin/_venv
echo "Installing dependencies..."
pip3 install -r requirements.txt --no-cache-dir
if [ $? -eq 0 ]; then
echo -e "\n----------------------------"
echo -e "Almost done! \\o/\n"
echo "Run 'PYTHON_ENV=[development|prodcution] ./bin/docker-entry.sh .env.example' to start the server!"
echo -e "\nIf you want to make changes to the python packages, e.g. 'pip3 install ...', activate the venv, first: '. .venv/bin/activate'"
fi
if [ "$1" != "" ] && [ -f $1 ]; then
ENV_FILE=$1
else
ENV_FILE=.env
fi
echo 'Stopping service'
pkill -f twitter-auth-key
if [ $? -ne 0 ]; then
echo "Service not running"
fi
if [ "$1" == "-k" ]; then
echo "Deleting logs"
rm ./logs/*
fi
echo 'Starting service'
./docker-entry.sh $ENV_FILE
version: '3.8'
services:
# api
api:
image: shadowban-tester
container_name: shadowban-tester-api
# restart: always
build:
context: .
depends_on:
- mongo
ports:
- '${PORT}:${PORT}'
volumes:
- type: bind
source: '.'
target: '/app/'
# MongoDB
mongo:
image: mongo
container_name: shadowban-tester-mongo
restart: always
environment:
MONGO_INITDB_ROOT_USERNAME: ${MONGO_USERNAME}
MONGO_INITDB_ROOT_PASSWORD: ${MONGO_PASSWORD}
ports:
- 27017:27017
volumes:
- ./mongo/data:/data/db
- ./mongo/config:/data/configdb
#!/usr/bin/env bash
if [ "$1" != "" ] && [ -f $1 ]; then
echo "Using provided .env file: $1"
export $(cat .env| grep -v "#" | xargs)
shift
fi
CMD="python3 -u ./backend.py"
if [ "$1" == "mprof" ]; then
shift
CMD="mprof run $@ ./backend.py"
echo -e "\nRecording memory profile\n"
fi
SERVICE_ARGS="\
--account-file $ACCOUNT_FILE \
--cookie-dir $COOKIE_DIR \
--log $LOG_FILE \
--port "$PORT" \
--host "$HOST" \
--mongo-host $MONGO_HOST \
--mongo-port $MONGO_PORT \
--mongo-db $MONGO_DB \
--mongo-username $MONGO_USERNAME \
--mongo-password $MONGO_PASSWORD \
--twitter-auth-key $TWITTER_AUTH_KEY \
--cors-allow $CORS_ALLOW \
--guest-sessions $GUEST_SESSIONS \
"
if [ $DEBUG -eq 1 ]; then
SERVICE_ARGS="$SERVICE_ARGS --debug"
fi
CMD="$CMD $SERVICE_ARGS $@"
echo -n "Starting server: "
if [ -n "$DEBUG" ]; then
echo $CMD
else
echo ""
fi
$CMD
......@@ -93,7 +93,7 @@ class TwitterSession:
self.next_refresh = time.time() + 3600
self._headers['X-Guest-Token'] = self._guest_token
async def login(self, username = None, password = None, email = None, cookie_dir=None):
async def login(self, username=None, password=None, cookie_dir=None):
self._session = aiohttp.ClientSession()
if password is not None:
......@@ -116,7 +116,7 @@ class TwitterSession:
form_data = {}
soup = BeautifulSoup(login_page, 'html.parser')
form_data["authenticity_token"] = soup.find('input', {'name': 'authenticity_token'}).get('value')
form_data["session[username_or_email]"] = email
form_data["session[username_or_email]"] = username
form_data["session[password]"] = password
form_data["remember_me"] = "1"
async with self._session.post('https://twitter.com/sessions', data=form_data, headers=self._headers) as r:
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment