From 60aa00238ca4de546bdf8f4a375b794096db0f48 Mon Sep 17 00:00:00 2001 From: npalaska Date: Wed, 2 Dec 2020 16:42:19 -0500 Subject: [PATCH] Initial pbench user authentication model implementation This implements 5 basic user APIs 1. Register User Handles Pbench User registration via JSON request POST /v1/register 2. Login User: POST /v1/login Returns a valid pbench auth token User is allowed to issue multiple login requests and thus generating multiple auth tokens, Each token is stored in a active_tokens table and has its own expiry User is authenticated for subsequest API calls if token not expired and present in the active_tokens table 3. Logout user: POST /v1/logout Deletes the auth_token from the active_tokens table. Once logged out user can not use the same auth token for other API access. 4. Get User: GET /v1/user/ Returns the user's self information that was registered, the username must be provided in the url If the Auth header does not belong to the username, reject the request unless auth token belongs to the admin 5. Delete User: DELETE /v1/user/" An API for a user to delete himself from the pbench database. A user can only perform delete action on himself unless the auth token belongs to the admin user. 6. Updare User: PUT /v1/user/ An API for updating the User registration fields, the username must be provided in the url Update is not allowed on registerd_on field --- lib/pbench/server/api/__init__.py | 58 +- lib/pbench/server/api/auth.py | 121 +++++ lib/pbench/server/api/resources/users_api.py | 504 ++++++++++++++++++ lib/pbench/server/database/__init__.py | 0 lib/pbench/server/database/alembic.ini | 85 +++ lib/pbench/server/database/alembic/README | 20 + lib/pbench/server/database/alembic/env.py | 108 ++++ .../server/database/alembic/script.py.mako | 24 + lib/pbench/server/database/database.py | 44 ++ lib/pbench/server/database/models/__init__.py | 0 .../server/database/models/active_tokens.py | 49 ++ lib/pbench/server/database/models/users.py | 126 +++++ lib/pbench/test/unit/server/conftest.py | 3 + lib/pbench/test/unit/server/test_user_auth.py | 425 +++++++++++++++ server/lib/config/pbench-server-default.cfg | 7 + server/requirements.txt | 9 + 16 files changed, 1575 insertions(+), 8 deletions(-) create mode 100644 lib/pbench/server/api/auth.py create mode 100644 lib/pbench/server/api/resources/users_api.py create mode 100644 lib/pbench/server/database/__init__.py create mode 100644 lib/pbench/server/database/alembic.ini create mode 100644 lib/pbench/server/database/alembic/README create mode 100644 lib/pbench/server/database/alembic/env.py create mode 100644 lib/pbench/server/database/alembic/script.py.mako create mode 100644 lib/pbench/server/database/database.py create mode 100644 lib/pbench/server/database/models/__init__.py create mode 100644 lib/pbench/server/database/models/active_tokens.py create mode 100644 lib/pbench/server/database/models/users.py create mode 100644 lib/pbench/test/unit/server/test_user_auth.py diff --git a/lib/pbench/server/api/__init__.py b/lib/pbench/server/api/__init__.py index 49070d2811..b6d5026b37 100644 --- a/lib/pbench/server/api/__init__.py +++ b/lib/pbench/server/api/__init__.py @@ -5,6 +5,7 @@ """ import os +import sys from flask import Flask from flask_restful import Api @@ -17,7 +18,16 @@ from pbench.common.logger import get_pbench_logger from pbench.server.api.resources.query_apis.elasticsearch_api import Elasticsearch from pbench.server.api.resources.query_apis.query_controllers import QueryControllers +from pbench.server.database.database import Database from pbench.server.api.resources.query_apis.query_month_indices import QueryMonthIndices +from pbench.server.api.auth import Auth + +from pbench.server.api.resources.users_api import ( + RegisterUser, + Login, + Logout, + UserAPI, +) def register_endpoints(api, app, config): @@ -25,33 +35,54 @@ def register_endpoints(api, app, config): to make the APIs active.""" base_uri = config.rest_uri - app.logger.info("Registering service endpoints with base URI {}", base_uri) + logger = app.logger + + # Init the the authentication module + token_auth = Auth() + Auth.set_logger(logger) + + logger.info("Registering service endpoints with base URI {}", base_uri) api.add_resource( Upload, f"{base_uri}/upload/ctrl/", - resource_class_args=(config, app.logger), + resource_class_args=(config, logger), ) api.add_resource( - HostInfo, f"{base_uri}/host_info", resource_class_args=(config, app.logger), + HostInfo, f"{base_uri}/host_info", resource_class_args=(config, logger), ) api.add_resource( Elasticsearch, f"{base_uri}/elasticsearch", - resource_class_args=(config, app.logger), + resource_class_args=(config, logger), ) api.add_resource( - GraphQL, f"{base_uri}/graphql", resource_class_args=(config, app.logger), + GraphQL, f"{base_uri}/graphql", resource_class_args=(config, logger), ) api.add_resource( QueryControllers, f"{base_uri}/controllers/list", - resource_class_args=(config, app.logger), + resource_class_args=(config, logger), ) api.add_resource( QueryMonthIndices, f"{base_uri}/controllers/months", - resource_class_args=(config, app.logger), + resource_class_args=(config, logger), + ) + + api.add_resource( + RegisterUser, f"{base_uri}/register", resource_class_args=(config, logger), + ) + api.add_resource( + Login, f"{base_uri}/login", resource_class_args=(config, logger, token_auth), + ) + api.add_resource( + Logout, f"{base_uri}/logout", resource_class_args=(config, logger, token_auth), + ) + api.add_resource( + UserAPI, + f"{base_uri}/user/", + resource_class_args=(logger, token_auth), ) @@ -74,7 +105,6 @@ def create_app(server_config): """Create Flask app with defined resource endpoints.""" app = Flask("api-server") - api = Api(app) CORS(app, resources={r"/api/*": {"origins": "*"}}) app.logger = get_pbench_logger(__name__, server_config) @@ -82,6 +112,18 @@ def create_app(server_config): app.config["DEBUG"] = False app.config["TESTING"] = False + api = Api(app) + register_endpoints(api, app, server_config) + try: + Database.init_db(server_config=server_config, logger=app.logger) + except Exception: + app.logger.exception("Exception while initializing sqlalchemy database") + sys.exit(1) + + @app.teardown_appcontext + def shutdown_session(exception=None): + Database.db_session.remove() + return app diff --git a/lib/pbench/server/api/auth.py b/lib/pbench/server/api/auth.py new file mode 100644 index 0000000000..2c2b136f00 --- /dev/null +++ b/lib/pbench/server/api/auth.py @@ -0,0 +1,121 @@ +import jwt +import os +import datetime +from flask import request, abort +from flask_httpauth import HTTPTokenAuth +from pbench.server.database.models.users import User +from pbench.server.database.models.active_tokens import ActiveTokens + + +class Auth: + token_auth = HTTPTokenAuth("Bearer") + + @staticmethod + def set_logger(logger): + # Logger gets set at the time of auth module initialization + Auth.logger = logger + + def encode_auth_token(self, token_expire_duration, user_id): + """ + Generates the Auth Token + :return: jwt token string + """ + current_utc = datetime.datetime.utcnow() + payload = { + "iat": current_utc, + "exp": current_utc + datetime.timedelta(minutes=int(token_expire_duration)), + "sub": user_id, + } + + # Get jwt key + jwt_key = self.get_secret_key() + return jwt.encode(payload, jwt_key, algorithm="HS256") + + def get_secret_key(self): + try: + return os.getenv("SECRET_KEY", "my_precious") + except Exception as e: + Auth.logger.exception(f"{__name__}: ERROR: {e.__traceback__}") + + def verify_user(self, username): + """ + Check if the provided username belongs to the current user by + querying the Usermodel with the current user + :param username: + :param logger + :return: User (UserModel instance), verified status (boolean) + """ + user = User.query(id=self.token_auth.current_user().id) + # check if the current username matches with the one provided + verified = user is not None and user.username == username + Auth.logger.warning("verified status of user '{}' is '{}'", username, verified) + + return user, verified + + def get_auth_token(self, logger): + # get auth token + auth_header = request.headers.get("Authorization") + + if not auth_header: + logger.warning("Missing expected Authorization header") + abort( + 403, + message="Please add 'Authorization' token as Authorization: Bearer ", + ) + + try: + auth_schema, auth_token = auth_header.split() + except ValueError: + logger.warning("Malformed Auth header") + abort( + 401, + message="Malformed Authorization header, please add request header as Authorization: Bearer ", + ) + else: + if auth_schema.lower() != "bearer": + logger.warning( + "Expected authorization schema to be 'bearer', not '{}'", + auth_schema, + ) + abort( + 401, + message="Malformed Authorization header, request auth needs bearer token: Bearer ", + ) + return auth_token + + @staticmethod + @token_auth.verify_token + def verify_auth(auth_token): + """ + Validates the auth token + :param auth_token: + :return: User object/None + """ + try: + payload = jwt.decode( + auth_token, os.getenv("SECRET_KEY", "my_precious"), algorithms="HS256", + ) + user_id = payload["sub"] + if ActiveTokens.valid(auth_token): + user = User.query(id=user_id) + return user + except jwt.ExpiredSignatureError: + try: + ActiveTokens.delete(auth_token) + except Exception: + Auth.logger.error( + "User attempted Pbench expired token but we could not delete the expired auth token from the database. token: '{}'", + auth_token, + ) + return None + Auth.logger.warning( + "User attempted Pbench expired token '{}', Token deleted from the database and no longer tracked", + auth_token, + ) + except jwt.InvalidTokenError: + Auth.logger.warning("User attempted invalid Pbench token '{}'", auth_token) + except Exception: + Auth.logger.exception( + "Exception occurred while verifying the auth token '{}'", auth_token + ) + return None diff --git a/lib/pbench/server/api/resources/users_api.py b/lib/pbench/server/api/resources/users_api.py new file mode 100644 index 0000000000..43c8f4a8ec --- /dev/null +++ b/lib/pbench/server/api/resources/users_api.py @@ -0,0 +1,504 @@ +import jwt +from flask import request, jsonify, make_response +from flask_restful import Resource, abort +from flask_bcrypt import check_password_hash +from email_validator import EmailNotValidError +from sqlalchemy.exc import SQLAlchemyError, IntegrityError +from pbench.server.database.models.users import User +from pbench.server.database.models.active_tokens import ActiveTokens +from pbench.server.api.auth import Auth + + +class RegisterUser(Resource): + """ + Abstracted pbench API for registering a new user + """ + + def __init__(self, config, logger): + self.server_config = config + self.logger = logger + + def post(self): + """ + Post request for registering a new user. + This requires a JSON data with required user fields + { + "username": "username", + "password": "password", + "first_name": first_name, + "last_name": "last_name", + "email": "user@domain.com" + } + + Required headers include + + Content-Type: application/json + Accept: application/json + + :return: JSON Payload + if we succeed to add a user entry in the database, the returned response_object will look like the following: + response_object = { + "message": "Successfully registered."/"failure message", + } + To get the auth token user has to perform the login action + """ + # get the post data + user_data = request.get_json() + if not user_data: + self.logger.warning("Invalid json object: {}", request.url) + abort(400, message="Invalid json object in request") + + username = user_data.get("username") + if not username: + self.logger.warning("Missing username field") + abort(400, message="Missing username field") + username = username.lower() + if User.is_admin_username(username): + self.logger.warning("User tried to register with admin username") + abort( + 400, message="Please choose another username", + ) + + # check if provided username already exists + try: + user = User.query(username=user_data.get("username")) + except Exception: + self.logger.exception("Exception while querying username") + abort(500, message="INTERNAL ERROR") + if user: + self.logger.warning( + "A user tried to re-register. Username: {}", user.username + ) + abort(403, message="Provided username is already in use.") + + password = user_data.get("password") + if not password: + self.logger.warning("Missing password field") + abort(400, message="Missing password field") + + email_id = user_data.get("email") + if not email_id: + self.logger.warning("Missing email field") + abort(400, message="Missing email field") + # check if provided email already exists + try: + user = User.query(email=email_id) + except Exception: + self.logger.exception("Exception while querying user email") + abort(500, message="INTERNAL ERROR") + if user: + self.logger.warning("A user tried to re-register. Email: {}", user.email) + abort(403, message="Provided email is already in use") + + first_name = user_data.get("first_name") + if not first_name: + self.logger.warning("Missing firstName field") + abort(400, message="Missing firstName field") + + last_name = user_data.get("last_name") + if not last_name: + self.logger.warning("Missing lastName field") + abort(400, message="Missing lastName field") + + try: + user = User( + username=username, + password=password, + first_name=first_name, + last_name=last_name, + email=email_id, + ) + + # insert the user + user.add() + self.logger.info( + "New user registered, username: {}, email: {}", username, email_id + ) + + response_object = { + "message": "Successfully registered.", + } + response = jsonify(response_object) + response.status_code = 201 + return make_response(response, 201) + except EmailNotValidError: + self.logger.warning("Invalid email {}", email_id) + abort(400, message=f"Invalid email: {email_id}") + except Exception: + self.logger.exception("Exception while registering a user") + abort(500, message="INTERNAL ERROR") + + +class Login(Resource): + """ + Pbench API for User Login or generating an auth token + """ + + def __init__(self, config, logger, auth): + self.server_config = config + self.logger = logger + self.auth = auth + self.token_expire_duration = self.server_config.get( + "pbench-server", "token_expiration_duration" + ) + + @Auth.token_auth.login_required(optional=True) + def post(self): + """ + Post request for logging in user. + The user is allowed to re-login multiple times and each time a new valid auth token will be provided + + This requires a JSON data with required user metadata fields + { + "username": "username", + "password": "password", + } + + Required headers include + + Content-Type: application/json + Accept: application/json + + :return: JSON Payload + if we succeed to decrypt the password hash, the returned response_object will include the auth_token + response_object = { + "message": "Successfully logged in."/"failure message", + "auth_token": "", # Will not present if failed + } + """ + # get the post data + post_data = request.get_json() + if not post_data: + self.logger.warning("Invalid json object: {}", request.url) + abort(400, message="Invalid json object in request") + + username = post_data.get("username") + if not username: + self.logger.warning("Username not provided during the login process") + abort(400, message="Please provide a valid username") + + password = post_data.get("password") + if not password: + self.logger.warning("Password not provided during the login process") + abort(400, message="Please provide a valid password") + + try: + # fetch the user data + user = User.query(username=username) + except Exception: + self.logger.exception("Exception occurred during user login") + abort(500, message="INTERNAL ERROR") + + if not user: + self.logger.warning( + "No user found in the db for Username: {} while login", username + ) + abort(403, message="Bad login") + + # Validate the password + if not check_password_hash(user.password, password): + self.logger.warning("Wrong password for user: {} during login", username) + abort(401, message="Bad login") + + try: + auth_token = self.auth.encode_auth_token( + self.token_expire_duration, user.id + ) + except ( + jwt.InvalidIssuer, + jwt.InvalidIssuedAtError, + jwt.InvalidAlgorithmError, + jwt.PyJWTError, + ): + self.logger.exception( + "Could not encode the JWT auth token for user: {} while login", username + ) + abort( + 500, message="INTERNAL ERROR", + ) + + # check if the user is already logged in, in case the request has a an authorization header included + # We would still proceed to login and return a new auth token to the user + if user == self.auth.token_auth.current_user(): + self.logger.warning("User already logged in and trying to re-login") + if self.auth.token_auth.get_auth()["token"] == auth_token: + # If a user attempts to re-login immediately within a millisecond, + # the new auth token will be same as the one present in the header + # Therefore we will not fulfill the re-login request + self.logger.info("Attempted immediate re-login") + abort(403, message="Retry login after some time") + + # Add the new auth token to the database for later access + try: + token = ActiveTokens(token=auth_token) + # TODO: Decide on the auth token limit per user + user.update(auth_tokens=token) + + self.logger.info("New auth token registered for user {}", user.email) + except IntegrityError: + self.logger.warning( + "Duplicate auth token got created, user might have tried to re-login immediately" + ) + abort(409, message="Retry login after some time") + except SQLAlchemyError as e: + self.logger.error( + "SQLAlchemy Exception while logging in a user {}", type(e) + ) + abort(500, message="INTERNAL ERROR") + except Exception: + self.logger.exception("Exception while logging in a user") + abort(500, message="INTERNAL ERROR") + + response_object = { + "message": "Successfully logged in.", + "auth_token": auth_token, + "username": username, + } + return make_response(jsonify(response_object), 200) + + +class Logout(Resource): + """ + Pbench API for User logout and deleting an auth token + """ + + def __init__(self, config, logger, auth): + self.server_config = config + self.logger = logger + self.auth = auth + + @Auth.token_auth.login_required() + def post(self): + """ + post request for logging out a user for the current auth token. + This requires a Pbench authentication auth token in the header field + + Required headers include + Authorization: Bearer + + :return: JSON Payload + response_object = { + "message": "Successfully logged out."/"failure message", + } + """ + auth_token = self.auth.token_auth.get_auth()["token"] + user = self.auth.token_auth.current_user() + + try: + ActiveTokens.delete(auth_token) + self.logger.info( + "Auth token entry deleted for user with username {}", user.email + ) + except Exception: + self.logger.exception("Exception occurred during deleting the user entry") + abort(500, message="INTERNAL ERROR") + + response_object = { + "message": "Successfully logged out.", + } + return make_response(jsonify(response_object), 200) + + +class UserAPI(Resource): + """ + Abstracted pbench API to get user data + """ + + def __init__(self, logger, auth): + self.logger = logger + self.auth = auth + + @Auth.token_auth.login_required() + def get(self, username): + """ + Get request for getting user data. + This requires a Pbench auth token in the header field + + Required headers include + + Content-Type: application/json + Accept: application/json + Authorization: Bearer Pbench_auth_token (user received upon login) + + :return: JSON Payload + response_object = { + "message": "Success"/"failure message", + "data": { + "username": , + "firstName": , + "lastName": , + "registered_on": registered_on, + } + } + """ + try: + user, verified = self.auth.verify_user(username) + except Exception: + self.logger.exception("Exception occurred during verifying the user") + abort(500, message="INTERNAL ERROR") + + # TODO: Check if the user has the right privileges + if verified: + response_object = { + "status": "success", + "data": user.get_json(), + } + return make_response(jsonify(response_object), 200) + elif user.is_admin(): + try: + target_user = User.query(username=username) + response_object = { + "message": "success", + "data": target_user.get_json(), + } + return make_response(jsonify(response_object), 200) + except Exception: + self.logger.exception( + "Exception occurred while querying the user. Username: {}", username + ) + abort(500, message="INTERNAL ERROR") + else: + self.logger.warning( + "User {} is not authorized to delete user {}.", user.username, username, + ) + abort( + 403, + message=f"Not authorized to access information about user {username}", + ) + + @Auth.token_auth.login_required() + def put(self, username): + """ + PUT request for updating user data. + This requires a Pbench auth token in the header field + + This requires a JSON data with required user registration fields that needs an update + Example Json: + { + "first_name": "new_name", + "password": "password", + ... + } + + Required headers include + + Content-Type: application/json + Accept: application/json + Authorization: Bearer Pbench_auth_token (user received upon login) + + :return: JSON Payload + response_object = { + "message": "Success"/"failure message", + "data": { + "username": , + "first_name": , + "last_name": , + "registered_on": registered_on, + } + } + """ + post_data = request.get_json() + if not post_data: + self.logger.warning("Invalid json object: {}", request.url) + abort(400, message="Invalid json object in request") + + try: + user, verified = self.auth.verify_user(username) + except Exception: + self.logger.exception("Exception occurred while verifying the user") + abort(500, message="INTERNAL ERROR") + + # TODO: Check if the user has the right privileges + if not verified: + self.logger.warning( + "User {} is not authorized to delete user {}.", user.username, username, + ) + abort( + 403, + message=f"Not authorized to update information about user {username}", + ) + + # Check if the user payload contain fields that are either protected or + # are not present in the user db. If any key in the payload does not match + # with the column name we will abort the update request. + non_existent = set(post_data.keys()).difference( + set(User.__table__.columns.keys()) + ) + if non_existent: + self.logger.warning( + "User trying to update fields that are not present in the user database. Fields: {}", + non_existent, + ) + abort(400, message="Invalid fields in update request payload") + protected = set(post_data.keys()).intersection(set(User.get_protected())) + for field in protected: + if getattr(user, field) != post_data[field]: + self.logger.warning( + "User trying to update the non-updatable fields. {}: {}", + field, + post_data[field], + ) + abort(403, message="Invalid update request payload") + try: + user.update(**post_data) + except Exception: + self.logger.exception("Exception occurred during updating user object") + abort(500, message="INTERNAL ERROR") + + response_object = { + "message": "success", + "data": user.get_json(), + } + return make_response(jsonify(response_object), 200) + + @Auth.token_auth.login_required() + def delete(self, username): + """ + Delete request for deleting a user from database. + This requires a Pbench auth token in the header field + + Required headers include + + Content-Type: application/json + Accept: application/json + Authorization: Bearer Pbench_auth_token (user received upon login) + + :return: JSON Payload + response_object = { + "message": "Successfully deleted."/"failure message", + } + """ + try: + user, verified = self.auth.verify_user(username) + except Exception: + self.logger.exception("Exception occurred during the getUser {}", username) + abort(500, message="INTERNAL ERROR") + + # TODO: Check if the user has the right privileges + if not verified and not user.is_admin(): + self.logger.warning( + "User {} is not authorized to delete user {}.", user.username, username, + ) + abort(403, message="Not authorized to delete user") + + try: + user = User.query(username=username) + # Do not delete if the user is admin + if not user.is_admin(): + User.delete(username) + except Exception: + self.logger.exception( + "Exception occurred during deleting the user entry for user '{}'", + username, + ) + abort(500, message="INTERNAL ERROR") + else: + if user.is_admin(): + self.logger.warning("Admin attempted to delete admin user") + abort(403, message="Admin user can not be deleted") + self.logger.info("User entry deleted for user with username {}", username) + + response_object = { + "message": "Successfully deleted.", + } + return make_response(jsonify(response_object), 200) diff --git a/lib/pbench/server/database/__init__.py b/lib/pbench/server/database/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/lib/pbench/server/database/alembic.ini b/lib/pbench/server/database/alembic.ini new file mode 100644 index 0000000000..1c45b50df2 --- /dev/null +++ b/lib/pbench/server/database/alembic.ini @@ -0,0 +1,85 @@ +# A generic, single database configuration. + +[alembic] +# path to migration scripts +script_location = alembic + +# template used to generate migration files +# file_template = %%(rev)s_%%(slug)s + +# timezone to use when rendering the date +# within the migration file as well as the filename. +# string value is passed to dateutil.tz.gettz() +# leave blank for localtime +# timezone = + +# max length of characters to apply to the +# "slug" field +# truncate_slug_length = 40 + +# set to 'true' to run the environment during +# the 'revision' command, regardless of autogenerate +# revision_environment = false + +# set to 'true' to allow .pyc and .pyo files without +# a source .py file to be detected as revisions in the +# versions/ directory +# sourceless = false + +# version location specification; this defaults +# to alembic/versions. When using multiple version +# directories, initial revisions must be specified with --version-path +# version_locations = %(here)s/bar %(here)s/bat alembic/versions + +# the output encoding used when revision files +# are written from script.py.mako +# output_encoding = utf-8 + +sqlalchemy.url = driver://user:pass@localhost/dbname + + +[post_write_hooks] +# post_write_hooks defines scripts or Python functions that are run +# on newly generated revision scripts. See the documentation for further +# detail and examples + +# format using "black" - use the console_scripts runner, against the "black" entrypoint +# hooks=black +# black.type=console_scripts +# black.entrypoint=black +# black.options=-l 79 + +# Logging configuration +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console, fileHandler + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARN +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/lib/pbench/server/database/alembic/README b/lib/pbench/server/database/alembic/README new file mode 100644 index 0000000000..2df098c6d7 --- /dev/null +++ b/lib/pbench/server/database/alembic/README @@ -0,0 +1,20 @@ +Generic single-database configuration. + +Some useful commands to run migrations: + +Migration commit files are stored in alembic/versions folder. + +To create a db migration file +# alembic revision — autogenerate -m “First commit” + +Using the above command alembic generates our first migration commit file in versions folder. +File names are usually stored as revision_id_.py + +Once this file is generated we are ready for database migration. +# alembic upgrade head + +To upgrade to a specific revision +# alembic upgrade + +To downgrade to a specific revision +# alembic downgrade diff --git a/lib/pbench/server/database/alembic/env.py b/lib/pbench/server/database/alembic/env.py new file mode 100644 index 0000000000..3af3ad9484 --- /dev/null +++ b/lib/pbench/server/database/alembic/env.py @@ -0,0 +1,108 @@ +""" +This file is auto generated when we run `alembic init alembic` but modified according to our needs. +This Python script runs whenever the alembic migration tool is invoked. +It contains instructions to configure and generate a SQLAlchemy engine, +procure a connection from that engine along with a transaction, and then +invoke the migration engine, using the connection as a source of database connectivity. +""" +import sys +import logging +from logging.config import fileConfig + +from alembic import context + +from pbench.server.database.database import Database +from pbench.common.logger import get_pbench_logger +from pbench.server.api import get_server_config + +# this is the Alembic Config object, which provides +# access to the values within the .ini file in use. +config = context.config + +# Interpret the config file for Python logging. +# This line sets up loggers basically. +fileConfig(config.config_file_name) + +# Add syslog handler to send logs to journald +log = logging.getLogger("alembic") +handler = logging.handlers.SysLogHandler("/dev/log") +log.addHandler(handler) + +# add your model's MetaData object here for 'autogenerate' support: + +target_metadata = Database.Base.metadata + +# other values from the config, defined by the needs of env.py, +# can be acquired: +# my_important_option = config.get_main_option("my_important_option") +# ... etc. + + +def run_migrations_offline(): + """Run migrations in 'offline' mode. + + This configures the context with just a URL + and not an Engine, though an Engine is acceptable + here as well. By skipping the Engine creation + we don't even need a DBAPI to be available. + + Calls to context.execute() here emit the given string to the + script output. + + """ + try: + server_config = get_server_config() + logger = get_pbench_logger(__name__, server_config) + except Exception as e: + print(e) + sys.exit(1) + + url = Database.get_engine_uri(server_config, logger) + if url is None: + sys.exit(1) + + context.configure( + url=url, + target_metadata=target_metadata, + literal_binds=True, + dialect_opts={"paramstyle": "named"}, + ) + + with context.begin_transaction(): + context.run_migrations() + + +def run_migrations_online(): + """Run migrations in 'online' mode. + + In this scenario we need to create an Engine + and associate a connection with the context. + + """ + try: + server_config = get_server_config() + logger = get_pbench_logger(__name__, server_config) + except Exception as e: + print(e) + sys.exit(1) + + connectable = Database.init_engine(server_config, logger) + # connectable = engine_from_config( + # config.get_section(config.config_ini_section), + # prefix="sqlalchemy.", + # poolclass=pool.NullPool, + # ) + + with connectable.connect() as connection: + context.configure(connection=connection, target_metadata=target_metadata) + + with context.begin_transaction(): + context.run_migrations() + + +if context.is_offline_mode(): + print("running migration offline") + run_migrations_offline() +else: + print("running migration online") + run_migrations_online() diff --git a/lib/pbench/server/database/alembic/script.py.mako b/lib/pbench/server/database/alembic/script.py.mako new file mode 100644 index 0000000000..2c0156303a --- /dev/null +++ b/lib/pbench/server/database/alembic/script.py.mako @@ -0,0 +1,24 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} + +""" +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +# revision identifiers, used by Alembic. +revision = ${repr(up_revision)} +down_revision = ${repr(down_revision)} +branch_labels = ${repr(branch_labels)} +depends_on = ${repr(depends_on)} + + +def upgrade(): + ${upgrades if upgrades else "pass"} + + +def downgrade(): + ${downgrades if downgrades else "pass"} diff --git a/lib/pbench/server/database/database.py b/lib/pbench/server/database/database.py new file mode 100644 index 0000000000..e8fcf93805 --- /dev/null +++ b/lib/pbench/server/database/database.py @@ -0,0 +1,44 @@ +from sqlalchemy import create_engine +from sqlalchemy.orm import scoped_session, sessionmaker +from sqlalchemy.ext.declarative import declarative_base +from pbench.server import NoOptionError, NoSectionError + + +class Database: + # Create declarative base model that our model can inherit from + Base = declarative_base() + # Initialize the db scoped session + db_session = scoped_session(sessionmaker(autocommit=False, autoflush=False)) + + @staticmethod + def get_engine_uri(config, logger): + try: + psql = config.get("Postgres", "db_uri") + return psql + except (NoSectionError, NoOptionError): + logger.error( + "Failed to find an [Postgres] section" " in configuration file." + ) + return None + + # return f"postgresql+{psql_driver}://{psql_username}:{psql_password}@{psql_host}:{psql_port}/{psql_db}" + + @staticmethod + def init_engine(server_config, logger): + try: + return create_engine(Database.get_engine_uri(server_config, logger)) + except Exception: + logger.exception("Exception while creating a sqlalchemy engine") + return None + + @staticmethod + def init_db(server_config, logger): + # Make sure all the models are imported before this function gets called + # so that they will be registered properly on the metadata. Otherwise + # metadata will not have any tables and create_all functionality will do nothing + + Database.Base.query = Database.db_session.query_property() + + engine = create_engine(Database.get_engine_uri(server_config, logger)) + Database.Base.metadata.create_all(bind=engine) + Database.db_session.configure(bind=engine) diff --git a/lib/pbench/server/database/models/__init__.py b/lib/pbench/server/database/models/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/lib/pbench/server/database/models/active_tokens.py b/lib/pbench/server/database/models/active_tokens.py new file mode 100644 index 0000000000..b78e27ed35 --- /dev/null +++ b/lib/pbench/server/database/models/active_tokens.py @@ -0,0 +1,49 @@ +import datetime +from sqlalchemy import Column, Integer, String, DateTime, ForeignKey +from pbench.server.database.database import Database + + +class ActiveTokens(Database.Base): + """Token model for storing the active auth tokens at any given time""" + + __tablename__ = "active_tokens" + id = Column(Integer, primary_key=True, autoincrement=True) + token = Column(String(500), unique=True, nullable=False, index=True) + created = Column(DateTime, nullable=False) + user_id = Column( + Integer, + ForeignKey("users.id", ondelete="CASCADE"), + nullable=False, + # no need to add index=True, all FKs have indexes + ) + + def __init__(self, token): + self.token = token + self.created = datetime.datetime.now() + + @staticmethod + def query(token): + # We currently only query token database with given token + token_model = ( + Database.db_session.query(ActiveTokens).filter_by(token=token).first() + ) + return token_model + + @staticmethod + def delete(auth_token): + """ + Deletes the given auth token + :param auth_token: + :return: + """ + try: + Database.db_session.query(ActiveTokens).filter_by(token=auth_token).delete() + Database.db_session.commit() + except Exception: + Database.db_session.rollback() + raise + + @staticmethod + def valid(auth_token): + # check whether auth token is in the active database + return bool(ActiveTokens.query(auth_token)) diff --git a/lib/pbench/server/database/models/users.py b/lib/pbench/server/database/models/users.py new file mode 100644 index 0000000000..94e9c10b12 --- /dev/null +++ b/lib/pbench/server/database/models/users.py @@ -0,0 +1,126 @@ +import datetime +from flask_bcrypt import generate_password_hash +from email_validator import validate_email +from sqlalchemy import Column, Integer, String, DateTime, LargeBinary +from pbench.server.database.database import Database +from sqlalchemy.orm import relationship, validates + + +class User(Database.Base): + """ User Model for storing user related details """ + + __tablename__ = "users" + + id = Column(Integer, primary_key=True, autoincrement=True) + username = Column(String(255), unique=True, nullable=False) + first_name = Column(String(255), unique=False, nullable=False) + last_name = Column(String(255), unique=False, nullable=False) + password = Column(LargeBinary(128), nullable=False) + registered_on = Column(DateTime, nullable=False, default=datetime.datetime.now()) + email = Column(String(255), unique=True, nullable=False) + auth_tokens = relationship("ActiveTokens", backref="users") + + def __str__(self): + return f"User, id: {self.id}, username: {self.username}" + + def get_json(self): + return { + "username": self.username, + "email": self.email, + "first_name": self.first_name, + "last_name": self.last_name, + "registered_on": self.registered_on, + } + + @staticmethod + def get_protected(): + return ["registered_on", "id"] + + @staticmethod + def query(id=None, username=None, email=None): + # Currently we would only query with single argument. Argument need to be either username/id/email + if username: + user = Database.db_session.query(User).filter_by(username=username).first() + elif id: + user = Database.db_session.query(User).filter_by(id=id).first() + elif email: + user = Database.db_session.query(User).filter_by(email=email).first() + else: + user = None + + return user + + def add(self): + """ + Add the current user object to the database + """ + try: + Database.db_session.add(self) + Database.db_session.commit() + except Exception: + Database.db_session.rollback() + raise + + @validates("password") + def evaluate_password(self, key, value): + return generate_password_hash(value) + + # validate the email field + @validates("email") + def evaluate_email(self, key, value): + valid = validate_email(value) + email = valid.email + + return email + + def update(self, **kwargs): + """ + Update the current user object with given keyword arguments + """ + try: + for key, value in kwargs.items(): + if key == "auth_tokens": + # Insert the auth token + self.auth_tokens.append(value) + Database.db_session.add(value) + else: + setattr(self, key, value) + Database.db_session.commit() + except Exception: + Database.db_session.rollback() + raise + + @staticmethod + def delete(username): + """ + Delete the user with a given username except admin + :param username: + :return: + """ + try: + user_query = Database.db_session.query(User).filter_by(username=username) + user_query.delete() + Database.db_session.commit() + return True + except Exception: + Database.db_session.rollback() + raise + + def is_admin(self): + # TODO: Add notion of Admin user + """this method would always return false for now until we add a notion of Admin user/group. + Once we know the admin credentials this method can check against those credentials to determine + whether the user is privileged to do more. + + This can be extended to groups as well for example a user belonging to certain group has only those + privileges that are assigned to the group. + """ + return False + + @staticmethod + def is_admin_username(username): + # TODO: Need to add an interface to fetch admins list instead of hard-coding the names, preferably via sql query + admins = ["admin"] + return username in admins + + # TODO: Add password recovery mechanism diff --git a/lib/pbench/test/unit/server/conftest.py b/lib/pbench/test/unit/server/conftest.py index 43a2f3f5a0..974fa94e51 100644 --- a/lib/pbench/test/unit/server/conftest.py +++ b/lib/pbench/test/unit/server/conftest.py @@ -11,6 +11,9 @@ [pbench-server] pbench-top-dir = {TMP}/srv/pbench +[Postgres] +db_uri = sqlite:///:memory: + [elasticsearch] host = elasticsearch.example.com port = 7080 diff --git a/lib/pbench/test/unit/server/test_user_auth.py b/lib/pbench/test/unit/server/test_user_auth.py new file mode 100644 index 0000000000..52fb08cd5b --- /dev/null +++ b/lib/pbench/test/unit/server/test_user_auth.py @@ -0,0 +1,425 @@ +import time +import datetime +from pbench.server.database.models.users import User +from pbench.server.database.models.active_tokens import ActiveTokens +from pbench.server.database.database import Database + + +def register_user( + client, server_config, email, username, password, firstname, lastname +): + return client.post( + f"{server_config.rest_uri}/register", + json={ + "email": email, + "password": password, + "username": username, + "first_name": firstname, + "last_name": lastname, + }, + ) + + +def login_user(client, server_config, username, password): + return client.post( + f"{server_config.rest_uri}/login", + json={"username": username, "password": password}, + ) + + +class TestUserAuthentication: + @staticmethod + def test_registration(client, server_config, pytestconfig): + client.config["SESSION_FILE_DIR"] = pytestconfig.cache.get("TMP", None) + """ Test for user registration """ + with client: + response = register_user( + client, + server_config, + username="user", + firstname="firstname", + lastname="lastName", + email="user@domain.com", + password="12345", + ) + data = response.json + assert data["message"] == "Successfully registered." + assert response.content_type == "application/json" + assert response.status_code, 201 + + @staticmethod + def test_registration_missing_fields(client, server_config): + """ Test for user registration missing fields""" + with client: + response = client.post( + f"{server_config.rest_uri}/register", + json={ + "email": "user@domain.com", + "password": "12345", + "username": "user", + }, + ) + data = response.json + assert data["message"] == "Missing firstName field" + assert response.content_type == "application/json" + assert response.status_code == 400 + + @staticmethod + def test_registration_email_validity(client, server_config): + """ Test for validating an email field during registration""" + with client: + response = register_user( + client, + server_config, + username="user", + firstname="firstname", + lastname="lastName", + email="user@domain,com", + password="12345", + ) + data = response.json + assert data["message"] == "Invalid email: user@domain,com" + assert response.content_type == "application/json" + assert response.status_code == 400 + + @staticmethod + def test_registration_with_registered_user(client, server_config): + """ Test registration with already registered email""" + user = User( + email="user@domain.com", + password="12345", + username="user", + first_name="firstname", + last_name="lastname", + ) + Database.db_session.add(user) + Database.db_session.commit() + with client: + response = register_user( + client, + server_config, + username="user", + firstname="firstname", + lastname="lastName", + email="user@domain.com", + password="12345", + ) + data = response.json + assert data["message"] == "Provided username is already in use." + assert response.content_type == "application/json" + assert response.status_code == 403 + + @staticmethod + def test_user_login(client, server_config): + """ Test for login of registered-user login """ + with client: + # user registration + resp_register = register_user( + client, + server_config, + username="user", + firstname="firstname", + lastname="lastName", + email="user@domain.com", + password="12345", + ) + data_register = resp_register.json + assert data_register["message"] == "Successfully registered." + # registered user login + response = login_user(client, server_config, "user", "12345") + data = response.json + assert data["message"] == "Successfully logged in." + assert data["auth_token"] + assert data["username"] == "user" + assert response.content_type == "application/json" + assert response.status_code == 200 + + @staticmethod + def test_user_relogin(client, server_config): + """ Test for login of registered-user login """ + with client: + # user registration + resp_register = register_user( + client, + server_config, + username="user", + firstname="firstname", + lastname="lastName", + email="user@domain.com", + password="12345", + ) + data_register = resp_register.json + assert data_register["message"] == "Successfully registered." + # registered user login + response = login_user(client, server_config, "user", "12345") + data = response.json + assert data["message"] == "Successfully logged in." + assert data["auth_token"] + assert response.content_type == "application/json" + assert response.status_code == 200 + + # Re-login with auth header + time.sleep(1) + response = client.post( + f"{server_config.rest_uri}/login", + headers=dict(Authorization="Bearer " + data["auth_token"]), + json={"username": "user", "password": "12345"}, + ) + data = response.json + assert data["message"] == "Successfully logged in." + assert response.status_code == 200 + + # Re-login without auth header + time.sleep(1) + response = client.post( + f"{server_config.rest_uri}/login", + json={"username": "user", "password": "12345"}, + ) + data = response.json + assert data["message"] == "Successfully logged in." + assert response.status_code == 200 + + @staticmethod + def test_user_login_with_wrong_password(client, server_config): + """ Test for login of registered-user login """ + with client: + # user registration + resp_register = register_user( + client, + server_config, + username="user", + firstname="firstname", + lastname="lastName", + email="user@domain.com", + password="12345", + ) + data_register = resp_register.json + assert data_register["message"] == "Successfully registered." + # registered user login + response = login_user(client, server_config, "user", "123456") + data = response.json + assert data["message"] == "Bad login" + assert response.content_type == "application/json" + assert response.status_code == 401 + + @staticmethod + def test_login_without_password(client, server_config): + """ Test for login of non-registered user """ + with client: + response = client.post( + f"{server_config.rest_uri}/login", json={"username": "username"}, + ) + data = response.json + assert data["message"] == "Please provide a valid password" + assert response.status_code == 400 + + @staticmethod + def test_non_registered_user_login(client, server_config): + """ Test for login of non-registered user """ + with client: + response = login_user(client, server_config, "username", "12345") + data = response.json + assert data["message"] == "Bad login" + assert response.status_code == 403 + + @staticmethod + def test_get_user(client, server_config): + """ Test for get user api""" + with client: + resp_register = register_user( + client, + server_config, + username="username", + firstname="firstname", + lastname="lastName", + email="user@domain.com", + password="12345", + ) + data_register = resp_register.json + assert data_register["message"] == "Successfully registered." + + response = login_user(client, server_config, "username", "12345") + data_login = response.json + assert data_login["message"] == "Successfully logged in." + response = client.get( + f"{server_config.rest_uri}/user/username", + headers=dict(Authorization="Bearer " + data_login["auth_token"]), + ) + data = response.json + assert data["data"] is not None + assert data["data"]["email"] == "user@domain.com" + assert data["data"]["username"] == "username" + assert data["data"]["first_name"] == "firstname" + assert response.status_code == 200 + + @staticmethod + def test_update_user(client, server_config): + """ Test for get user api""" + with client: + resp_register = register_user( + client, + server_config, + username="username", + firstname="firstname", + lastname="lastName", + email="user@domain.com", + password="12345", + ) + data_register = resp_register.json + assert data_register["message"] == "Successfully registered." + + response = login_user(client, server_config, "username", "12345") + data_login = response.json + assert data_login["message"] == "Successfully logged in." + + new_registration_time = datetime.datetime.now() + response = client.put( + f"{server_config.rest_uri}/user/username", + json={"registered_on": new_registration_time, "first_name": "newname"}, + headers=dict(Authorization="Bearer " + data_login["auth_token"]), + ) + data = response.json + assert response.status_code == 403 + assert data["message"] == "Invalid update request payload" + + # Test password update + response = client.put( + f"{server_config.rest_uri}/user/username", + json={"password": "newpass"}, + headers=dict(Authorization="Bearer " + data_login["auth_token"]), + ) + data = response.json + assert response.status_code == 200 + assert data["data"]["first_name"] == "firstname" + assert data["data"]["email"] == "user@domain.com" + time.sleep(1) + response = login_user(client, server_config, "username", "newpass") + data_login = response.json + assert response.status_code == 200 + assert data_login["message"] == "Successfully logged in." + + @staticmethod + def test_malformed_auth_token(client, server_config): + """ Test for user status for malformed auth token""" + with client: + resp_register = register_user( + client, + server_config, + username="username", + firstname="firstname", + lastname="lastName", + email="user@domain.com", + password="12345", + ) + assert resp_register.json["message"] == "Successfully registered." + response = client.get( + f"{server_config.rest_uri}/user/username", + headers=dict(Authorization="Bearer" + "malformed"), + ) + data = response.json + assert data is None + + @staticmethod + def test_valid_logout(client, server_config): + """ Test for logout before token expires """ + with client: + # user registration + resp_register = register_user( + client, + server_config, + username="user", + firstname="firstname", + lastname="lastName", + email="user@domain.com", + password="12345", + ) + data_register = resp_register.json + assert data_register["message"] == "Successfully registered." + # user login + resp_login = login_user(client, server_config, "user", "12345") + data_login = resp_login.json + assert data_login["message"] == "Successfully logged in." + assert data_login["auth_token"] + # valid token logout + response = client.post( + f"{server_config.rest_uri}/logout", + headers=dict(Authorization="Bearer " + data_login["auth_token"]), + ) + data = response.json + assert data["message"] == "Successfully logged out." + # Check if the token has been successfully removed from the database + assert ( + not Database.db_session.query(ActiveTokens) + .filter_by(token=data_login["auth_token"]) + .first() + ) + assert response.status_code == 200 + + @staticmethod + def test_invalid_logout(client, server_config): + """ Testing logout after the token expires """ + with client: + # user registration + resp_register = register_user( + client, + server_config, + username="username", + firstname="firstname", + lastname="lastName", + email="user@domain.com", + password="12345", + ) + data_register = resp_register.json + assert data_register["message"] == "Successfully registered." + assert resp_register.status_code == 201 + # user login + resp_login = login_user(client, server_config, "username", "12345") + data_login = resp_login.json + assert data_login["message"] == "Successfully logged in." + assert data_login["auth_token"] + assert resp_login.status_code == 200 + + # log out with the current token + logout_response = client.post( + f"{server_config.rest_uri}/logout", + headers=dict(Authorization="Bearer " + data_login["auth_token"]), + ) + assert logout_response.json["message"] == "Successfully logged out." + + # invalid token logout + response = client.post( + f"{server_config.rest_uri}/logout", + headers=dict(Authorization="Bearer " + data_login["auth_token"]), + ) + data = response.json + assert data is None + + @staticmethod + def test_delete_user(client, server_config): + """ Test for user status for malformed auth token""" + with client: + resp_register = register_user( + client, + server_config, + username="username", + firstname="firstname", + lastname="lastName", + email="user@domain.com", + password="12345", + ) + data_register = resp_register.json + assert data_register["message"] == "Successfully registered." + + # user login + resp_login = login_user(client, server_config, "username", "12345") + data_login = resp_login.json + assert data_login["message"] == "Successfully logged in." + assert data_login["auth_token"] + + response = client.delete( + f"{server_config.rest_uri}/user/username", + headers=dict(Authorization="Bearer " + data_login["auth_token"]), + ) + data = response.json + assert data["message"] == "Successfully deleted." + assert response.status_code == 200 diff --git a/server/lib/config/pbench-server-default.cfg b/server/lib/config/pbench-server-default.cfg index 3dd08a6636..79b6958290 100644 --- a/server/lib/config/pbench-server-default.cfg +++ b/server/lib/config/pbench-server-default.cfg @@ -29,6 +29,9 @@ mailto=%(admin-email)s mailfrom=%(user)s@%(host)s commit_id=unknown +# Token expiration duration in minutes, can be overridden in the main config file, defaults to 60 mins +token_expiration_duration = 60 + # Maximum number of days an unpacked tar ball directory hierarchy will be # kept around. max-unpacked-age = 30 @@ -136,6 +139,10 @@ lowerbound = 820 # host = # port = +# # These should be overridden in the env-specific config file. +# [postgres] +# db_uri = driver://user:password@hostname/dbname + # We need to install some stuff in the apache document root so we # either get it directly or look in the config file. # diff --git a/server/requirements.txt b/server/requirements.txt index cdfb7ce0ed..79c107e8d0 100644 --- a/server/requirements.txt +++ b/server/requirements.txt @@ -1,10 +1,19 @@ boto3 python-dateutil elasticsearch +email-validator flask +flask-bcrypt +Flask-HTTPAuth flask-restful flask_cors +flask-jwt-extended +flask-sqlalchemy +flask-migrate gunicorn humanize pyesbulk>=2.0.1 +PyJwt +psycopg2-binary requests +sqlalchemy