diff --git a/.env.sample b/.env.sample index 2b01856..de2b222 100644 --- a/.env.sample +++ b/.env.sample @@ -1,3 +1,4 @@ +GAMCFGDIR=/home/compiler/.config/compiler-admin/gam HARVEST_CLIENT_NAME=Client1 HARVEST_DATA=data/harvest-sample.csv TOGGL_DATA=data/toggl-sample.csv diff --git a/README.md b/README.md index 0c6881a..152d831 100644 --- a/README.md +++ b/README.md @@ -10,17 +10,19 @@ Built on top of [GAMADV-XTD3](https://github.com/taers232c/GAMADV-XTD3) and [GYB ```bash $ compiler-admin -h -usage: compiler-admin [-h] [-v] {info,init,user} ... +usage: compiler-admin [-h] [-v] {info,init,time,user} ... positional arguments: - {info,init,user} The command to run - info Print configuration and debugging information. - init Initialize a new admin project. This command should be run once before any others. - user Work with users in the Compiler org. + {info,init,time,user} + The command to run + info Print configuration and debugging information. + init Initialize a new admin project. This command should be run once before any others. + time Work with Compiler time entries. + user Work with users in the Compiler org. options: - -h, --help show this help message and exit - -v, --version show program's version number and exit + -h, --help show this help message and exit + -v, --version show program's version number and exit ``` ## Getting started @@ -54,6 +56,37 @@ The `init` commands follows the steps in the [GAMADV-XTD3 Wiki](https://github.c Additionally, GYB is used for Gmail backup/restore. See the [GYB Wiki](https://github.com/GAM-team/got-your-back/wiki) for more information. +## Working with time entires + +The `time` command provides an interface for working with time entries from Compiler's various systems: + +```bash +$ compiler-admin time -h +usage: compiler-admin time [-h] {convert} ... + +positional arguments: + {convert} The time command to run. + convert Convert a time report from one format into another. + +options: + -h, --help show this help message and exit +``` + +### Converting an hours report + +With a CSV exported from either Harvest or Toggl, use this command to convert to the opposite format: + +```bash +$ compiler-admin time convert -h +usage: compiler-admin time convert [-h] [--input INPUT] [--output OUTPUT] [--client CLIENT] + +options: + -h, --help show this help message and exit + --input INPUT The path to the source data for conversion. Defaults to stdin. + --output OUTPUT The path to the file where converted data should be written. Defaults to stdout. + --client CLIENT The name of the client to use in converted data. +``` + ## Working with users The following commands are available to work with users in the Compiler domain: diff --git a/compiler_admin/commands/time/__init__.py b/compiler_admin/commands/time/__init__.py new file mode 100644 index 0000000..b1f9bc7 --- /dev/null +++ b/compiler_admin/commands/time/__init__.py @@ -0,0 +1,15 @@ +from argparse import Namespace + +from compiler_admin.commands.time.convert import convert # noqa: F401 + + +def time(args: Namespace, *extra): + # try to call the subcommand function directly from global (module) symbols + # if the subcommand function was imported above, it should exist in globals() + global_env = globals() + + if args.subcommand in global_env: + cmd_func = global_env[args.subcommand] + cmd_func(args, *extra) + else: + raise NotImplementedError(f"Unknown time subcommand: {args.subcommand}") diff --git a/compiler_admin/commands/time/convert.py b/compiler_admin/commands/time/convert.py new file mode 100644 index 0000000..4f57a7e --- /dev/null +++ b/compiler_admin/commands/time/convert.py @@ -0,0 +1,26 @@ +from argparse import Namespace + +import pandas as pd + +from compiler_admin import RESULT_SUCCESS +from compiler_admin.services.harvest import INPUT_COLUMNS as TOGGL_COLUMNS, convert_to_toggl +from compiler_admin.services.toggl import INPUT_COLUMNS as HARVEST_COLUMNS, convert_to_harvest + + +def _get_source_converter(source): + columns = pd.read_csv(source, nrows=0).columns.tolist() + + if set(HARVEST_COLUMNS) <= set(columns): + return convert_to_harvest + elif set(TOGGL_COLUMNS) <= set(columns): + return convert_to_toggl + else: + raise NotImplementedError("A converter for the given source data does not exist.") + + +def convert(args: Namespace, *extras): + converter = _get_source_converter(args.input) + + converter(args.input, args.output, args.client) + + return RESULT_SUCCESS diff --git a/compiler_admin/commands/user/__init__.py b/compiler_admin/commands/user/__init__.py index 5445211..759c28c 100644 --- a/compiler_admin/commands/user/__init__.py +++ b/compiler_admin/commands/user/__init__.py @@ -10,9 +10,12 @@ def user(args: Namespace, *extra): - # try to call the subcommand function directly from local symbols - # if the subcommand function was imported above, it should exist in locals() - if args.subcommand in locals(): - locals()[args.subcommand](args, *extra) + # try to call the subcommand function directly from global (module) symbols + # if the subcommand function was imported above, it should exist in globals() + global_env = globals() + + if args.subcommand in global_env: + cmd_func = global_env[args.subcommand] + cmd_func(args, *extra) else: - raise ValueError(f"Unknown user subcommand: {args.subcommand}") + raise NotImplementedError(f"Unknown user subcommand: {args.subcommand}") diff --git a/compiler_admin/main.py b/compiler_admin/main.py index 3d8f500..a5d9fa5 100644 --- a/compiler_admin/main.py +++ b/compiler_admin/main.py @@ -4,78 +4,106 @@ from compiler_admin import __version__ as version from compiler_admin.commands.info import info from compiler_admin.commands.init import init +from compiler_admin.commands.time import time from compiler_admin.commands.user import user from compiler_admin.commands.user.convert import ACCOUNT_TYPE_OU +def add_sub_cmd_parser(parser: ArgumentParser, dest="subcommand", help=None): + """Helper adds a subparser for the given dest.""" + return parser.add_subparsers(dest=dest, help=help) + + def add_sub_cmd(cmd: _SubParsersAction, subcmd, help) -> ArgumentParser: """Helper creates a new subcommand parser.""" return cmd.add_parser(subcmd, help=help) -def add_sub_cmd_username(cmd: _SubParsersAction, subcmd, help) -> ArgumentParser: +def add_sub_cmd_with_username_arg(cmd: _SubParsersAction, subcmd, help) -> ArgumentParser: """Helper creates a new subcommand parser with a required username arg.""" - return add_username_arg(add_sub_cmd(cmd, subcmd, help=help)) - - -def add_username_arg(cmd: ArgumentParser) -> ArgumentParser: - cmd.add_argument("username", help="A Compiler user account name, sans domain.") - return cmd - + sub_cmd = add_sub_cmd(cmd, subcmd, help=help) + sub_cmd.add_argument("username", help="A Compiler user account name, sans domain.") + return sub_cmd -def main(argv=None): - argv = argv if argv is not None else sys.argv[1:] - parser = ArgumentParser(prog="compiler-admin") - - # https://stackoverflow.com/a/8521644/812183 - parser.add_argument( - "-v", - "--version", - action="version", - version=f"%(prog)s {version}", - ) - - cmd_parsers = parser.add_subparsers(dest="command", help="The command to run") +def setup_info_command(cmd_parsers: _SubParsersAction): info_cmd = add_sub_cmd(cmd_parsers, "info", help="Print configuration and debugging information.") info_cmd.set_defaults(func=info) - init_cmd = add_sub_cmd_username( + +def setup_init_command(cmd_parsers: _SubParsersAction): + init_cmd = add_sub_cmd_with_username_arg( cmd_parsers, "init", help="Initialize a new admin project. This command should be run once before any others." ) init_cmd.add_argument("--gam", action="store_true", help="If provided, initialize a new GAM project.") init_cmd.add_argument("--gyb", action="store_true", help="If provided, initialize a new GYB project.") init_cmd.set_defaults(func=init) + +def setup_time_command(cmd_parsers: _SubParsersAction): + time_cmd = add_sub_cmd(cmd_parsers, "time", help="Work with Compiler time entries.") + time_cmd.set_defaults(func=time) + time_subcmds = add_sub_cmd_parser(time_cmd, help="The time command to run.") + + time_convert = add_sub_cmd(time_subcmds, "convert", help="Convert a time report from one format into another.") + time_convert.add_argument( + "--input", default=sys.stdin, help="The path to the source data for conversion. Defaults to stdin." + ) + time_convert.add_argument( + "--output", default=sys.stdout, help="The path to the file where converted data should be written. Defaults to stdout." + ) + time_convert.add_argument("--client", default=None, help="The name of the client to use in converted data.") + + +def setup_user_command(cmd_parsers: _SubParsersAction): user_cmd = add_sub_cmd(cmd_parsers, "user", help="Work with users in the Compiler org.") user_cmd.set_defaults(func=user) - user_subcmds = user_cmd.add_subparsers(dest="subcommand", help="The user command to run.") + user_subcmds = add_sub_cmd_parser(user_cmd, help="The user command to run.") - user_create = add_sub_cmd_username(user_subcmds, "create", help="Create a new user in the Compiler domain.") + user_create = add_sub_cmd_with_username_arg(user_subcmds, "create", help="Create a new user in the Compiler domain.") user_create.add_argument("--notify", help="An email address to send the newly created account info.") - user_convert = add_sub_cmd_username(user_subcmds, "convert", help="Convert a user account to a new type.") + user_convert = add_sub_cmd_with_username_arg(user_subcmds, "convert", help="Convert a user account to a new type.") user_convert.add_argument("account_type", choices=ACCOUNT_TYPE_OU.keys(), help="Target account type for this conversion.") - user_delete = add_sub_cmd_username(user_subcmds, "delete", help="Delete a user account.") + user_delete = add_sub_cmd_with_username_arg(user_subcmds, "delete", help="Delete a user account.") user_delete.add_argument("--force", action="store_true", default=False, help="Don't ask for confirmation before deletion.") - user_offboard = add_sub_cmd_username(user_subcmds, "offboard", help="Offboard a user account.") + user_offboard = add_sub_cmd_with_username_arg(user_subcmds, "offboard", help="Offboard a user account.") user_offboard.add_argument("--alias", help="Account to assign username as an alias.") user_offboard.add_argument( "--force", action="store_true", default=False, help="Don't ask for confirmation before offboarding." ) - user_reset = add_sub_cmd_username( + user_reset = add_sub_cmd_with_username_arg( user_subcmds, "reset-password", help="Reset a user's password to a randomly generated string." ) user_reset.add_argument("--notify", help="An email address to send the newly generated password.") - add_sub_cmd_username(user_subcmds, "restore", help="Restore an email backup from a prior offboarding.") + add_sub_cmd_with_username_arg(user_subcmds, "restore", help="Restore an email backup from a prior offboarding.") - user_signout = add_sub_cmd_username(user_subcmds, "signout", help="Signs a user out from all active sessions.") + user_signout = add_sub_cmd_with_username_arg(user_subcmds, "signout", help="Signs a user out from all active sessions.") user_signout.add_argument("--force", action="store_true", default=False, help="Don't ask for confirmation before signout.") + +def main(argv=None): + argv = argv if argv is not None else sys.argv[1:] + parser = ArgumentParser(prog="compiler-admin") + + # https://stackoverflow.com/a/8521644/812183 + parser.add_argument( + "-v", + "--version", + action="version", + version=f"%(prog)s {version}", + ) + + cmd_parsers = add_sub_cmd_parser(parser, dest="command", help="The command to run") + setup_info_command(cmd_parsers) + setup_init_command(cmd_parsers) + setup_time_command(cmd_parsers) + setup_user_command(cmd_parsers) + if len(argv) == 0: argv = ["info"] diff --git a/compiler_admin/services/files.py b/compiler_admin/services/files.py new file mode 100644 index 0000000..d9df718 --- /dev/null +++ b/compiler_admin/services/files.py @@ -0,0 +1,25 @@ +import json + +import pandas as pd + + +def read_csv(file_path, **kwargs) -> pd.DataFrame: + """Read a file path or buffer of CSV data into a pandas.DataFrame.""" + return pd.read_csv(file_path, **kwargs) + + +def read_json(file_path: str): + """Read a file path of JSON data into a python object.""" + with open(file_path, "r") as f: + return json.load(f) + + +def write_csv(file_path, data: pd.DataFrame, columns: list[str] = None): + """Write a pandas.DataFrame as CSV to the given path or buffer, with an optional list of columns to write.""" + data.to_csv(file_path, columns=columns, index=False) + + +def write_json(file_path: str, data): + """Write a python object as JSON to the given path.""" + with open(file_path, "w") as f: + json.dump(data, f, indent=2) diff --git a/compiler_admin/services/harvest.py b/compiler_admin/services/harvest.py new file mode 100644 index 0000000..562fe95 --- /dev/null +++ b/compiler_admin/services/harvest.py @@ -0,0 +1,91 @@ +from datetime import datetime, timedelta +import os +import sys +from typing import TextIO + +import pandas as pd + +import compiler_admin.services.files as files + +# input CSV columns needed for conversion +INPUT_COLUMNS = ["Date", "Client", "Project", "Notes", "Hours", "First name", "Last name"] + +# default output CSV columns +OUTPUT_COLUMNS = ["Email", "Start date", "Start time", "Duration", "Project", "Task", "Client", "Billable", "Description"] + + +def _calc_start_time(group: pd.DataFrame): + """Start time is offset by the previous record's duration, with a default of 0 offset for the first record.""" + group["Start time"] = group["Start time"] + group["Duration"].shift(fill_value=pd.to_timedelta("00:00:00")).cumsum() + return group + + +def _duration_str(duration: timedelta) -> str: + """Use total seconds to convert to a datetime and format as a string e.g. 01:30.""" + return datetime.fromtimestamp(duration.total_seconds()).strftime("%H:%M") + + +def _toggl_client_name(): + """Gets the value of the TOGGL_CLIENT_NAME env var.""" + return os.environ.get("TOGGL_CLIENT_NAME") + + +def convert_to_toggl( + source_path: str | TextIO = sys.stdin, + output_path: str | TextIO = sys.stdout, + client_name: str = None, + output_cols: list[str] = OUTPUT_COLUMNS, +): + """Convert Harvest formatted entries in source_path to equivalent Toggl formatted entries. + + Args: + source_path: The path to a readable CSV file of Harvest time entries; or a readable buffer of the same. + + output_cols (list[str]): A list of column names for the output + + output_path: The path to a CSV file where Toggl time entries will be written; or a writeable buffer for the same. + + Returns: + None. Either prints the resulting CSV data or writes to output_path. + """ + if client_name is None: + client_name = _toggl_client_name() + + # read CSV file, parsing dates + source = files.read_csv(source_path, usecols=INPUT_COLUMNS, parse_dates=["Date"], cache_dates=True) + + # rename columns that can be imported as-is + source.rename(columns={"Project": "Task", "Notes": "Description", "Date": "Start date"}, inplace=True) + + # update static calculated columns + source["Client"] = client_name + source["Project"] = client_name + source["Billable"] = "Yes" + + # add the Email column + source["Email"] = source["First name"].apply(lambda x: f"{x.lower()}@compiler.la") + + # Convert numeric Hours to timedelta Duration + source["Duration"] = source["Hours"].apply(pd.to_timedelta, unit="hours") + + # Default start time to 09:00 + source["Start time"] = pd.to_timedelta("09:00:00") + + user_days = ( + source + # sort and group by email and date + .sort_values(["Email", "Start date"]).groupby(["Email", "Start date"], observed=False) + # calculate a start time within each group (excluding the groupby columns) + .apply(_calc_start_time, include_groups=False) + ) + + # convert timedeltas to duration strings + user_days["Duration"] = user_days["Duration"].apply(_duration_str) + user_days["Start time"] = user_days["Start time"].apply(_duration_str) + + # re-sort by start date/time and user + # reset the index to get rid of the group multi index and fold the group columns back down + output_data = pd.DataFrame(data=user_days).reset_index() + output_data.sort_values(["Start date", "Start time", "Email"], inplace=True) + + files.write_csv(output_path, output_data, output_cols) diff --git a/compiler_admin/services/toggl.py b/compiler_admin/services/toggl.py new file mode 100644 index 0000000..685ea04 --- /dev/null +++ b/compiler_admin/services/toggl.py @@ -0,0 +1,129 @@ +import os +import sys +from typing import TextIO + +import pandas as pd + +from compiler_admin.services.google import user_info as google_user_info +import compiler_admin.services.files as files + +# cache of previously seen project information, keyed on Toggl project name +PROJECT_INFO = {} + +# cache of previously seen user information, keyed on email +USER_INFO = {} +NOT_FOUND = "NOT FOUND" + +# input CSV columns needed for conversion +INPUT_COLUMNS = ["Email", "Task", "Client", "Start date", "Start time", "Duration", "Description"] + +# default output CSV columns +OUTPUT_COLUMNS = ["Date", "Client", "Project", "Task", "Notes", "Hours", "First name", "Last name"] + + +def _harvest_client_name(): + """Gets the value of the HARVEST_CLIENT_NAME env var.""" + return os.environ.get("HARVEST_CLIENT_NAME") + + +def _get_info(obj: dict, key: str, env_key: str): + """Read key from obj, populating obj once from a file path at env_key.""" + if obj == {}: + file_path = os.environ.get(env_key) + if file_path: + file_info = files.read_json(file_path) + obj.update(file_info) + return obj.get(key) + + +def _toggl_project_info(project: str): + """Return the cached project for the given project key.""" + return _get_info(PROJECT_INFO, project, "TOGGL_PROJECT_INFO") + + +def _toggl_user_info(email: str): + """Return the cached user for the given email.""" + return _get_info(USER_INFO, email, "TOGGL_USER_INFO") + + +def _get_first_name(email: str) -> str: + """Get cached first name or derive from email.""" + user = _toggl_user_info(email) + first_name = user.get("First Name") if user else None + if first_name is None: + parts = email.split("@") + first_name = parts[0].capitalize() + data = {"First Name": first_name} + if email in USER_INFO: + USER_INFO[email].update(data) + else: + USER_INFO[email] = data + return first_name + + +def _get_last_name(email: str): + """Get cached last name or query from Google.""" + user = _toggl_user_info(email) + last_name = user.get("Last Name") if user else None + if last_name is None: + user = google_user_info(email) + last_name = user.get("Last Name") if user else None + if email in USER_INFO: + USER_INFO[email].update(user) + else: + USER_INFO[email] = user + return last_name + + +def _str_timedelta(td): + """Convert a string formatted duration (e.g. 01:30) to a timedelta.""" + return pd.to_timedelta(pd.to_datetime(td, format="%H:%M:%S").strftime("%H:%M:%S")) + + +def convert_to_harvest( + source_path: str | TextIO = sys.stdin, + output_path: str | TextIO = sys.stdout, + client_name: str = None, + output_cols: list[str] = OUTPUT_COLUMNS, +): + """Convert Toggl formatted entries in source_path to equivalent Harvest formatted entries. + + Args: + source_path: The path to a readable CSV file of Toggl time entries; or a readable buffer of the same. + + client_name (str): The value to assign in the output "Client" field + + output_cols (list[str]): A list of column names for the output + + output_path: The path to a CSV file where Harvest time entries will be written; or a writeable buffer for the same. + + Returns: + None. Either prints the resulting CSV data or writes to output_path. + """ + if client_name is None: + client_name = _harvest_client_name() + + # read CSV file, parsing dates and times + source = files.read_csv(source_path, usecols=INPUT_COLUMNS, parse_dates=["Start date"], cache_dates=True) + source["Start time"] = source["Start time"].apply(_str_timedelta) + source["Duration"] = source["Duration"].apply(_str_timedelta) + source.sort_values(["Start date", "Start time", "Email"], inplace=True) + + # rename columns that can be imported as-is + source.rename(columns={"Task": "Project", "Description": "Notes", "Start date": "Date"}, inplace=True) + + # update static calculated columns + source["Client"] = client_name + source["Task"] = "Project Consulting" + + # get cached project name if any + source["Project"] = source["Project"].apply(lambda x: _toggl_project_info(x) or x) + + # assign First and Last name + source["First name"] = source["Email"].apply(_get_first_name) + source["Last name"] = source["Email"].apply(_get_last_name) + + # calculate hours as a decimal from duration timedelta + source["Hours"] = (source["Duration"].dt.total_seconds() / 3600).round(2) + + files.write_csv(output_path, source, columns=output_cols) diff --git a/compose.yaml b/compose.yaml index 5a23c22..bd3cf83 100644 --- a/compose.yaml +++ b/compose.yaml @@ -4,8 +4,8 @@ services: context: . dockerfile: .devcontainer/Dockerfile image: compiler_admin:dev - environment: - GAMCFGDIR: /home/compiler/.config/compiler-admin/gam + env_file: + - .env entrypoint: sleep infinity volumes: - .:/home/compiler/admin diff --git a/notebooks/data/harvest-sample.csv b/notebooks/data/harvest-sample.csv index 771b61b..1914afc 100644 --- a/notebooks/data/harvest-sample.csv +++ b/notebooks/data/harvest-sample.csv @@ -1,4 +1,4 @@ -Date,Client,Project,Project Code,Task,Notes,Hours,Billable?,Invoiced?,Approved?,First Name,Last Name,Roles,Employee?,External Reference URL +Date,Client,Project,Project Code,Task,Notes,Hours,Billable?,Invoiced?,Approved?,First name,Last name,Roles,Employee?,External Reference URL 2023-01-21,Client1,Project5,PROJECT,Project Consulting,nascetur ridiculus mus etiam vel augue vestibulum rutrum rutrum neque aenean auctor gravida sem praesent id massa id nisl venenatis lacinia aenean sit,2.51,true,false,false,Hetti,Becken,Team Member,true, 2023-01-07,Client1,Project4,PROJECT,Project Consulting,quam a odio in hac habitasse platea dictumst maecenas ut massa quis augue luctus tincidunt nulla mollis molestie lorem quisque ut erat curabitur,1.84,false,true,false,Sawyer,Berrey,Team Member,false, 2023-01-25,Client1,Project5,PROJECT,Project Consulting,odio odio elementum eu interdum eu tincidunt in leo maecenas pulvinar lobortis est,2.11,true,false,true,Silas,Idenden,Team Member,true, diff --git a/notebooks/verify-harvest.ipynb b/notebooks/verify-harvest.ipynb new file mode 100644 index 0000000..bb39a43 --- /dev/null +++ b/notebooks/verify-harvest.ipynb @@ -0,0 +1,179 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "\n", + "import pandas as pd" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "google_file = os.environ[\"HARVEST_GOOGLE_DATA\"]\n", + "local_file = os.environ[\"HARVEST_DATA\"]\n", + "\n", + "df_google = pd.read_csv(google_file)\n", + "df_local = pd.read_csv(local_file)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df_google.info()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df_local.info()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df_google.columns.to_list()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df_local.columns.to_list()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df_local.columns.equals(df_google.columns)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df_google[\"Hours\"].sum()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df_local[\"Hours\"].sum()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "len(df_google)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "len(df_local)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "google_groups = df_google.groupby([\"Project\", \"Last name\"])\n", + "google_sums = {}\n", + "for group in google_groups.groups:\n", + " project, last_name = group\n", + " group_df = google_groups.get_group(group)\n", + " google_sums[group] = group_df['Hours'].sum()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "local_groups = df_local.groupby([\"Project\", \"Last name\"])\n", + "local_sums = {}\n", + "for group in local_groups.groups:\n", + " project, last_name = group\n", + " group_df = local_groups.get_group(group)\n", + " local_sums[group] = group_df['Hours'].sum()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "len(local_sums) == len(google_sums)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "for k,v in local_sums.items():\n", + " print(k)\n", + " assert k in google_sums\n", + " assert v == google_sums[k]" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.6" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/pyproject.toml b/pyproject.toml index 2c1640a..fdf9e49 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,7 +11,8 @@ authors = [ ] requires-python = ">=3.11" dependencies = [ - "advanced-gam-for-google-workspace @ git+https://github.com/taers232c/GAMADV-XTD3.git@v6.71.15#subdirectory=src" + "advanced-gam-for-google-workspace @ git+https://github.com/taers232c/GAMADV-XTD3.git@v6.71.15#subdirectory=src", + "pandas==2.2.2", ] [project.urls] @@ -24,7 +25,6 @@ dev = [ "build", "flake8", "ipykernel", - "pandas", "pre-commit", "setuptools_scm>=8" ] diff --git a/tests/commands/time/__init__.py b/tests/commands/time/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/commands/time/test__init__.py b/tests/commands/time/test__init__.py new file mode 100644 index 0000000..db439fb --- /dev/null +++ b/tests/commands/time/test__init__.py @@ -0,0 +1,25 @@ +from argparse import Namespace + +import pytest + +from compiler_admin.commands.time import time + + +def test_time_subcommand_exists(mocker): + args = Namespace(subcommand="subcmd") + subcmd = mocker.patch("compiler_admin.commands.time.globals", return_value={"subcmd": mocker.Mock()}) + + time(args, 1, 2, 3) + + subcmd.assert_called_once() + subcmd.return_value["subcmd"].assert_called_once_with(args, 1, 2, 3) + + +def test_time_subcommand_doesnt_exists(mocker): + args = Namespace(subcommand="subcmd") + subcmd = mocker.patch("compiler_admin.commands.time.globals", return_value={}) + + with pytest.raises(NotImplementedError, match="Unknown time subcommand: subcmd"): + time(args, 1, 2, 3) + + subcmd.assert_called_once() diff --git a/tests/commands/time/test_convert.py b/tests/commands/time/test_convert.py new file mode 100644 index 0000000..b3fbc98 --- /dev/null +++ b/tests/commands/time/test_convert.py @@ -0,0 +1,45 @@ +from argparse import Namespace +from io import StringIO +import pytest + +from compiler_admin import RESULT_SUCCESS +from compiler_admin.commands.time.convert import ( + __name__ as MODULE, + _get_source_converter, + convert_to_harvest, + convert_to_toggl, + convert, +) + + +@pytest.fixture +def mock_get_source_converter(mocker): + return mocker.patch(f"{MODULE}._get_source_converter") + + +def test_get_source_converter_match_toggl(toggl_file): + result = _get_source_converter(toggl_file) + + assert result == convert_to_harvest + + +def test_get_source_converter_match_harvest(harvest_file): + result = _get_source_converter(harvest_file) + + assert result == convert_to_toggl + + +def test_get_source_converter_mismatch(): + data = StringIO("one,two,three\n1,2,3") + + with pytest.raises(NotImplementedError, match="A converter for the given source data does not exist."): + _get_source_converter(data) + + +def test_convert(mock_get_source_converter): + args = Namespace(input="input", output="output", client="client") + res = convert(args) + + assert res == RESULT_SUCCESS + mock_get_source_converter.assert_called_once_with(args.input) + mock_get_source_converter.return_value.assert_called_once_with(args.input, args.output, args.client) diff --git a/tests/commands/user/__init__.py b/tests/commands/user/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/commands/user/test__init__.py b/tests/commands/user/test__init__.py new file mode 100644 index 0000000..fd55421 --- /dev/null +++ b/tests/commands/user/test__init__.py @@ -0,0 +1,25 @@ +from argparse import Namespace + +import pytest + +from compiler_admin.commands.user import user + + +def test_user_subcommand_exists(mocker): + args = Namespace(subcommand="subcmd") + subcmd = mocker.patch("compiler_admin.commands.user.globals", return_value={"subcmd": mocker.Mock()}) + + user(args, 1, 2, 3) + + subcmd.assert_called() + subcmd.return_value["subcmd"].assert_called_once_with(args, 1, 2, 3) + + +def test_time_subcommand_doesnt_exists(mocker): + args = Namespace(subcommand="subcmd") + subcmd = mocker.patch("compiler_admin.commands.user.globals", return_value={}) + + with pytest.raises(NotImplementedError, match="Unknown user subcommand: subcmd"): + user(args, 1, 2, 3) + + subcmd.assert_called_once() diff --git a/tests/conftest.py b/tests/conftest.py index fa10dff..a24ad3c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -82,6 +82,12 @@ def mock_commands_signout(mock_module_name): return mock_module_name("signout") +@pytest.fixture +def mock_commands_time(mock_module_name): + """Fixture returns a function that patches the time command function in a given module.""" + return mock_module_name("time") + + @pytest.fixture def mock_commands_user(mock_module_name): """Fixture returns a function that patches the user command function in a given module.""" @@ -167,3 +173,13 @@ def _mock_NamedTemporaryFile(module, readlines=[""], **kwargs): return patched return _mock_NamedTemporaryFile + + +@pytest.fixture +def harvest_file(): + return "notebooks/data/harvest-sample.csv" + + +@pytest.fixture +def toggl_file(): + return "notebooks/data/toggl-sample.csv" diff --git a/tests/services/test_files.py b/tests/services/test_files.py new file mode 100644 index 0000000..0d4b72c --- /dev/null +++ b/tests/services/test_files.py @@ -0,0 +1,83 @@ +import json +from tempfile import NamedTemporaryFile +import pytest + +import compiler_admin.services.files +from compiler_admin.services.files import pd, read_csv, read_json, write_csv, write_json + + +@pytest.fixture +def sample_data(): + return {"one": [2, 3, 4], "two": [4, 6, 8], "three": [6, 9, 12]} + + +@pytest.fixture +def sample_csv_lines(sample_data): + one, two, three = sample_data["one"], sample_data["two"], sample_data["three"] + expected_cols = list(sample_data.keys()) + + # create CSV data from sample_data + lines = [f"{','.join(expected_cols)}\n"] + for i in range(len(one)): + lines.append(f"{','.join([str(one[i]), str(two[i]), str(three[i])])}\n") + + return lines + + +@pytest.fixture +def spy_pandas(mocker): + return mocker.patch.object(compiler_admin.services.files, "pd", wraps=pd) + + +@pytest.fixture +def temp_file(): + temp = NamedTemporaryFile() + + yield temp + + if not temp.closed: + temp.close() + + +def test_read_csv(sample_data, sample_csv_lines, spy_pandas, temp_file): + one, two, three = sample_data["one"], sample_data["two"], sample_data["three"] + expected_cols = list(sample_data.keys()) + + with open(temp_file.name, "wt") as f: + f.writelines(sample_csv_lines) + + df = read_csv(temp_file.name, usecols=expected_cols) + + spy_pandas.read_csv.assert_called_once_with(temp_file.name, usecols=expected_cols) + + assert df.columns.to_list() == expected_cols + assert df["one"].to_list() == one + assert df["two"].to_list() == two + assert df["three"].to_list() == three + + +def test_read_json(sample_data, temp_file): + with open(temp_file.name, "wt") as f: + json.dump(sample_data, f) + + assert read_json(temp_file.name) == sample_data + + +def test_write_csv(sample_data, sample_csv_lines, mocker, temp_file): + df = pd.DataFrame(data=sample_data) + expected_columns = df.columns.to_list() + spy_df = mocker.patch.object(df, "to_csv", wraps=df.to_csv) + + write_csv(temp_file.name, df, columns=expected_columns) + + spy_df.assert_called_once_with(temp_file.name, columns=expected_columns, index=False) + + with open(temp_file.name, "rt") as f: + assert f.readlines() == sample_csv_lines + + +def test_write_json(sample_data, temp_file): + write_json(temp_file.name, sample_data) + + with open(temp_file.name, "rt") as f: + assert json.load(f) == sample_data diff --git a/tests/services/test_harvest.py b/tests/services/test_harvest.py new file mode 100644 index 0000000..29c0f1d --- /dev/null +++ b/tests/services/test_harvest.py @@ -0,0 +1,111 @@ +import sys +from datetime import timedelta +from io import StringIO + +import numpy as np +import pandas as pd +import pytest + +import compiler_admin.services.harvest +from compiler_admin.services.harvest import ( + __name__ as MODULE, + files, + INPUT_COLUMNS, + OUTPUT_COLUMNS, + _calc_start_time, + _duration_str, + _toggl_client_name, + convert_to_toggl, +) + + +@pytest.fixture(autouse=True) +def mock_environment(monkeypatch): + monkeypatch.setenv("TOGGL_CLIENT_NAME", "Test_Client") + + +@pytest.fixture +def spy_files(mocker): + return mocker.patch.object(compiler_admin.services.harvest, "files", wraps=files) + + +@pytest.fixture +def mock_toggl_client_name(mocker): + return mocker.patch(f"{MODULE}._toggl_client_name") + + +def test_calc_start_time(): + durations = pd.to_timedelta(np.arange(1, 6), unit="m") + df = pd.DataFrame(data={"Duration": durations, "Start time": [pd.to_timedelta("09:00:00") for d in durations]}) + + calc_df = _calc_start_time(df) + + assert calc_df.columns.equals(df.columns) + assert calc_df["Duration"].equals(df["Duration"]) + assert calc_df["Start time"].to_list() == [ + # offset = 0, cumsum = 0 + pd.to_timedelta("09:00:00"), + # offset = 1, cumsum = 1 + pd.to_timedelta("09:01:00"), + # offset = 2, cumsum = 3 + pd.to_timedelta("09:03:00"), + # offset = 3, cumsum = 6 + pd.to_timedelta("09:06:00"), + # offset = 4, cumsum = 10 + pd.to_timedelta("09:10:00"), + ] + + +def test_duration_str(): + td = timedelta(hours=1, minutes=30, seconds=15) + + result = _duration_str(td) + + assert isinstance(result, str) + assert result == "01:30" + + +def test_toggl_client_name(monkeypatch): + assert _toggl_client_name() == "Test_Client" + + monkeypatch.setenv("TOGGL_CLIENT_NAME", "New Test Client") + + assert _toggl_client_name() == "New Test Client" + + +def test_convert_to_toggl_mocked(harvest_file, spy_files, mock_toggl_client_name): + convert_to_toggl(harvest_file, client_name=None) + + mock_toggl_client_name.assert_called_once() + + spy_files.read_csv.assert_called_once() + call_args = spy_files.read_csv.call_args + assert (harvest_file,) in call_args + assert call_args.kwargs["usecols"] == INPUT_COLUMNS + assert call_args.kwargs["parse_dates"] == ["Date"] + assert call_args.kwargs["cache_dates"] is True + + spy_files.write_csv.assert_called_once() + call_args = spy_files.write_csv.call_args + assert call_args[0][0] == sys.stdout + assert call_args[0][2] == OUTPUT_COLUMNS + + +def test_convert_to_toggl_sample(harvest_file, toggl_file): + output = None + + with StringIO() as output_data: + convert_to_toggl(harvest_file, output_data, "Test Client 123") + output = output_data.getvalue() + + assert output + assert isinstance(output, str) + assert ",".join(OUTPUT_COLUMNS) in output + + order = ["Start date", "Start time", "Email"] + sample_output_df = pd.read_csv(toggl_file).sort_values(order) + output_df = pd.read_csv(StringIO(output)).sort_values(order) + + assert set(output_df.columns.to_list()) <= set(sample_output_df.columns.to_list()) + assert output_df["Client"].eq("Test Client 123").all() + assert output_df["Project"].eq("Test Client 123").all() diff --git a/tests/services/test_toggl.py b/tests/services/test_toggl.py new file mode 100644 index 0000000..b14621d --- /dev/null +++ b/tests/services/test_toggl.py @@ -0,0 +1,214 @@ +import sys +from datetime import timedelta +from io import StringIO +from tempfile import NamedTemporaryFile + +import pandas as pd +import pytest + +import compiler_admin.services.toggl +from compiler_admin.services.toggl import ( + __name__ as MODULE, + files, + INPUT_COLUMNS, + OUTPUT_COLUMNS, + PROJECT_INFO, + USER_INFO, + _harvest_client_name, + _get_info, + _toggl_project_info, + _toggl_user_info, + _get_first_name, + _get_last_name, + _str_timedelta, + convert_to_harvest, +) + + +@pytest.fixture(autouse=True) +def mock_environment(monkeypatch): + monkeypatch.setenv("HARVEST_CLIENT_NAME", "Test_Client") + monkeypatch.setenv("TOGGL_PROJECT_INFO", "notebooks/data/toggl-project-info-sample.json") + monkeypatch.setenv("TOGGL_USER_INFO", "notebooks/data/toggl-user-info-sample.json") + + +@pytest.fixture(autouse=True) +def reset_USER_INFO(): + USER_INFO.clear() + + +@pytest.fixture +def spy_files(mocker): + return mocker.patch.object(compiler_admin.services.toggl, "files", wraps=files) + + +@pytest.fixture +def mock_harvest_client_name(mocker): + return mocker.patch(f"{MODULE}._harvest_client_name") + + +@pytest.fixture +def mock_get_info(mocker): + return mocker.patch(f"{MODULE}._get_info") + + +@pytest.fixture +def mock_google_user_info(mocker): + return mocker.patch(f"{MODULE}.google_user_info") + + +def test_harvest_client_name(monkeypatch): + assert _harvest_client_name() == "Test_Client" + + monkeypatch.setenv("HARVEST_CLIENT_NAME", "New Test Client") + + assert _harvest_client_name() == "New Test Client" + + +def test_get_info(monkeypatch): + with NamedTemporaryFile("w") as temp: + monkeypatch.setenv("INFO_FILE", temp.name) + temp.write('{"key": "value"}') + temp.seek(0) + + obj = {} + result = _get_info(obj, "key", "INFO_FILE") + + assert result == "value" + assert obj["key"] == "value" + + +def test_get_info_no_file(): + obj = {} + result = _get_info(obj, "key", "INFO_FILE") + + assert result is None + assert "key" not in obj + + +def test_toggl_project_info(mock_get_info): + _toggl_project_info("project") + + mock_get_info.assert_called_once_with(PROJECT_INFO, "project", "TOGGL_PROJECT_INFO") + + +def test_toggl_user_info(mock_get_info): + _toggl_user_info("user") + + mock_get_info.assert_called_once_with(USER_INFO, "user", "TOGGL_USER_INFO") + + +def test_get_first_name_matching(mock_get_info): + mock_get_info.return_value = {"First Name": "User"} + + result = _get_first_name("email") + + assert result == "User" + + +def test_get_first_name_calcuated_with_record(mock_get_info): + email = "user@email.com" + mock_get_info.return_value = {} + USER_INFO[email] = {"Data": 1234} + + result = _get_first_name(email) + + assert result == "User" + assert USER_INFO[email]["First Name"] == "User" + assert USER_INFO[email]["Data"] == 1234 + + +def test_get_first_name_calcuated_without_record(mock_get_info): + email = "user@email.com" + mock_get_info.return_value = {} + + result = _get_first_name(email) + + assert result == "User" + assert USER_INFO[email]["First Name"] == "User" + assert list(USER_INFO[email].keys()) == ["First Name"] + + +def test_get_last_name_matching(mock_get_info, mock_google_user_info): + mock_get_info.return_value = {"Last Name": "User"} + + result = _get_last_name("email") + + assert result == "User" + mock_google_user_info.assert_not_called() + + +def test_get_last_name_lookup_with_record(mock_get_info, mock_google_user_info): + email = "user@email.com" + mock_get_info.return_value = {} + USER_INFO[email] = {"Data": 1234} + mock_google_user_info.return_value = {"Last Name": "User"} + + result = _get_last_name(email) + + assert result == "User" + assert USER_INFO[email]["Last Name"] == "User" + assert USER_INFO[email]["Data"] == 1234 + mock_google_user_info.assert_called_once_with(email) + + +def test_get_last_name_lookup_without_record(mock_get_info, mock_google_user_info): + email = "user@email.com" + mock_get_info.return_value = {} + mock_google_user_info.return_value = {"Last Name": "User"} + + result = _get_last_name(email) + + assert result == "User" + assert USER_INFO[email]["Last Name"] == "User" + assert list(USER_INFO[email].keys()) == ["Last Name"] + mock_google_user_info.assert_called_once_with(email) + + +def test_str_timedelta(): + dt = "01:30:15" + + result = _str_timedelta(dt) + + assert isinstance(result, timedelta) + assert result.total_seconds() == (1 * 60 * 60) + (30 * 60) + 15 + + +def test_convert_to_harvest_mocked(toggl_file, spy_files, mock_harvest_client_name, mock_google_user_info): + mock_google_user_info.return_value = {} + + convert_to_harvest(toggl_file, client_name=None) + + mock_harvest_client_name.assert_called_once() + + spy_files.read_csv.assert_called_once() + call_args = spy_files.read_csv.call_args + assert (toggl_file,) in call_args + assert call_args.kwargs["usecols"] == INPUT_COLUMNS + assert call_args.kwargs["parse_dates"] == ["Start date"] + assert call_args.kwargs["cache_dates"] is True + + spy_files.write_csv.assert_called_once() + call_args = spy_files.write_csv.call_args + assert sys.stdout in call_args[0] + assert call_args.kwargs["columns"] == OUTPUT_COLUMNS + + +def test_convert_to_harvest_sample(toggl_file, harvest_file, mock_google_user_info): + mock_google_user_info.return_value = {} + output = None + + with StringIO() as output_data: + convert_to_harvest(toggl_file, output_data, "Test Client 123") + output = output_data.getvalue() + + assert output + assert isinstance(output, str) + assert ",".join(OUTPUT_COLUMNS) in output + + order = ["Date", "First name", "Hours"] + sample_output_df = pd.read_csv(harvest_file).sort_values(order) + output_df = pd.read_csv(StringIO(output)).sort_values(order) + + assert set(output_df.columns.to_list()) <= set(sample_output_df.columns.to_list()) + assert output_df["Client"].eq("Test Client 123").all() diff --git a/tests/test_main.py b/tests/test_main.py index 3d64f03..4ae00d3 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -1,5 +1,6 @@ from argparse import Namespace import subprocess +import sys import pytest @@ -18,6 +19,11 @@ def mock_commands_init(mock_commands_init): return mock_commands_init(MODULE) +@pytest.fixture +def mock_commands_time(mock_commands_time): + return mock_commands_time(MODULE) + + @pytest.fixture def mock_commands_user(mock_commands_user): return mock_commands_user(MODULE) @@ -65,6 +71,73 @@ def test_main_init_no_username(mock_commands_init): assert mock_commands_init.call_count == 0 +def test_main_time_convert_default(mock_commands_time): + main(argv=["time", "convert"]) + + mock_commands_time.assert_called_once() + call_args = mock_commands_time.call_args.args + assert ( + Namespace( + func=mock_commands_time, command="time", subcommand="convert", client=None, input=sys.stdin, output=sys.stdout + ) + in call_args + ) + + +def test_main_time_convert_client(mock_commands_time): + main(argv=["time", "convert", "--client", "client123"]) + + mock_commands_time.assert_called_once() + call_args = mock_commands_time.call_args.args + assert ( + Namespace( + func=mock_commands_time, + command="time", + subcommand="convert", + client="client123", + input=sys.stdin, + output=sys.stdout, + ) + in call_args + ) + + +def test_main_time_convert_input(mock_commands_time): + main(argv=["time", "convert", "--input", "file.csv"]) + + mock_commands_time.assert_called_once() + call_args = mock_commands_time.call_args.args + assert ( + Namespace( + func=mock_commands_time, + command="time", + subcommand="convert", + client=None, + input="file.csv", + output=sys.stdout, + ) + in call_args + ) + + +def test_main_time_convert_output(mock_commands_time): + main(argv=["time", "convert", "--output", "file.csv"]) + + mock_commands_time.assert_called_once() + call_args = mock_commands_time.call_args.args + assert ( + Namespace( + func=mock_commands_time, + command="time", + subcommand="convert", + client=None, + input=sys.stdin, + output="file.csv", + ) + in call_args + ) + + def test_main_user_create(mock_commands_user): main(argv=["user", "create", "username"])