From 203d474bbbd958724f82a5e1e570cb529d094501 Mon Sep 17 00:00:00 2001 From: Josseline Perdomo Date: Aug 19 2021 20:24:27 +0000 Subject: [PATCH 1/5] Added filters and move utils into dashboard package Signed-off-by: Josseline Perdomo --- diff --git a/contributor_trends/dashboard/utils/__init__.py b/contributor_trends/dashboard/utils/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/contributor_trends/dashboard/utils/__init__.py diff --git a/contributor_trends/dashboard/utils/exceptions.py b/contributor_trends/dashboard/utils/exceptions.py new file mode 100644 index 0000000..cc54088 --- /dev/null +++ b/contributor_trends/dashboard/utils/exceptions.py @@ -0,0 +1,2 @@ +class InvalidDatasetFormat(Exception): + pass diff --git a/contributor_trends/dashboard/utils/filters.py b/contributor_trends/dashboard/utils/filters.py new file mode 100644 index 0000000..c141295 --- /dev/null +++ b/contributor_trends/dashboard/utils/filters.py @@ -0,0 +1,16 @@ +from datetime import datetime +from typing import Optional + +import pytz +from dash.development.base_component import Component + +from contributor_trends.dashboard.datasets import Dataset + + +def last_updated(date: datetime) -> str: + localized_date = pytz.utc.localize(date) + return f"Last updated at {localized_date.strftime('%B %d %Y, %X %Z')}." + + +def default_component(dataset: Optional[Dataset] = None) -> Component: + return Component.UNDEFINED From 0566e6639efdccf25dd4a0fe9ab9476a71d173c5 Mon Sep 17 00:00:00 2001 From: Josseline Perdomo Date: Aug 19 2021 20:27:16 +0000 Subject: [PATCH 2/5] Refactor typing and added properties to be used in the components Signed-off-by: Josseline Perdomo --- diff --git a/contributor_trends/dashboard/datasets.py b/contributor_trends/dashboard/datasets.py index 09a6b00..b13520e 100644 --- a/contributor_trends/dashboard/datasets.py +++ b/contributor_trends/dashboard/datasets.py @@ -1,20 +1,22 @@ import abc -from datetime import datetime from dataclasses import dataclass +from datetime import datetime from enum import Enum -from typing import Tuple, Union +from typing import Tuple, Union, NoReturn from urllib.parse import urljoin import pandas as pd import requests -from contributor_trends.utils.exceptions import InvalidDatasetFormat +from contributor_trends.dashboard.utils.exceptions import InvalidDatasetFormat + +__all__ = ["Dataset", "ActiveContributors"] @dataclass class DatasetMeta: filepath: str - updated_at: datetime + updated_at: Union[datetime, str] def __post_init__(self): if type(self.updated_at) == str: @@ -29,13 +31,10 @@ class ReportFrequency(Enum): class Dataset(abc.ABC): - def __init__(self) -> None: - self._source = "" - self._meta = DatasetMeta(filepath="", updated_at=datetime.min) - self._data = pd.DataFrame() - - def __str__(self) -> str: - return f"Last updated: {self.last_updated}" + def __init__(self) -> NoReturn: + self._source: str = "" + self._meta: DatasetMeta = DatasetMeta(filepath="", updated_at=datetime.min) + self._data: pd.DataFrame = pd.DataFrame() @property def id(self) -> str: @@ -46,6 +45,14 @@ class Dataset(abc.ABC): raise NotImplementedError @property + def last_updated(self) -> datetime: + return self._meta.updated_at + + @property + def description(self) -> str: + return "" + + @property def data(self) -> pd.DataFrame: return self._data @@ -57,10 +64,6 @@ class Dataset(abc.ABC): def y(self) -> Union[pd.Series, str]: raise NotImplementedError - @property - def last_updated(self) -> datetime: - return self._meta.updated_at - @abc.abstractmethod def _prepare_data(self, dataframe: pd.DataFrame) -> pd.DataFrame: # It should be implemented by the inheritance classes @@ -86,7 +89,7 @@ class Dataset(abc.ABC): return updated_status, meta - def load(self, source: str) -> None: + def load(self, source: str) -> NoReturn: self._source = source meta_updated, meta = self._get_meta() @@ -96,7 +99,7 @@ class Dataset(abc.ABC): self._meta = meta self._data = self._get_data() - def update(self): + def update(self) -> NoReturn: # TODO: Add logging to check when there is an update request meta_updated, meta = self._get_meta() if meta_updated: @@ -104,8 +107,10 @@ class Dataset(abc.ABC): self._data = self._get_data() +# TODO: Maybe add a config file (json or yaml) to setup static fields in the dataset presentation +# (like titles, and descriptions) class ActiveContributors(Dataset): - def __init__(self): + def __init__(self) -> NoReturn: super().__init__() self.report_frequency = ReportFrequency.WEEKLY.value @@ -118,6 +123,14 @@ class ActiveContributors(Dataset): return "Active Contributors" @property + def description(self) -> str: + return ( + "Stacked graph of contributors with measured activity each week — and at least 13 weeks total in the last " + "year. “Old school” contributors have been active for longer than two years; new contributors, less than " + "one. Blue line shows all contributors active this week regardless of amount of other activity." + ) + + @property def x(self) -> Union[pd.Index, pd.Series, str]: return self._data.index @@ -125,13 +138,10 @@ class ActiveContributors(Dataset): def y(self) -> Union[pd.Series, str]: return "ActiveUsers" - def load(self, base_url: str) -> None: + def load(self, base_url: str) -> NoReturn: endpoint = urljoin(base_url, "reports/contributors") super(ActiveContributors, self).load(endpoint) def _prepare_data(self, dataframe: pd.DataFrame) -> pd.DataFrame: dataframe.Date = pd.to_datetime(dataframe.Date, format="%m/%d/%y") return dataframe.resample(self.report_frequency, on="Date").sum() - - -active_contributors = ActiveContributors() From 46dcca48da60918a9730b4ab0c734345a3383394 Mon Sep 17 00:00:00 2001 From: Josseline Perdomo Date: Aug 19 2021 20:29:01 +0000 Subject: [PATCH 3/5] Refactor figures package concept, it will handle only plotly express charts Signed-off-by: Josseline Perdomo --- diff --git a/contributor_trends/dashboard/figures.py b/contributor_trends/dashboard/figures.py index 2b3fc30..0f12b39 100644 --- a/contributor_trends/dashboard/figures.py +++ b/contributor_trends/dashboard/figures.py @@ -1,17 +1,9 @@ -from typing import Type, Callable, Tuple - import plotly.express as px from plotly.graph_objs import Figure from contributor_trends.dashboard.datasets import Dataset -def area_chart(dataset: Type[Dataset]) -> Figure: - return px.area(dataset.data, x=dataset.x, y=dataset.y) - - -def update( - dataset: Type[Dataset], figure: Callable[[Type[Dataset]], Figure] -) -> Tuple[Figure, str]: - dataset.update() - return figure(dataset), str(dataset) +def area_chart(dataset: Dataset) -> Figure: + fig = px.area(dataset.data, x=dataset.x, y=dataset.y) + return fig From 6add06aad7f2e7700be516a85aca56ef6ff6d86b Mon Sep 17 00:00:00 2001 From: Josseline Perdomo Date: Aug 19 2021 20:32:55 +0000 Subject: [PATCH 4/5] Replaced components functions into class components Signed-off-by: Josseline Perdomo --- diff --git a/contributor_trends/app.py b/contributor_trends/app.py index cc1b5b5..1b3ceca 100644 --- a/contributor_trends/app.py +++ b/contributor_trends/app.py @@ -1,22 +1,16 @@ from flask import Flask -from contributor_trends.dashboard import create_dashboard - def create_app() -> Flask: - app = Flask(__name__, instance_relative_config=False) - if app.env == "development": - app.config.from_object("config.DevelopmentConfig") + server = Flask(__name__, instance_relative_config=False) + if server.env == "development": + server.config.from_object("config.DevelopmentConfig") else: - app.config.from_object("config.ProductionConfig") + server.config.from_object("config.ProductionConfig") - with app.app_context(): - from contributor_trends.extensions import cache + with server.app_context(): + from contributor_trends.dashboard import create_dashboard - meta_viewport = [ - {"name": "viewport", "content": "width=device-width, initial-scale=1"} - ] - app = create_dashboard(app, meta_tags=meta_viewport) - cache.init_app(app.server, config=app.server.config["CACHE_CONFIG"]) + app = create_dashboard(server) return app.server diff --git a/contributor_trends/config.py b/contributor_trends/config.py index b155218..9d3d61c 100644 --- a/contributor_trends/config.py +++ b/contributor_trends/config.py @@ -1,16 +1,13 @@ import os +BASE_DIR = os.path.dirname(__file__) -class Config: - BASE_DIR = os.path.dirname(__file__) +class Config: APP_TITLE = "Fedora Contributor Trends" + APP_DESCRIPTION = "Analyze the behavior of the Fedora contributors and their activities since 2012." APP_DATASOURCE_URL = os.environ.get("APP_DATASOURCE_URL") - CACHE_CONFIG = { - "CACHE_TYPE": "SimpleCache", - } - class DevelopmentConfig(Config): APP_UPDATE_TIME = 30000 # 30 seconds in milliseconds diff --git a/contributor_trends/dashboard/__init__.py b/contributor_trends/dashboard/__init__.py index 9b2763e..66b99c5 100644 --- a/contributor_trends/dashboard/__init__.py +++ b/contributor_trends/dashboard/__init__.py @@ -1,34 +1,30 @@ -from typing import Optional, List +from typing import Optional from dash import Dash from flask import Flask -from contributor_trends.dashboard import layouts -from contributor_trends.dashboard.callbacks import register_callbacks - -def create_dashboard( - server: Flask, - url_base_pathname: Optional[str] = "/", - assets_folder: Optional[str] = "assets", - meta_tags: Optional[List[dict]] = None, -) -> Dash: +def create_dashboard(server: Flask, url_base_pathname: Optional[str] = "/") -> Dash: dash_app = Dash( __name__, server=server, url_base_pathname=url_base_pathname, - assets_folder=assets_folder, - meta_tags=meta_tags, + meta_tags=[ + {"name": "viewport", "content": "width=device-width, initial-scale=1"} + ], + external_stylesheets=[server.config["ASSETS_STYLES_URL"]], suppress_callback_exceptions=True, ) with server.app_context(): - dash_app.title = server.config["APP_TITLE"] - dash_app.layout = layouts.index_layout - register_callbacks(dash_app) + from contributor_trends.dashboard.callbacks import register_callbacks + from contributor_trends.dashboard.components import create_components + from contributor_trends.dashboard.layouts import app_layout - from contributor_trends.dashboard.datasets import active_contributors + dash_app.title = server.config["APP_TITLE"] + dash_app.layout = app_layout - active_contributors.load(base_url=server.config["APP_DATASOURCE_URL"]) + create_components(dash_app.server) + register_callbacks(dash_app) return dash_app diff --git a/contributor_trends/dashboard/callbacks.py b/contributor_trends/dashboard/callbacks.py index 3a80cf8..b70eb02 100644 --- a/contributor_trends/dashboard/callbacks.py +++ b/contributor_trends/dashboard/callbacks.py @@ -1,39 +1,36 @@ +from typing import Union, List, NoReturn + import dash -import dash_html_components as html from dash.dependencies import Input, Output +from dash.development.base_component import Component -from contributor_trends.dashboard import figures -from contributor_trends.dashboard.datasets import active_contributors -from contributor_trends.dashboard.layouts import ( - chart_layout, - not_found_layout, -) +from contributor_trends.dashboard.components import active_contributors_chart +from contributor_trends.dashboard.layouts import index_layout, not_found_layout -def register_callbacks(dash_app: dash.Dash): +def register_callbacks(dash_app: dash.Dash) -> NoReturn: @dash_app.callback( Output(component_id="page-content", component_property="children"), Input(component_id="url", component_property="pathname"), ) - def page_content(pathname: str) -> html.Div: + def page_content(pathname: str) -> Union[List[Component], Component]: if pathname == "/": - # TODO: quit directly the chart_layout call, is better add a container - return chart_layout(active_contributors) + return index_layout() return not_found_layout(pathname) @dash_app.callback( Output( - component_id=f"{active_contributors.id}-chart", + component_id=f"{active_contributors_chart.id}-chart", component_property="children", ), Output( - component_id=f"{active_contributors.id}-chart-meta", + component_id=f"{active_contributors_chart.id}-chart-meta", component_property="children", ), Input( - component_id=f"{active_contributors.id}-interval", + component_id=f"{active_contributors_chart.id}-interval", component_property="n_intervals", ), ) def update_active_contributors(i: int): - return figures.update(active_contributors, figures.area_chart) + return active_contributors_chart.update() diff --git a/contributor_trends/dashboard/components.py b/contributor_trends/dashboard/components.py new file mode 100644 index 0000000..a6ccc36 --- /dev/null +++ b/contributor_trends/dashboard/components.py @@ -0,0 +1,100 @@ +from typing import Callable, Tuple, NoReturn, Optional +from urllib.parse import urljoin + +import dash_core_components as dcc +import dash_html_components as html +from dash.development.base_component import Component +from flask import current_app, Flask +from plotly.graph_objs import Figure + +from contributor_trends.dashboard import figures +from contributor_trends.dashboard.datasets import ActiveContributors, Dataset +from contributor_trends.dashboard.utils import filters + +__all__ = ["create_components", "header", "active_contributors_chart"] + + +class Chart: + def __init__(self) -> NoReturn: + self._dataset: Optional[Dataset] = None + self._figure: Callable[[Dataset], Figure] = filters.default_component + + @property + def id(self): + return self._dataset.id + + def init_state( + self, dataset: Dataset, figure: Callable[[Dataset], Figure] + ) -> NoReturn: + self._dataset = dataset + self._figure = figure + + def render(self) -> Component: + return html.Div( + [ + html.H4(self._dataset.title, className="chart-title"), + dcc.Graph( + id=f"{self._dataset.id}-chart", + figure=self._figure(self._dataset), + ), + # TODO: add a description checker here, description could be empty :D + html.Div( + [ + html.P(self._dataset.description), + html.Small( + filters.last_updated(self._dataset.last_updated), + id=f"{self._dataset.id}-chart-meta", + className="chart-footnote", + ), + ], + className="chart-meta", + ), + dcc.Interval( + id=f"{self._dataset.id}-interval", + interval=current_app.config["APP_UPDATE_TIME"], # in milliseconds + ), + ], + className="chart-container", + ) + + def update(self) -> Tuple[Figure, str]: + self._dataset.update() + return self._figure(self._dataset), filters.last_updated( + self._dataset.last_updated + ) + + +class Header: + @staticmethod + def render() -> Component: + return html.Header( + [ + html.Img( + src=urljoin( + current_app.config["ASSETS_IMAGES_URL"], + "fedora-logo-sprite.svg", + ), + alt="Fedora Logo", + className="header-logo", + ), + html.H1(current_app.config["APP_TITLE"], className="header-title"), + html.P( + current_app.config["APP_DESCRIPTION"], + className="header-description", + ), + ], + className="header", + ) + + +header = Header() +active_contributors_chart = Chart() + + +# TODO: Maybe it will be better in a different module +def create_components(server: Flask) -> NoReturn: + active_contributors = ActiveContributors() + active_contributors.load(base_url=server.config["APP_DATASOURCE_URL"]) + active_contributors_chart.init_state( + dataset=active_contributors, figure=figures.area_chart + ) diff --git a/contributor_trends/dashboard/layouts.py b/contributor_trends/dashboard/layouts.py index 4d2584f..75429f1 100644 --- a/contributor_trends/dashboard/layouts.py +++ b/contributor_trends/dashboard/layouts.py @@ -1,48 +1,25 @@ -from typing import Type +from typing import List import dash_core_components as dcc import dash_html_components as html from dash.development.base_component import Component -from flask import current_app - -from contributor_trends.dashboard import figures -from contributor_trends.dashboard.datasets import Dataset - - -def chart_layout(dataset: Type[Dataset]) -> Component: - return html.Div( - [ - html.H2(dataset.title, className="chart-title"), - html.Div( - [ - dcc.Graph( - id=f"{dataset.id}-chart", figure=figures.area_chart(dataset) - ), - html.P(str(dataset), id=f"{dataset.id}-chart-meta"), - ], - className="chart-content", - ), - dcc.Interval( - id=f"{dataset.id}-interval", - interval=current_app.config["APP_UPDATE_TIME"], # in milliseconds - ), - ], - id=f"{dataset.id}-container", - className="chart-container", - ) - - -def index_layout() -> Component: - return html.Div( - [dcc.Location(id="url", refresh=False), html.Div(id="page-content")] - ) - - -def not_found_layout(url) -> Component: - return html.Div( - [ - html.H1("404: Not found"), - html.Hr(), - html.P(f"The pathname {url} was not recognised."), - ] - ) + +from contributor_trends.dashboard.components import header, active_contributors_chart + + +def app_layout() -> Component: + return html.Div([dcc.Location(id="url", refresh=False)], id="page-content") + + +def index_layout() -> List[Component]: + # TODO: extends to more than 1 dataset + return [header.render(), active_contributors_chart.render()] + + +def not_found_layout(url) -> List[Component]: + # TODO: adding more classes to make a better design + return [ + html.H1("404: Not found"), + html.Hr(), + html.P(f"The pathname {url} was not recognised."), + ] From 720696fb9beebe79f0441ccdc385e82b8c64ec35 Mon Sep 17 00:00:00 2001 From: Josseline Perdomo Date: Aug 19 2021 20:50:42 +0000 Subject: [PATCH 5/5] Deleted older utils package Signed-off-by: Josseline Perdomo --- diff --git a/contributor_trends/utils/__init__.py b/contributor_trends/utils/__init__.py deleted file mode 100644 index e69de29..0000000 --- a/contributor_trends/utils/__init__.py +++ /dev/null diff --git a/contributor_trends/utils/exceptions.py b/contributor_trends/utils/exceptions.py deleted file mode 100644 index cc54088..0000000 --- a/contributor_trends/utils/exceptions.py +++ /dev/null @@ -1,2 +0,0 @@ -class InvalidDatasetFormat(Exception): - pass