From 529905d7ab7ac780704ade6122abe4c219e159e8 Mon Sep 17 00:00:00 2001 From: Ulincsys Date: Mon, 3 Jul 2023 18:47:03 -0500 Subject: [PATCH 01/34] Upgrade SQLAlchemy version - Switch to SQLAlchemy ORM syntax for repo filtering - Will replace the large SQL string in repo_load_controller.py - Current implementation located in augur/api/view/utils.py - Add clearer error message when page number is incorrect on repo view - Upgrade SQLAlchemy to v2.0.17 - Required for new syntax that the old version does not support - Specifically `ilike` and `regexp_replace` Signed-off-by: Ulincsys --- augur/api/view/routes.py | 16 ++++--- augur/api/view/utils.py | 70 +++++++++++++++++++++++++++--- augur/application/util.py | 3 -- augur/templates/repos-table.j2 | 4 ++ augur/util/repo_load_controller.py | 1 - setup.py | 2 +- 6 files changed, 80 insertions(+), 16 deletions(-) diff --git a/augur/api/view/routes.py b/augur/api/view/routes.py index 3e62dbfd94..889076fcb0 100644 --- a/augur/api/view/routes.py +++ b/augur/api/view/routes.py @@ -74,14 +74,18 @@ def repo_table_view(): pagination_offset = config.get_value("frontend", "pagination_offset") if current_user.is_authenticated: - data = current_user.get_repos(page = page, sort = sorting, direction = direction, search=query)[0] - page_count = (current_user.get_repo_count(search = query)[0] or 0) // pagination_offset + data = load_repos_test(user = current_user, search = query, page = page, sort = sorting, direction = direction, source = "user") + page_count = load_repos_test(user = current_user, search = query, count = True, source = "user") + # data = current_user.get_repos(page = page, sort = sorting, direction = direction, search=query)[0] + # page_count = (current_user.get_repo_count(search = query)[0] or 0) // pagination_offset else: - data = get_all_repos(page = page, sort = sorting, direction = direction, search=query)[0] - page_count = (get_all_repos_count(search = query)[0] or 0) // pagination_offset + data = load_repos_test(search = query, page = page, sort = sorting, direction = direction) + page_count = load_repos_test(search = query, count = True) + # data = get_all_repos(page = page, sort = sorting, direction = direction, search=query)[0] + # page_count = (get_all_repos_count(search = query)[0] or 0) // pagination_offset - #if not cacheFileExists("repos.json"): - # return renderLoading("repos/views/table", query, "repos.json") + if not data.count(): + data = None # return renderRepos("table", query, data, sorting, rev, page, True) return render_module("repos-table", title="Repos", repos=data, query_key=query, activePage=page, pages=page_count, offset=pagination_offset, PS="repo_table_view", reverse = rev, sorting = sorting) diff --git a/augur/api/view/utils.py b/augur/api/view/utils.py index a8e6998f9a..3d8f5bcff6 100644 --- a/augur/api/view/utils.py +++ b/augur/api/view/utils.py @@ -2,10 +2,17 @@ from concurrent.futures import ThreadPoolExecutor from flask import render_template, flash, url_for, Flask from .init import * -from .server import User from ..server import app, db_session from augur.application.config import AugurConfig -import urllib.request, urllib.error, json, os, math, yaml, urllib3, time, logging, re +import urllib.request, urllib.error, json, os, math, yaml, urllib3, time, logging, re, math + +from augur.application.db.session import DatabaseSession +from augur.application.db.engine import DatabaseEngine +from augur.application.db.models import User, Repo, RepoGroup, UserGroup, UserRepo +from sqlalchemy import Column, Table, Integer, MetaData, or_, Label +from sqlalchemy.sql.operators import ilike_op, distinct_op +from sqlalchemy.sql.functions import coalesce +from augur.application.db.models.base import Base init_logging() @@ -141,9 +148,6 @@ def getSetting(key, section = "View"): loadSettings() -User.api = getSetting("serving") -User.logger = logger - version_check(settings) """ ---------------------------------------------------------------- @@ -456,3 +460,59 @@ def render_module(module, **args): def renderLoading(dest, query, request): cache_files_requested.append(request) return render_template('index.j2', body="loading", title="Loading", d=dest, query_key=query, api_url=getSetting('serving')) + +with DatabaseEngine() as engine: + augur_data_schema = MetaData(schema = "augur_data") + augur_data_schema.reflect(bind = engine, views = True) + + commits_materialized_view: Table = augur_data_schema.tables["augur_data.api_get_all_repos_commits"] + issues_materialized_view: Table = augur_data_schema.tables["augur_data.api_get_all_repos_issues"] + +""" ---------------------------------------------------------------- +""" +def load_repos_test(count = False, source = None, **kwargs): + columns: list[Label] = [ + Repo.repo_id.distinct().label("repo_id"), + Repo.description.label("description"), + Repo.repo_git.label("url"), + coalesce(commits_materialized_view.columns.commits_all_time, 0).label("commits_all_time"), + coalesce(issues_materialized_view.columns.issues_all_time, 0).label("issues_all_time"), + RepoGroup.rg_name.label("rg_name"), + Repo.repo_git.regexp_replace('.*github\.com\/[A-Za-z0-9 \- _]+\/([A-Za-z0-9 \- _ .]+)$', "\\1").label("repo_name"), + Repo.repo_git.regexp_replace('.*github\.com\/([A-Za-z0-9 \- _]+)\/[A-Za-z0-9 \- _ .]+$', "\\1").label("repo_owner"), + RepoGroup.repo_group_id.label("repo_group_id") + ] + + def get_colum_by_label(label: str)-> Label: + for column in columns: + if column.name == label: + return column + + repos = db_session.query(*columns)\ + .outerjoin(commits_materialized_view, Repo.repo_id == commits_materialized_view.columns.repo_id)\ + .outerjoin(issues_materialized_view, Repo.repo_id == issues_materialized_view.columns.repo_id)\ + .join(RepoGroup, Repo.repo_group_id == RepoGroup.repo_group_id) + + user: User = kwargs.get("user") + if user: + repos = repos.join(UserRepo, Repo.repo_id == UserRepo.repo_id)\ + .join(UserGroup, UserGroup.group_id == UserRepo.group_id)\ + .filter(UserGroup.user_id == user.user_id) + + search = kwargs.get("search") + qkey = kwargs.get("query_key") or ["repo_name", "repo_owner"] + if search: + if isinstance(qkey, list) and len(qkey) > 0: + repos = repos.filter(or_(ilike_op(get_colum_by_label(filter_column), f"%{search}%") for filter_column in qkey)) + else: + repos = repos.filter(ilike_op(get_colum_by_label(qkey), f"%{search}%")) + + page_size: int = kwargs.get("page_size") or 25 + if count: + c = repos.count() + return math.ceil(c / page_size) - 1 + + page: int = kwargs.get("page") or 0 + offset = page * page_size + + return repos.slice(offset, offset + page_size) \ No newline at end of file diff --git a/augur/application/util.py b/augur/application/util.py index 1915abdeb0..03e591df98 100644 --- a/augur/application/util.py +++ b/augur/application/util.py @@ -25,6 +25,3 @@ def get_all_repos_count(**kwargs): result = controller.get_repo_count(source="all", **kwargs) return result - - - diff --git a/augur/templates/repos-table.j2 b/augur/templates/repos-table.j2 index a0593c73e1..fce58eca05 100644 --- a/augur/templates/repos-table.j2 +++ b/augur/templates/repos-table.j2 @@ -86,6 +86,10 @@ {% elif query_key %}

Your search did not match any repositories

+{% elif pages %} +

Pagination Error

+

Something went wrong displaying the requested section of results.

+

Please go back and try again.

{% elif current_user.is_authenticated %}

No Repos Tracked

Add repos to your personal tracker in your profile page

diff --git a/augur/util/repo_load_controller.py b/augur/util/repo_load_controller.py index 3436167aee..d95a0817ab 100644 --- a/augur/util/repo_load_controller.py +++ b/augur/util/repo_load_controller.py @@ -305,7 +305,6 @@ def generate_repo_query(self, source, count, **kwargs): query_args["page_size"] = page_size query_args["offset"] = offset - query_args["order_by"] = order_by return query, query_args, {"status": "success"} diff --git a/setup.py b/setup.py index 9177110f17..6328288354 100644 --- a/setup.py +++ b/setup.py @@ -38,7 +38,7 @@ "alembic==1.8.1", # 1.8.1 "coloredlogs==15.0", # 15.0.1 "Beaker==1.11.0", # 1.11.0 - "SQLAlchemy==1.3.23", # 1.4.40 + "SQLAlchemy==2.0.17", # 1.4.40 "itsdangerous==2.0.1", # 2.1.2 'Jinja2~=3.0.3', "Flask==2.0.2", # 2.2.2 From f2620ce73cf4bff7aea38c5908eebfaa3a5d423a Mon Sep 17 00:00:00 2001 From: sgoggins Date: Sat, 8 Jul 2023 17:48:21 -0500 Subject: [PATCH 02/34] fixing order --- augur/tasks/start_tasks.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/augur/tasks/start_tasks.py b/augur/tasks/start_tasks.py index 1918efbf8a..c8f5dc59ff 100644 --- a/augur/tasks/start_tasks.py +++ b/augur/tasks/start_tasks.py @@ -88,7 +88,8 @@ def primary_repo_collect_phase(repo_git): #facade_phase(logger,repo_git), collect_linux_badge_info.si(repo_git), collect_releases.si(repo_git), - grab_comitters.si(repo_git) + grab_comitters.si(repo_git), + process_ossf_dependency_metrics.si(repo_git) ) return repo_task_group @@ -102,7 +103,6 @@ def secondary_repo_collect_phase(repo_git): repo_task_group = group( process_pull_request_files.si(repo_git), process_pull_request_commits.si(repo_git), - process_ossf_dependency_metrics.si(repo_git), chain(collect_pull_request_reviews.si(repo_git), collect_pull_request_review_comments.si(repo_git)) ) From 005d3452b67c660fab35c66752e1a35f1fc56ae3 Mon Sep 17 00:00:00 2001 From: "Sean P. Goggins" Date: Sun, 9 Jul 2023 00:05:02 +0000 Subject: [PATCH 03/34] email test Signed-off-by: Sean P. Goggins --- sendgridtest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sendgridtest.py b/sendgridtest.py index 0aea1e25c0..8e3139f2a0 100644 --- a/sendgridtest.py +++ b/sendgridtest.py @@ -7,7 +7,7 @@ message = Mail( from_email='metrix@goggins.com', to_emails='gogginss@missouri.edu', - subject='Sending with Twilio SendGrid is Fun', + subject='Hi, Attached is your report of recommendations provided as a part of your DEI Badge! Contained are specific suggestions you may consider for making your project more open and welcoming to newcomers from diverse backgrounds. .... etc.', html_content='and easy to do anywhere, even with Python') try: sg = SendGridAPIClient(os.environ.get('SENDGRID_API_KEY')) From 8e2876f85228e48742fb84deb099f07de1e26eec Mon Sep 17 00:00:00 2001 From: "Sean P. Goggins" Date: Sun, 9 Jul 2023 00:06:19 +0000 Subject: [PATCH 04/34] test Signed-off-by: Sean P. Goggins --- sendgridtest.py | 19 ------------------- 1 file changed, 19 deletions(-) delete mode 100644 sendgridtest.py diff --git a/sendgridtest.py b/sendgridtest.py deleted file mode 100644 index 8e3139f2a0..0000000000 --- a/sendgridtest.py +++ /dev/null @@ -1,19 +0,0 @@ -# using SendGrid's Python Library -# https://github.com/sendgrid/sendgrid-python -import os -from sendgrid import SendGridAPIClient -from sendgrid.helpers.mail import Mail - -message = Mail( - from_email='metrix@goggins.com', - to_emails='gogginss@missouri.edu', - subject='Hi, Attached is your report of recommendations provided as a part of your DEI Badge! Contained are specific suggestions you may consider for making your project more open and welcoming to newcomers from diverse backgrounds. .... etc.', - html_content='and easy to do anywhere, even with Python') -try: - sg = SendGridAPIClient(os.environ.get('SENDGRID_API_KEY')) - response = sg.send(message) - print(response.status_code) - print(response.body) - print(response.headers) -except Exception as e: - print(e.message) From 0954a837ac1ea4bec7074a91868f22c05e216f36 Mon Sep 17 00:00:00 2001 From: "Sean P. Goggins" Date: Sun, 9 Jul 2023 00:08:21 +0000 Subject: [PATCH 05/34] update version Signed-off-by: Sean P. Goggins --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 811157100a..ce06f34625 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Augur NEW Release v0.51.1 +# Augur NEW Release v0.51.2 [![first-timers-only](https://img.shields.io/badge/first--timers--only-friendly-blue.svg?style=flat-square)](https://www.firsttimersonly.com/) We follow the [First Timers Only](https://www.firsttimersonly.com/) philosophy of tagging issues for first timers only, and walking one newcomer through the resolution process weekly. [You can find these issues tagged with "first timers only" on our issues list.](https://github.com/chaoss/augur/labels/first-timers-only). @@ -8,7 +8,7 @@ ### [If you want to jump right in, updated docker build/compose and bare metal installation instructions are available here](docs/new-install.md) -Augur is now releasing a dramatically improved new version to the main branch. It is also available here: https://github.com/chaoss/augur/releases/tag/v0.51.1 +Augur is now releasing a dramatically improved new version to the main branch. It is also available here: https://github.com/chaoss/augur/releases/tag/v0.51.2 - The `main` branch is a stable version of our new architecture, which features: - Dramatic improvement in the speed of large scale data collection (100,000+ repos). All data is obtained for 100k+ repos within 2 weeks. - A new job management architecture that uses Celery and Redis to manage queues, and enables users to run a Flower job monitoring dashboard From b8e08923a70cb87a7ff64d4a10cde5eca3e654aa Mon Sep 17 00:00:00 2001 From: sgoggins Date: Thu, 13 Jul 2023 10:55:43 -0500 Subject: [PATCH 06/34] updating materialized view refresh code. --- augur/tasks/db/refresh_materialized_views.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/augur/tasks/db/refresh_materialized_views.py b/augur/tasks/db/refresh_materialized_views.py index 53b29ddbd2..5c592645fb 100644 --- a/augur/tasks/db/refresh_materialized_views.py +++ b/augur/tasks/db/refresh_materialized_views.py @@ -18,12 +18,14 @@ def refresh_materialized_views(): refresh_view_query = s.sql.text(""" REFRESH MATERIALIZED VIEW augur_data.api_get_all_repos_issues with data; REFRESH MATERIALIZED VIEW augur_data.explorer_commits_and_committers_daily_count with data; + REFRESH MATERIALIZED VIEW augur_data.api_get_all_repos_prs with data; REFRESH MATERIALIZED VIEW augur_data.api_get_all_repos_commits with data; REFRESH MATERIALIZED VIEW augur_data.augur_new_contributors with data; REFRESH MATERIALIZED VIEW augur_data.explorer_contributor_actions with data; REFRESH MATERIALIZED VIEW augur_data.explorer_libyear_all with data; REFRESH MATERIALIZED VIEW augur_data.explorer_libyear_detail with data; REFRESH MATERIALIZED VIEW augur_data.explorer_new_contributors with data; + REFRESH MATERIALIZED VIEW augur_data.explorer_entry_list with data; REFRESH MATERIALIZED VIEW augur_data.explorer_libyear_summary with data; """) From b81a07e3906b66434dadbfbff1b8fb8a63e2083d Mon Sep 17 00:00:00 2001 From: sgoggins Date: Thu, 13 Jul 2023 11:38:59 -0500 Subject: [PATCH 07/34] materialized view refresh script: Started. Needs to be parameterized to use db.config.json --- scripts/control/refresh-matviews.sh | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 scripts/control/refresh-matviews.sh diff --git a/scripts/control/refresh-matviews.sh b/scripts/control/refresh-matviews.sh new file mode 100644 index 0000000000..576466b2df --- /dev/null +++ b/scripts/control/refresh-matviews.sh @@ -0,0 +1,11 @@ +#!/bin/sh +psql -U augur -h localhost -p 5432 -d padres -c 'REFRESH MATERIALIZED VIEW augur_data.api_get_all_repos_issues with data;' +psql -U augur -h localhost -p 5432 -d padres -c 'REFRESH MATERIALIZED VIEW augur_data.explorer_commits_and_committers_daily_count with data;' +psql -U augur -h localhost -p 5432 -d padres -c 'REFRESH MATERIALIZED VIEW augur_data.api_get_all_repo_prs with data;' +psql -U augur -h localhost -p 5432 -d padres -c 'REFRESH MATERIALIZED VIEW augur_data.api_get_all_repos_commits with data;' +psql -U augur -h localhost -p 5432 -d padres -c 'REFRESH MATERIALIZED VIEW augur_data.augur_new_contributors with data;' +psql -U augur -h localhost -p 5432 -d padres -c 'REFRESH MATERIALIZED VIEW augur_data.explorer_contributor_actions with data;' +psql -U augur -h localhost -p 5432 -d padres -c 'REFRESH MATERIALIZED VIEW augur_data.explorer_libyear_all with data;' +psql -U augur -h localhost -p 5432 -d padres -c 'REFRESH MATERIALIZED VIEW augur_data.explorer_libyear_detail with data;' +psql -U augur -h localhost -p 5432 -d padres -c 'REFRESH MATERIALIZED VIEW augur_data.explorer_new_contributors with data;' +psql -U augur -h localhost -p 5432 -d padres -c 'REFRESH MATERIALIZED VIEW augur_data.explorer_entry_list with data;' \ No newline at end of file From 7df2d08162384cc527894be08173f929f75757ee Mon Sep 17 00:00:00 2001 From: Isaac Milarsky Date: Tue, 25 Jul 2023 17:39:59 -0500 Subject: [PATCH 08/34] fix fetchall_data_from_sql_text for sqlalchemy 2.x Signed-off-by: Isaac Milarsky --- augur/application/db/models/augur_data.py | 10 ++++++---- augur/application/db/models/augur_operations.py | 3 ++- augur/application/db/session.py | 4 ++-- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/augur/application/db/models/augur_data.py b/augur/application/db/models/augur_data.py index 676a71deec..ee9f561a2a 100644 --- a/augur/application/db/models/augur_data.py +++ b/augur/application/db/models/augur_data.py @@ -1195,10 +1195,12 @@ class Commit(Base): "Contributor", primaryjoin="Commit.cmt_author_platform_username == Contributor.cntrb_login", ) - contributor1 = relationship( - "Contributor", - primaryjoin="Commit.cmt_author_platform_username == Contributor.cntrb_login", - ) + + # Redundant? + #contributor1 = relationship( + # "Contributor", + # primaryjoin="Commit.cmt_author_platform_username == Contributor.cntrb_login", + #) repo = relationship("Repo") diff --git a/augur/application/db/models/augur_operations.py b/augur/application/db/models/augur_operations.py index e2100196dc..25bb05eda5 100644 --- a/augur/application/db/models/augur_operations.py +++ b/augur/application/db/models/augur_operations.py @@ -275,7 +275,8 @@ class User(Base): {"schema": "augur_operations"} ) - groups = relationship("UserGroup") + # Redundant. + #groups = relationship("UserGroup") tokens = relationship("UserSessionToken") applications = relationship("ClientApplication") diff --git a/augur/application/db/session.py b/augur/application/db/session.py index 2212c1fdc1..7e9a6da54c 100644 --- a/augur/application/db/session.py +++ b/augur/application/db/session.py @@ -95,8 +95,8 @@ def fetchall_data_from_sql_text(self,sql_text): with self.engine.connect() as connection: - result = connection.execute(sql_text).fetchall() - return [dict(zip(row.keys(), row)) for row in result] + result = connection.execute(sql_text) + return [dict(row) for row in result.mappings()] def insert_data(self, data: Union[List[dict], dict], table, natural_keys: List[str], return_columns: Optional[List[str]] = None, string_fields: Optional[List[str]] = None, on_conflict_update:bool = True) -> Optional[List[dict]]: From 6d01398d8c60c9e3baaa1297dc352367e6afddf2 Mon Sep 17 00:00:00 2001 From: Isaac Milarsky Date: Tue, 25 Jul 2023 17:50:08 -0500 Subject: [PATCH 09/34] new method of converting row to dict in sqlalchemy 2.x Signed-off-by: Isaac Milarsky --- augur/tasks/github/facade_github/tasks.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/augur/tasks/github/facade_github/tasks.py b/augur/tasks/github/facade_github/tasks.py index 04ccc16e12..f0e8845fb2 100644 --- a/augur/tasks/github/facade_github/tasks.py +++ b/augur/tasks/github/facade_github/tasks.py @@ -251,8 +251,8 @@ def insert_facade_contributors(repo_id): """).bindparams(repo_id=repo_id) #Execute statement with session. - result = manifest.augur_db.execute_sql(new_contrib_sql).fetchall() - new_contribs = [dict(zip(row.keys(), row)) for row in result] + result = manifest.augur_db.execute_sql(new_contrib_sql) + new_contribs = [dict(row) for row in result.mappings()] #print(new_contribs) @@ -302,8 +302,8 @@ def insert_facade_contributors(repo_id): #existing_cntrb_emails = json.loads(pd.read_sql(resolve_email_to_cntrb_id_sql, self.db, params={ # 'repo_id': repo_id}).to_json(orient="records")) - result = session.execute_sql(resolve_email_to_cntrb_id_sql).fetchall() - existing_cntrb_emails = [dict(zip(row.keys(), row)) for row in result] + result = session.execute_sql(resolve_email_to_cntrb_id_sql) + existing_cntrb_emails = [dict(row) for row in result.mappings()] print(existing_cntrb_emails) link_commits_to_contributor(session,list(existing_cntrb_emails)) From fb5091ef053c2549722043d51bda15a1698dfde2 Mon Sep 17 00:00:00 2001 From: Isaac Milarsky Date: Mon, 7 Aug 2023 11:27:49 -0500 Subject: [PATCH 10/34] deal with redundant relationship definition Signed-off-by: Isaac Milarsky --- augur/application/db/models/augur_operations.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/augur/application/db/models/augur_operations.py b/augur/application/db/models/augur_operations.py index e1228d8b55..8507133b06 100644 --- a/augur/application/db/models/augur_operations.py +++ b/augur/application/db/models/augur_operations.py @@ -740,7 +740,8 @@ class UserRepo(Base): ForeignKey("augur_data.repo.repo_id", name="user_repo_user_id_fkey"), primary_key=True, nullable=False ) - repo = relationship("Repo") + #made redundant by user_repo relationship in Repo orm class. + #repo = relationship("Repo") group = relationship("UserGroup") @staticmethod From 20e2f26d82e726080d3006c4d531153bd59f7683 Mon Sep 17 00:00:00 2001 From: Isaac Milarsky Date: Mon, 7 Aug 2023 11:39:44 -0500 Subject: [PATCH 11/34] patch contributor breadth worker to use 2.x version of sqlalchemy execute() Signed-off-by: Isaac Milarsky --- .../contributor_breadth_worker.py | 7 +++++-- augur/tasks/db/refresh_materialized_views.py | 4 +++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/augur/tasks/data_analysis/contributor_breadth_worker/contributor_breadth_worker.py b/augur/tasks/data_analysis/contributor_breadth_worker/contributor_breadth_worker.py index 44a6761cf9..9fdcf434ac 100644 --- a/augur/tasks/data_analysis/contributor_breadth_worker/contributor_breadth_worker.py +++ b/augur/tasks/data_analysis/contributor_breadth_worker/contributor_breadth_worker.py @@ -33,7 +33,8 @@ def contributor_breadth_model() -> None: WHERE gh_login IS NOT NULL """) - result = engine.execute(cntrb_login_query) + with engine.connect() as connection: + result = connection.execute(cntrb_login_query) current_cntrb_logins = [dict(row) for row in result] @@ -45,7 +46,9 @@ def contributor_breadth_model() -> None: GROUP BY c.gh_login; """) - cntrb_newest_events_list = engine.execute(cntrb_newest_events_query) + with engine.connect() as connection: + cntrb_newest_events_list = connection.execute(cntrb_newest_events_query) + cntrb_newest_events_list = [dict(row) for row in cntrb_newest_events_list] cntrb_newest_events_map = {} diff --git a/augur/tasks/db/refresh_materialized_views.py b/augur/tasks/db/refresh_materialized_views.py index 5c592645fb..74f6361928 100644 --- a/augur/tasks/db/refresh_materialized_views.py +++ b/augur/tasks/db/refresh_materialized_views.py @@ -15,10 +15,12 @@ def refresh_materialized_views(): logger = logging.getLogger(refresh_materialized_views.__name__) + #Removed to make it work: + # REFRESH MATERIALIZED VIEW augur_data.api_get_all_repos_prs with data; refresh_view_query = s.sql.text(""" REFRESH MATERIALIZED VIEW augur_data.api_get_all_repos_issues with data; REFRESH MATERIALIZED VIEW augur_data.explorer_commits_and_committers_daily_count with data; - REFRESH MATERIALIZED VIEW augur_data.api_get_all_repos_prs with data; + REFRESH MATERIALIZED VIEW augur_data.api_get_all_repos_commits with data; REFRESH MATERIALIZED VIEW augur_data.augur_new_contributors with data; REFRESH MATERIALIZED VIEW augur_data.explorer_contributor_actions with data; From fd2b4c5b226ee526838e530db023b077bd547203 Mon Sep 17 00:00:00 2001 From: Isaac Milarsky Date: Mon, 7 Aug 2023 13:34:25 -0500 Subject: [PATCH 12/34] fix insert_data to work with sqlalchemy 2.x by using new begin method with connection object. This allows a commit to be implicit within the resulting context manager and emulates the functionality of the now removed autocommit Signed-off-by: Isaac Milarsky --- augur/application/db/session.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/augur/application/db/session.py b/augur/application/db/session.py index 7e9a6da54c..bcc988991d 100644 --- a/augur/application/db/session.py +++ b/augur/application/db/session.py @@ -174,7 +174,9 @@ def insert_data(self, data: Union[List[dict], dict], table, natural_keys: List[s while attempts < 10: try: - with EngineConnection(self.engine) as connection: + #begin keyword is needed for sqlalchemy 2.x + #this is because autocommit support was removed in 2.0 + with EngineConnection(self.engine).begin() as connection: connection.execute(stmnt) break except OperationalError as e: From 9e4445cda42dff9c2a4b9cfac9088b8402b8d136 Mon Sep 17 00:00:00 2001 From: Isaac Milarsky Date: Mon, 7 Aug 2023 14:34:11 -0500 Subject: [PATCH 13/34] fix missing argument in splitting insert_data calls in recursion in insert_data method. fix missing .begin() call in block to deal with return columns. fix dict conversion for list of rows in sqlalchemy 2.x in insert_data and contributor breadth worker Signed-off-by: Isaac Milarsky --- augur/application/db/session.py | 17 ++++++++++------- .../contributor_breadth_worker.py | 2 +- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/augur/application/db/session.py b/augur/application/db/session.py index bcc988991d..23cf8abc85 100644 --- a/augur/application/db/session.py +++ b/augur/application/db/session.py @@ -176,7 +176,7 @@ def insert_data(self, data: Union[List[dict], dict], table, natural_keys: List[s try: #begin keyword is needed for sqlalchemy 2.x #this is because autocommit support was removed in 2.0 - with EngineConnection(self.engine).begin() as connection: + with self.engine.begin() as connection: connection.execute(stmnt) break except OperationalError as e: @@ -193,14 +193,15 @@ def insert_data(self, data: Union[List[dict], dict], table, natural_keys: List[s raise e except Exception as e: + #self.logger.info(e) if(len(data) == 1): raise e else: first_half = data[:len(data)//2] second_half = data[len(data)//2:] - self.insert_data(first_half, natural_keys, return_columns, string_fields, on_conflict_update) - self.insert_data(second_half, natural_keys, return_columns, string_fields, on_conflict_update) + self.insert_data(first_half, table,natural_keys, return_columns, string_fields, on_conflict_update) + self.insert_data(second_half,table, natural_keys, return_columns, string_fields, on_conflict_update) else: self.logger.error("Unable to insert data in 10 attempts") @@ -215,7 +216,7 @@ def insert_data(self, data: Union[List[dict], dict], table, natural_keys: List[s # othewise it gets the requested return columns and returns them as a list of dicts while attempts < 10: try: - with EngineConnection(self.engine) as connection: + with self.engine.begin() as connection: return_data_tuples = connection.execute(stmnt).fetchall() break except OperationalError as e: @@ -246,9 +247,11 @@ def insert_data(self, data: Union[List[dict], dict], table, natural_keys: List[s if deadlock_detected is True: self.logger.error("Made it through even though Deadlock was detected") - return_data = [] - for data_tuple in return_data_tuples: - return_data.append(dict(data_tuple)) + return_data = [dict(row) for row in return_data_tuples.mappings()] + + #no longer working in sqlalchemy 2.x + #for data_tuple in return_data_tuples: + # return_data.append(dict(data_tuple)) # using on confilict do nothing does not return the # present values so this does gets the return values diff --git a/augur/tasks/data_analysis/contributor_breadth_worker/contributor_breadth_worker.py b/augur/tasks/data_analysis/contributor_breadth_worker/contributor_breadth_worker.py index 9fdcf434ac..19053b2f05 100644 --- a/augur/tasks/data_analysis/contributor_breadth_worker/contributor_breadth_worker.py +++ b/augur/tasks/data_analysis/contributor_breadth_worker/contributor_breadth_worker.py @@ -36,7 +36,7 @@ def contributor_breadth_model() -> None: with engine.connect() as connection: result = connection.execute(cntrb_login_query) - current_cntrb_logins = [dict(row) for row in result] + current_cntrb_logins = [dict(row) for row in result.mappings()] cntrb_newest_events_query = s.sql.text(""" From a39eb0b7edba1bc8df8427bdc7c184d67b29e7b3 Mon Sep 17 00:00:00 2001 From: Isaac Milarsky Date: Mon, 7 Aug 2023 14:45:00 -0500 Subject: [PATCH 14/34] update mappings syntax Signed-off-by: Isaac Milarsky --- augur/application/db/session.py | 2 +- .../contributor_breadth_worker/contributor_breadth_worker.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/augur/application/db/session.py b/augur/application/db/session.py index 23cf8abc85..cd5788df9f 100644 --- a/augur/application/db/session.py +++ b/augur/application/db/session.py @@ -217,7 +217,7 @@ def insert_data(self, data: Union[List[dict], dict], table, natural_keys: List[s while attempts < 10: try: with self.engine.begin() as connection: - return_data_tuples = connection.execute(stmnt).fetchall() + return_data_tuples = connection.execute(stmnt) break except OperationalError as e: if isinstance(e.orig, DeadlockDetected): diff --git a/augur/tasks/data_analysis/contributor_breadth_worker/contributor_breadth_worker.py b/augur/tasks/data_analysis/contributor_breadth_worker/contributor_breadth_worker.py index 19053b2f05..bca949df82 100644 --- a/augur/tasks/data_analysis/contributor_breadth_worker/contributor_breadth_worker.py +++ b/augur/tasks/data_analysis/contributor_breadth_worker/contributor_breadth_worker.py @@ -49,7 +49,7 @@ def contributor_breadth_model() -> None: with engine.connect() as connection: cntrb_newest_events_list = connection.execute(cntrb_newest_events_query) - cntrb_newest_events_list = [dict(row) for row in cntrb_newest_events_list] + cntrb_newest_events_list = [dict(row) for row in cntrb_newest_events_list.mappings()] cntrb_newest_events_map = {} for cntrb_event in cntrb_newest_events_list: From c390c93042e5075b40a4efa33f47850043384ca1 Mon Sep 17 00:00:00 2001 From: Isaac Milarsky Date: Mon, 7 Aug 2023 15:57:43 -0500 Subject: [PATCH 15/34] update alembic scripts to work with sqlalchemy 2.x Signed-off-by: Isaac Milarsky --- augur/application/db/session.py | 4 ++-- .../schema/alembic/versions/1_augur_new_changes.py | 10 ++++++---- .../schema/alembic/versions/3_oauth_and_user_groups.py | 4 ++-- .../versions/9_add_repo_group_for_frontend_repos.py | 2 +- 4 files changed, 11 insertions(+), 9 deletions(-) diff --git a/augur/application/db/session.py b/augur/application/db/session.py index cd5788df9f..f1d1e64dd0 100644 --- a/augur/application/db/session.py +++ b/augur/application/db/session.py @@ -85,7 +85,7 @@ def __del__(self): def execute_sql(self, sql_text): - with self.engine.connect() as connection: + with self.engine.begin() as connection: return_data = connection.execute(sql_text) @@ -93,7 +93,7 @@ def execute_sql(self, sql_text): def fetchall_data_from_sql_text(self,sql_text): - with self.engine.connect() as connection: + with self.engine.begin() as connection: result = connection.execute(sql_text) return [dict(row) for row in result.mappings()] diff --git a/augur/application/schema/alembic/versions/1_augur_new_changes.py b/augur/application/schema/alembic/versions/1_augur_new_changes.py index 0be3780a36..2e8440294f 100644 --- a/augur/application/schema/alembic/versions/1_augur_new_changes.py +++ b/augur/application/schema/alembic/versions/1_augur_new_changes.py @@ -300,8 +300,9 @@ def change_cntrb_id_to_uuid_5(upgrade=True): """ INSERT INTO "augur_data"."contributors"("cntrb_id", "cntrb_login", "cntrb_email", "cntrb_company", "cntrb_created_at", "cntrb_type", "cntrb_fake", "cntrb_deleted", "cntrb_long", "cntrb_lat", "cntrb_country_code", "cntrb_state", "cntrb_city", "cntrb_location", "cntrb_canonical", "gh_user_id", "gh_login", "gh_url", "gh_html_url", "gh_node_id", "gh_avatar_url", "gh_gravatar_id", "gh_followers_url", "gh_following_url", "gh_gists_url", "gh_starred_url", "gh_subscriptions_url", "gh_organizations_url", "gh_repos_url", "gh_events_url", "gh_received_events_url", "gh_type", "gh_site_admin", "tool_source", "tool_version", "data_source", "data_collection_date") VALUES (:cntrb_uuid, 'not-provided', NULL, NULL, '2019-06-13 11:33:39', NULL, 0, 0, NULL, NULL, NULL, NULL, NULL, NULL, NULL, 1, 'nobody', 'http://fake.me', 'http://fake.me', 'x', 'http://fake.me', NULL, 'http://fake.me', 'http://fake.me', 'http://fake.me', 'http://fake.me', 'http://fake.me', 'http://fake.me', 'http://fake.me', 'http://fake.me', NULL, NULL, NULL, NULL, NULL, NULL, '2019-06-13 16:35:25'); """ - ), - cntrb_uuid=UnresolvableUUID().to_UUID() + ).bindparams( + cntrb_uuid=UnresolvableUUID().to_UUID() + ) ) conn.execute( @@ -309,8 +310,9 @@ def change_cntrb_id_to_uuid_5(upgrade=True): """ INSERT INTO "augur_data"."contributors" ("cntrb_id", "cntrb_login", "cntrb_email", "cntrb_full_name", "cntrb_company", "cntrb_created_at", "cntrb_type", "cntrb_fake", "cntrb_deleted", "cntrb_long", "cntrb_lat", "cntrb_country_code", "cntrb_state", "cntrb_city", "cntrb_location", "cntrb_canonical", "cntrb_last_used", "gh_user_id", "gh_login", "gh_url", "gh_html_url", "gh_node_id", "gh_avatar_url", "gh_gravatar_id", "gh_followers_url", "gh_following_url", "gh_gists_url", "gh_starred_url", "gh_subscriptions_url", "gh_organizations_url", "gh_repos_url", "gh_events_url", "gh_received_events_url", "gh_type", "gh_site_admin", "gl_web_url", "gl_avatar_url", "gl_state", "gl_username", "gl_full_name", "gl_id", "tool_source", "tool_version", "data_source", "data_collection_date") VALUES (:cntrb_uuid, 'nan', 'kannayoshihiro@gmail.com', 'KANNA Yoshihiro', 'UTMC', '2009-04-17 12:43:58', NULL, 0, 0, NULL, NULL, NULL, NULL, NULL, NULL, 'kannayoshihiro@gmail.com', '2021-01-28 21:56:10-06', 74832, 'nan', 'https://api.github.com/users/nan', 'https://github.com/nan', 'MDQ6VXNlcjc0ODMy', 'https://avatars.githubusercontent.com/u/74832?v=4', '', 'https://api.github.com/users/nan/followers', 'https://api.github.com/users/nan/following{/other_user}', 'https://api.github.com/users/nan/gists{/gist_id}', 'https://api.github.com/users/nan/starred{/owner}{/repo}', 'https://api.github.com/users/nan/subscriptions', 'https://api.github.com/users/nan/orgs', 'https://api.github.com/users/nan/repos', 'https://api.github.com/users/nan/events{/privacy}', 'https://api.github.com/users/nan/received_events', 'User', 'false', NULL, NULL, NULL, NULL, NULL, NULL, 'GitHub API Worker', '1.0.0', 'GitHub API', '2021-10-28 15:23:46'); """ - ), - cntrb_uuid=GithubUUID().to_UUID() + ).bindparams( + cntrb_uuid=GithubUUID().to_UUID() + ) ) else: diff --git a/augur/application/schema/alembic/versions/3_oauth_and_user_groups.py b/augur/application/schema/alembic/versions/3_oauth_and_user_groups.py index 8d75b7a709..0d9c6d744a 100644 --- a/augur/application/schema/alembic/versions/3_oauth_and_user_groups.py +++ b/augur/application/schema/alembic/versions/3_oauth_and_user_groups.py @@ -85,9 +85,9 @@ def upgrade(): table_changes = """ - ALTER TABLE user_repos + ALTER TABLE augur_operations.user_repos ADD COLUMN group_id BIGINT, - ADD CONSTRAINT user_repos_group_id_fkey FOREIGN KEY (group_id) REFERENCES user_groups(group_id), + ADD CONSTRAINT user_repos_group_id_fkey FOREIGN KEY (group_id) REFERENCES augur_operations.user_groups(group_id), DROP COLUMN user_id, ADD PRIMARY KEY (group_id, repo_id); """ diff --git a/augur/application/schema/alembic/versions/9_add_repo_group_for_frontend_repos.py b/augur/application/schema/alembic/versions/9_add_repo_group_for_frontend_repos.py index 288f584cf5..52a6e017db 100644 --- a/augur/application/schema/alembic/versions/9_add_repo_group_for_frontend_repos.py +++ b/augur/application/schema/alembic/versions/9_add_repo_group_for_frontend_repos.py @@ -25,7 +25,7 @@ def upgrade(): conn = op.get_bind() result = conn.execute(text(f"""SELECT * FROM "augur_data"."repo_groups" WHERE rg_name='{repo_group_name}';""")).fetchall() if len(result) == 0: - conn.execute(f"""INSERT INTO "augur_data"."repo_groups" ("rg_name", "rg_description", "rg_website", "rg_recache", "rg_last_modified", "rg_type", "tool_source", "tool_version", "data_source", "data_collection_date") VALUES ('{repo_group_name}', 'DO NOT DELETE OR FRONTEND REPOS WILL BREAK', '', 0, '2023-02-17 15:00:00', NULL, NULL, NULL, NULL, NULL);""") + conn.execute(text(f"""INSERT INTO "augur_data"."repo_groups" ("rg_name", "rg_description", "rg_website", "rg_recache", "rg_last_modified", "rg_type", "tool_source", "tool_version", "data_source", "data_collection_date") VALUES ('{repo_group_name}', 'DO NOT DELETE OR FRONTEND REPOS WILL BREAK', '', 0, '2023-02-17 15:00:00', NULL, NULL, NULL, NULL, NULL);""")) # ### end Alembic commands ### From 98c0a5140cc4951245c1fee9e16307d6c8f165f3 Mon Sep 17 00:00:00 2001 From: Isaac Milarsky Date: Tue, 8 Aug 2023 11:41:29 -0500 Subject: [PATCH 16/34] get rid of redundancies in relationship definitions in orm Signed-off-by: Isaac Milarsky --- .../application/db/models/augur_operations.py | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/augur/application/db/models/augur_operations.py b/augur/application/db/models/augur_operations.py index 8507133b06..54db0bb99f 100644 --- a/augur/application/db/models/augur_operations.py +++ b/augur/application/db/models/augur_operations.py @@ -271,8 +271,7 @@ class User(Base): {"schema": "augur_operations"} ) - # Redundant. - #groups = relationship("UserGroup") + groups = relationship("UserGroup") tokens = relationship("UserSessionToken") applications = relationship("ClientApplication") @@ -629,7 +628,7 @@ class UserGroup(Base): {"schema": "augur_operations"} ) - user = relationship("User") + #user = relationship("User") repos = relationship("UserRepo") @staticmethod @@ -742,7 +741,7 @@ class UserRepo(Base): #made redundant by user_repo relationship in Repo orm class. #repo = relationship("Repo") - group = relationship("UserGroup") + #group = relationship("UserGroup") @staticmethod def insert(session, repo_id: int, group_id:int = 1) -> bool: @@ -951,7 +950,7 @@ class UserSessionToken(Base): application_id = Column(ForeignKey("augur_operations.client_applications.id", name="user_session_token_application_id_fkey"), nullable=False) created_at = Column(BigInteger) - user = relationship("User") + #user = relationship("User") application = relationship("ClientApplication") refresh_tokens = relationship("RefreshToken") @@ -993,8 +992,8 @@ class ClientApplication(Base): redirect_url = Column(String, nullable=False) api_key = Column(String, nullable=False) - user = relationship("User") - sessions = relationship("UserSessionToken") + #user = relationship("User") + #sessions = relationship("UserSessionToken") subscriptions = relationship("Subscription") def __eq__(self, other): @@ -1015,8 +1014,8 @@ class Subscription(Base): application_id = Column(ForeignKey("augur_operations.client_applications.id", name="subscriptions_application_id_fkey"), primary_key=True) type_id = Column(ForeignKey("augur_operations.subscription_types.id", name="subscriptions_type_id_fkey"), primary_key=True) - application = relationship("ClientApplication") - type = relationship("SubscriptionType") + #application = relationship("ClientApplication") + #type = relationship("SubscriptionType") class SubscriptionType(Base): __tablename__ = "subscription_types" @@ -1042,7 +1041,7 @@ class RefreshToken(Base): id = Column(String, primary_key=True) user_session_token = Column(ForeignKey("augur_operations.user_session_tokens.token", name="refresh_token_session_token_id_fkey"), nullable=False) - user_session = relationship("UserSessionToken") + #user_session = relationship("UserSessionToken") @staticmethod def create(session, user_session_token_id): From e92023b2721272f0965a90ec452feadce085dbd9 Mon Sep 17 00:00:00 2001 From: Isaac Milarsky Date: Tue, 8 Aug 2023 12:35:20 -0500 Subject: [PATCH 17/34] use begin() syntax to implicitly commit results of sqlalchemy 2.x transactions Signed-off-by: Isaac Milarsky --- augur/application/cli/db.py | 4 ++-- augur/application/schema/alembic/env.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/augur/application/cli/db.py b/augur/application/cli/db.py index f09aaabbd2..42d57ecc6b 100644 --- a/augur/application/cli/db.py +++ b/augur/application/cli/db.py @@ -99,7 +99,7 @@ def add_repo_groups(filename): """ Create new repo groups in Augur's database """ - with DatabaseEngine() as engine, engine.connect() as connection: + with DatabaseEngine() as engine, engine.begin() as connection: df = pd.read_sql( s.sql.text("SELECT repo_group_id FROM augur_data.repo_groups"), @@ -248,7 +248,7 @@ def update_api_key(api_key): """ ) - with DatabaseEngine() as engine, engine.connect() as connection: + with DatabaseEngine() as engine, engine.begin() as connection: connection.execute(update_api_key_sql, api_key=api_key) logger.info(f"Updated Augur API key to: {api_key}") diff --git a/augur/application/schema/alembic/env.py b/augur/application/schema/alembic/env.py index d170ef243f..85b02eac9c 100644 --- a/augur/application/schema/alembic/env.py +++ b/augur/application/schema/alembic/env.py @@ -60,7 +60,7 @@ def run_migrations_online(): """ - with DatabaseEngine() as connectable, connectable.connect() as connection: + with DatabaseEngine() as connectable, connectable.begin() as connection: context.configure( connection=connection, target_metadata=target_metadata, From 82ba6f87780e5ef8575f7d2c3f774343f6713a81 Mon Sep 17 00:00:00 2001 From: Isaac Milarsky Date: Fri, 11 Aug 2023 12:48:17 -0500 Subject: [PATCH 18/34] missed a dictionary conversion that used the 1.4 method instead of 2.x method sqlalchemy Signed-off-by: Isaac Milarsky --- augur/tasks/github/pull_requests/files_model/core.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/augur/tasks/github/pull_requests/files_model/core.py b/augur/tasks/github/pull_requests/files_model/core.py index e7ebcd9457..81b4c4397a 100644 --- a/augur/tasks/github/pull_requests/files_model/core.py +++ b/augur/tasks/github/pull_requests/files_model/core.py @@ -20,8 +20,8 @@ def pull_request_files_model(repo_id,logger, augur_db, key_auth): pr_numbers = [] #pd.read_sql(pr_number_sql, self.db, params={}) - result = augur_db.execute_sql(pr_number_sql).fetchall() - pr_numbers = [dict(zip(row.keys(), row)) for row in result] + result = augur_db.execute_sql(pr_number_sql)#.fetchall() + pr_numbers = [dict(row) for row in result.mappings()] query = augur_db.session.query(Repo).filter(Repo.repo_id == repo_id) repo = execute_session_query(query, 'one') From b31bd78a3c764200b1f14db4bc9c16aa40d7bc6a Mon Sep 17 00:00:00 2001 From: Isaac Milarsky Date: Mon, 14 Aug 2023 10:50:05 -0500 Subject: [PATCH 19/34] remove call to undefined method in gunicorn start that was there for some reason Signed-off-by: Isaac Milarsky --- augur/api/view/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/augur/api/view/utils.py b/augur/api/view/utils.py index ea5ec80bd0..76551a6ab9 100644 --- a/augur/api/view/utils.py +++ b/augur/api/view/utils.py @@ -74,7 +74,7 @@ def getSetting(key, section = "View"): loadSettings() -version_check(settings) +#version_check(settings) """ ---------------------------------------------------------------- """ From 34fbeee2250877aecda2600c2655691897c71b17 Mon Sep 17 00:00:00 2001 From: Shiva953 Date: Sat, 7 Oct 2023 11:55:37 +0530 Subject: [PATCH 20/34] add pull_requests_new API endpoint Signed-off-by: Shiva953 --- augur/api/metrics/pull_request.py | 47 +++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/augur/api/metrics/pull_request.py b/augur/api/metrics/pull_request.py index 9fbcc61757..e26e7103a5 100644 --- a/augur/api/metrics/pull_request.py +++ b/augur/api/metrics/pull_request.py @@ -10,6 +10,53 @@ from ..server import engine +@register_metric() +def pull_requests_new(repo_group_id, repo_id=None, period='day', begin_date=None, end_date=None): + """ + Returns a time series of the number of new Pull Requests opened during a certain period. + + :param repo_id: The repository's id + :param repo_group_id: The repository's group id + :param period: To set the periodicity to 'day', 'week', 'month' or 'year', defaults to 'day' + :param begin_date: Specifies the begin date, defaults to '1970-1-1 00:00:01' + :param end_date: Specifies the end date, defaults to datetime.now() + :return: DataFrame of new Pull Requests/period + """ + if not begin_date: + begin_date = '1970-1-1 00:00:01' + if not end_date: + end_date = datetime.now().strftime('%Y-%m-%d %H:%M:%S') + + if repo_id: + new_pull_requests_query = s.sql.text(""" + SELECT DATE_TRUNC(:period, pr_created_at) AS created_date, + COUNT(pr_id) AS new_pull_requests + FROM pull_requests + WHERE repo_id = :repo_id + AND pr_created_at BETWEEN :begin_date AND :end_date + GROUP BY created_date + """) + + results = pd.read_sql(new_pull_requests_query, engine, params={'repo_id': repo_id, 'period': period, + 'begin_date': begin_date, + 'end_date': end_date}) + else: + new_pull_requests_query = s.sql.text(""" + SELECT DATE_TRUNC(:period, pr_created_at) AS created_date, + COUNT(pr_id) AS new_pull_requests + FROM pull_requests + WHERE repo_id IN (SELECT repo_id FROM repo WHERE repo_group_id = :repo_group_id) + AND pr_created_at BETWEEN :begin_date AND :end_date + GROUP BY created_date + """) + + results = pd.read_sql(new_pull_requests_query, engine, + params={'repo_group_id': repo_group_id, 'period': period, + 'begin_date': begin_date, + 'end_date': end_date}) + + return results + @register_metric() def pull_requests_merge_contributor_new(repo_group_id, repo_id=None, period='day', begin_date=None, end_date=None): """ From ed6915fd372365df5bcffc25797d6c044d18d01e Mon Sep 17 00:00:00 2001 From: Shiva953 Date: Mon, 16 Oct 2023 18:51:06 +0530 Subject: [PATCH 21/34] added API documentation for the endpoints Signed-off-by: Shiva953 --- docs/source/rest-api/spec.yml | 84 +++++++++++++++++++++++++++++++++++ 1 file changed, 84 insertions(+) diff --git a/docs/source/rest-api/spec.yml b/docs/source/rest-api/spec.yml index ae583dbbbb..e60135e8d9 100644 --- a/docs/source/rest-api/spec.yml +++ b/docs/source/rest-api/spec.yml @@ -1678,6 +1678,90 @@ paths: type: array tags: - experimental + /repo-groups/:repo_group_id/pull-requests-new: + get: + description: Returns a time series of the number of new Pull Requests opened during a certain period for a specific repository group. + operationId: Pull Requests New (Repo Group) + parameters: + - name: repo_group_id + in: path + description: The repository group's id. + required: true + schema: + type: string + - name: period + in: query + description: Sets the periodicity to 'day', 'week', 'month', or 'year'. + schema: + type: string + - name: begin_date + in: query + description: Specifies the begin date. + schema: + type: string + - name: end_date + in: query + description: Specifies the end date. + schema: + type: string + responses: + '200': + description: OK + content: + application/json: + schema: + type: object + properties: + created_date: + type: string + description: Date in the format 'YYYY-MM-DD'. + new_pull_requests: + type: integer + description: The number of new pull requests opened on the date. + tags: + - experimental + /repo-groups/:repo_id/pull-requests-new: + get: + description: Returns a time series of the number of new Pull Requests opened during a certain period for a specific repository. + operationId: Pull Requests New (Repo) + parameters: + - name: repo_id + in: path + description: The repository's id. + required: true + schema: + type: string + - name: period + in: query + description: Sets the periodicity to 'day', 'week', 'month', or 'year'. + schema: + type: string + - name: begin_date + in: query + description: Specifies the begin date. + schema: + type: string + - name: end_date + in: query + description: Specifies the end date. + schema: + type: string + responses: + '200': + description: OK + content: + application/json: + schema: + type: object + properties: + created_date: + type: string + description: Date in the format 'YYYY-MM-DD'. + new_pull_requests: + type: integer + description: The number of new pull requests opened on the date. + tags: + - experimental /repo-groups/:repo_group_id/pull-requests-closed-no-merge: get: description: Timeseries of pull request which were closed but not merged From ec5d543c24018840aa1242efc3e1b6fcfdd178ca Mon Sep 17 00:00:00 2001 From: Isaac Milarsky Date: Thu, 26 Oct 2023 17:19:17 -0500 Subject: [PATCH 22/34] fix legacy alembic upgrade Signed-off-by: Isaac Milarsky --- augur/application/schema/alembic/env.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/augur/application/schema/alembic/env.py b/augur/application/schema/alembic/env.py index 85b02eac9c..83c1ee17cf 100644 --- a/augur/application/schema/alembic/env.py +++ b/augur/application/schema/alembic/env.py @@ -5,7 +5,9 @@ from alembic import context from augur.application.db.models.base import Base -from augur.application.db.engine import DatabaseEngine +from augur.application.db.engine import DatabaseEngine, get_database_string +from sqlalchemy import create_engine +from sqlalchemy.pool import NullPool # this is the Alembic Config object, which provides # access to the values within the .ini file in use. @@ -59,8 +61,10 @@ def run_migrations_online(): and associate a connection with the context. """ + url = get_database_string() + engine = create_engine(url) - with DatabaseEngine() as connectable, connectable.begin() as connection: + with engine.connect() as connection: context.configure( connection=connection, target_metadata=target_metadata, From 740847789e899e7d0a2427959da601ec112d9db7 Mon Sep 17 00:00:00 2001 From: Isaac Milarsky Date: Thu, 26 Oct 2023 17:51:37 -0500 Subject: [PATCH 23/34] upgrade sqlalchemy version and give context for schemas for alembic Signed-off-by: Isaac Milarsky --- augur/application/schema/alembic/env.py | 12 +++++++++++- setup.py | 2 +- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/augur/application/schema/alembic/env.py b/augur/application/schema/alembic/env.py index 83c1ee17cf..94127a43be 100644 --- a/augur/application/schema/alembic/env.py +++ b/augur/application/schema/alembic/env.py @@ -6,7 +6,7 @@ from alembic import context from augur.application.db.models.base import Base from augur.application.db.engine import DatabaseEngine, get_database_string -from sqlalchemy import create_engine +from sqlalchemy import create_engine, event from sqlalchemy.pool import NullPool # this is the Alembic Config object, which provides @@ -64,6 +64,16 @@ def run_migrations_online(): url = get_database_string() engine = create_engine(url) + @event.listens_for(engine, "connect", insert=True) + def set_search_path(dbapi_connection, connection_record): + existing_autocommit = dbapi_connection.autocommit + dbapi_connection.autocommit = True + cursor = dbapi_connection.cursor() + cursor.execute("SET SESSION search_path=public,augur_data,augur_operations,spdx") + cursor.close() + dbapi_connection.autocommit = existing_autocommit + + with engine.connect() as connection: context.configure( connection=connection, diff --git a/setup.py b/setup.py index f48474c3dd..c8b0da67cc 100644 --- a/setup.py +++ b/setup.py @@ -41,7 +41,7 @@ "alembic==1.8.1", # 1.8.1 "coloredlogs==15.0", # 15.0.1 "Beaker==1.11.0", # 1.11.0 - "SQLAlchemy==2.0.17", # 1.4.40 + "SQLAlchemy==2.0.22", # 1.4.40 "itsdangerous==2.0.1", # 2.1.2 'Jinja2~=3.0.3', "Flask==2.0.2", # 2.2.2 From 740e4184fec8ce93ffaaa7bf91d574e3ab520b01 Mon Sep 17 00:00:00 2001 From: "Sean P. Goggins" Date: Thu, 26 Oct 2023 17:56:56 -0500 Subject: [PATCH 24/34] fixing go install on OSX where brew is used Signed-off-by: Sean P. Goggins --- scripts/install/workers.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/install/workers.sh b/scripts/install/workers.sh index 4f6dd7a62b..b0fdb6d2d7 100755 --- a/scripts/install/workers.sh +++ b/scripts/install/workers.sh @@ -39,7 +39,7 @@ do done -if [ -f "/usr/local/go/bin/go" ] || [ -f "/usr/local/bin/go" ] || [ -f "/usr/bin/go" ] || [ -f "/snap/bin/go" ]; then +if [ -f "/usr/local/go/bin/go" ] || [ -f "/usr/local/bin/go" ] || [ -f "/usr/bin/go" ] || [ -f "/snap/bin/go" ] || [ -f "/opt/homebrew/bin/go" ]; then echo "found go!" else echo "Installing go!" @@ -96,4 +96,4 @@ cd $CURRENT_DIR cd $HOME/scc; go build; echo "scc build done" -cd $CURRENT_DIR \ No newline at end of file +cd $CURRENT_DIR From 5d2b51576f7636d1c6f89af66125f72b972de3ff Mon Sep 17 00:00:00 2001 From: "Sean P. Goggins" Date: Thu, 26 Oct 2023 18:16:31 -0500 Subject: [PATCH 25/34] library version updates and fixed OSSF Scorecard Cloning (I think) Signed-off-by: Sean P. Goggins --- .../data_analysis/message_insights/setup.py | 8 +++---- scripts/install/workers.sh | 24 +++++++++---------- setup.py | 2 +- 3 files changed, 17 insertions(+), 17 deletions(-) diff --git a/augur/tasks/data_analysis/message_insights/setup.py b/augur/tasks/data_analysis/message_insights/setup.py index 311eb9b6f9..0dc5dfab4e 100644 --- a/augur/tasks/data_analysis/message_insights/setup.py +++ b/augur/tasks/data_analysis/message_insights/setup.py @@ -36,10 +36,10 @@ def read(filename): 'nltk==3.6.6', 'pandas==1.3.5', 'emoji==1.2.0', - 'Keras<2.9.0rc0', - 'Keras-Preprocessing==1.1.2', - 'tensorflow==2.8.0', - 'h5py~=3.6.0', + 'Keras==2.13.1', + 'Keras-Preprocessing', + 'tensorflow==2.13.1', + 'h5py==3.10.0', 'scikit-image==0.19.1', 'joblib==1.0.1', 'xgboost', diff --git a/scripts/install/workers.sh b/scripts/install/workers.sh index b0fdb6d2d7..3401509e13 100755 --- a/scripts/install/workers.sh +++ b/scripts/install/workers.sh @@ -52,23 +52,23 @@ fi if [ -d "$HOME/scorecard" ]; then echo " Scorecard already exists, skipping cloning ..." echo " Updating Scorecard ... " - rm -rf $HOME/scorecard - echo "Cloning OSSF Scorecard to generate scorecard data ..." - git clone https://github.com/ossf/scorecard $HOME/scorecard - cd $HOME/scorecard + #rm -rf $HOME/scorecard + #echo "Cloning OSSF Scorecard to generate scorecard data ..." + #git clone https://github.com/ossf/scorecard $HOME/scorecard + #cd $HOME/scorecard CURRENT_DIR=$PWD; cd $CURRENT_DIR cd $HOME/scorecard; - go build; - echo "scorecard build done" - cd $CURRENT_DIR + #go build; + #echo "scorecard build done" + #cd $CURRENT_DIR #CURRENT_DIR=$PWD; #cd $HOME/scorecard; - #git pull; - #go mod tidy; - #go build; - #echo "Scorecard build done." - #cd $CURRENT_DIR + git pull; + go mod tidy; + go build; + echo "Scorecard build done." + cd $CURRENT_DIR else echo "Cloning OSSF Scorecard to generate scorecard data ..." git clone https://github.com/ossf/scorecard $HOME/scorecard diff --git a/setup.py b/setup.py index c8b0da67cc..3eaa27eb46 100644 --- a/setup.py +++ b/setup.py @@ -65,7 +65,7 @@ "partd >= 0.3.10", # 1.3.0 "distributed >= 2021.03.0", # 2022.8.1 "nltk==3.6.6", # 3.7 - "h5py~=3.6.0", # 3.7 + "h5py==3.10.0", # 3.7 "scipy==1.7.3", # 1.9.0 "blinker==1.4", # 1.5 "protobuf<3.22", # 4.21.5 From 523753feb51340945a9c5c378317d5ab0e997eb7 Mon Sep 17 00:00:00 2001 From: kaxada Date: Mon, 30 Oct 2023 15:42:51 +0300 Subject: [PATCH 26/34] fixed focus field when 'register' is clicked --- augur/templates/login.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/augur/templates/login.j2 b/augur/templates/login.j2 index c71d02d50f..faaab620ea 100644 --- a/augur/templates/login.j2 +++ b/augur/templates/login.j2 @@ -108,7 +108,7 @@