Merge pull request #2078 from chaoss/spg-api-patch

Updates to API, Docs, Materialized Views

README.md

@@ -1,12 +1,12 @@
-# Augur NEW Release v0.43.3
+# Augur NEW Release v0.43.5
 [![first-timers-only](https://img.shields.io/badge/first--timers--only-friendly-blue.svg?style=flat-square)](https://www.firsttimersonly.com/) We follow the [First Timers Only](https://www.firsttimersonly.com/) philosophy of tagging issues for first timers only, and walking one newcomer through the resolution process weekly. [You can find these issues tagged with "first timers only" on our issues list.](https://github.com/chaoss/augur/labels/first-timers-only).
 
 [![standard-readme compliant](https://img.shields.io/badge/standard--readme-OK-green.svg?style=flat-square)](https://github.com/RichardLitt/standard-readme) [![Build Docker images](https://github.com/chaoss/augur/actions/workflows/build_docker.yml/badge.svg)](https://github.com/chaoss/augur/actions/workflows/build_docker.yml) [![Hits-of-Code](https://hitsofcode.com/github/chaoss/augur?branch=main)](https://hitsofcode.com/github/chaoss/augur/view?branch=main) [![CII Best Practices](https://bestpractices.coreinfrastructure.org/projects/2788/badge)](https://bestpractices.coreinfrastructure.org/projects/2788)
 
 ## NEW RELEASE ALERT!
 [If you want to jump right in, updated docker build/compose and bare metal installation instructions are available here](docs/new-install.md)
 
-Augur is now releasing a dramatically improved new version to the main branch. It is also available here: https://github.com/chaoss/augur/releases/tag/v0.43.3
+Augur is now releasing a dramatically improved new version to the main branch. It is also available here: https://github.com/chaoss/augur/releases/tag/v0.43.5
 - The `main` branch is a stable version of our new architecture, which features:
   - Dramatic improvement in the speed of large scale data collection (10,000+ repos). All data is obtained for 10k+ repos within a week
   - A new job management architecture that uses Celery and Redis to manage queues, and enables users to run a Flower job monitoring dashboard

augur/api/metrics/insight.py

@@ -26,7 +26,7 @@ def top_insights(repo_group_id, num_repos=6):
             SELECT repo_id
             FROM repo
             WHERE repo_group_id = :repo_group_id
-            AND repo_id IN (SELECT repo_id FROM repo_insights GROUP BY repo_id, ri_id HAVING 304 > count(repo_insights.repo_id) ORDER BY ri_id desc)
+            AND repo_id IN (SELECT repo_id FROM repo_insights GROUP BY repo_id, ri_id HAVING 304 > count(repo_insights.repo_id) ORDER BY ri_id desc))
             LIMIT :num_repos
         )
     """)

augur/api/metrics/message.py

@@ -1,4 +1,98 @@
 #SPDX-License-Identifier: MIT
+
 """
 Metrics that provide data about messages (of any form) & their associated activity
 """
+
+import datetime
+import sqlalchemy as s
+import pandas as pd
+from augur.api.util import register_metric
+
+from augur.application.db.engine import create_database_engine
+engine = create_database_engine()
+
+
+@register_metric()
+def repo_messages(repo_group_id, repo_id=None, period='day', begin_date=None, end_date=None):
+    """
+    Returns a timeseries of the count of persons opening an issue for the first time.
+
+    :param repo_id: The repository's id
+    :param repo_group_id: The repository's group id
+    :param period: To set the periodicity to 'day', 'week', 'month' or 'year', defaults to 'day'
+    :param begin_date: Specifies the begin date, defaults to '1970-1-1 00:00:00'
+    :param end_date: Specifies the end date, defaults to datetime.now()
+    :return: DataFrame of persons/period
+    """
+
+    if not begin_date:
+        begin_date = '1970-1-1 00:00:01'
+    if not end_date:
+        end_date = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
+
+    repomessagesSQL = None
+
+
+    if repo_id:
+
+        repomessagesSQL = s.sql.text("""
+            SELECT
+                date_trunc( :period, message.msg_timestamp :: DATE ) AS message_date,
+                COUNT ( * ),
+                repo_name 
+            FROM
+                augur_data.repo,
+                augur_data.message 
+            WHERE
+                augur_data.repo.repo_id = augur_data.message.repo_id 
+                AND
+                augur_data.repo.repo_id = :repo_id 
+                AND 
+                message.msg_timestamp BETWEEN :begin_date AND :end_date
+            GROUP BY
+                message_date,
+                repo_name 
+            ORDER BY
+                repo_name,
+                message_date
+
+        """)
+
+
+        results = pd.read_sql(repomessagesSQL, engine, params={'repo_id': repo_id, 'period': period,
+                                                                    'begin_date': begin_date, 'end_date': end_date})
+    else: 
+
+        repomessagesSQL = s.sql.text("""
+
+            SELECT
+                date_trunc( :period, message.msg_timestamp :: DATE ) AS message_date,
+                COUNT ( * ),
+                rg_name 
+            FROM
+                augur_data.repo,
+                augur_data.repo_groups,
+                augur_data.message 
+            WHERE
+                augur_data.repo.repo_id = augur_data.message.repo_id 
+                AND augur_data.repo_groups.repo_group_id = repo.repo_group_id 
+                AND
+                augur_data.repo_groups.repo_group_id = :repo_group_id 
+                AND 
+                message.msg_timestamp BETWEEN :begin_date AND :end_date
+            GROUP BY
+                message_date,
+                rg_name 
+            ORDER BY
+                rg_name,
+                message_date
+        """)
+
+        results = pd.read_sql(repomessagesSQL, engine,
+                              params={'repo_group_id': repo_group_id, 'period': period,
+                                      'begin_date': begin_date, 'end_date': end_date})
+
+    return results
+
+

augur/api/routes/util.py

@@ -39,7 +39,8 @@ def get_all_repos():
                 repo.repo_git AS url,
                 repo.repo_status,
                 a.commits_all_time,
-                b.issues_all_time ,
+                b.issues_all_time,
+                c.pull_requests_all_time,
                 rg_name,
                 repo.repo_group_id
             FROM
@@ -51,6 +52,9 @@ def get_all_repos():
                 (select * from api_get_all_repos_issues) b
                 on
                 repo.repo_id = b.repo_id
+                left outer join 
+                (select * from api_get_all_repo_prs) c 
+                on repo.repo_id=c.repo_id 
                 JOIN repo_groups ON repo_groups.repo_group_id = repo.repo_group_id
             order by repo_name
         """)
@@ -77,7 +81,8 @@ def get_repos_in_repo_group(repo_group_id):
                 repo.repo_git AS url,
                 repo.repo_status,
                 a.commits_all_time,
-                b.issues_all_time
+                b.issues_all_time,
+                c.pull_requests_all_time
             FROM
                 repo
                 left outer join
@@ -87,6 +92,9 @@ def get_repos_in_repo_group(repo_group_id):
                 (select repo_id, count ( issues.issue_id) as issues_all_time from issues where issues.pull_request IS NULL group by repo_id) b
                 on
                 repo.repo_id = b.repo_id
+                left outer join 
+                (select * from api_get_all_repo_prs) c 
+                on repo.repo_id=c.repo_id                 
                 JOIN repo_groups ON repo_groups.repo_group_id = repo.repo_group_id
             WHERE
                 repo_groups.repo_group_id = :repo_group_id

augur/application/cli/backend.py

@@ -142,7 +142,7 @@ def kill():
 def clear_redis_caches():
     """Clears the redis databases that celery and redis use."""
 
-    logger.info("Flusing all redis databases this instance was using")
+    logger.info("Flushing all redis databases this instance was using")
     celery_purge_command = "celery -A augur.tasks.init.celery_app.celery_app purge -f"
     subprocess.call(celery_purge_command.split(" "))
     redis_connection.flushdb()