Implement pagination on transfer job listing (#402)

* implement pagination backend

* implement pagination on frontend

* fix setter

* fixes to pagination UI, use keyword args

* update models for api documentation

* add index on copy_job.owner

* set index=True on owner field

* merge

* update down revision on db migration

src/backend/api/managers/copy_job_manager.py

@@ -10,15 +10,16 @@
 
 
 @token_required
-def list(page_size=50, offset=0):
+def list(page_size=50, page=1):
     owner = get_logged_in_user(request)
     try:
-        copy_jobs = (CopyJob.query
-            .filter_by(owner=owner)
-            .order_by(CopyJob.id.desc())
-            .limit(page_size)
-            .all()
-        )
+        query = (CopyJob.query
+                 .filter_by(owner=owner)
+                 .order_by(CopyJob.id.desc())
+                 .paginate(page=page,
+                           per_page=page_size,
+                           error_out=False)
+                 )
     except Exception as e:
         import envelopes
         import os
@@ -35,11 +36,17 @@ def list(page_size=50, offset=0):
             body=str(e)
         )
         envelope.send(server, port,
-          login=os.getenv("MOTUZ_SMTP_USER"),
-          password=os.getenv("MOTUZ_SMTP_PASSWORD"), tls=use_ssl)
+                      login=os.getenv("MOTUZ_SMTP_USER"),
+                      password=os.getenv("MOTUZ_SMTP_PASSWORD"), tls=use_ssl)
         logging.exception(e, exc_info=True)
         raise HTTP_500_INTERNAL_SERVER_ERROR(str(e))
-    return copy_jobs
+
+    return {
+        'data': query.items,
+        'total': query.total,
+        'page': query.page,
+        'pages': query.pages
+    }
 
 
 @token_required
@@ -85,7 +92,7 @@ def retrieve(id):
     if copy_job.owner != owner:
         raise HTTP_404_NOT_FOUND('Copy Job with id {} not found'.format(id))
 
-    for _ in range(2): # Sometimes rabbitmq closes the connection!
+    for _ in range(2):  # Sometimes rabbitmq closes the connection!
         try:
             task = tasks.copy_job.AsyncResult(str(copy_job.id))
             copy_job.progress_text = task.info.get('text', '')
@@ -105,7 +112,7 @@ def stop(id):
     task = tasks.copy_job.AsyncResult(str(copy_job.id))
     task.revoke(terminate=True)
 
-    copy_job = CopyJob.query.get(id) # Avoid race conditions
+    copy_job = CopyJob.query.get(id)  # Avoid race conditions
     if copy_job.progress_state == 'PROGRESS':
         copy_job.progress_state = 'STOPPED'
         db.session.commit()

src/backend/api/models/copy_job.py

@@ -18,7 +18,7 @@ class CopyJob(db.Model, TimestampMixin):
     copy_links = db.Column(db.Boolean)
     notification_email = db.Column(db.String)
 
-    owner = db.Column(db.String)
+    owner = db.Column(db.String, index=True)
 
     progress_state = db.Column(db.String, nullable=True)
     progress_current = db.Column(db.Integer, nullable=True)

src/backend/api/views/copy_job_views.py

@@ -1,18 +1,14 @@
 import logging
-import random
 
 from flask import request
-from flask_restplus import Resource, Namespace, fields
+from flask_restplus import Resource, Namespace, fields, reqparse
 
-from ..managers import copy_job_manager
-from .. import tasks
 from ..exceptions import HTTP_EXCEPTION
-
-
+from ..managers import copy_job_manager
 
 api = Namespace('copy-jobs', description='CopyJob related operations')
 
-dto = api.model('copy-job', {
+job_dto = api.model('copy-job', {
     'id': fields.Integer(readonly=True, example=1234),
     'description': fields.String(required=True, example='Task Description'),
     'src_cloud_id': fields.Integer(required=False, example=1),
@@ -34,27 +30,39 @@
     'progress_execution_time': fields.Integer(readonly=True, example=3600),
 })
 
+list_dto = api.model('copy-job-list', {
+    'data': fields.List(fields.Nested(job_dto)),
+    'total': fields.Integer(example=100),
+    'page': fields.Integer(example=1),
+    'pages': fields.Integer(example=10)
+})
+
+list_arg_parser = reqparse.RequestParser()
+list_arg_parser.add_argument('page', help='Current page', type=int, default=1)
+list_arg_parser.add_argument('page_size', help='Maximum number of records to return per page', type=int, default=50)
+
 
 @api.route('/')
 class CopyJobList(Resource):
-
-    @api.marshal_list_with(dto)
+    @api.marshal_list_with(list_dto)
+    @api.expect(list_arg_parser)
     def get(self):
         """
         List all Copy Jobs
         """
         try:
-            return copy_job_manager.list()
+            return copy_job_manager.list(
+                page_size=int(request.args.get('page_size', 50)),
+                page=int(request.args.get('page', 1))
+            )
         except HTTP_EXCEPTION as e:
             api.abort(e.code, e.payload)
         except Exception as e:
             logging.exception(e, exc_info=True)
             api.abort(500, str(e))
 
-
-
-    @api.expect(dto, validate=True)
-    @api.marshal_with(dto, code=201)
+    @api.expect(job_dto, validate=True)
+    @api.marshal_with(job_dto, code=201)
     def post(self):
         """
         Create a new Copy Job
@@ -68,13 +76,11 @@ def post(self):
             api.abort(500, str(e))
 
 
-
 @api.route('/<id>')
 @api.param('id', 'The Copy Job Identifier')
 @api.response(404, 'Copy Job not found.')
 class CopyJob(Resource):
-
-    @api.marshal_with(dto, code=200)
+    @api.marshal_with(job_dto, code=200)
     def get(self, id):
         """
         Get a specific Copy Job
@@ -88,13 +94,11 @@ def get(self, id):
             api.abort(500, str(e))
 
 
-
 @api.route('/<id>/stop/')
 @api.param('id', 'The Copy Job Identifier')
 @api.response(404, 'Copy Job not found.')
 class CopyJob(Resource):
-
-    @api.marshal_with(dto, code=202)
+    @api.marshal_with(job_dto, code=202)
     def put(self, id):
         """
         Stop the Copy Job

src/backend/migrations/versions/20220608_180830_cb321d9a4a6c_.py

@@ -0,0 +1,28 @@
+"""empty message
+
+Revision ID: cb321d9a4a6c
+Revises: 51633c8a4305
+Create Date: 2022-06-08 18:08:30.379479
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = 'cb321d9a4a6c'
+down_revision = '51633c8a4305'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_index(op.f('ix_copy_job_owner'), 'copy_job', ['owner'], unique=False)
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_index(op.f('ix_copy_job_owner'), table_name='copy_job')
+    # ### end Alembic commands ###

src/frontend/js/actions/apiActions.jsx

@@ -147,9 +147,9 @@ export const _makeDirectory = (data) => ({
 });
 
 
-export const listCopyJobs = () => ({
+export const listCopyJobs = (page) => ({
     [RSAA]: {
-        endpoint: `/api/copy-jobs/`,
+        endpoint: `/api/copy-jobs/?page=${page}`,
         method: 'GET',
         headers: withAuth({ 'Content-Type': 'application/json' }),
         types: [ LIST_COPY_JOBS_REQUEST, LIST_COPY_JOBS_SUCCESS, LIST_COPY_JOBS_FAILURE ],

src/frontend/js/reducers/apiReducer.jsx

@@ -6,6 +6,7 @@ const initialState = {
     clouds: [],
     cloudErrors: {},
     jobs: [],
+    jobPages: 1,
     hashsumJobs: [],
     jobErrors: {},
     cloudConnectionVerification: {
@@ -21,11 +22,13 @@ export default (state=initialState, action) => {
         return state;
     }
     case api.LIST_COPY_JOBS_SUCCESS: {
-        const jobs = action.payload;
+        const jobs = action.payload.data;
+        const jobPages = action.payload.pages;
         jobs.sort((a, b) => b.id - a.id)
         return {
             ...state,
             jobs,
+            jobPages,
         }
     }
     case api.LIST_COPY_JOBS_FAILURE: {

src/frontend/js/views/App/CopyJobSection/CopyJobTable.jsx

@@ -1,5 +1,5 @@
 import React from 'react';
-import { ProgressBar } from 'react-bootstrap';
+import {Pagination, ProgressBar, Table} from 'react-bootstrap';
 
 import UriResource from 'components/UriResource.jsx'
 import parseTime from 'utils/parseTime.jsx'
@@ -9,7 +9,8 @@ class CopyJobTable extends React.Component {
     constructor(props) {
         super(props);
         this.timeout = null;
-        this.previousJobsInProgress = new Set()
+        this.previousJobsInProgress = new Set();
+        this.page = 1;
     }
 
     render() {
@@ -143,20 +144,64 @@ class CopyJobTable extends React.Component {
             this.scheduleRefresh(currentJobsInProgress);
         }
 
+        const pageItems = [];
+        const pages = this.props.jobPages;
+        const cutoff = 3;
+        const endCutoff = pages - cutoff;
+        const maxPages = 5;
+        const showEllipses = pages > maxPages;
+
+        if (showEllipses && this.page > cutoff) {
+            pageItems.push(
+                <Pagination.First onClick={() => this.onPageClick(1)} />,
+                <Pagination.Prev onClick={() => this.onPageClick(this.page - 1)} />,
+                <Pagination.Ellipsis disabled />,
+            )
+        }
+
+        const startPage = showEllipses ? Math.max(1, this.page - 2) : 1;
+        const endPage = showEllipses ? Math.min(pages, this.page + 2) : pages;
+        for (let page = startPage; page <= endPage; page++) {
+            pageItems.push(
+                <Pagination.Item
+                    key={page}
+                    active={page === this.page}
+                    onClick={() => this.onPageClick(page)}
+                >
+                    {page}
+                </Pagination.Item>
+            )
+        }
+
+        if (showEllipses && this.page <= endCutoff) {
+            pageItems.push(
+                <Pagination.Ellipsis disabled />,
+                <Pagination.Next onClick={() => this.onPageClick(this.page + 1)} />,
+                <Pagination.Last onClick={() => this.onPageClick(pages)} />,
+            )
+        }
+
         return (
-            <table className='table table-sm table-striped table-hover text-left'>
-                <thead>
-                    <tr>{tableHeaders}</tr>
-                </thead>
-                <tbody>
-                    {tableRows}
-                </tbody>
-            </table>
+            <>
+                {pages > 1 && (
+                    <Pagination size="sm" className='justify-content-end mr-3'>
+                        {React.Children.toArray(pageItems)}
+                    </Pagination>
+                )}
+                <Table striped hover size="sm" className='text-left'>
+                    <thead>
+                        <tr>{tableHeaders}</tr>
+                    </thead>
+                    <tbody>
+                        {tableRows}
+                    </tbody>
+                </Table>
+            </>
         );
     }
 
     componentDidMount() {
-        this.props.fetchData();
+        this.props.fetchData(this.page);
     }
 
     componentWillUnmount() {
@@ -167,10 +212,15 @@ class CopyJobTable extends React.Component {
         const refreshDelay = 1000; // 1s
         this._clearTimeout();
         this.timeout = setTimeout(() => {
-            this.props.fetchData();
+            this.props.fetchData(this.page);
         }, refreshDelay)
     }
 
+    onPageClick(page) {
+        this.page = page;
+        this.props.fetchData(this.page);
+    }
+
     _onSelectJob(selectedJob) {
         this.props.onShowDetails(selectedJob);
     }
@@ -186,8 +236,9 @@ class CopyJobTable extends React.Component {
 CopyJobTable.defaultProps = {
     id: '',
     jobs: [],
+    jobPages: 1,
     connections: [],
-    fetchData: () => {},
+    fetchData: (page) => {},
     refreshPanes: () => {},
     onShowDetails: (copyJob) => {},
 }
@@ -199,11 +250,12 @@ import { refreshPanes } from 'actions/paneActions.jsx';
 
 const mapStateToProps = state => ({
     jobs: state.api.jobs,
+    jobPages: state.api.jobPages,
     connections: state.api.clouds,
 });
 
 const mapDispatchToProps = dispatch => ({
-    fetchData: () => dispatch(listCopyJobs()),
+    fetchData: (page) => dispatch(listCopyJobs(page)),
     refreshPanes: () => dispatch(refreshPanes()),
     onShowDetails: (copyJob) => dispatch(showEditCopyJobDialog(copyJob)),
 });