Merge pull request #370 from ror-community/staging

Merge staging to prod: Add dump index endpoint

rorapi/common/matching.py

@@ -308,8 +308,6 @@ def match_by_type(text, matching_type, countries, version):
         elif matching_type == MATCHING_TYPE_HEURISTICS:
             q.add_common_query(fields, normalize(text))
     queries = [q.get_query() for q in queries]
-    for q in queries:
-        print(q.to_dict())
     matched = [
         match_by_query(t, matching_type, q, countries, version)
         for t, q in zip(substrings, queries)
@@ -404,7 +402,6 @@ def __init__(self, affiliation, version):
             # do not perform search if substring exactly matches a country name or ISO code
             if do_not_match == False:
                 n = MatchingNode(part_cleaned, self.version)
-                print(n)
                 self.nodes.append(n)
 
     def remove_low_scores(self, min_score):

rorapi/common/urls.py

@@ -3,7 +3,7 @@
 from rest_framework.documentation import include_docs_urls
 
 from  . import views
-from rorapi.common.views import HeartbeatView,GenerateAddress,GenerateId,IndexData
+from rorapi.common.views import HeartbeatView,GenerateAddress,GenerateId,IndexData,IndexDataDump
 
 urlpatterns = [
     # Health check
@@ -13,6 +13,7 @@
     path('generateaddress/<str:geonamesid>', GenerateAddress.as_view()),
     url(r"^generateid$", GenerateId.as_view()),
     path('indexdata/<str:branch>', IndexData.as_view()),
+    url(r"^(?P<version>(v1|v2))\/indexdatadump\/(?P<filename>v(\d+\.)?(\d+\.)?(\*|\d+)-\d{4}-\d{2}-\d{2}-ror-data)\/(?P<dataenv>(test|prod))$", IndexDataDump.as_view()),
     url(r"^(?P<version>(v1|v2))\/", include(views.organizations_router.urls)),
     url(r"^", include(views.organizations_router.urls)),
     url(r"^docs/", include_docs_urls(title="Research Organization Registry")),

rorapi/common/views.py

@@ -31,8 +31,12 @@
 import os
 import update_address as ua
 from rorapi.management.commands.generaterorid import check_ror_id
+from rorapi.management.commands.generaterorid import check_ror_id
 from rorapi.management.commands.indexror import process_files
 
+from django.core import management
+import rorapi.management.commands.indexrordump
+
 
 class OrganizationViewSet(viewsets.ViewSet):
     lookup_value_regex = r"((https?(:\/\/|%3A%2F%2F))?ror\.org(\/|%2F))?.*"
@@ -128,9 +132,9 @@ class GenerateId(APIView):
 
     def get(self, request, version=REST_FRAMEWORK["DEFAULT_VERSION"]):
         id = check_ror_id(version)
+        print("Generated ID: {}".format(id))
         return Response({"id": id})
 
-
 class IndexData(APIView):
     permission_classes = [OurTokenPermission]
 
@@ -140,3 +144,21 @@ def get(self, request, branch):
         if msg["status"] == "ERROR":
             st = 400
         return Response({"status": msg["status"], "msg": msg["msg"]}, status=st)
+
+
+class IndexDataDump(APIView):
+    permission_classes = [OurTokenPermission]
+
+    def get(self, request, filename, dataenv, version=REST_FRAMEWORK["DEFAULT_VERSION"]):
+        schema = 1
+        testdata = True
+        st = 200
+        if version == 'v2':
+            schema = 2
+        if dataenv == 'prod':
+            testdata = False
+        msg = management.call_command("setup", filename, schema=schema, testdata=testdata)
+        if 'ERROR' in msg:
+            st = 400
+
+        return Response({"status": msg}, status=st)

rorapi/management/commands/setup.py

@@ -37,6 +37,7 @@ def add_arguments(self, parser):
         parser.add_argument('-t', '--testdata', action='store_true', help='Set flag to pull data dump from ror-data-test instead of ror-data')
 
     def handle(self, *args, **options):
+        msg = None
         # make sure ROR dump file exists
         filename = options['filename']
         use_test_data = options['testdata']
@@ -51,10 +52,11 @@ def handle(self, *args, **options):
             DeleteIndexCommand().handle(*args, **options)
             CreateIndexCommand().handle(*args, **options)
             IndexRorDumpCommand().handle(*args, **options)
-
+            msg = 'SUCCESS: ROR dataset {} indexed in version {}. Using test repo: {}'.format(filename, str(options['schema']), str(use_test_data))
         else:
-            self.stdout.write('ROR dataset for file {} not found. '.
-                              format(filename) +
-                              'Please generate the data dump first.')
-            return
+            msg = 'ERROR: ROR dataset for file {} not found. '.format(filename) \
+                +'Please generate the data dump first.'
+            self.stdout.write(msg)
+
+        return msg
 

rorapi/tests/tests_unit/tests_views_v1.py

@@ -247,4 +247,32 @@ def test_heartbeat_success(self, search_mock):
         search_mock.return_value = \
             IterableAttrDict(self.test_data, self.test_data['hits']['hits'])
         response = self.client.get('/heartbeat')
-        self.assertEquals(response.status_code, 200)
+        self.assertEquals(response.status_code, 200)
+
+class IndexRorDumpViewTestCase(SimpleTestCase):
+    def setUp(self):
+        self.success_msg = "SUCCESS: ROR dataset vX.XX-XXXX-XX-XX-ror-data indexed in version X. Using test repo: X"
+        self.error_msg = "ERROR: ROR dataset for file vX.XX-XXXX-XX-XX-ror-data not found. Please generate the data dump first."
+        self.maxDiff = None
+
+    @mock.patch('rorapi.common.views.OurTokenPermission.has_permission')
+    @mock.patch('django.core.management.call_command')
+    def test_index_ror_success(self, index_mock, permission_mock):
+        index_mock.return_value = self.success_msg
+        permission_mock.return_value = True
+        response = self.client.get('/v1/indexdatadump/v1.1-1111-11-11-ror-data/prod')
+        self.assertEquals(response.status_code, 200)
+
+    @mock.patch('rorapi.common.views.OurTokenPermission.has_permission')
+    @mock.patch('django.core.management.call_command')
+    def test_index_ror_fail_error(self, index_mock, permission_mock):
+        index_mock.return_value = self.error_msg
+        permission_mock.return_value = True
+        response = self.client.get('/v1/indexdatadump/v1.1-1111-11-11-ror-data/prod')
+        self.assertEquals(response.status_code, 400)
+
+    @mock.patch('rorapi.common.views.OurTokenPermission.has_permission')
+    def test_index_ror_fail_no_permission(self, permission_mock):
+        permission_mock.return_value = False
+        response = self.client.get('/v1/indexdatadump/v1.1-1111-11-11-ror-data/prod')
+        self.assertEquals(response.status_code, 403)

rorapi/tests/tests_unit/tests_views_v2.py

@@ -243,4 +243,32 @@ def test_heartbeat_success(self, search_mock):
         search_mock.return_value = \
             IterableAttrDict(self.test_data, self.test_data['hits']['hits'])
         response = self.client.get('/v2/heartbeat')
-        self.assertEquals(response.status_code, 200)
+        self.assertEquals(response.status_code, 200)
+
+class IndexRorDumpViewTestCase(SimpleTestCase):
+    def setUp(self):
+        self.success_msg = "SUCCESS: ROR dataset vX.XX-XXXX-XX-XX-ror-data indexed in version X. Using test repo: X"
+        self.error_msg = "ERROR: ROR dataset for file vX.XX-XXXX-XX-XX-ror-data not found. Please generate the data dump first."
+        self.maxDiff = None
+
+    @mock.patch('rorapi.common.views.OurTokenPermission.has_permission')
+    @mock.patch('django.core.management.call_command')
+    def test_index_ror_success(self, index_mock, permission_mock):
+        index_mock.return_value = self.success_msg
+        permission_mock.return_value = True
+        response = self.client.get('/v2/indexdatadump/v1.1-1111-11-11-ror-data/prod')
+        self.assertEquals(response.status_code, 200)
+
+    @mock.patch('rorapi.common.views.OurTokenPermission.has_permission')
+    @mock.patch('django.core.management.call_command')
+    def test_index_ror_fail_error(self, index_mock, permission_mock):
+        index_mock.return_value = self.error_msg
+        permission_mock.return_value = True
+        response = self.client.get('/v2/indexdatadump/v1.1-1111-11-11-ror-data/prod')
+        self.assertEquals(response.status_code, 400)
+
+    @mock.patch('rorapi.common.views.OurTokenPermission.has_permission')
+    def test_index_ror_fail_no_permission(self, permission_mock):
+        permission_mock.return_value = False
+        response = self.client.get('/v2/indexdatadump/v1.1-1111-11-11-ror-data/prod')
+        self.assertEquals(response.status_code, 403)