From 5c8bd7e476922f8b40b8202f5c453292ba46e0c2 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Fri, 23 Apr 2021 16:13:23 -0700 Subject: [PATCH 01/18] added support for audit log proto --- google/cloud/logging_v2/entries.py | 8 +++++++- setup.py | 1 + 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/google/cloud/logging_v2/entries.py b/google/cloud/logging_v2/entries.py index 87e04201..7e641425 100644 --- a/google/cloud/logging_v2/entries.py +++ b/google/cloud/logging_v2/entries.py @@ -27,6 +27,7 @@ from google.cloud._helpers import _rfc3339_nanos_to_datetime from google.cloud._helpers import _datetime_to_rfc3339 +import google.cloud.audit.audit_log_pb2 _GLOBAL_RESOURCE = Resource(type="global", labels={}) @@ -322,7 +323,12 @@ def payload_json(self): def to_api_repr(self): """API repr (JSON format) for entry.""" info = super(ProtobufEntry, self).to_api_repr() - info["protoPayload"] = MessageToDict(self.payload) + proto_payload = None + if self.payload_json: + proto_payload = dict(self.payload_json) + elif self.payload_pb: + proto_payload = MessageToDict(self.payload_pb) + info["protoPayload"] = proto_payload return info def parse_message(self, message): diff --git a/setup.py b/setup.py index c88b4ebd..63460ee0 100644 --- a/setup.py +++ b/setup.py @@ -44,6 +44,7 @@ release_status = "Development Status :: 5 - Production/Stable" dependencies = [ "google-api-core[grpc] >= 1.22.2, < 2.0.0dev", + "google-cloud-audit-log >= 0.1.0", "google-cloud-core >= 1.4.1, < 2.0dev", "proto-plus >= 1.11.0", ] From b15b20e699f28908e9232c911bfde2f35224fafe Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Mon, 26 Apr 2021 12:10:35 -0700 Subject: [PATCH 02/18] added system test for audit logs --- tests/system/test_system.py | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/tests/system/test_system.py b/tests/system/test_system.py index e6f5aa7c..caebafa7 100644 --- a/tests/system/test_system.py +++ b/tests/system/test_system.py @@ -142,16 +142,16 @@ def tearDown(self): def _logger_name(prefix): return prefix + unique_resource_id("-") - def test_list_entry_with_unregistered(self): + def test_list_entry_with_auditlog(self): from google.protobuf import any_pb2 from google.protobuf import descriptor_pool from google.cloud.logging_v2 import entries pool = descriptor_pool.Default() type_name = "google.cloud.audit.AuditLog" - # Make sure the descriptor is not known in the registry. - with self.assertRaises(KeyError): - pool.FindMessageTypeByName(type_name) + # Make sure the descriptor is known in the registry. + # Raises KeyError if unknown + pool.FindMessageTypeByName(type_name) type_url = "type.googleapis.com/" + type_name filter_ = self.TYPE_FILTER.format(type_url) + f" AND {_time_filter}" @@ -161,13 +161,11 @@ def test_list_entry_with_unregistered(self): protobuf_entry = retry(lambda: next(entry_iter))() self.assertIsInstance(protobuf_entry, entries.ProtobufEntry) - if Config.CLIENT._use_grpc: - self.assertIsNone(protobuf_entry.payload_json) - self.assertIsInstance(protobuf_entry.payload_pb, any_pb2.Any) - self.assertEqual(protobuf_entry.payload_pb.type_url, type_url) - else: - self.assertIsNone(protobuf_entry.payload_pb) - self.assertEqual(protobuf_entry.payload_json["@type"], type_url) + self.assertIsNone(protobuf_entry.payload_pb) + self.assertIsInstance(protobuf_entry.payload_json, dict) + self.assertEqual(protobuf_entry.payload_json["@type"], type_url) + self.assertEqual(protobuf_entry.to_api_repr()['protoPayload']['@type'], type_url) + def test_log_text(self): TEXT_PAYLOAD = "System test: test_log_text" From f389c336fbc115ef95a5fa95482440d3203800f0 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Mon, 26 Apr 2021 14:49:48 -0700 Subject: [PATCH 03/18] changed payload_json logic --- google/cloud/logging_v2/entries.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/google/cloud/logging_v2/entries.py b/google/cloud/logging_v2/entries.py index 7e641425..12ee3764 100644 --- a/google/cloud/logging_v2/entries.py +++ b/google/cloud/logging_v2/entries.py @@ -317,7 +317,7 @@ def payload_pb(self): @property def payload_json(self): - if not isinstance(self.payload, Any): + if isinstance(self.payload, dict): return self.payload def to_api_repr(self): From 7a75fcab86513622d1eb5e40330a5ee656773341 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Mon, 26 Apr 2021 15:03:26 -0700 Subject: [PATCH 04/18] tests should still pass of audit log can't be found --- tests/system/test_system.py | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/tests/system/test_system.py b/tests/system/test_system.py index caebafa7..a77775dc 100644 --- a/tests/system/test_system.py +++ b/tests/system/test_system.py @@ -153,21 +153,26 @@ def test_list_entry_with_auditlog(self): # Raises KeyError if unknown pool.FindMessageTypeByName(type_name) + # retrieve log type_url = "type.googleapis.com/" + type_name filter_ = self.TYPE_FILTER.format(type_url) + f" AND {_time_filter}" entry_iter = iter(Config.CLIENT.list_entries(page_size=1, filter_=filter_)) - retry = RetryErrors(TooManyRequests) - protobuf_entry = retry(lambda: next(entry_iter))() - - self.assertIsInstance(protobuf_entry, entries.ProtobufEntry) - self.assertIsNone(protobuf_entry.payload_pb) - self.assertIsInstance(protobuf_entry.payload_json, dict) - self.assertEqual(protobuf_entry.payload_json["@type"], type_url) - self.assertEqual(protobuf_entry.to_api_repr()['protoPayload']['@type'], type_url) - - - def test_log_text(self): + try: + retry = RetryErrors(TooManyRequests) + protobuf_entry = retry(lambda: next(entry_iter))() + + self.assertIsInstance(protobuf_entry, entries.ProtobufEntry) + self.assertIsNone(protobuf_entry.payload_pb) + self.assertIsInstance(protobuf_entry.payload_json, dict) + self.assertEqual(protobuf_entry.payload_json["@type"], type_url) + self.assertEqual(protobuf_entry.to_api_repr()['protoPayload']['@type'], type_url) + except StopIteration: + # AuditLog not found in project + # We can't write these ourselves, so just pass the test + pass + + def test_log_text(self): TEXT_PAYLOAD = "System test: test_log_text" logger = Config.CLIENT.logger(self._logger_name("log_text")) self.to_delete.append(logger) From 646f717880bfd8218857cecb1d8e2c21fc8aa440 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Tue, 27 Apr 2021 12:10:26 -0700 Subject: [PATCH 05/18] system test actually creates audit log --- tests/system/test_system.py | 75 ++++++++++++++++++++++++++++--------- 1 file changed, 57 insertions(+), 18 deletions(-) diff --git a/tests/system/test_system.py b/tests/system/test_system.py index a77775dc..d90b1044 100644 --- a/tests/system/test_system.py +++ b/tests/system/test_system.py @@ -16,6 +16,7 @@ from datetime import timedelta from datetime import timezone import logging +import numbers import os import pytest import unittest @@ -36,6 +37,8 @@ from google.cloud.logging_v2 import client from google.cloud.logging_v2.resource import Resource +from google.protobuf.struct_pb2 import Struct, Value, ListValue + from test_utils.retry import RetryErrors from test_utils.retry import RetryResult from test_utils.system import unique_resource_id @@ -142,6 +145,27 @@ def tearDown(self): def _logger_name(prefix): return prefix + unique_resource_id("-") + @staticmethod + def _to_value(data): + if data is None: + return Value(null_value=NullValue.NULL_VALUE) + elif isinstance(data, numbers.Number): + return Value(number_value=data) + elif isinstance(data, str): + return Value(string_value=data) + elif isinstance(data, bool): + return Value(bool_value=data) + elif isinstance(data, (list, tuple, set)): + return Value(list_value=ListValue(values=(TestLogging._to_value(e) for e in data))) + elif isinstance(data, dict): + return Value(struct_value=TestLogging._dict_to_struct(data)) + else: + raise TypeError('Unknown data type: %r' % type(data)) + + @staticmethod + def _dict_to_struct(data): + return Struct(fields={k: TestLogging._to_value(v) for k, v in data.items()}) + def test_list_entry_with_auditlog(self): from google.protobuf import any_pb2 from google.protobuf import descriptor_pool @@ -149,30 +173,45 @@ def test_list_entry_with_auditlog(self): pool = descriptor_pool.Default() type_name = "google.cloud.audit.AuditLog" + type_url = "type.googleapis.com/" + type_name # Make sure the descriptor is known in the registry. # Raises KeyError if unknown pool.FindMessageTypeByName(type_name) + # create log + audit_dict = { + "@type": type_url, + "methodName": "test", + "requestMetadata": { + "callerIp": "::1", + "callerSuppliedUserAgent": "test" + }, + "resourceName": "test", + "serviceName": "test", + "status": {"code": 0} + } + audit_struct = self._dict_to_struct(audit_dict) + + logger = Config.CLIENT.logger("audit-proto") + logger.log_proto(audit_struct) + # retrieve log - type_url = "type.googleapis.com/" + type_name filter_ = self.TYPE_FILTER.format(type_url) + f" AND {_time_filter}" - entry_iter = iter(Config.CLIENT.list_entries(page_size=1, filter_=filter_)) - - try: - retry = RetryErrors(TooManyRequests) - protobuf_entry = retry(lambda: next(entry_iter))() - - self.assertIsInstance(protobuf_entry, entries.ProtobufEntry) - self.assertIsNone(protobuf_entry.payload_pb) - self.assertIsInstance(protobuf_entry.payload_json, dict) - self.assertEqual(protobuf_entry.payload_json["@type"], type_url) - self.assertEqual(protobuf_entry.to_api_repr()['protoPayload']['@type'], type_url) - except StopIteration: - # AuditLog not found in project - # We can't write these ourselves, so just pass the test - pass - - def test_log_text(self): + entry_iter = iter(logger.list_entries(page_size=1, filter_=filter_)) + + retry = RetryErrors(TooManyRequests) + protobuf_entry = retry(lambda: next(entry_iter))() + + self.assertIsInstance(protobuf_entry, entries.ProtobufEntry) + self.assertIsNone(protobuf_entry.payload_pb) + self.assertIsInstance(protobuf_entry.payload_json, dict) + self.assertEqual(protobuf_entry.payload_json["@type"], type_url) + self.assertEqual(protobuf_entry.payload_json["methodName"], audit_dict['methodName']) + self.assertEqual(protobuf_entry.to_api_repr()["protoPayload"]["@type"], type_url) + self.assertEqual(protobuf_entry.to_api_repr()["protoPayload"]["methodName"], audit_dict['methodName']) + + + def test_log_text(self): TEXT_PAYLOAD = "System test: test_log_text" logger = Config.CLIENT.logger(self._logger_name("log_text")) self.to_delete.append(logger) From 06cf4f5a728990f135aadad7a70c4d587cae982e Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Tue, 27 Apr 2021 12:37:43 -0700 Subject: [PATCH 06/18] improved tests --- google/cloud/logging_v2/entries.py | 2 +- tests/system/test_system.py | 2 +- tests/unit/test_entries.py | 16 +++++++++++++++- 3 files changed, 17 insertions(+), 3 deletions(-) diff --git a/google/cloud/logging_v2/entries.py b/google/cloud/logging_v2/entries.py index 12ee3764..0dd67ce2 100644 --- a/google/cloud/logging_v2/entries.py +++ b/google/cloud/logging_v2/entries.py @@ -317,7 +317,7 @@ def payload_pb(self): @property def payload_json(self): - if isinstance(self.payload, dict): + if isinstance(self.payload, collections.abc.Mapping): return self.payload def to_api_repr(self): diff --git a/tests/system/test_system.py b/tests/system/test_system.py index d90b1044..d94df2d8 100644 --- a/tests/system/test_system.py +++ b/tests/system/test_system.py @@ -199,7 +199,7 @@ def test_list_entry_with_auditlog(self): filter_ = self.TYPE_FILTER.format(type_url) + f" AND {_time_filter}" entry_iter = iter(logger.list_entries(page_size=1, filter_=filter_)) - retry = RetryErrors(TooManyRequests) + retry = RetryErrors(TooManyRequests, StopIteration) protobuf_entry = retry(lambda: next(entry_iter))() self.assertIsInstance(protobuf_entry, entries.ProtobufEntry) diff --git a/tests/unit/test_entries.py b/tests/unit/test_entries.py index ef90b815..dfa478d8 100644 --- a/tests/unit/test_entries.py +++ b/tests/unit/test_entries.py @@ -503,6 +503,21 @@ def test_to_api_repr_defaults(self): } self.assertEqual(entry.to_api_repr(), expected) + + def test_to_api_repr_struct(self): + from google.protobuf.struct_pb2 import Struct, Value + from google.cloud.logging_v2.logger import _GLOBAL_RESOURCE + + LOG_NAME = "struct.log" + message = Struct(fields={"foo": Value(bool_value=True)}) + entry = self._make_one(log_name=LOG_NAME, payload=message) + expected = { + "logName": LOG_NAME, + "jsonPayload": message, + "resource": _GLOBAL_RESOURCE._to_dict(), + } + self.assertEqual(entry.to_api_repr(), expected) + def test_to_api_repr_explicit(self): import datetime from google.cloud.logging import Resource @@ -637,7 +652,6 @@ def test_to_api_repr_proto_defaults(self): from google.cloud.logging_v2.logger import _GLOBAL_RESOURCE from google.protobuf.struct_pb2 import Struct from google.protobuf.struct_pb2 import Value - LOG_NAME = "test.log" message = Struct(fields={"foo": Value(bool_value=True)}) From 549566e20038f67254145db8cd2704297cf7466d Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Tue, 27 Apr 2021 12:45:00 -0700 Subject: [PATCH 07/18] fixed warnings --- tests/system/test_system.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/system/test_system.py b/tests/system/test_system.py index d94df2d8..afcb82f9 100644 --- a/tests/system/test_system.py +++ b/tests/system/test_system.py @@ -330,7 +330,7 @@ def test_log_handler_async(self): cloud_logger = logging.getLogger(handler.name) cloud_logger.addHandler(handler) - cloud_logger.warn(LOG_MESSAGE) + cloud_logger.warning(LOG_MESSAGE) handler.flush() entries = _list_entries(logger) expected_payload = {"message": LOG_MESSAGE, "python_logger": handler.name} @@ -352,7 +352,7 @@ def test_log_handler_sync(self): LOGGER_NAME = "mylogger" cloud_logger = logging.getLogger(LOGGER_NAME) cloud_logger.addHandler(handler) - cloud_logger.warn(LOG_MESSAGE) + cloud_logger.warning(LOG_MESSAGE) entries = _list_entries(logger) expected_payload = {"message": LOG_MESSAGE, "python_logger": LOGGER_NAME} @@ -382,7 +382,7 @@ def test_handlers_w_extras(self): "resource": Resource(type="cloudiot_device", labels={}), "labels": {"test-label": "manual"}, } - cloud_logger.warn(LOG_MESSAGE, extra=extra) + cloud_logger.warning(LOG_MESSAGE, extra=extra) entries = _list_entries(logger) self.assertEqual(len(entries), 1) @@ -403,7 +403,7 @@ def test_log_root_handler(self): self.to_delete.append(logger) google.cloud.logging.handlers.handlers.setup_logging(handler) - logging.warn(LOG_MESSAGE) + logging.warning(LOG_MESSAGE) entries = _list_entries(logger) expected_payload = {"message": LOG_MESSAGE, "python_logger": "root"} From fc360536643a4f935e4794ee28216e0015ed3bd7 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Tue, 27 Apr 2021 12:46:43 -0700 Subject: [PATCH 08/18] fixed lint issues --- tests/system/test_system.py | 27 ++++++++++++++++----------- tests/unit/test_entries.py | 2 +- 2 files changed, 17 insertions(+), 12 deletions(-) diff --git a/tests/system/test_system.py b/tests/system/test_system.py index afcb82f9..89e6c86b 100644 --- a/tests/system/test_system.py +++ b/tests/system/test_system.py @@ -156,11 +156,13 @@ def _to_value(data): elif isinstance(data, bool): return Value(bool_value=data) elif isinstance(data, (list, tuple, set)): - return Value(list_value=ListValue(values=(TestLogging._to_value(e) for e in data))) + return Value( + list_value=ListValue(values=(TestLogging._to_value(e) for e in data)) + ) elif isinstance(data, dict): return Value(struct_value=TestLogging._dict_to_struct(data)) else: - raise TypeError('Unknown data type: %r' % type(data)) + raise TypeError("Unknown data type: %r" % type(data)) @staticmethod def _dict_to_struct(data): @@ -182,13 +184,10 @@ def test_list_entry_with_auditlog(self): audit_dict = { "@type": type_url, "methodName": "test", - "requestMetadata": { - "callerIp": "::1", - "callerSuppliedUserAgent": "test" - }, + "requestMetadata": {"callerIp": "::1", "callerSuppliedUserAgent": "test"}, "resourceName": "test", "serviceName": "test", - "status": {"code": 0} + "status": {"code": 0}, } audit_struct = self._dict_to_struct(audit_dict) @@ -206,10 +205,16 @@ def test_list_entry_with_auditlog(self): self.assertIsNone(protobuf_entry.payload_pb) self.assertIsInstance(protobuf_entry.payload_json, dict) self.assertEqual(protobuf_entry.payload_json["@type"], type_url) - self.assertEqual(protobuf_entry.payload_json["methodName"], audit_dict['methodName']) - self.assertEqual(protobuf_entry.to_api_repr()["protoPayload"]["@type"], type_url) - self.assertEqual(protobuf_entry.to_api_repr()["protoPayload"]["methodName"], audit_dict['methodName']) - + self.assertEqual( + protobuf_entry.payload_json["methodName"], audit_dict["methodName"] + ) + self.assertEqual( + protobuf_entry.to_api_repr()["protoPayload"]["@type"], type_url + ) + self.assertEqual( + protobuf_entry.to_api_repr()["protoPayload"]["methodName"], + audit_dict["methodName"], + ) def test_log_text(self): TEXT_PAYLOAD = "System test: test_log_text" diff --git a/tests/unit/test_entries.py b/tests/unit/test_entries.py index dfa478d8..b8795b8c 100644 --- a/tests/unit/test_entries.py +++ b/tests/unit/test_entries.py @@ -503,7 +503,6 @@ def test_to_api_repr_defaults(self): } self.assertEqual(entry.to_api_repr(), expected) - def test_to_api_repr_struct(self): from google.protobuf.struct_pb2 import Struct, Value from google.cloud.logging_v2.logger import _GLOBAL_RESOURCE @@ -652,6 +651,7 @@ def test_to_api_repr_proto_defaults(self): from google.cloud.logging_v2.logger import _GLOBAL_RESOURCE from google.protobuf.struct_pb2 import Struct from google.protobuf.struct_pb2 import Value + LOG_NAME = "test.log" message = Struct(fields={"foo": Value(bool_value=True)}) From 568bfcfb3f572febf0b38f4342551a3eb6c73340 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Tue, 27 Apr 2021 13:15:33 -0700 Subject: [PATCH 09/18] fixed lint issues --- google/cloud/logging_v2/entries.py | 2 -- tests/system/test_system.py | 3 +-- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/google/cloud/logging_v2/entries.py b/google/cloud/logging_v2/entries.py index 0dd67ce2..0c3c8640 100644 --- a/google/cloud/logging_v2/entries.py +++ b/google/cloud/logging_v2/entries.py @@ -27,8 +27,6 @@ from google.cloud._helpers import _rfc3339_nanos_to_datetime from google.cloud._helpers import _datetime_to_rfc3339 -import google.cloud.audit.audit_log_pb2 - _GLOBAL_RESOURCE = Resource(type="global", labels={}) diff --git a/tests/system/test_system.py b/tests/system/test_system.py index 89e6c86b..b2bfce28 100644 --- a/tests/system/test_system.py +++ b/tests/system/test_system.py @@ -37,7 +37,7 @@ from google.cloud.logging_v2 import client from google.cloud.logging_v2.resource import Resource -from google.protobuf.struct_pb2 import Struct, Value, ListValue +from google.protobuf.struct_pb2 import Struct, Value, ListValue, NullValue from test_utils.retry import RetryErrors from test_utils.retry import RetryResult @@ -169,7 +169,6 @@ def _dict_to_struct(data): return Struct(fields={k: TestLogging._to_value(v) for k, v in data.items()}) def test_list_entry_with_auditlog(self): - from google.protobuf import any_pb2 from google.protobuf import descriptor_pool from google.cloud.logging_v2 import entries From 05f32835b776a0cf71ef8d8ad72052a3647b410b Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Tue, 27 Apr 2021 14:33:55 -0700 Subject: [PATCH 10/18] fixed test issues --- google/cloud/logging_v2/entries.py | 2 ++ tests/system/test_system.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/google/cloud/logging_v2/entries.py b/google/cloud/logging_v2/entries.py index 0c3c8640..70aa1f1d 100644 --- a/google/cloud/logging_v2/entries.py +++ b/google/cloud/logging_v2/entries.py @@ -27,6 +27,8 @@ from google.cloud._helpers import _rfc3339_nanos_to_datetime from google.cloud._helpers import _datetime_to_rfc3339 +import google.cloud.audit.audit_log_pb2 # noqa: F401 + _GLOBAL_RESOURCE = Resource(type="global", labels={}) diff --git a/tests/system/test_system.py b/tests/system/test_system.py index b2bfce28..0a18d14c 100644 --- a/tests/system/test_system.py +++ b/tests/system/test_system.py @@ -197,7 +197,7 @@ def test_list_entry_with_auditlog(self): filter_ = self.TYPE_FILTER.format(type_url) + f" AND {_time_filter}" entry_iter = iter(logger.list_entries(page_size=1, filter_=filter_)) - retry = RetryErrors(TooManyRequests, StopIteration) + retry = RetryErrors((TooManyRequests, StopIteration), max_tries=6) protobuf_entry = retry(lambda: next(entry_iter))() self.assertIsInstance(protobuf_entry, entries.ProtobufEntry) From 83e311102c6256d696e95fe324bfb814b0a16d7c Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Thu, 3 Jun 2021 11:54:58 -0700 Subject: [PATCH 11/18] added more proto dependencies --- google/cloud/logging_v2/entries.py | 5 ++++- setup.py | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/google/cloud/logging_v2/entries.py b/google/cloud/logging_v2/entries.py index 70aa1f1d..b32860a5 100644 --- a/google/cloud/logging_v2/entries.py +++ b/google/cloud/logging_v2/entries.py @@ -27,7 +27,10 @@ from google.cloud._helpers import _rfc3339_nanos_to_datetime from google.cloud._helpers import _datetime_to_rfc3339 -import google.cloud.audit.audit_log_pb2 # noqa: F401 +import google.cloud.audit.audit_log_pb2 # noqa: F401 +import google.cloud.appengine_logging # noqa: F401 +import google.cloud.bigquery_logging # noqa: F401 +import google.cloud.iam_logging # noqa: F401 _GLOBAL_RESOURCE = Resource(type="global", labels={}) diff --git a/setup.py b/setup.py index 63460ee0..041b68dd 100644 --- a/setup.py +++ b/setup.py @@ -44,8 +44,11 @@ release_status = "Development Status :: 5 - Production/Stable" dependencies = [ "google-api-core[grpc] >= 1.22.2, < 2.0.0dev", - "google-cloud-audit-log >= 0.1.0", + "google-cloud-appengine-logging >= 0.1.0, < 1.0.0dev", + "google-cloud-audit-log >= 0.1.0, < 1.0.0dev", + "google-cloud-bigquery-logging >- 0.1.0, < 1.0.0dev", "google-cloud-core >= 1.4.1, < 2.0dev", + "google-cloud-iam-logging >= 0.1.0, <1.0.0dev", "proto-plus >= 1.11.0", ] extras = {} From 5ce683fbbf556e4c64ffdcc4fccdc7bbddc50c0c Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Thu, 3 Jun 2021 12:45:43 -0700 Subject: [PATCH 12/18] added new system test --- setup.py | 2 +- tests/system/test_system.py | 49 +++++++++++++++++++++++++++++++++++-- 2 files changed, 48 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index 041b68dd..64d62f1c 100644 --- a/setup.py +++ b/setup.py @@ -46,7 +46,7 @@ "google-api-core[grpc] >= 1.22.2, < 2.0.0dev", "google-cloud-appengine-logging >= 0.1.0, < 1.0.0dev", "google-cloud-audit-log >= 0.1.0, < 1.0.0dev", - "google-cloud-bigquery-logging >- 0.1.0, < 1.0.0dev", + "google-cloud-bigquery-logging >= 0.1.0, < 1.0.0dev", "google-cloud-core >= 1.4.1, < 2.0dev", "google-cloud-iam-logging >= 0.1.0, <1.0.0dev", "proto-plus >= 1.11.0", diff --git a/tests/system/test_system.py b/tests/system/test_system.py index 0a18d14c..a254a9ae 100644 --- a/tests/system/test_system.py +++ b/tests/system/test_system.py @@ -197,7 +197,7 @@ def test_list_entry_with_auditlog(self): filter_ = self.TYPE_FILTER.format(type_url) + f" AND {_time_filter}" entry_iter = iter(logger.list_entries(page_size=1, filter_=filter_)) - retry = RetryErrors((TooManyRequests, StopIteration), max_tries=6) + retry = RetryErrors((TooManyRequests, StopIteration), max_tries=10) protobuf_entry = retry(lambda: next(entry_iter))() self.assertIsInstance(protobuf_entry, entries.ProtobufEntry) @@ -215,6 +215,52 @@ def test_list_entry_with_auditlog(self): audit_dict["methodName"], ) + def test_list_entry_with_requestlog(self): + from google.protobuf import descriptor_pool + from google.cloud.logging_v2 import entries + + pool = descriptor_pool.Default() + type_name = "google.appengine.logging.v1.RequestLog" + type_url = "type.googleapis.com/" + type_name + # Make sure the descriptor is known in the registry. + # Raises KeyError if unknown + pool.FindMessageTypeByName(type_name) + + # create log + req_dict = { + "@type": type_url, + "ip": "0.0.0.0", + "appId": "test", + "versionId": "test", + "requestId": "12345", + "startTime": "2021-06-02T23:15:41.225062Z", + "endTime": "2021-06-02T23:16:41.225062Z", + "latency": "500.0s", + "method": "GET", + "status": 500, + "resource": "test", + "httpVersion": "HTTP/1.1" + } + req_struct = self._dict_to_struct(req_dict) + + logger = Config.CLIENT.logger("req-proto") + logger.log_proto(req_struct) + + # retrieve log + filter_ = self.TYPE_FILTER.format(type_url) + f" AND {_time_filter}" + entry_iter = iter(logger.list_entries(page_size=1, filter_=filter_)) + + retry = RetryErrors((TooManyRequests, StopIteration), max_tries=10) + protobuf_entry = retry(lambda: next(entry_iter))() + + self.assertIsInstance(protobuf_entry, entries.ProtobufEntry) + self.assertIsNone(protobuf_entry.payload_pb) + self.assertIsInstance(protobuf_entry.payload_json, dict) + self.assertEqual(protobuf_entry.payload_json["@type"], type_url) + self.assertEqual( + protobuf_entry.to_api_repr()["protoPayload"]["@type"], type_url + ) + def test_log_text(self): TEXT_PAYLOAD = "System test: test_log_text" logger = Config.CLIENT.logger(self._logger_name("log_text")) @@ -645,7 +691,6 @@ def test_update_sink(self): self.assertEqual(sink.filter_, UPDATED_FILTER) self.assertEqual(sink.destination, dataset_uri) - class _DeleteWrapper(object): def __init__(self, publisher, topic_path): self.publisher = publisher From dc90e72b8a886c121b1886ddb1a066f6486f3457 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Thu, 3 Jun 2021 13:10:31 -0700 Subject: [PATCH 13/18] removed extra protos --- google/cloud/logging_v2/entries.py | 3 +-- setup.py | 2 -- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/google/cloud/logging_v2/entries.py b/google/cloud/logging_v2/entries.py index b32860a5..36355466 100644 --- a/google/cloud/logging_v2/entries.py +++ b/google/cloud/logging_v2/entries.py @@ -27,10 +27,9 @@ from google.cloud._helpers import _rfc3339_nanos_to_datetime from google.cloud._helpers import _datetime_to_rfc3339 +# import officially supported proto definitions import google.cloud.audit.audit_log_pb2 # noqa: F401 import google.cloud.appengine_logging # noqa: F401 -import google.cloud.bigquery_logging # noqa: F401 -import google.cloud.iam_logging # noqa: F401 _GLOBAL_RESOURCE = Resource(type="global", labels={}) diff --git a/setup.py b/setup.py index 64d62f1c..c1728507 100644 --- a/setup.py +++ b/setup.py @@ -46,9 +46,7 @@ "google-api-core[grpc] >= 1.22.2, < 2.0.0dev", "google-cloud-appengine-logging >= 0.1.0, < 1.0.0dev", "google-cloud-audit-log >= 0.1.0, < 1.0.0dev", - "google-cloud-bigquery-logging >= 0.1.0, < 1.0.0dev", "google-cloud-core >= 1.4.1, < 2.0dev", - "google-cloud-iam-logging >= 0.1.0, <1.0.0dev", "proto-plus >= 1.11.0", ] extras = {} From 66d9c62917d4147e587696c6f7a87533cd7d94f1 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Thu, 3 Jun 2021 14:53:16 -0700 Subject: [PATCH 14/18] fixed lint issues --- google/cloud/logging_v2/entries.py | 4 ++-- tests/system/test_system.py | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/google/cloud/logging_v2/entries.py b/google/cloud/logging_v2/entries.py index 36355466..fa7e5d9d 100644 --- a/google/cloud/logging_v2/entries.py +++ b/google/cloud/logging_v2/entries.py @@ -28,8 +28,8 @@ from google.cloud._helpers import _datetime_to_rfc3339 # import officially supported proto definitions -import google.cloud.audit.audit_log_pb2 # noqa: F401 -import google.cloud.appengine_logging # noqa: F401 +import google.cloud.audit.audit_log_pb2 # noqa: F401 +import google.cloud.appengine_logging # noqa: F401 _GLOBAL_RESOURCE = Resource(type="global", labels={}) diff --git a/tests/system/test_system.py b/tests/system/test_system.py index a254a9ae..e1242bf7 100644 --- a/tests/system/test_system.py +++ b/tests/system/test_system.py @@ -239,7 +239,7 @@ def test_list_entry_with_requestlog(self): "method": "GET", "status": 500, "resource": "test", - "httpVersion": "HTTP/1.1" + "httpVersion": "HTTP/1.1", } req_struct = self._dict_to_struct(req_dict) @@ -691,6 +691,7 @@ def test_update_sink(self): self.assertEqual(sink.filter_, UPDATED_FILTER) self.assertEqual(sink.destination, dataset_uri) + class _DeleteWrapper(object): def __init__(self, publisher, topic_path): self.publisher = publisher From afddbf990b83245a284c182c89f61c9f103e56d3 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Thu, 3 Jun 2021 14:53:23 -0700 Subject: [PATCH 15/18] added test comments --- tests/system/test_system.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/system/test_system.py b/tests/system/test_system.py index e1242bf7..d841136f 100644 --- a/tests/system/test_system.py +++ b/tests/system/test_system.py @@ -169,6 +169,9 @@ def _dict_to_struct(data): return Struct(fields={k: TestLogging._to_value(v) for k, v in data.items()}) def test_list_entry_with_auditlog(self): + """ + Test emitting and listing logs containing a google.cloud.audit.AuditLog proto message + """ from google.protobuf import descriptor_pool from google.cloud.logging_v2 import entries @@ -216,6 +219,9 @@ def test_list_entry_with_auditlog(self): ) def test_list_entry_with_requestlog(self): + """ + Test emitting and listing logs containing a google.appengine.logging.v1.RequestLog proto message + """ from google.protobuf import descriptor_pool from google.cloud.logging_v2 import entries From 2aa35ca4a79bb0f2268b775738d5703c218cf76b Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Tue, 8 Jun 2021 16:44:26 -0700 Subject: [PATCH 16/18] adjusted system test parameters --- tests/system/test_system.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/system/test_system.py b/tests/system/test_system.py index 14a81265..7550b199 100644 --- a/tests/system/test_system.py +++ b/tests/system/test_system.py @@ -200,7 +200,7 @@ def test_list_entry_with_auditlog(self): filter_ = self.TYPE_FILTER.format(type_url) + f" AND {_time_filter}" entry_iter = iter(logger.list_entries(page_size=1, filter_=filter_)) - retry = RetryErrors((TooManyRequests, StopIteration), max_tries=10) + retry = RetryErrors((TooManyRequests, StopIteration), max_tries=7) protobuf_entry = retry(lambda: next(entry_iter))() self.assertIsInstance(protobuf_entry, entries.ProtobufEntry) @@ -239,8 +239,6 @@ def test_list_entry_with_requestlog(self): "appId": "test", "versionId": "test", "requestId": "12345", - "startTime": "2021-06-02T23:15:41.225062Z", - "endTime": "2021-06-02T23:16:41.225062Z", "latency": "500.0s", "method": "GET", "status": 500, @@ -256,7 +254,7 @@ def test_list_entry_with_requestlog(self): filter_ = self.TYPE_FILTER.format(type_url) + f" AND {_time_filter}" entry_iter = iter(logger.list_entries(page_size=1, filter_=filter_)) - retry = RetryErrors((TooManyRequests, StopIteration), max_tries=10) + retry = RetryErrors((TooManyRequests, StopIteration), max_tries=7) protobuf_entry = retry(lambda: next(entry_iter))() self.assertIsInstance(protobuf_entry, entries.ProtobufEntry) From 2d89fa17e39a2a29fb795cecb47ac23c9f4bffbe Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Tue, 8 Jun 2021 17:12:09 -0700 Subject: [PATCH 17/18] use uuids in system tests --- tests/system/test_system.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/system/test_system.py b/tests/system/test_system.py index 7550b199..39ce0303 100644 --- a/tests/system/test_system.py +++ b/tests/system/test_system.py @@ -20,6 +20,7 @@ import os import pytest import unittest +import uuid from google.api_core.exceptions import BadGateway from google.api_core.exceptions import Conflict @@ -193,11 +194,11 @@ def test_list_entry_with_auditlog(self): } audit_struct = self._dict_to_struct(audit_dict) - logger = Config.CLIENT.logger("audit-proto") + logger = Config.CLIENT.logger(f"audit-proto-{uuid.uuid1()}") logger.log_proto(audit_struct) # retrieve log - filter_ = self.TYPE_FILTER.format(type_url) + f" AND {_time_filter}" + filter_ = self.TYPE_FILTER.format(type_url) entry_iter = iter(logger.list_entries(page_size=1, filter_=filter_)) retry = RetryErrors((TooManyRequests, StopIteration), max_tries=7) @@ -247,11 +248,11 @@ def test_list_entry_with_requestlog(self): } req_struct = self._dict_to_struct(req_dict) - logger = Config.CLIENT.logger("req-proto") + logger = Config.CLIENT.logger(f"req-proto-{uuid.uuid1()}") logger.log_proto(req_struct) # retrieve log - filter_ = self.TYPE_FILTER.format(type_url) + f" AND {_time_filter}" + filter_ = self.TYPE_FILTER.format(type_url) entry_iter = iter(logger.list_entries(page_size=1, filter_=filter_)) retry = RetryErrors((TooManyRequests, StopIteration), max_tries=7) From 2c8e1363ad3f61c6f5ac996dd6d6846981401096 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Wed, 9 Jun 2021 12:16:22 -0700 Subject: [PATCH 18/18] changed retry logic --- tests/system/test_system.py | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/tests/system/test_system.py b/tests/system/test_system.py index 39ce0303..81de866e 100644 --- a/tests/system/test_system.py +++ b/tests/system/test_system.py @@ -198,11 +198,8 @@ def test_list_entry_with_auditlog(self): logger.log_proto(audit_struct) # retrieve log - filter_ = self.TYPE_FILTER.format(type_url) - entry_iter = iter(logger.list_entries(page_size=1, filter_=filter_)) - - retry = RetryErrors((TooManyRequests, StopIteration), max_tries=7) - protobuf_entry = retry(lambda: next(entry_iter))() + retry = RetryErrors((TooManyRequests, StopIteration), max_tries=8) + protobuf_entry = retry(lambda: next(logger.list_entries()))() self.assertIsInstance(protobuf_entry, entries.ProtobufEntry) self.assertIsNone(protobuf_entry.payload_pb) @@ -252,11 +249,8 @@ def test_list_entry_with_requestlog(self): logger.log_proto(req_struct) # retrieve log - filter_ = self.TYPE_FILTER.format(type_url) - entry_iter = iter(logger.list_entries(page_size=1, filter_=filter_)) - - retry = RetryErrors((TooManyRequests, StopIteration), max_tries=7) - protobuf_entry = retry(lambda: next(entry_iter))() + retry = RetryErrors((TooManyRequests, StopIteration), max_tries=8) + protobuf_entry = retry(lambda: next(logger.list_entries()))() self.assertIsInstance(protobuf_entry, entries.ProtobufEntry) self.assertIsNone(protobuf_entry.payload_pb)