From d0a1cf45fba4daafc7cf7da1259357f19ed25551 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20W=C3=BCrl?= Date: Sun, 22 Sep 2024 22:39:53 +0200 Subject: [PATCH] fix some issues --- blitzortung/__init__.py | 2 +- blitzortung/{types.py => base.py} | 0 blitzortung/cache.py | 23 ++++++------ blitzortung/data.py | 52 +++++++++++++-------------- blitzortung/dataimport/strike.py | 2 +- blitzortung/service/general.py | 3 +- blitzortung/util.py | 3 +- tests/db/test_db_mapper.py | 4 +-- tests/db/test_db_query.py | 2 +- tests/db/test_db_query_builder.py | 4 +-- tests/{test_types.py => test_base.py} | 8 ++--- tests/test_builder.py | 6 ++-- tests/test_cache.py | 15 ++++++++ tests/test_data.py | 2 +- tests/test_data_benchmark.py | 2 +- tests/test_dataimport.py | 8 ++--- tests/test_geom.py | 24 ++++++------- tests/test_util.py | 3 +- 18 files changed, 89 insertions(+), 74 deletions(-) rename blitzortung/{types.py => base.py} (100%) rename tests/{test_types.py => test_base.py} (90%) diff --git a/blitzortung/__init__.py b/blitzortung/__init__.py index e882f10..f72022b 100644 --- a/blitzortung/__init__.py +++ b/blitzortung/__init__.py @@ -55,7 +55,7 @@ class Error(Exception): from . import db from . import geom from . import util -from . import types +from . import base INJECTOR = injector.Injector( [config.ConfigModule(), db.DbModule()]) diff --git a/blitzortung/types.py b/blitzortung/base.py similarity index 100% rename from blitzortung/types.py rename to blitzortung/base.py diff --git a/blitzortung/cache.py b/blitzortung/cache.py index 7b1a3ce..54e6998 100644 --- a/blitzortung/cache.py +++ b/blitzortung/cache.py @@ -74,21 +74,14 @@ def get(self, cached_object_creator, *args, **kwargs): current_time = int(time.time()) if cache_key in self.cache: - count = 0 - if self.size is not None: - count = self.keys[cache_key] - del self.keys[cache_key] entry = self.cache[cache_key] if entry.is_valid(current_time): if self.size is not None: - self.keys[cache_key] = count + 1 + self.track_usage(cache_key) self.total_hit_count += 1 return entry.get_payload() - elif self.size is not None: - if len(self.keys) >= self.size: - expired_key = next(iter(self.keys)) - del self.keys[expired_key] - del self.cache[expired_key] + elif self.size is not None and len(self.keys) >= self.size: + self.remove_oldest_entry() expires = current_time + self.__ttl_seconds payload = cached_object_creator(*args, **kwargs) @@ -99,6 +92,16 @@ def get(self, cached_object_creator, *args, **kwargs): return entry.get_payload() + def remove_oldest_entry(self): + expired_key = next(iter(self.keys)) + del self.keys[expired_key] + del self.cache[expired_key] + + def track_usage(self, cache_key): + count = self.keys[cache_key] + del self.keys[cache_key] + self.keys[cache_key] = count + 1 + def clear(self): self.total_count = 0 self.total_hit_count = 0 diff --git a/blitzortung/data.py b/blitzortung/data.py index b7ce0f2..dcc5567 100644 --- a/blitzortung/data.py +++ b/blitzortung/data.py @@ -21,21 +21,21 @@ import datetime import math -from . import types +from . import base from .geom import GridElement -class Timestamp(types.EqualityAndHash): +class Timestamp(base.EqualityAndHash): timestamp_string_minimal_fractional_seconds_length = 20 timestamp_string_microseconds_length = 26 __slots__ = ['datetime', 'nanosecond'] - def __init__(self, date_time=datetime.datetime.utcnow().replace(tzinfo=datetime.timezone.utc), nanosecond=0): - if type(date_time) == str: + def __init__(self, date_time=datetime.datetime.now(datetime.timezone.utc), nanosecond=0): + if isinstance(date_time, str): date_time, date_time_nanosecond = Timestamp.from_timestamp(date_time) nanosecond += date_time_nanosecond - elif type(date_time) == int: + elif isinstance(date_time, int): date_time, date_time_nanosecond = Timestamp.from_nanoseconds(date_time) nanosecond += date_time_nanosecond @@ -73,7 +73,7 @@ def from_nanoseconds(total_nanoseconds): return datetime.datetime(1970, 1, 1, tzinfo=datetime.timezone.utc) + \ datetime.timedelta(seconds=total_seconds, microseconds=residual_microseconds), \ - residual_nanoseconds + residual_nanoseconds @property def year(self): @@ -107,7 +107,7 @@ def microsecond(self): def tzinfo(self): return self.datetime.tzinfo - epoch = datetime.datetime.utcfromtimestamp(0).replace(tzinfo=datetime.timezone.utc) + epoch = datetime.datetime.fromtimestamp(0, datetime.timezone.utc) @property def value(self): @@ -122,48 +122,48 @@ def __ne__(self, other): return self.datetime != other.datetime or self.nanosecond != other.nanosecond def __lt__(self, other): - if type(other) == datetime.datetime: + if isinstance(other, datetime.datetime): return self.datetime < other else: return self.datetime < other.datetime or ( self.datetime == other.datetime and self.nanosecond < other.nanosecond) def __le__(self, other): - if type(other) == datetime.datetime: + if isinstance(other, datetime.datetime): return self.datetime <= other else: return self.datetime < other.datetime or ( self.datetime == other.datetime and self.nanosecond <= other.nanosecond) def __gt__(self, other): - if type(other) == datetime.datetime: + if isinstance(other, datetime.datetime): return self.datetime > other else: return self.datetime > other.datetime or ( self.datetime == other.datetime and self.nanosecond > other.nanosecond) def __ge__(self, other): - if type(other) == datetime.datetime: + if isinstance(other, datetime.datetime): return self.datetime >= other else: return self.datetime > other.datetime or ( self.datetime == other.datetime and self.nanosecond >= other.nanosecond) def __add__(self, other): - if type(other) == Timedelta: + if isinstance(other, Timedelta): return Timestamp(self.datetime + other.timedelta, self.nanosecond + other.nanodelta) - elif type(other) == datetime.timedelta: + elif isinstance(other, datetime.timedelta): return Timestamp(self.datetime + other, self.nanosecond) - elif type(other) == int: + elif isinstance(other, int): return Timestamp(self.datetime, self.nanosecond + other) return NotImplemented def __sub__(self, other): - if type(other) == Timestamp: + if isinstance(other, Timestamp): return Timedelta(self.datetime - other.datetime, self.nanosecond - other.nanosecond) - elif type(other) == datetime.timedelta: + elif isinstance(other, datetime.timedelta): return Timestamp(self.datetime - other, self.nanosecond) - elif type(other) == int: + elif isinstance(other, int): return Timestamp(self.datetime, self.nanosecond - other) return NotImplemented @@ -185,7 +185,7 @@ def __repr__(self): NaT = Timestamp(None) -class Timedelta(types.EqualityAndHash): +class Timedelta(base.EqualityAndHash): def __init__(self, timedelta=datetime.timedelta(), nanodelta=0): if nanodelta < 0 or nanodelta > 999: microdelta = nanodelta // 1000 @@ -206,7 +206,7 @@ def __repr__(self): return "Timedelta({}, {})".format(self.timedelta, self.nanodelta) -class Event(types.Point): +class Event(base.Point): time_format = '%Y-%m-%d %H:%M:%S' time_format_fractional_seconds = time_format + '.%f' @@ -234,10 +234,10 @@ def has_same_location(self, other): @property def is_valid(self): - return (self.x != 0.0 or self.y != 0.0) \ - and -180 <= self.x <= 180 \ - and -90 < self.y < 90 \ - and self.has_valid_timestamp + return (not math.isclose(self.x, 0.0, rel_tol=1e-09, abs_tol=1e-09) or not math.isclose(self.y, 0.0, rel_tol=1e-09)) \ + and -180 <= self.x <= 180 \ + and -90 < self.y < 90 \ + and self.has_valid_timestamp @property def has_valid_timestamp(self): @@ -257,12 +257,12 @@ def __str__(self): timestamp_string = "NaT" return "%s %.4f %.4f" \ - % (timestamp_string, self.x, self.y) + % (timestamp_string, self.x, self.y) @property def uuid(self): return "%s-%05.0f-%05.0f" \ - % (str(self.timestamp().value), self.x * 100, self.y * 100) + % (str(self.timestamp().value), self.x * 100, self.y * 100) class RawWaveformEvent(Event): @@ -314,7 +314,7 @@ def __ne__(self, other): @property def is_valid(self): return super().is_valid \ - and self.number > 0 + and self.number > 0 @property def is_offline(self): diff --git a/blitzortung/dataimport/strike.py b/blitzortung/dataimport/strike.py index 1cdfc1b..25e7fb9 100644 --- a/blitzortung/dataimport/strike.py +++ b/blitzortung/dataimport/strike.py @@ -44,7 +44,7 @@ def __init__(self, data_transport: HttpFileTransport, data_url: BlitzortungDataP def get_strikes_since(self, latest_strike=None, region=1): latest_strike = latest_strike if latest_strike else \ - (datetime.datetime.utcnow() - datetime.timedelta(hours=6)).replace(tzinfo=datetime.timezone.utc) + datetime.datetime.now(datetime.timezone.utc) - datetime.timedelta(hours=6) logger.debug("import strikes since %s" % latest_strike) for url_path in self.url_path_generator.get_paths(latest_strike): diff --git a/blitzortung/service/general.py b/blitzortung/service/general.py index 96dcdd2..6ab2ed2 100644 --- a/blitzortung/service/general.py +++ b/blitzortung/service/general.py @@ -25,8 +25,7 @@ def create_time_interval(minute_length, minute_offset): - end_time = datetime.datetime.utcnow() - end_time = end_time.replace(tzinfo=datetime.timezone.utc) + end_time = datetime.datetime.now(datetime.timezone.utc) end_time = end_time.replace(microsecond=0) end_time += datetime.timedelta(minutes=minute_offset) start_time = end_time - datetime.timedelta(minutes=minute_length) diff --git a/blitzortung/util.py b/blitzortung/util.py index 0e2476e..a5485e2 100644 --- a/blitzortung/util.py +++ b/blitzortung/util.py @@ -90,8 +90,7 @@ def time_intervals(start_time, duration, end_time=None): """ current_time = round_time(start_time, duration) if not end_time: - end_time = datetime.datetime.utcnow() - end_time = end_time.replace(tzinfo=datetime.timezone.utc) + end_time = datetime.datetime.now(datetime.timezone.utc) end_time = round_time(end_time, duration) while current_time <= end_time: diff --git a/tests/db/test_db_mapper.py b/tests/db/test_db_mapper.py index 9094ab6..986c767 100644 --- a/tests/db/test_db_mapper.py +++ b/tests/db/test_db_mapper.py @@ -39,7 +39,7 @@ def setUp(self): self.strike_builder = Mock(name="strike_builder", spec=blitzortung.builder.Strike) self.strike_mapper = blitzortung.db.mapper.Strike(self.strike_builder) - self.timestamp = datetime.datetime.utcnow().replace(tzinfo=datetime.timezone.utc) + self.timestamp = datetime.datetime.now(datetime.timezone.utc) self.result = { 'id': 12, 'timestamp': self.timestamp, @@ -92,7 +92,7 @@ def setUp(self): self.station_builder = Mock(name="station_builder", spec=blitzortung.builder.Station) self.strike_mapper = blitzortung.db.mapper.Station(self.station_builder) - self.timestamp = datetime.datetime.utcnow().replace(tzinfo=datetime.timezone.utc) + self.timestamp = datetime.datetime.now(datetime.timezone.utc) self.result = { 'number': 31, 'user': '', diff --git a/tests/db/test_db_query.py b/tests/db/test_db_query.py index c1c11d6..fca138b 100644 --- a/tests/db/test_db_query.py +++ b/tests/db/test_db_query.py @@ -95,7 +95,7 @@ def test_exception_when_start_is_not_integer(self): def test_exception_when_end_is_not_integer(self): with pytest.raises(ValueError): - blitzortung.db.query.TimeInterval(datetime.datetime.utcnow(), "asdf") + blitzortung.db.query.TimeInterval(datetime.datetime.now(datetime.timezone.utc), "asdf") class QueryTest(TestCase): diff --git a/tests/db/test_db_query_builder.py b/tests/db/test_db_query_builder.py index 28e09a3..b454fae 100644 --- a/tests/db/test_db_query_builder.py +++ b/tests/db/test_db_query_builder.py @@ -31,7 +31,7 @@ class StrikeTest(object): def setUp(self): self.query_builder = blitzortung.db.query_builder.Strike() - self.end_time = datetime.datetime.utcnow().replace(tzinfo=datetime.timezone.utc, second=0, microsecond=0) + self.end_time = datetime.datetime.now(datetime.timezone.utc).replace(second=0, microsecond=0) self.start_time = self.end_time - datetime.timedelta(minutes=10) self.srid = 1234 @@ -97,7 +97,7 @@ def test_grid_query_with_count_threshold(self): class StrikeClusterTest(object): def setUp(self): self.query_builder = blitzortung.db.query_builder.StrikeCluster() - self.end_time = datetime.datetime.utcnow().replace(tzinfo=datetime.timezone.utc, second=0, microsecond=0) + self.end_time = datetime.datetime.now(datetime.timezone.utc).replace(second=0, microsecond=0) self.interval_duration = datetime.timedelta(minutes=10) self.srid = "" diff --git a/tests/test_types.py b/tests/test_base.py similarity index 90% rename from tests/test_types.py rename to tests/test_base.py index 6cdf480..d20a9ab 100644 --- a/tests/test_types.py +++ b/tests/test_base.py @@ -20,14 +20,14 @@ import math -import blitzortung.types +import blitzortung.base class PointTest(object): def setUp(self): - self.point1 = blitzortung.types.Point(11, 49) - self.point2 = blitzortung.types.Point(12, 49) - self.point3 = blitzortung.types.Point(11, 50) + self.point1 = blitzortung.base.Point(11, 49) + self.point2 = blitzortung.base.Point(12, 49) + self.point3 = blitzortung.base.Point(11, 50) self.radians_factor = math.pi / 180 def test_get_coordinate_components(self): diff --git a/tests/test_builder.py b/tests/test_builder.py index a7cb7cc..a3897eb 100644 --- a/tests/test_builder.py +++ b/tests/test_builder.py @@ -151,7 +151,7 @@ def test_set_id(self): self.builder.set_x(0.0) self.builder.set_y(0.0) - self.builder.set_timestamp(datetime.datetime.utcnow()) + self.builder.set_timestamp(datetime.datetime.now(datetime.timezone.utc)) self.builder.set_amplitude(1.0) self.builder.set_lateral_error(5.0) self.builder.set_station_count(10) @@ -159,7 +159,7 @@ def test_set_id(self): assert_that(self.builder.build().id).is_equal_to(1234) def test_set_timestamp(self): - timestamp = datetime.datetime.utcnow() + timestamp = datetime.datetime.now(datetime.timezone.utc) self.builder.set_timestamp(timestamp) self.builder.set_x(0.0) @@ -270,7 +270,7 @@ def test_build_station_offline(self): self.builder.set_id(364) self.builder.set_number(123) - end = datetime.datetime.utcnow() - datetime.timedelta(hours=1) + end = datetime.datetime.now(datetime.timezone.utc) - datetime.timedelta(hours=1) begin = end - datetime.timedelta(hours=1) self.builder.set_begin(begin) self.builder.set_end(end) diff --git a/tests/test_cache.py b/tests/test_cache.py index 9c1dca8..8e092bd 100644 --- a/tests/test_cache.py +++ b/tests/test_cache.py @@ -194,3 +194,18 @@ def test_auto_expiry(self): time.sleep(1) _ = self.cache.get(CachedObject, name="bar") assert_that(self.cache.get_size()).is_equal_to(1) + +def test_bench_object_cache_get(benchmark): + cache = ObjectCache() + benchmark.pedantic(cache.get, args=(CachedObject, "foo"), rounds=1000, iterations=100) + +def test_bench_object_cache_with_size_get(benchmark): + cache = ObjectCache(size=2) + benchmark.pedantic(cache.get, args=(CachedObject, "foo"), rounds=1000, iterations=100) + +def test_bench_object_cache_generate_cache_key(benchmark): + cache = ObjectCache() + benchmark.pedantic(cache.generate_cache_key, args=(CachedObject, ("foo", "bar"), {"baz": "asdf", "qux": "quux"}), rounds=1000, iterations=100) + + print("hit count", cache.total_hit_count) + diff --git a/tests/test_data.py b/tests/test_data.py index 5ba8df2..09c309c 100644 --- a/tests/test_data.py +++ b/tests/test_data.py @@ -223,7 +223,7 @@ def test_get_status(self): class TestGridData(object): def setup_method(self): - self.reference_time = datetime.datetime.utcnow() + self.reference_time = datetime.datetime.now(datetime.timezone.utc) self.grid = blitzortung.geom.Grid(-5, 4, -3, 2, 0.5, 1.25) self.grid_data = blitzortung.data.GridData(self.grid) diff --git a/tests/test_data_benchmark.py b/tests/test_data_benchmark.py index 100290c..fbb4f80 100644 --- a/tests/test_data_benchmark.py +++ b/tests/test_data_benchmark.py @@ -2,7 +2,7 @@ from blitzortung.data import Timestamp, Event from blitzortung.geom import Envelope -from blitzortung.types import Point +from blitzortung.base import Point @pytest.fixture diff --git a/tests/test_dataimport.py b/tests/test_dataimport.py index 44d2f83..3bc1374 100644 --- a/tests/test_dataimport.py +++ b/tests/test_dataimport.py @@ -139,7 +139,7 @@ def setUp(self): self.provider.read_data = Mock() def test_get_strikes_since(self): - now = datetime.datetime.utcnow() + now = datetime.datetime.now(datetime.timezone.utc) latest_strike_timestamp = now - datetime.timedelta(hours=1) self.url_generator.get_paths.return_value = ['path1', 'path2'] strike_data1 = {'one': 1} @@ -157,7 +157,7 @@ def test_get_strikes_since(self): assert_that(list(strikes)).contains(strike2) def test_get_strikes_since_with_builder_error(self): - now = datetime.datetime.utcnow() + now = datetime.datetime.now(datetime.timezone.utc) latest_strike_timestamp = now - datetime.timedelta(hours=1) self.url_generator.get_paths.return_value = ['path'] strike_data = {'one': 1} @@ -170,7 +170,7 @@ def test_get_strikes_since_with_builder_error(self): assert_that(strikes).is_empty() def test_get_strikes_since_with_generic_exception(self): - now = datetime.datetime.utcnow() + now = datetime.datetime.now(datetime.timezone.utc) latest_strike_timestamp = now - datetime.timedelta(hours=1) self.url_generator.get_url_paths.return_value = ['path'] strike_data = {'one': 1} @@ -236,7 +236,7 @@ def setUp(self): ) def test_get_raw_data_since(self): - last_data = datetime.datetime.utcnow() - datetime.timedelta(hours=1) + last_data = datetime.datetime.now(datetime.timezone.utc) - datetime.timedelta(hours=1) self.data_url.build_path.side_effect = ['full_url1', 'full_url2'] self.url_path_generator.get_paths.return_value = ['url_path1', 'url_path2'] diff --git a/tests/test_geom.py b/tests/test_geom.py index 13c6be2..9206376 100644 --- a/tests/test_geom.py +++ b/tests/test_geom.py @@ -26,7 +26,7 @@ from assertpy import assert_that import blitzortung.geom -import blitzortung.types +import blitzortung.base class GeometryForTest(blitzortung.geom.Geometry): @@ -75,21 +75,21 @@ def test_get_envelope_parameters(self): assert_that(self.envelope.y_delta).is_equal_to(5) def test_contains_point_inside_envelope(self): - self.assertTrue(self.envelope.contains(blitzortung.types.Point(0, 0))) - self.assertTrue(self.envelope.contains(blitzortung.types.Point(1, 1.5))) - self.assertTrue(self.envelope.contains(blitzortung.types.Point(-1, -1.5))) + self.assertTrue(self.envelope.contains(blitzortung.base.Point(0, 0))) + self.assertTrue(self.envelope.contains(blitzortung.base.Point(1, 1.5))) + self.assertTrue(self.envelope.contains(blitzortung.base.Point(-1, -1.5))) def test_contains_point_on_border(self): - self.assertTrue(self.envelope.contains(blitzortung.types.Point(0, -3))) - self.assertTrue(self.envelope.contains(blitzortung.types.Point(0, 2))) - self.assertTrue(self.envelope.contains(blitzortung.types.Point(-5, 0))) - self.assertTrue(self.envelope.contains(blitzortung.types.Point(4, 0))) + self.assertTrue(self.envelope.contains(blitzortung.base.Point(0, -3))) + self.assertTrue(self.envelope.contains(blitzortung.base.Point(0, 2))) + self.assertTrue(self.envelope.contains(blitzortung.base.Point(-5, 0))) + self.assertTrue(self.envelope.contains(blitzortung.base.Point(4, 0))) def test_does_not_contain_point_outside_border(self): - self.assertFalse(self.envelope.contains(blitzortung.types.Point(0, -3.0001))) - self.assertFalse(self.envelope.contains(blitzortung.types.Point(0, 2.0001))) - self.assertFalse(self.envelope.contains(blitzortung.types.Point(-5.0001, 0))) - self.assertFalse(self.envelope.contains(blitzortung.types.Point(4.0001, 0))) + self.assertFalse(self.envelope.contains(blitzortung.base.Point(0, -3.0001))) + self.assertFalse(self.envelope.contains(blitzortung.base.Point(0, 2.0001))) + self.assertFalse(self.envelope.contains(blitzortung.base.Point(-5.0001, 0))) + self.assertFalse(self.envelope.contains(blitzortung.base.Point(4.0001, 0))) def test_get_env(self): expected_env = shapely.geometry.LinearRing([(-5, -3), (-5, 2), (4, 2), (4, -3)]) diff --git a/tests/test_util.py b/tests/test_util.py index 62e5b48..93d14e6 100644 --- a/tests/test_util.py +++ b/tests/test_util.py @@ -69,8 +69,7 @@ def test_time_intervals_generator_before_start_of_interval(self): ) def test_time_intervals_generator_at_current_time(self): - end_time = datetime.datetime.utcnow() - end_time = end_time.replace(tzinfo=datetime.timezone.utc) + end_time = datetime.datetime.now(datetime.timezone.utc) start_time = end_time - self.duration times = [time for time in blitzortung.util.time_intervals(start_time, self.duration)]