From 8057c701e56e392385fbc06df719f60310d0bbe8 Mon Sep 17 00:00:00 2001 From: Maciej Moscicki Date: Wed, 21 Aug 2024 13:10:32 +0200 Subject: [PATCH] Synchronize zookeeper clusters with UI (#1888) --- hermes-console/json-server/db.json | 26 ++ hermes-console/json-server/routes.json | 1 + hermes-console/json-server/server.ts | 16 + hermes-console/src/api/hermes-client/index.ts | 42 ++ .../composables/sync/use-sync/useSync.spec.ts | 175 +++++++++ .../src/composables/sync/use-sync/useSync.ts | 116 ++++++ .../src/dummy/groupInconsistency.ts | 31 ++ hermes-console/src/i18n/en-US/index.ts | 12 + hermes-console/src/mocks/handlers.ts | 44 +++ .../consistency/useConsistencyStore.spec.ts | 43 +- .../store/consistency/useConsistencyStore.ts | 23 ++ .../inconsistent-group/InconsistentGroup.vue | 10 + .../InconsistentMetadata.spec.ts | 31 +- .../InconsistentMetadata.vue | 26 ++ .../inconsistent-topic/InconsistentTopic.vue | 30 +- .../management/api/ConsistencyEndpoint.java | 36 ++ .../consistency/DcConsistencyService.java | 100 +++++ .../consistency/SynchronizationException.java | 7 + .../domain/consistency/StorageSyncSpec.groovy | 369 ++++++++++++++++++ 19 files changed, 1134 insertions(+), 4 deletions(-) create mode 100644 hermes-console/src/composables/sync/use-sync/useSync.spec.ts create mode 100644 hermes-console/src/composables/sync/use-sync/useSync.ts create mode 100644 hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/consistency/SynchronizationException.java create mode 100644 hermes-management/src/test/groovy/pl/allegro/tech/hermes/management/domain/consistency/StorageSyncSpec.groovy diff --git a/hermes-console/json-server/db.json b/hermes-console/json-server/db.json index 8c2720872d..23dab13ee0 100644 --- a/hermes-console/json-server/db.json +++ b/hermes-console/json-server/db.json @@ -91,6 +91,32 @@ ] } ], + "inconsistentGroups3":[ + { + "name": "pl.allegro.public.group", + "inconsistentMetadata": [], + "inconsistentTopics": [ + { + "name": "pl.allegro.public.group.DummyEvent", + "inconsistentMetadata": [], + "inconsistentSubscriptions": [ + { + "name": "pl.allegro.public.group.DummyEvent$foobar-service", + "inconsistentMetadata": [ + { + "datacenter": "DC1" + }, + { + "datacenter": "DC2", + "content": "{\n \"id\": \"foobar-service\",\n \"topicName\": \"pl.allegro.public.group.DummyEvent\",\n \"name\": \"foobar-service\",\n \"endpoint\": \"service://foobar-service/events/dummy-event\",\n \"state\": \"ACTIVE\",\n \"description\": \"Test Hermes endpoint\",\n \"subscriptionPolicy\": {\n \"rate\": 10,\n \"messageTtl\": 60,\n \"messageBackoff\": 100,\n \"requestTimeout\": 1000,\n \"socketTimeout\": 0,\n \"sendingDelay\": 0,\n \"backoffMultiplier\": 1.0,\n \"backoffMaxIntervalInSec\": 600,\n \"retryClientErrors\": true,\n \"backoffMaxIntervalMillis\": 600000\n },\n \"trackingEnabled\": false,\n \"trackingMode\": \"trackingOff\",\n \"owner\": {\n \"source\": \"Service Catalog\",\n \"id\": \"42\"\n },\n \"monitoringDetails\": {\n \"severity\": \"NON_IMPORTANT\",\n \"reaction\": \"\"\n },\n \"contentType\": \"JSON\",\n \"deliveryType\": \"SERIAL\",\n \"filters\": [\n {\n \"type\": \"avropath\",\n \"path\": \"foobar\",\n \"matcher\": \"^FOO_BAR$|^BAZ_BAR$\",\n \"matchingStrategy\": \"any\"\n },\n {\n \"type\": \"avropath\",\n \"path\": \".foo.bar.baz\",\n \"matcher\": \"true\",\n \"matchingStrategy\": \"all\"\n }\n ],\n \"mode\": \"ANYCAST\",\n \"headers\": [\n {\n \"name\": \"X-My-Header\",\n \"value\": \"boobar\"\n },\n {\n \"name\": \"X-Another-Header\",\n \"value\": \"foobar\"\n }\n ],\n \"endpointAddressResolverMetadata\": {\n \"additionalMetadata\": false,\n \"nonSupportedProperty\": 2\n },\n \"http2Enabled\": false,\n \"subscriptionIdentityHeadersEnabled\": false,\n \"autoDeleteWithTopicEnabled\": false,\n \"createdAt\": 1579507131.238,\n \"modifiedAt\": 1672140855.813\n}" + } + ] + } + ] + } + ] + } + ], "topicNames": [ "pl.allegro.public.offer.product.ProductEventV1", "pl.allegro.public.offer.product.ProductEventV2", diff --git a/hermes-console/json-server/routes.json b/hermes-console/json-server/routes.json index eb0b5ae948..0a741b6490 100644 --- a/hermes-console/json-server/routes.json +++ b/hermes-console/json-server/routes.json @@ -3,6 +3,7 @@ "/consistency/groups": "/consistencyGroups", "/consistency/inconsistencies/groups?groupNames=pl.allegro.public.offer*": "/inconsistentGroups", "/consistency/inconsistencies/groups?groupNames=pl.allegro.public.group2*": "/inconsistentGroups2", + "/consistency/inconsistencies/groups?groupNames=pl.allegro.public.group": "/inconsistentGroups3", "/groups": "/groups", "/owners/sources/Service%20Catalog/:id": "/topicsOwners/:id", "/readiness/datacenters": "/readinessDatacenters", diff --git a/hermes-console/json-server/server.ts b/hermes-console/json-server/server.ts index 6c1daab3d7..0a47cb6361 100644 --- a/hermes-console/json-server/server.ts +++ b/hermes-console/json-server/server.ts @@ -87,6 +87,22 @@ server.put( }, ); +server.post( + '/consistency/sync/topics/pl.allegro.public.group.DummyEvent/subscriptions/barbaz-service*', + (req, res) => { + res.sendStatus(200); + }, +); + +server.post( + '/consistency/sync/topics/pl.allegro.public.group.DummyEvent*', + (req, res) => { + res.status(404).jsonp({ + message: 'Group pl.allegro.public.group not found', + }); + }, +); + server.post('/filters/:topic', (req, res) => { res.jsonp(filterDebug); }); diff --git a/hermes-console/src/api/hermes-client/index.ts b/hermes-console/src/api/hermes-client/index.ts index 4c4f6f89a4..23b7ca55ef 100644 --- a/hermes-console/src/api/hermes-client/index.ts +++ b/hermes-console/src/api/hermes-client/index.ts @@ -464,3 +464,45 @@ export function verifyFilters( }, ); } + +export function syncGroup( + groupName: string, + primaryDatacenter: string, +): ResponsePromise { + return axios.post(`/consistency/sync/groups/${groupName}`, null, { + params: { + primaryDatacenter: primaryDatacenter, + }, + }); +} + +export function syncTopic( + topicQualifiedName: string, + primaryDatacenter: string, +): ResponsePromise { + return axios.post( + `/consistency/sync/topics/${topicQualifiedName}`, + null, + { + params: { + primaryDatacenter: primaryDatacenter, + }, + }, + ); +} + +export function syncSubscription( + topicQualifiedName: string, + subscriptionName: string, + primaryDatacenter: string, +): ResponsePromise { + return axios.post( + `/consistency/sync/topics/${topicQualifiedName}/subscriptions/${subscriptionName}`, + null, + { + params: { + primaryDatacenter: primaryDatacenter, + }, + }, + ); +} diff --git a/hermes-console/src/composables/sync/use-sync/useSync.spec.ts b/hermes-console/src/composables/sync/use-sync/useSync.spec.ts new file mode 100644 index 0000000000..e9c57240e4 --- /dev/null +++ b/hermes-console/src/composables/sync/use-sync/useSync.spec.ts @@ -0,0 +1,175 @@ +import { afterEach, describe, expect } from 'vitest'; +import { createTestingPinia } from '@pinia/testing'; +import { + expectNotificationDispatched, + notificationStoreSpy, +} from '@/utils/test-utils'; +import { setActivePinia } from 'pinia'; +import { setupServer } from 'msw/node'; +import { + syncGroupHandler, + syncSubscriptionHandler, + syncTopicHandler, +} from '@/mocks/handlers'; +import { useSync } from '@/composables/sync/use-sync/useSync'; +import { waitFor } from '@testing-library/vue'; + +describe('useSync', () => { + const server = setupServer(); + + const pinia = createTestingPinia({ + fakeApp: true, + }); + + beforeEach(() => { + setActivePinia(pinia); + }); + + afterEach(() => { + server.resetHandlers(); + }); + + it('should show error notification when group sync fails', async () => { + // given + const groupName = 'group'; + server.use(syncGroupHandler({ groupName, statusCode: 500 })); + server.listen(); + + const notificationStore = notificationStoreSpy(); + + // when + const { syncGroup } = useSync(); + const result = await syncGroup(groupName, 'DC1'); + + // then + expect(result).toBeFalsy(); + + await waitFor(() => { + expectNotificationDispatched(notificationStore, { + type: 'error', + title: 'notifications.consistency.sync.failure', + }); + }); + }); + + it('should show error notification when topic sync fails', async () => { + // given + const topicName = 'group.topic'; + server.use(syncTopicHandler({ topicName, statusCode: 500 })); + server.listen(); + + const notificationStore = notificationStoreSpy(); + + // when + const { syncTopic } = useSync(); + const result = await syncTopic(topicName, 'DC1'); + + // then + expect(result).toBeFalsy(); + + await waitFor(() => { + expectNotificationDispatched(notificationStore, { + type: 'error', + title: 'notifications.consistency.sync.failure', + }); + }); + }); + + it('should show error notification when subscription sync fails', async () => { + // given + const topicName = 'group.topic'; + const subscriptionName = 'subscription'; + server.use( + syncSubscriptionHandler({ topicName, subscriptionName, statusCode: 500 }), + ); + server.listen(); + + const notificationStore = notificationStoreSpy(); + + // when + const { syncSubscription } = useSync(); + const result = await syncSubscription(topicName, subscriptionName, 'DC1'); + + // then + expect(result).toBeFalsy(); + + await waitFor(() => { + expectNotificationDispatched(notificationStore, { + type: 'error', + title: 'notifications.consistency.sync.failure', + }); + }); + }); + + it('should show success notification when group sync is successful', async () => { + const groupName = 'group'; + + server.use(syncGroupHandler({ groupName, statusCode: 200 })); + server.listen(); + + const notificationStore = notificationStoreSpy(); + + // when + const { syncGroup } = useSync(); + const result = await syncGroup(groupName, 'DC1'); + + // then + expect(result).toBeTruthy(); + + await waitFor(() => { + expectNotificationDispatched(notificationStore, { + type: 'success', + text: 'notifications.consistency.sync.success', + }); + }); + }); + + it('should show success notification when topic sync is successful', async () => { + // given + const topicName = 'group.topic'; + server.use(syncTopicHandler({ topicName, statusCode: 200 })); + server.listen(); + + const notificationStore = notificationStoreSpy(); + + // when + const { syncTopic } = useSync(); + const result = await syncTopic(topicName, 'DC1'); + + // then + expect(result).toBeTruthy(); + + await waitFor(() => { + expectNotificationDispatched(notificationStore, { + type: 'success', + text: 'notifications.consistency.sync.success', + }); + }); + }); + + it('should show success notification when subscription sync is successful', async () => { + // given + const topicName = 'group.topic'; + const subscriptionName = 'subscription'; + server.use( + syncSubscriptionHandler({ topicName, subscriptionName, statusCode: 200 }), + ); + server.listen(); + + const notificationStore = notificationStoreSpy(); + + // when + const { syncSubscription } = useSync(); + const result = await syncSubscription(topicName, subscriptionName, 'DC1'); + + // then + expect(result).toBeTruthy(); + + await waitFor(() => { + expectNotificationDispatched(notificationStore, { + type: 'success', + text: 'notifications.consistency.sync.success', + }); + }); + }); +}); diff --git a/hermes-console/src/composables/sync/use-sync/useSync.ts b/hermes-console/src/composables/sync/use-sync/useSync.ts new file mode 100644 index 0000000000..ff6ed46f9e --- /dev/null +++ b/hermes-console/src/composables/sync/use-sync/useSync.ts @@ -0,0 +1,116 @@ +import { dispatchErrorNotification } from '@/utils/notification-utils'; +import { + syncGroup as doSyncGroup, + syncSubscription as doSyncSubscription, + syncTopic as doSyncTopic, +} from '@/api/hermes-client'; +import { groupName } from '@/utils/topic-utils/topic-utils'; +import { useConsistencyStore } from '@/store/consistency/useConsistencyStore'; +import { useGlobalI18n } from '@/i18n'; +import { useNotificationsStore } from '@/store/app-notifications/useAppNotifications'; + +export interface UseSync { + syncGroup: (groupName: string, primaryDatacenter: string) => Promise; + syncTopic: ( + topicQualifiedName: string, + primaryDatacenter: string, + ) => Promise; + syncSubscription: ( + topicQualifiedName: string, + subscriptionName: string, + primaryDatacenter: string, + ) => Promise; +} + +export function useSync(): UseSync { + const notificationStore = useNotificationsStore(); + const consistencyStore = useConsistencyStore(); + + const syncGroup = async (groupName: string, primaryDatacenter: string) => { + try { + await doSyncGroup(groupName, primaryDatacenter); + await notificationStore.dispatchNotification({ + text: useGlobalI18n().t('notifications.consistency.sync.success', { + group: groupName, + }), + type: 'success', + }); + await consistencyStore.refresh(groupName); + return true; + } catch (e: any) { + dispatchErrorNotification( + e, + notificationStore, + useGlobalI18n().t('notifications.consistency.sync.failure', { + group: groupName, + }), + ); + return false; + } + }; + + const syncTopic = async ( + topicQualifiedName: string, + primaryDatacenter: string, + ) => { + const group = groupName(topicQualifiedName); + try { + await doSyncTopic(topicQualifiedName, primaryDatacenter); + await notificationStore.dispatchNotification({ + text: useGlobalI18n().t('notifications.consistency.sync.success', { + group, + }), + type: 'success', + }); + await consistencyStore.refresh(group); + return true; + } catch (e: any) { + dispatchErrorNotification( + e, + notificationStore, + useGlobalI18n().t('notifications.consistency.sync.failure', { + group, + }), + ); + return false; + } + }; + + const syncSubscription = async ( + topicQualifiedName: string, + subscriptionName: string, + primaryDatacenter: string, + ) => { + const group = groupName(topicQualifiedName); + try { + await doSyncSubscription( + topicQualifiedName, + subscriptionName, + primaryDatacenter, + ); + await notificationStore.dispatchNotification({ + text: useGlobalI18n().t('notifications.consistency.sync.success', { + group, + }), + type: 'success', + }); + await consistencyStore.refresh(group); + return true; + } catch (e: any) { + dispatchErrorNotification( + e, + notificationStore, + useGlobalI18n().t('notifications.consistency.sync.failure', { + group, + }), + ); + return false; + } + }; + + return { + syncGroup, + syncSubscription, + syncTopic, + }; +} diff --git a/hermes-console/src/dummy/groupInconsistency.ts b/hermes-console/src/dummy/groupInconsistency.ts index ae114b1938..2e06da4fc7 100644 --- a/hermes-console/src/dummy/groupInconsistency.ts +++ b/hermes-console/src/dummy/groupInconsistency.ts @@ -58,3 +58,34 @@ export const dummyGroupInconsistency3: InconsistentGroup[] = [ inconsistentTopics: [], }, ]; + +export const dummyGroupInconsistency4: InconsistentGroup[] = [ + { + name: 'pl.allegro.public.group', + inconsistentMetadata: [ + { + datacenter: 'DC1', + content: '{"lorem": "ipsum"}', + }, + { + datacenter: 'DC2', + content: '{"lorem": "ipsum"}', + }, + ], + inconsistentTopics: [], + }, + { + name: 'pl.allegro.public.group2', + inconsistentMetadata: [ + { + datacenter: 'DC1', + content: '{"lorem": "ipsum"}', + }, + { + datacenter: 'DC2', + content: '{"lorem": "ipsum"}', + }, + ], + inconsistentTopics: [], + }, +]; diff --git a/hermes-console/src/i18n/en-US/index.ts b/hermes-console/src/i18n/en-US/index.ts index 37ce1b5b14..01ec3f08b8 100644 --- a/hermes-console/src/i18n/en-US/index.ts +++ b/hermes-console/src/i18n/en-US/index.ts @@ -47,6 +47,12 @@ const en_US = { confirmText: "Type 'prod' to confirm action.", }, consistency: { + sync: { + header: 'Sync datacenters', + explanation: + 'Pick DC which contains correct data. Data from that DC will be propagated to other DCs.', + cta: 'Correct data is in DC:', + }, connectionError: { title: 'Connection error', text: 'Could not fetch information about consistency', @@ -708,6 +714,12 @@ const en_US = { failure: "Couldn't delete topic {topic}", }, }, + consistency: { + sync: { + success: 'Synchronization of {group} succeeded', + failure: 'Synchronization of {group} failed', + }, + }, subscription: { create: { success: 'Subscription {subscriptionName} successfully created', diff --git a/hermes-console/src/mocks/handlers.ts b/hermes-console/src/mocks/handlers.ts index d2eceac040..ebcccbb847 100644 --- a/hermes-console/src/mocks/handlers.ts +++ b/hermes-console/src/mocks/handlers.ts @@ -975,3 +975,47 @@ export const subscriptionFilterVerificationErrorHandler = ({ status: 500, }); }); + +export const syncGroupHandler = ({ + groupName, + statusCode, +}: { + groupName: string; + statusCode: number; +}) => + http.post(`${url}/consistency/sync/groups/${groupName}`, () => { + return new HttpResponse(undefined, { + status: statusCode, + }); + }); + +export const syncTopicHandler = ({ + topicName, + statusCode, +}: { + topicName: string; + statusCode: number; +}) => + http.post(`${url}/consistency/sync/topics/${topicName}`, () => { + return new HttpResponse(undefined, { + status: statusCode, + }); + }); + +export const syncSubscriptionHandler = ({ + topicName, + statusCode, + subscriptionName, +}: { + topicName: string; + subscriptionName: string; + statusCode: number; +}) => + http.post( + `${url}/consistency/sync/topics/${topicName}/subscriptions/${subscriptionName}`, + () => { + return new HttpResponse(undefined, { + status: statusCode, + }); + }, + ); diff --git a/hermes-console/src/store/consistency/useConsistencyStore.spec.ts b/hermes-console/src/store/consistency/useConsistencyStore.spec.ts index bb5e1b73d1..52dd62e082 100644 --- a/hermes-console/src/store/consistency/useConsistencyStore.spec.ts +++ b/hermes-console/src/store/consistency/useConsistencyStore.spec.ts @@ -1,6 +1,10 @@ import { beforeEach, describe, expect } from 'vitest'; import { createPinia, setActivePinia } from 'pinia'; -import { dummyGroupInconsistency } from '@/dummy/groupInconsistency'; +import { + dummyGroupInconsistency, + dummyGroupInconsistency2, + dummyGroupInconsistency4, +} from '@/dummy/groupInconsistency'; import { dummyInconsistentGroups } from '@/dummy/inconsistentGroups'; import { fetchConsistencyGroupsErrorHandler, @@ -86,4 +90,41 @@ describe('useConsistencyStore', () => { expect(consistencyStore.error.fetchError).not.toBeNull(); }); + + it('should remove group from store when refresh returns that it is consistent', async () => { + // given + server.use(fetchGroupInconsistenciesHandler({ groupsInconsistency: [] })); + server.listen(); + const consistencyStore = useConsistencyStore(); + consistencyStore.groups = dummyGroupInconsistency4; + // make a copy of initial state + const expected = JSON.parse(JSON.stringify(dummyGroupInconsistency4[1])); + + // when + await consistencyStore.refresh(dummyGroupInconsistency4[0].name); + + // then + expect(consistencyStore.groups.length).toEqual(1); + expect(consistencyStore.groups[0]).toEqual(expected); + }); + + it('should update group in store when refresh returns different value', async () => { + // given + server.use( + fetchGroupInconsistenciesHandler({ + groupsInconsistency: dummyGroupInconsistency2, + }), + ); + server.listen(); + const consistencyStore = useConsistencyStore(); + consistencyStore.groups = dummyGroupInconsistency; + // make a copy of initial state + + // when + await consistencyStore.refresh(dummyGroupInconsistency[0].name); + + // then + expect(consistencyStore.groups.length).toEqual(1); + expect(consistencyStore.groups).toEqual(dummyGroupInconsistency2); + }); }); diff --git a/hermes-console/src/store/consistency/useConsistencyStore.ts b/hermes-console/src/store/consistency/useConsistencyStore.ts index 1d18182c20..3a6bbf4a52 100644 --- a/hermes-console/src/store/consistency/useConsistencyStore.ts +++ b/hermes-console/src/store/consistency/useConsistencyStore.ts @@ -52,6 +52,29 @@ export const useConsistencyStore = defineStore('consistency', { this.fetchInProgress = false; } }, + async refresh(group: string) { + this.fetchInProgress = true; + try { + const refreshedGroup = (await fetchInconsistentGroups([group])).data; + let groupIndex = -1; + for (let i = 0; i < this.groups.length; i++) { + if (this.groups[i].name == group) { + groupIndex = i; + break; + } + } + if (groupIndex == -1) return; + if (refreshedGroup.length == 0) { + this.groups.splice(groupIndex, 1); + } else { + this.groups[groupIndex] = refreshedGroup[0]; + } + } catch (e) { + this.error.fetchError = e as Error; + } finally { + this.fetchInProgress = false; + } + }, }, getters: { group( diff --git a/hermes-console/src/views/admin/consistency/inconsistent-group/InconsistentGroup.vue b/hermes-console/src/views/admin/consistency/inconsistent-group/InconsistentGroup.vue index 8352004610..56c7b25696 100644 --- a/hermes-console/src/views/admin/consistency/inconsistent-group/InconsistentGroup.vue +++ b/hermes-console/src/views/admin/consistency/inconsistent-group/InconsistentGroup.vue @@ -2,6 +2,7 @@ import { useConsistencyStore } from '@/store/consistency/useConsistencyStore'; import { useI18n } from 'vue-i18n'; import { useRouter } from 'vue-router'; + import { useSync } from '@/composables/sync/use-sync/useSync'; import InconsistentMetadata from '@/views/admin/consistency/inconsistent-metadata/InconsistentMetadata.vue'; const router = useRouter(); @@ -13,6 +14,7 @@ >; const consistencyStore = useConsistencyStore(); + const { syncGroup } = useSync(); const group = consistencyStore.group(groupId); @@ -33,6 +35,13 @@ title: groupId, }, ]; + + async function sync(datacenter: string) { + const succeeded = await syncGroup(groupId, datacenter); + if (succeeded) { + router.push('/ui/consistency'); + } + }