diff --git a/metis-authentication/metis-authentication-common/pom.xml b/metis-authentication/metis-authentication-common/pom.xml index cb7145848..2c4e708fa 100644 --- a/metis-authentication/metis-authentication-common/pom.xml +++ b/metis-authentication/metis-authentication-common/pom.xml @@ -4,7 +4,7 @@ metis-authentication eu.europeana.metis - 3 + 4 metis-authentication-common @@ -19,11 +19,6 @@ metis-common-zoho ${project.version} - - eu.europeana.metis - metis-common-network - ${project.version} - javax.annotation javax.annotation-api @@ -41,12 +36,10 @@ org.junit.jupiter junit-jupiter-api - test org.junit.jupiter junit-jupiter-engine - test \ No newline at end of file diff --git a/metis-authentication/metis-authentication-common/src/main/java/eu/europeana/metis/authentication/utils/MetisZohoOAuthPSQLHandler.java b/metis-authentication/metis-authentication-common/src/main/java/eu/europeana/metis/authentication/utils/MetisZohoOAuthPSQLHandler.java index 2f5ba324d..8d6018f10 100644 --- a/metis-authentication/metis-authentication-common/src/main/java/eu/europeana/metis/authentication/utils/MetisZohoOAuthPSQLHandler.java +++ b/metis-authentication/metis-authentication-common/src/main/java/eu/europeana/metis/authentication/utils/MetisZohoOAuthPSQLHandler.java @@ -1,7 +1,7 @@ package eu.europeana.metis.authentication.utils; -import static eu.europeana.metis.network.SonarqubeNullcheckAvoidanceUtils.performAction; -import static eu.europeana.metis.network.SonarqubeNullcheckAvoidanceUtils.performThrowingFunction; +import static eu.europeana.metis.utils.SonarqubeNullcheckAvoidanceUtils.performAction; +import static eu.europeana.metis.utils.SonarqubeNullcheckAvoidanceUtils.performThrowingFunction; import com.zoho.oauth.client.ZohoPersistenceHandler; import com.zoho.oauth.common.ZohoOAuthException; diff --git a/metis-authentication/metis-authentication-rest-client/pom.xml b/metis-authentication/metis-authentication-rest-client/pom.xml index af12396ee..473839d91 100644 --- a/metis-authentication/metis-authentication-rest-client/pom.xml +++ b/metis-authentication/metis-authentication-rest-client/pom.xml @@ -4,7 +4,7 @@ metis-authentication eu.europeana.metis - 3 + 4 metis-authentication-rest-client @@ -42,13 +42,10 @@ org.junit.jupiter junit-jupiter-api - test org.mockito mockito-core - ${version.mockito.core} - test \ No newline at end of file diff --git a/metis-authentication/metis-authentication-rest/pom.xml b/metis-authentication/metis-authentication-rest/pom.xml index a9020aebd..ef9621d8b 100644 --- a/metis-authentication/metis-authentication-rest/pom.xml +++ b/metis-authentication/metis-authentication-rest/pom.xml @@ -4,7 +4,7 @@ metis-authentication eu.europeana.metis - 3 + 4 metis-authentication-rest war @@ -67,13 +67,10 @@ org.mockito mockito-core - ${version.mockito.core} - test org.junit.jupiter junit-jupiter-api - test org.hamcrest @@ -89,7 +86,6 @@ com.jayway.jsonpath json-path - ${version.json.path} test diff --git a/metis-authentication/metis-authentication-rest/src/main/java/eu/europeana/metis/authentication/rest/config/Application.java b/metis-authentication/metis-authentication-rest/src/main/java/eu/europeana/metis/authentication/rest/config/Application.java index 277917ae9..b52a40ea1 100644 --- a/metis-authentication/metis-authentication-rest/src/main/java/eu/europeana/metis/authentication/rest/config/Application.java +++ b/metis-authentication/metis-authentication-rest/src/main/java/eu/europeana/metis/authentication/rest/config/Application.java @@ -1,6 +1,6 @@ package eu.europeana.metis.authentication.rest.config; -import static eu.europeana.metis.network.SonarqubeNullcheckAvoidanceUtils.performAction; +import static eu.europeana.metis.utils.SonarqubeNullcheckAvoidanceUtils.performAction; import com.zoho.oauth.common.ZohoOAuthException; import eu.europeana.metis.authentication.dao.PsqlMetisUserDao; diff --git a/metis-authentication/metis-authentication-rest/src/main/resources/hibernate.cfg.xml.example b/metis-authentication/metis-authentication-rest/src/main/resources/hibernate.cfg.xml.example index d066ad979..edbd123c0 100644 --- a/metis-authentication/metis-authentication-rest/src/main/resources/hibernate.cfg.xml.example +++ b/metis-authentication/metis-authentication-rest/src/main/resources/hibernate.cfg.xml.example @@ -5,7 +5,9 @@ org.postgresql.Driver - jdbc:postgresql://exampl.location.of.psql.server:port/database + + jdbc:postgresql://exampl.location.of.psql.server:port/database?ssl=true&sslfactory=org.postgresql.ssl.DefaultJavaSSLFactory&sslmode=verify-ca org.hibernate.dialect.PostgreSQL82Dialect username password diff --git a/metis-authentication/metis-authentication-service/pom.xml b/metis-authentication/metis-authentication-service/pom.xml index 1008fe943..ca739c740 100644 --- a/metis-authentication/metis-authentication-service/pom.xml +++ b/metis-authentication/metis-authentication-service/pom.xml @@ -4,7 +4,7 @@ metis-authentication eu.europeana.metis - 3 + 4 metis-authentication-service @@ -27,18 +27,14 @@ org.mockito mockito-core - ${version.mockito.core} - test org.junit.jupiter junit-jupiter-api - test org.junit.jupiter junit-jupiter-engine - test \ No newline at end of file diff --git a/metis-authentication/metis-authentication-service/src/main/java/eu/europeana/metis/authentication/dao/PsqlMetisUserDao.java b/metis-authentication/metis-authentication-service/src/main/java/eu/europeana/metis/authentication/dao/PsqlMetisUserDao.java index 9983a8b3d..d61ac6c0c 100644 --- a/metis-authentication/metis-authentication-service/src/main/java/eu/europeana/metis/authentication/dao/PsqlMetisUserDao.java +++ b/metis-authentication/metis-authentication-service/src/main/java/eu/europeana/metis/authentication/dao/PsqlMetisUserDao.java @@ -1,7 +1,7 @@ package eu.europeana.metis.authentication.dao; -import static eu.europeana.metis.network.SonarqubeNullcheckAvoidanceUtils.performAction; -import static eu.europeana.metis.network.SonarqubeNullcheckAvoidanceUtils.performFunction; +import static eu.europeana.metis.utils.SonarqubeNullcheckAvoidanceUtils.performAction; +import static eu.europeana.metis.utils.SonarqubeNullcheckAvoidanceUtils.performFunction; import eu.europeana.metis.authentication.user.AccountRole; import eu.europeana.metis.authentication.user.MetisUserAccessToken; diff --git a/metis-authentication/pom.xml b/metis-authentication/pom.xml index 5506a603f..adec231e2 100644 --- a/metis-authentication/pom.xml +++ b/metis-authentication/pom.xml @@ -4,7 +4,7 @@ metis-framework eu.europeana.metis - 3 + 4 metis-authentication pom diff --git a/metis-common/metis-common-mongo/pom.xml b/metis-common/metis-common-mongo/pom.xml index b133ba975..4770897b6 100644 --- a/metis-common/metis-common-mongo/pom.xml +++ b/metis-common/metis-common-mongo/pom.xml @@ -4,11 +4,16 @@ metis-common eu.europeana.metis - 3 + 4 metis-common-mongo + + eu.europeana.metis + metis-common-utils + ${project.version} + eu.europeana.metis metis-common-network @@ -19,11 +24,6 @@ eu.europeana.corelib corelib-storage - - org.mongodb - mongodb-driver-core - ${version.mongodb.driver.core} - dev.morphia.morphia morphia-core diff --git a/metis-common/metis-common-mongo/src/main/java/eu/europeana/metis/mongo/connection/MongoClientProvider.java b/metis-common/metis-common-mongo/src/main/java/eu/europeana/metis/mongo/connection/MongoClientProvider.java index b7ebeea58..7b6ae4333 100644 --- a/metis-common/metis-common-mongo/src/main/java/eu/europeana/metis/mongo/connection/MongoClientProvider.java +++ b/metis-common/metis-common-mongo/src/main/java/eu/europeana/metis/mongo/connection/MongoClientProvider.java @@ -31,6 +31,10 @@ * Whether writes should be retried if they fail due to a network error is defaulted to * {@value #DEFAULT_RETRY_WRITES}. * + *
  • + * The application name (used among other things for server logging) is defaulted to + * {@value #DEFAULT_APPLICATION_NAME}. + *
    • * * These defaults can be overridden or additional (default) settings can be set upon * construction. To facilitate this, this class offers access to the default settings by means of @@ -43,6 +47,7 @@ public class MongoClientProvider { private static final ReadPreference DEFAULT_READ_PREFERENCE = ReadPreference.secondaryPreferred(); private static final int DEFAULT_MAX_CONNECTION_IDLE_MILLIS = 30_000; private static final boolean DEFAULT_RETRY_WRITES = false; + private static final String DEFAULT_APPLICATION_NAME = "Europeana Application Suite"; private final MongoClientCreator creator; private final String authenticationDatabase; @@ -74,14 +79,15 @@ public MongoClientProvider(String connectionUri, Function exceptionCr } /** - * Constructor from a {@link MongoProperties} object. The caller can provide settings that will - * override the default settings (i.e. the default settings will not be used). + * Constructor from a {@link MongoProperties} object. The caller needs to provide settings that + * will be used instead of the default settings. * * @param properties The properties of the Mongo connection. Note that if the passed properties - * object is changed after calling this method, those changes will not be reflected when calling + * object is changed after calling this method, those changes will not be reflected when creating + * mongo clients. * @param clientSettingsBuilder The settings to be applied. The default settings will not be used. - * The caller can incorporate the default settings by using an client settings builder obtained - * from {@link #getDefaultClientSettingsBuilder()}. {@link #createMongoClient()}. + * The caller can however choose to incorporate the default settings as needed by using a client + * settings builder obtained from {@link #getDefaultClientSettingsBuilder()} as input. * @throws E In case the properties are wrong */ public MongoClientProvider(MongoProperties properties, Builder clientSettingsBuilder) @@ -101,6 +107,8 @@ public MongoClientProvider(MongoProperties properties, Builder clientSettings if (mongoCredential != null) { clientSettingsBuilder.credential(mongoCredential); } + Optional.ofNullable(properties.getApplicationName()).filter(name -> !name.isBlank()) + .ifPresent(clientSettingsBuilder::applicationName); final MongoClientSettings mongoClientSettings = clientSettingsBuilder.build(); this.creator = () -> MongoClients.create(mongoClientSettings); @@ -126,9 +134,11 @@ public MongoClientProvider(MongoProperties properties) throws E { public static Builder getDefaultClientSettingsBuilder() { return MongoClientSettings.builder() // TODO: 7/16/20 Remove default retry writes after upgrade to mongo server version 4.2 - .retryWrites(DEFAULT_RETRY_WRITES).applyToConnectionPoolSettings(builder -> builder - .maxConnectionIdleTime(DEFAULT_MAX_CONNECTION_IDLE_MILLIS, TimeUnit.MILLISECONDS)) - .readPreference(DEFAULT_READ_PREFERENCE); + .retryWrites(DEFAULT_RETRY_WRITES) + .applyToConnectionPoolSettings(builder -> builder + .maxConnectionIdleTime(DEFAULT_MAX_CONNECTION_IDLE_MILLIS, TimeUnit.MILLISECONDS)) + .readPreference(DEFAULT_READ_PREFERENCE) + .applicationName(DEFAULT_APPLICATION_NAME); } /** diff --git a/metis-common/metis-common-mongo/src/main/java/eu/europeana/metis/mongo/connection/MongoProperties.java b/metis-common/metis-common-mongo/src/main/java/eu/europeana/metis/mongo/connection/MongoProperties.java index efeb9c0ae..7678327d1 100644 --- a/metis-common/metis-common-mongo/src/main/java/eu/europeana/metis/mongo/connection/MongoProperties.java +++ b/metis-common/metis-common-mongo/src/main/java/eu/europeana/metis/mongo/connection/MongoProperties.java @@ -28,6 +28,7 @@ public class MongoProperties { private MongoCredential mongoCredentials; private boolean mongoEnableSsl; private ReadPreferenceValue readPreferenceValue; + private String applicationName; /** * Constructor. @@ -39,7 +40,7 @@ public MongoProperties(Function exceptionCreator) { } /** - * Setter for all properties. + * Setter for multiple properties. * * @param hosts The hosts. This cannot be null or empty. * @param ports The ports. This cannot be null or empty. Must contain either the same number of @@ -66,7 +67,7 @@ public void setAllProperties(String[] hosts, int[] ports, String authenticationD } /** - * Setter for all properties. + * Setter for multiple properties. * * @param hosts The hosts. This cannot be null or empty. * @param ports The ports. This cannot be null or empty. Must contain either the same number of @@ -79,12 +80,37 @@ public void setAllProperties(String[] hosts, int[] ports, String authenticationD * @param readPreferenceValue The read preference. Can be null, where then the default applies * @throws E In case either of the arrays is null, or their lengths don't match. */ + @Deprecated public void setAllProperties(String[] hosts, int[] ports, String authenticationDatabase, String username, String password, boolean enableSsl, ReadPreferenceValue readPreferenceValue) throws E { + setAllProperties(hosts, ports, authenticationDatabase, username, password, enableSsl, + readPreferenceValue, null); + } + + /** + * Setter for multiple properties. + * + * @param hosts The hosts. This cannot be null or empty. + * @param ports The ports. This cannot be null or empty. Must contain either the same number of + * elements as the hosts array, or exactly 1 element (which will then apply to all hosts). + * @param authenticationDatabase The name of the authentication database. Can be null, in which + * case no authentication takes place. + * @param username The username. Can be null, in which case no authentication takes place. + * @param password The password. Can be null, in which case no authentication takes place. + * @param enableSsl Whether to enable SSL connections. + * @param readPreferenceValue The read preference. Can be null, where then the default applies + * @param applicationName The name of the application. Can be null, where then the default applies + * @throws E In case either of the arrays is null, or their lengths don't match. + */ + public void setAllProperties(String[] hosts, int[] ports, String authenticationDatabase, + String username, String password, boolean enableSsl, ReadPreferenceValue readPreferenceValue, + String applicationName) + throws E { setAllProperties(hosts, ports, authenticationDatabase, username, password); this.mongoEnableSsl = enableSsl; setReadPreferenceValue(readPreferenceValue); + setApplicationName(applicationName); } /** @@ -137,7 +163,7 @@ public void setMongoEnableSsl() { } /** - * Set the read preference value. Can be null, where then the default applies + * Set the read preference value. Can be null, in which case the default applies. * * @param readPreferenceValue the read preference value (null for the default). */ @@ -145,6 +171,16 @@ public void setReadPreferenceValue(ReadPreferenceValue readPreferenceValue) { this.readPreferenceValue = readPreferenceValue; } + /** + * Set the application name. Can be null, in which case a default generic application name is + * to be used. + * + * @param applicationName The application name, or null for the default. + */ + public void setApplicationName(String applicationName) { + this.applicationName = applicationName; + } + private T nonNull(T value, String fieldName) throws E { if (value == null) { throw exceptionCreator.apply(String.format("Value '%s' cannot be null.", fieldName)); @@ -199,14 +235,23 @@ public boolean mongoEnableSsl() { } /** - * This method returns the value of the read preference (or null for the default behavior) + * This method returns the value of the read preference (or null for the default behavior). * - * @return the read preference set + * @return The read preference. */ public ReadPreferenceValue getReadPreferenceValue() { return readPreferenceValue; } + /** + * This method returns the value of the application name (or null for the default). + * + * @return The application name. + */ + public String getApplicationName() { + return applicationName; + } + /** * Enum for read preference values */ diff --git a/metis-common/metis-common-mongo/src/main/java/eu/europeana/metis/mongo/embedded/EmbeddedLocalhostMongo.java b/metis-common/metis-common-mongo/src/main/java/eu/europeana/metis/mongo/embedded/EmbeddedLocalhostMongo.java index 92a9f6beb..b1313d39f 100644 --- a/metis-common/metis-common-mongo/src/main/java/eu/europeana/metis/mongo/embedded/EmbeddedLocalhostMongo.java +++ b/metis-common/metis-common-mongo/src/main/java/eu/europeana/metis/mongo/embedded/EmbeddedLocalhostMongo.java @@ -3,11 +3,11 @@ import de.flapdoodle.embed.mongo.Command; import de.flapdoodle.embed.mongo.MongodExecutable; import de.flapdoodle.embed.mongo.MongodStarter; -import de.flapdoodle.embed.mongo.config.MongodConfigBuilder; +import de.flapdoodle.embed.mongo.config.Defaults; +import de.flapdoodle.embed.mongo.config.MongodConfig; import de.flapdoodle.embed.mongo.config.Net; -import de.flapdoodle.embed.mongo.config.RuntimeConfigBuilder; import de.flapdoodle.embed.mongo.distribution.Version; -import de.flapdoodle.embed.process.config.IRuntimeConfig; +import de.flapdoodle.embed.process.config.RuntimeConfig; import de.flapdoodle.embed.process.config.io.ProcessOutput; import de.flapdoodle.embed.process.runtime.Network; import eu.europeana.metis.network.NetworkUtil; @@ -39,14 +39,17 @@ public void start() { if (mongodExecutable == null) { try { mongoPort = NetworkUtil.getAvailableLocalPort(); - IRuntimeConfig runtimeConfig = new RuntimeConfigBuilder() - .defaultsWithLogger(Command.MongoD, LOGGER) + RuntimeConfig runtimeConfig = Defaults.runtimeConfigFor(Command.MongoD, LOGGER) .processOutput(ProcessOutput.getDefaultInstanceSilent()) .build(); + MongodConfig mongodConfig = MongodConfig.builder() + .version(Version.V4_0_12) + .net(new Net(DEFAULT_MONGO_HOST, mongoPort, Network.localhostIsIPv6())) + .build(); + MongodStarter runtime = MongodStarter.getInstance(runtimeConfig); - mongodExecutable = runtime.prepare(new MongodConfigBuilder().version(Version.V3_6_5) - .net(new Net(DEFAULT_MONGO_HOST, mongoPort, Network.localhostIsIPv6())).build()); + mongodExecutable = runtime.prepare(mongodConfig); mongodExecutable.start(); } catch (IOException e) { LOGGER.error("Exception when starting embedded mongo", e); diff --git a/metis-common/metis-common-mongo/src/main/java/eu/europeana/metis/mongo/utils/MorphiaUtils.java b/metis-common/metis-common-mongo/src/main/java/eu/europeana/metis/mongo/utils/MorphiaUtils.java index 1ba733185..af2eab0fe 100644 --- a/metis-common/metis-common-mongo/src/main/java/eu/europeana/metis/mongo/utils/MorphiaUtils.java +++ b/metis-common/metis-common-mongo/src/main/java/eu/europeana/metis/mongo/utils/MorphiaUtils.java @@ -6,7 +6,7 @@ import dev.morphia.query.Query; import dev.morphia.query.internal.MorphiaCursor; import eu.europeana.metis.network.ExternalRequestUtil; -import eu.europeana.metis.network.SonarqubeNullcheckAvoidanceUtils; +import eu.europeana.metis.utils.SonarqubeNullcheckAvoidanceUtils; import java.util.List; import java.util.Optional; import java.util.function.BiFunction; diff --git a/metis-common/metis-common-network/pom.xml b/metis-common/metis-common-network/pom.xml index 464d2ec48..64a14cfd3 100644 --- a/metis-common/metis-common-network/pom.xml +++ b/metis-common/metis-common-network/pom.xml @@ -4,7 +4,7 @@ metis-common eu.europeana.metis - 3 + 4 metis-common-network diff --git a/metis-common/metis-common-solr/pom.xml b/metis-common/metis-common-solr/pom.xml index 93840adc5..10ae9d154 100644 --- a/metis-common/metis-common-solr/pom.xml +++ b/metis-common/metis-common-solr/pom.xml @@ -4,7 +4,7 @@ metis-common eu.europeana.metis - 3 + 4 metis-common-solr @@ -34,12 +34,10 @@ org.junit.jupiter junit-jupiter-api - test org.junit.jupiter junit-jupiter-engine - test     diff --git a/metis-common/metis-common-utils/pom.xml b/metis-common/metis-common-utils/pom.xml index 891336661..4c603eb55 100644 --- a/metis-common/metis-common-utils/pom.xml +++ b/metis-common/metis-common-utils/pom.xml @@ -4,7 +4,7 @@ metis-common eu.europeana.metis - 3 + 4 metis-common-utils @@ -31,18 +31,14 @@ org.junit.jupiter junit-jupiter-api - test org.junit.jupiter junit-jupiter-engine - test org.mockito mockito-core - ${version.mockito.core} - test diff --git a/metis-common/metis-common-utils/src/main/java/eu/europeana/metis/exception/StructuredExceptionWrapper.java b/metis-common/metis-common-utils/src/main/java/eu/europeana/metis/exception/StructuredExceptionWrapper.java index 84e646300..892855f0f 100644 --- a/metis-common/metis-common-utils/src/main/java/eu/europeana/metis/exception/StructuredExceptionWrapper.java +++ b/metis-common/metis-common-utils/src/main/java/eu/europeana/metis/exception/StructuredExceptionWrapper.java @@ -18,6 +18,10 @@ public class StructuredExceptionWrapper { @XmlElement private String errorMessage; + public StructuredExceptionWrapper() { + // Required for serialization and deserialization + } + /** * Constructor that is json friendly and is displayed to the client if an exception was thrown. * diff --git a/metis-common/metis-common-utils/src/main/java/eu/europeana/metis/utils/RestEndpoints.java b/metis-common/metis-common-utils/src/main/java/eu/europeana/metis/utils/RestEndpoints.java index 02f5bd8b4..51cd90b69 100644 --- a/metis-common/metis-common-utils/src/main/java/eu/europeana/metis/utils/RestEndpoints.java +++ b/metis-common/metis-common-utils/src/main/java/eu/europeana/metis/utils/RestEndpoints.java @@ -45,12 +45,12 @@ public final class RestEndpoints { //ORCHESTRATION public static final String ORCHESTRATOR_WORKFLOWS_DATASETID = "/orchestrator/workflows/{datasetId}"; public static final String ORCHESTRATOR_WORKFLOWS_DATASETID_EXECUTE = "/orchestrator/workflows/{datasetId}/execute"; - public static final String ORCHESTRATOR_WORKFLOWS_DATASETID_EXECUTE_DIRECT = "/orchestrator/workflows/{datasetId}/execute/direct"; public static final String ORCHESTRATOR_WORKFLOWS_SCHEDULE = "/orchestrator/workflows/schedule"; public static final String ORCHESTRATOR_WORKFLOWS_SCHEDULE_DATASETID = "/orchestrator/workflows/schedule/{datasetId}"; public static final String ORCHESTRATOR_WORKFLOWS_EXECUTIONS_EXECUTIONID = "/orchestrator/workflows/executions/{executionId}"; public static final String ORCHESTRATOR_WORKFLOWS_EXECUTIONS_EXECUTIONID_PLUGINS_DATA_AVAILABILITY = "/orchestrator/workflows/executions/{executionId}/plugins/data-availability"; public static final String ORCHESTRATOR_WORKFLOWS_EXECUTIONS_DATASET_DATASETID = "/orchestrator/workflows/executions/dataset/{datasetId}"; + public static final String ORCHESTRATOR_WORKFLOWS_EXECUTIONS_DATASET_DATASETID_ALLOWED_INCREMENTAL = "/orchestrator/workflows/executions/dataset/{datasetId}/allowed_incremental"; public static final String ORCHESTRATOR_WORKFLOWS_EXECUTIONS_DATASET_DATASETID_ALLOWED_PLUGIN = "/orchestrator/workflows/executions/dataset/{datasetId}/allowed_plugin"; public static final String ORCHESTRATOR_WORKFLOWS_EXECUTIONS_DATASET_DATASETID_HISTORY = "/orchestrator/workflows/executions/dataset/{datasetId}/history"; public static final String ORCHESTRATOR_WORKFLOWS_EXECUTIONS_DATASET_DATASETID_INFORMATION = "/orchestrator/workflows/executions/dataset/{datasetId}/information"; @@ -79,10 +79,6 @@ public final class RestEndpoints { /* METIS SCHEMA VALIDATION ENDPOINT */ public static final String SCHEMA_VALIDATE = "/schema/validate/{schema}"; public static final String SCHEMA_BATCH_VALIDATE = "/schema/validate/batch/{schema}"; - public static final String SCHEMAS_DOWNLOAD_BY_NAME = "/schemas/download/schema/{name}/{version}"; - public static final String SCHEMAS_MANAGE_BY_NAME = "/schemas/schema/{name}/{version}"; - public static final String SCHEMAS_UPDATE_BY_NAME = "/schemas/schema/update/{name}/{version}"; - public static final String SCHEMAS_ALL = "/schemas/all"; /* METIS PREVIEW SERVICE ENDPOINT*/ public static final String DATA_CHECKER_UPLOAD = "/upload"; diff --git a/metis-common/metis-common-network/src/main/java/eu/europeana/metis/network/SonarqubeNullcheckAvoidanceUtils.java b/metis-common/metis-common-utils/src/main/java/eu/europeana/metis/utils/SonarqubeNullcheckAvoidanceUtils.java similarity index 98% rename from metis-common/metis-common-network/src/main/java/eu/europeana/metis/network/SonarqubeNullcheckAvoidanceUtils.java rename to metis-common/metis-common-utils/src/main/java/eu/europeana/metis/utils/SonarqubeNullcheckAvoidanceUtils.java index b906e520c..69027bf09 100644 --- a/metis-common/metis-common-network/src/main/java/eu/europeana/metis/network/SonarqubeNullcheckAvoidanceUtils.java +++ b/metis-common/metis-common-utils/src/main/java/eu/europeana/metis/utils/SonarqubeNullcheckAvoidanceUtils.java @@ -1,4 +1,4 @@ -package eu.europeana.metis.network; +package eu.europeana.metis.utils; import java.util.function.Consumer; import java.util.function.Function; diff --git a/metis-common/metis-common-zoho/pom.xml b/metis-common/metis-common-zoho/pom.xml index 5bf712006..46f3a46de 100644 --- a/metis-common/metis-common-zoho/pom.xml +++ b/metis-common/metis-common-zoho/pom.xml @@ -4,7 +4,7 @@ metis-common eu.europeana.metis - 3 + 4 metis-common-zoho diff --git a/metis-common/pom.xml b/metis-common/pom.xml index 394beccba..3c4a53d65 100644 --- a/metis-common/pom.xml +++ b/metis-common/pom.xml @@ -4,7 +4,7 @@ metis-framework eu.europeana.metis - 3 + 4 metis-common pom diff --git a/metis-core/metis-core-common/pom.xml b/metis-core/metis-core-common/pom.xml index 0f7542a81..741949c57 100644 --- a/metis-core/metis-core-common/pom.xml +++ b/metis-core/metis-core-common/pom.xml @@ -4,19 +4,17 @@ metis-core eu.europeana.metis - 3 + 4 metis-core-common org.junit.jupiter junit-jupiter-api - test org.junit.jupiter junit-jupiter-engine - test eu.europeana.metis @@ -68,11 +66,6 @@ spring-web ${version.spring} - - org.mongodb - mongodb-driver-core - ${version.mongodb.driver.core} - org.apache.commons commons-lang3 diff --git a/metis-core/metis-core-common/src/main/java/eu/europeana/metis/core/common/TransformationParameters.java b/metis-core/metis-core-common/src/main/java/eu/europeana/metis/core/common/TransformationParameters.java new file mode 100644 index 000000000..7bf576648 --- /dev/null +++ b/metis-core/metis-core-common/src/main/java/eu/europeana/metis/core/common/TransformationParameters.java @@ -0,0 +1,39 @@ +package eu.europeana.metis.core.common; + +import eu.europeana.metis.core.dataset.Dataset; +import java.util.Locale; + +/** + * This class is to be used to create the transformation parameters based on a provided {@link + * Dataset}, so that there is a centralized location of how those parameters should be created. + */ +public class TransformationParameters { + + private final String datasetName; + private final String edmCountry; + private final String edmLanguage; + + /** + * Constructor that initializes all final fields. + * + * @param dataset the provided dataset + */ + public TransformationParameters(Dataset dataset) { + //DatasetName in Transformation should be a concatenation datasetId_datasetName + datasetName = dataset.getDatasetId() + "_" + dataset.getDatasetName(); + edmCountry = dataset.getCountry().getName(); + edmLanguage = dataset.getLanguage().name().toLowerCase(Locale.US); + } + + public String getDatasetName() { + return datasetName; + } + + public String getEdmCountry() { + return edmCountry; + } + + public String getEdmLanguage() { + return edmLanguage; + } +} diff --git a/metis-core/metis-core-common/src/main/java/eu/europeana/metis/core/rest/IncrementalHarvestingAllowedView.java b/metis-core/metis-core-common/src/main/java/eu/europeana/metis/core/rest/IncrementalHarvestingAllowedView.java new file mode 100644 index 000000000..d91db4ac4 --- /dev/null +++ b/metis-core/metis-core-common/src/main/java/eu/europeana/metis/core/rest/IncrementalHarvestingAllowedView.java @@ -0,0 +1,17 @@ +package eu.europeana.metis.core.rest; + +/** + * An object wrapping a boolean indicating whether incremental harvesting is allowed. + */ +public class IncrementalHarvestingAllowedView { + + private final boolean incrementalHarvestingAllowed; + + public IncrementalHarvestingAllowedView(boolean incrementalHarvestingAllowed) { + this.incrementalHarvestingAllowed = incrementalHarvestingAllowed; + } + + public boolean isIncrementalHarvestingAllowed() { + return incrementalHarvestingAllowed; + } +} diff --git a/metis-core/metis-core-common/src/main/java/eu/europeana/metis/core/workflow/plugins/OaipmhHarvestPlugin.java b/metis-core/metis-core-common/src/main/java/eu/europeana/metis/core/workflow/plugins/OaipmhHarvestPlugin.java index 0e54937b3..e9485643f 100644 --- a/metis-core/metis-core-common/src/main/java/eu/europeana/metis/core/workflow/plugins/OaipmhHarvestPlugin.java +++ b/metis-core/metis-core-common/src/main/java/eu/europeana/metis/core/workflow/plugins/OaipmhHarvestPlugin.java @@ -68,10 +68,10 @@ DpsTask prepareDpsTask(String datasetId, EcloudBasePluginParameters ecloudBasePl OAIPMHHarvestingDetails oaipmhHarvestingDetails = new OAIPMHHarvestingDetails(); if (StringUtils.isNotEmpty(metadataFormat)) { - oaipmhHarvestingDetails.setSchemas(new HashSet<>(Collections.singletonList(metadataFormat))); + oaipmhHarvestingDetails.setSchema(metadataFormat); } if (StringUtils.isNotEmpty(setSpec)) { - oaipmhHarvestingDetails.setSets(new HashSet<>(Collections.singletonList(setSpec))); + oaipmhHarvestingDetails.setSet(setSpec); } oaipmhHarvestingDetails.setDateFrom(fromDate); oaipmhHarvestingDetails.setDateUntil(untilDate); diff --git a/metis-core/metis-core-common/src/main/java/eu/europeana/metis/core/workflow/plugins/OaipmhHarvestPluginMetadata.java b/metis-core/metis-core-common/src/main/java/eu/europeana/metis/core/workflow/plugins/OaipmhHarvestPluginMetadata.java index f082739bb..147ac4bd3 100644 --- a/metis-core/metis-core-common/src/main/java/eu/europeana/metis/core/workflow/plugins/OaipmhHarvestPluginMetadata.java +++ b/metis-core/metis-core-common/src/main/java/eu/europeana/metis/core/workflow/plugins/OaipmhHarvestPluginMetadata.java @@ -16,6 +16,7 @@ public class OaipmhHarvestPluginMetadata extends AbstractExecutablePluginMetadat private String url; private String metadataFormat; private String setSpec; + private boolean incrementalHarvest; // Default: false (i.e. full harvest) @JsonFormat(pattern = CommonStringValues.DATE_FORMAT) private Date fromDate; @JsonFormat(pattern = CommonStringValues.DATE_FORMAT) @@ -53,6 +54,14 @@ public void setSetSpec(String setSpec) { this.setSpec = setSpec; } + public void setIncrementalHarvest(boolean incrementalHarvest) { + this.incrementalHarvest = incrementalHarvest; + } + + public boolean isIncrementalHarvest() { + return incrementalHarvest; + } + public Date getFromDate() { return fromDate == null ? null : new Date(fromDate.getTime()); } diff --git a/metis-core/metis-core-common/src/test/java/eu/europeana/metis/core/common/TransformationParametersTest.java b/metis-core/metis-core-common/src/test/java/eu/europeana/metis/core/common/TransformationParametersTest.java new file mode 100644 index 000000000..a3edc1e49 --- /dev/null +++ b/metis-core/metis-core-common/src/test/java/eu/europeana/metis/core/common/TransformationParametersTest.java @@ -0,0 +1,23 @@ +package eu.europeana.metis.core.common; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import eu.europeana.metis.core.dataset.Dataset; +import java.util.Locale; +import org.junit.jupiter.api.Test; + +class TransformationParametersTest { + + @Test + void testTransformationParametersConstruction() { + final Dataset dataset = new Dataset(); + dataset.setDatasetId("exampleDatasetId"); + dataset.setDatasetName("exampleDatasetName"); + dataset.setCountry(Country.GREECE); + dataset.setLanguage(Language.EL); + final TransformationParameters transformationParameters = new TransformationParameters(dataset); + assertEquals(dataset.getDatasetId() + "_" + dataset.getDatasetName(), transformationParameters.getDatasetName()); + assertEquals(dataset.getCountry().getName(), transformationParameters.getEdmCountry()); + assertEquals(dataset.getLanguage().name().toLowerCase(Locale.US), transformationParameters.getEdmLanguage()); + } +} \ No newline at end of file diff --git a/metis-core/metis-core-rest/pom.xml b/metis-core/metis-core-rest/pom.xml index 7fcccfdde..fee54b7a7 100644 --- a/metis-core/metis-core-rest/pom.xml +++ b/metis-core/metis-core-rest/pom.xml @@ -4,7 +4,7 @@ metis-core eu.europeana.metis - 3 + 4 metis-core-rest war @@ -24,6 +24,10 @@ corelib-web ${version.corelib} + + eu.europeana.corelib + corelib-definitions + wstx-asl org.codehaus.woodstox @@ -112,19 +116,15 @@ org.mockito mockito-core - ${version.mockito.core} - test org.junit.jupiter junit-jupiter-api - test org.junit.jupiter junit-jupiter-engine - test org.springframework @@ -140,13 +140,11 @@ com.jayway.jsonpath json-path - ${version.json.path} test com.jayway.jsonpath json-path-assert - ${version.json.path} test diff --git a/metis-core/metis-core-rest/src/main/java/eu/europeana/metis/core/rest/DatasetController.java b/metis-core/metis-core-rest/src/main/java/eu/europeana/metis/core/rest/DatasetController.java index c7a02df22..05819b362 100644 --- a/metis-core/metis-core-rest/src/main/java/eu/europeana/metis/core/rest/DatasetController.java +++ b/metis-core/metis-core-rest/src/main/java/eu/europeana/metis/core/rest/DatasetController.java @@ -616,10 +616,10 @@ public ResponseListWrapper getAllDatasetsByOrganizationName( MediaType.APPLICATION_JSON_VALUE, MediaType.APPLICATION_XML_VALUE}) @ResponseStatus(HttpStatus.OK) @ResponseBody - public List getDatasetsCountries( + public List getDatasetsCountries( @RequestHeader("Authorization") String authorization) throws GenericMetisException { authenticationClient.getUserByAccessTokenInHeader(authorization); - return Arrays.stream(Country.values()).map(CountryModel::new).collect(Collectors.toList()); + return Arrays.stream(Country.values()).map(CountryView::new).collect(Collectors.toList()); } /** @@ -638,10 +638,10 @@ public List getDatasetsCountries( MediaType.APPLICATION_JSON_VALUE, MediaType.APPLICATION_XML_VALUE}) @ResponseStatus(HttpStatus.OK) @ResponseBody - public List getDatasetsLanguages( + public List getDatasetsLanguages( @RequestHeader("Authorization") String authorization) throws GenericMetisException { authenticationClient.getUserByAccessTokenInHeader(authorization); - return Language.getLanguageListSortedByName().stream().map(LanguageModel::new) + return Language.getLanguageListSortedByName().stream().map(LanguageView::new) .collect(Collectors.toList()); } @@ -688,7 +688,7 @@ public ResponseListWrapper getDatasetSearch( return responseListWrapper; } - private static class CountryModel { + private static class CountryView { @JsonProperty("enum") private final String enumName; @@ -697,21 +697,21 @@ private static class CountryModel { @JsonProperty private final String isoCode; - CountryModel(Country country) { + CountryView(Country country) { this.enumName = country.name(); this.name = country.getName(); this.isoCode = country.getIsoCode(); } } - private static class LanguageModel { + private static class LanguageView { @JsonProperty("enum") private final String enumName; @JsonProperty private final String name; - LanguageModel(Language language) { + LanguageView(Language language) { this.enumName = language.name(); this.name = language.getName(); } diff --git a/metis-core/metis-core-rest/src/main/java/eu/europeana/metis/core/rest/OrchestratorController.java b/metis-core/metis-core-rest/src/main/java/eu/europeana/metis/core/rest/OrchestratorController.java index 87e91000f..449ba3b17 100644 --- a/metis-core/metis-core-rest/src/main/java/eu/europeana/metis/core/rest/OrchestratorController.java +++ b/metis-core/metis-core-rest/src/main/java/eu/europeana/metis/core/rest/OrchestratorController.java @@ -1,7 +1,5 @@ package eu.europeana.metis.core.rest; -import eu.europeana.metis.utils.CommonStringValues; -import eu.europeana.metis.utils.RestEndpoints; import eu.europeana.metis.authentication.rest.client.AuthenticationClient; import eu.europeana.metis.authentication.user.MetisUser; import eu.europeana.metis.core.common.DaoFieldNames; @@ -18,6 +16,8 @@ import eu.europeana.metis.core.workflow.plugins.PluginType; import eu.europeana.metis.exception.BadContentException; import eu.europeana.metis.exception.GenericMetisException; +import eu.europeana.metis.utils.CommonStringValues; +import eu.europeana.metis.utils.RestEndpoints; import java.util.Date; import java.util.Set; import org.slf4j.Logger; @@ -309,6 +309,33 @@ public WorkflowExecution getWorkflowExecutionByExecutionId( return workflowExecution; } + /** + * This method returns whether currently it is permitted/possible to perform incremental + * harvesting for the given dataset. + * + * @param authorization the authorization header with the access token + * @param datasetId The ID of the dataset for which to check. + * @return Whether we can perform incremental harvesting for the dataset. + * @throws GenericMetisException which can be one of: + *
      + *
    • {@link eu.europeana.metis.core.exceptions.NoDatasetFoundException} if the dataset + * identifier provided does not exist
      • + *
    • {@link eu.europeana.metis.exception.UserUnauthorizedException} if the user is not + * authenticated or authorized to perform this task
      • + *
    + */ + @GetMapping(value = RestEndpoints.ORCHESTRATOR_WORKFLOWS_EXECUTIONS_DATASET_DATASETID_ALLOWED_INCREMENTAL, produces = { + MediaType.APPLICATION_JSON_VALUE, MediaType.APPLICATION_XML_VALUE}) + @ResponseStatus(HttpStatus.OK) + @ResponseBody + public IncrementalHarvestingAllowedView isIncrementalHarvestingAllowed( + @RequestHeader("Authorization") String authorization, + @PathVariable("datasetId") String datasetId) throws GenericMetisException { + final MetisUser metisUser = authenticationClient.getUserByAccessTokenInHeader(authorization); + return new IncrementalHarvestingAllowedView( + orchestratorService.isIncrementalHarvestingAllowed(metisUser, datasetId)); + } + /** * Check if a specified {@code pluginType} is allowed for execution. This is checked based on, if * there was a previous successful finished plugin that follows a specific order (unless the {@code @@ -342,7 +369,7 @@ public MetisPlugin getLatestFinishedPluginWorkflowExecutionByDatasetIdIfPluginTy @RequestParam(value = "enforcedPluginType", required = false, defaultValue = "") ExecutablePluginType enforcedPredecessorType) throws GenericMetisException { MetisUser metisUser = authenticationClient.getUserByAccessTokenInHeader(authorization); - MetisPlugin latestFinishedPluginWorkflowExecutionByDatasetId = orchestratorService + MetisPlugin latestFinishedPluginWorkflowExecutionByDatasetId = orchestratorService .getLatestFinishedPluginByDatasetIdIfPluginTypeAllowedForExecution(metisUser, datasetId, pluginType, enforcedPredecessorType); if (latestFinishedPluginWorkflowExecutionByDatasetId == null) { diff --git a/metis-core/metis-core-rest/src/main/java/eu/europeana/metis/core/rest/config/ApplicationInitUtils.java b/metis-core/metis-core-rest/src/main/java/eu/europeana/metis/core/rest/config/ApplicationInitUtils.java index f6b0d60a7..e5d69d945 100644 --- a/metis-core/metis-core-rest/src/main/java/eu/europeana/metis/core/rest/config/ApplicationInitUtils.java +++ b/metis-core/metis-core-rest/src/main/java/eu/europeana/metis/core/rest/config/ApplicationInitUtils.java @@ -6,12 +6,16 @@ import eu.europeana.metis.utils.CustomTruststoreAppender; import eu.europeana.metis.utils.CustomTruststoreAppender.TrustStoreConfigurationException; import org.apache.commons.lang.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * This class is responsible for performing initializing tasks for the application. */ final class ApplicationInitUtils { + private static final Logger LOGGER = LoggerFactory.getLogger(ApplicationInitUtils.class); + private ApplicationInitUtils() { } @@ -30,6 +34,7 @@ static MongoClient initializeApplication(ConfigurationPropertiesHolder propertie CustomTruststoreAppender .appendCustomTrustoreToDefault(propertiesHolder.getTruststorePath(), propertiesHolder.getTruststorePassword()); + LOGGER.info("Custom truststore appended to default truststore"); } // Initialize the socks proxy. @@ -38,6 +43,7 @@ static MongoClient initializeApplication(ConfigurationPropertiesHolder propertie propertiesHolder .getSocksProxyUsername(), propertiesHolder.getSocksProxyPassword()).init(); + LOGGER.info("Socks proxy enabled"); } // Initialize the Mongo connection diff --git a/metis-core/metis-core-rest/src/main/java/eu/europeana/metis/core/rest/config/ConfigurationPropertiesHolder.java b/metis-core/metis-core-rest/src/main/java/eu/europeana/metis/core/rest/config/ConfigurationPropertiesHolder.java index babd98fd2..dafee7c30 100644 --- a/metis-core/metis-core-rest/src/main/java/eu/europeana/metis/core/rest/config/ConfigurationPropertiesHolder.java +++ b/metis-core/metis-core-rest/src/main/java/eu/europeana/metis/core/rest/config/ConfigurationPropertiesHolder.java @@ -2,6 +2,7 @@ import eu.europeana.metis.core.workflow.ValidationProperties; import eu.europeana.metis.mongo.connection.MongoProperties; +import eu.europeana.metis.mongo.connection.MongoProperties.ReadPreferenceValue; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.PropertySource; import org.springframework.stereotype.Component; @@ -44,14 +45,26 @@ public class ConfigurationPropertiesHolder { private String redisHost; @Value("${redis.port}") private int redisPort; + @Value("${redis.username}") + private String redisUsername; @Value("${redis.password}") private String redisPassword; @Value("${redis.enableSSL}") private boolean redisEnableSSL; @Value("${redis.enable.custom.truststore}") private boolean redisEnableCustomTruststore; + @Value("${redisson.connection.pool.size}") + private int redissonConnectionPoolSize; @Value("${redisson.lock.watchdog.timeout.in.secs}") private int redissonLockWatchdogTimeoutInSecs; + @Value("${redisson.connect.timeout.in.millisecs}") + private int redissonConnectTimeoutInMillisecs; + @Value("${redisson.dns.monitor.interval.in.millisecs}") + private int redissonDnsMonitorIntervalInMillisecs; + @Value("${redisson.idle.connection.timeout.in.millisecs}") + private int redissonIdleConnectionTimeoutInMillisecs; + @Value("${redisson.retry.attempts}") + private int redissonRetryAttempts; //RabbitMq @Value("${rabbitmq.host}") @@ -70,6 +83,8 @@ public class ConfigurationPropertiesHolder { private int rabbitmqHighestPriority; @Value("${rabbitmq.enableSSL}") private boolean rabbitmqEnableSSL; + @Value("${rabbitmq.enable.custom.truststore}") + private boolean rabbitmqEnableCustomTruststore; // Metis core configuration @Value("${metis.core.baseUrl}") @@ -120,6 +135,8 @@ public class ConfigurationPropertiesHolder { private String mongoDb; @Value("${mongo.enableSSL}") private boolean mongoEnableSSL; + @Value("${mongo.application.name}") + private String mongoApplicationName; //Validation @Value("${validation.internal.schema.zip}") @@ -199,6 +216,10 @@ public int getRedisPort() { return redisPort; } + public String getRedisUsername() { + return redisUsername; + } + public String getRedisPassword() { return redisPassword; } @@ -211,10 +232,30 @@ public boolean isRedisEnableCustomTruststore() { return redisEnableCustomTruststore; } + public int getRedissonConnectionPoolSize() { + return redissonConnectionPoolSize; + } + public int getRedissonLockWatchdogTimeoutInSecs() { return redissonLockWatchdogTimeoutInSecs; } + public int getRedissonConnectTimeoutInMillisecs() { + return redissonConnectTimeoutInMillisecs; + } + + public int getRedissonDnsMonitorIntervalInMillisecs() { + return redissonDnsMonitorIntervalInMillisecs; + } + + public int getRedissonIdleConnectionTimeoutInMillisecs() { + return redissonIdleConnectionTimeoutInMillisecs; + } + + public int getRedissonRetryAttempts() { + return redissonRetryAttempts; + } + public String getRabbitmqHost() { return rabbitmqHost; } @@ -239,6 +280,10 @@ public boolean isRabbitmqEnableSSL() { return rabbitmqEnableSSL; } + public boolean isRabbitmqEnableCustomTruststore() { + return rabbitmqEnableCustomTruststore; + } + public String getRabbitmqQueueName() { return rabbitmqQueueName; } @@ -303,7 +348,7 @@ public MongoProperties getMongoProperties() { final MongoProperties mongoProperties = new MongoProperties<>( IllegalArgumentException::new); mongoProperties.setAllProperties(mongoHosts, mongoPorts, mongoAuthenticationDb, mongoUsername, - mongoPassword, mongoEnableSSL, null); + mongoPassword, mongoEnableSSL, ReadPreferenceValue.PRIMARY_PREFERRED, mongoApplicationName); return mongoProperties; } diff --git a/metis-core/metis-core-rest/src/main/java/eu/europeana/metis/core/rest/config/OrchestratorConfig.java b/metis-core/metis-core-rest/src/main/java/eu/europeana/metis/core/rest/config/OrchestratorConfig.java index c88141338..c69fbd5b5 100644 --- a/metis-core/metis-core-rest/src/main/java/eu/europeana/metis/core/rest/config/OrchestratorConfig.java +++ b/metis-core/metis-core-rest/src/main/java/eu/europeana/metis/core/rest/config/OrchestratorConfig.java @@ -5,13 +5,14 @@ import eu.europeana.cloud.mcs.driver.DataSetServiceClient; import eu.europeana.cloud.mcs.driver.FileServiceClient; import eu.europeana.cloud.mcs.driver.RecordServiceClient; +import eu.europeana.metis.core.dao.DataEvolutionUtils; import eu.europeana.metis.core.dao.DatasetDao; import eu.europeana.metis.core.dao.DatasetXsltDao; import eu.europeana.metis.core.dao.DepublishRecordIdDao; import eu.europeana.metis.core.dao.ScheduledWorkflowDao; import eu.europeana.metis.core.dao.WorkflowDao; import eu.europeana.metis.core.dao.WorkflowExecutionDao; -import eu.europeana.metis.core.dao.WorkflowUtils; +import eu.europeana.metis.core.dao.WorkflowValidationUtils; import eu.europeana.metis.core.execution.SchedulerExecutor; import eu.europeana.metis.core.execution.SemaphoresPerPluginManager; import eu.europeana.metis.core.execution.WorkflowExecutionMonitor; @@ -82,19 +83,32 @@ RedissonClient getRedissonClient() throws MalformedURLException { singleServerConfig = config.useSingleServer().setAddress(String .format("rediss://%s:%s", propertiesHolder.getRedisHost(), propertiesHolder.getRedisPort())); + LOGGER.info("Redis enabled SSL"); if (propertiesHolder.isRedisEnableCustomTruststore()) { singleServerConfig .setSslTruststore(new File(propertiesHolder.getTruststorePath()).toURI().toURL()); singleServerConfig.setSslTruststorePassword(propertiesHolder.getTruststorePassword()); + LOGGER.info("Redis enabled SSL using custom Truststore"); } } else { singleServerConfig = config.useSingleServer().setAddress(String .format("redis://%s:%s", propertiesHolder.getRedisHost(), propertiesHolder.getRedisPort())); + LOGGER.info("Redis disabled SSL"); + } + if (StringUtils.isNotEmpty(propertiesHolder.getRedisUsername())) { + singleServerConfig.setUsername(propertiesHolder.getRedisUsername()); } if (StringUtils.isNotEmpty(propertiesHolder.getRedisPassword())) { singleServerConfig.setPassword(propertiesHolder.getRedisPassword()); } + + singleServerConfig.setConnectionPoolSize(propertiesHolder.getRedissonConnectionPoolSize()) + .setConnectionMinimumIdleSize(propertiesHolder.getRedissonConnectionPoolSize()) + .setConnectTimeout(propertiesHolder.getRedissonConnectTimeoutInMillisecs()) + .setDnsMonitoringInterval(propertiesHolder.getRedissonDnsMonitorIntervalInMillisecs()) + .setIdleConnectionTimeout(propertiesHolder.getRedissonIdleConnectionTimeoutInMillisecs()) + .setRetryAttempts(propertiesHolder.getRedissonRetryAttempts()); config.setLockWatchdogTimeout(TimeUnit.SECONDS.toMillis(propertiesHolder .getRedissonLockWatchdogTimeoutInSecs())); //Give some secs to unlock if connection lost, or if too long to unlock redissonClient = Redisson.create(config); @@ -103,23 +117,24 @@ RedissonClient getRedissonClient() throws MalformedURLException { @Bean public OrchestratorService getOrchestratorService(WorkflowDao workflowDao, - WorkflowExecutionDao workflowExecutionDao, WorkflowUtils workflowUtils, DatasetDao datasetDao, + WorkflowExecutionDao workflowExecutionDao, WorkflowValidationUtils workflowValidationUtils, + DataEvolutionUtils dataEvolutionUtils, DatasetDao datasetDao, WorkflowExecutionFactory workflowExecutionFactory, WorkflowExecutorManager workflowExecutorManager, Authorizer authorizer, DepublishRecordIdDao depublishRecordIdDao) { OrchestratorService orchestratorService = new OrchestratorService(workflowExecutionFactory, - workflowDao, workflowExecutionDao, workflowUtils, datasetDao, workflowExecutorManager, - redissonClient, authorizer, depublishRecordIdDao); + workflowDao, workflowExecutionDao, workflowValidationUtils, dataEvolutionUtils, datasetDao, + workflowExecutorManager, redissonClient, authorizer, depublishRecordIdDao); orchestratorService.setSolrCommitPeriodInMins(propertiesHolder.getSolrCommitPeriodInMins()); return orchestratorService; } @Bean public WorkflowExecutionFactory getWorkflowExecutionFactory( - WorkflowExecutionDao workflowExecutionDao, WorkflowUtils workflowUtils, + WorkflowExecutionDao workflowExecutionDao, DataEvolutionUtils dataEvolutionUtils, DatasetXsltDao datasetXsltDao, DepublishRecordIdDao depublishRecordIdDao) { WorkflowExecutionFactory workflowExecutionFactory = new WorkflowExecutionFactory(datasetXsltDao, - depublishRecordIdDao, workflowExecutionDao, workflowUtils); + depublishRecordIdDao, workflowExecutionDao, dataEvolutionUtils); workflowExecutionFactory .setValidationExternalProperties(propertiesHolder.getValidationExternalProperties()); workflowExecutionFactory @@ -191,9 +206,14 @@ public WorkflowExecutionDao getWorkflowExecutionDao( } @Bean - WorkflowUtils getWorkflowUtils(WorkflowExecutionDao workflowExecutionDao, + DataEvolutionUtils getDataEvolutionUtils(WorkflowExecutionDao workflowExecutionDao) { + return new DataEvolutionUtils(workflowExecutionDao); + } + + @Bean + WorkflowValidationUtils getWorkflowValidationUtils(DataEvolutionUtils dataEvolutionUtils, DepublishRecordIdDao depublishRecordIdDao) { - return new WorkflowUtils(workflowExecutionDao, depublishRecordIdDao); + return new WorkflowValidationUtils(depublishRecordIdDao, dataEvolutionUtils); } @Bean diff --git a/metis-core/metis-core-rest/src/main/java/eu/europeana/metis/core/rest/config/QueueConfig.java b/metis-core/metis-core-rest/src/main/java/eu/europeana/metis/core/rest/config/QueueConfig.java index c87f71b8f..7a2c0947f 100644 --- a/metis-core/metis-core-rest/src/main/java/eu/europeana/metis/core/rest/config/QueueConfig.java +++ b/metis-core/metis-core-rest/src/main/java/eu/europeana/metis/core/rest/config/QueueConfig.java @@ -9,12 +9,19 @@ import eu.europeana.metis.core.execution.WorkflowExecutorManager; import eu.europeana.metis.exception.GenericMetisException; import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Paths; import java.security.KeyManagementException; +import java.security.KeyStore; +import java.security.KeyStoreException; import java.security.NoSuchAlgorithmException; +import java.security.cert.CertificateException; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeoutException; import javax.annotation.PreDestroy; +import javax.net.ssl.SSLContext; +import javax.net.ssl.TrustManagerFactory; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -53,7 +60,7 @@ public QueueConfig(ConfigurationPropertiesHolder propertiesHolder) { @Bean Connection getConnection() - throws KeyManagementException, NoSuchAlgorithmException, IOException, TimeoutException { + throws KeyManagementException, NoSuchAlgorithmException, IOException, TimeoutException, KeyStoreException, CertificateException { ConnectionFactory connectionFactory = new ConnectionFactory(); connectionFactory.setHost(propertiesHolder.getRabbitmqHost()); connectionFactory.setPort(propertiesHolder.getRabbitmqPort()); @@ -64,7 +71,22 @@ Connection getConnection() connectionFactory.setPassword(propertiesHolder.getRabbitmqPassword()); connectionFactory.setAutomaticRecoveryEnabled(true); if (propertiesHolder.isRabbitmqEnableSSL()) { - connectionFactory.useSslProtocol(); + if (propertiesHolder.isRabbitmqEnableCustomTruststore()) { + //Load the ssl context with the provided truststore + KeyStore keyStore = KeyStore.getInstance(KeyStore.getDefaultType()); + keyStore.load(Files.newInputStream(Paths.get(propertiesHolder.getTruststorePath())), + propertiesHolder.getTruststorePassword().toCharArray()); + TrustManagerFactory trustManagerFactory = TrustManagerFactory + .getInstance(TrustManagerFactory.getDefaultAlgorithm()); + trustManagerFactory.init(keyStore); + SSLContext sslContext = SSLContext.getInstance("TLS"); + sslContext.init(null, trustManagerFactory.getTrustManagers(), null); + connectionFactory.useSslProtocol(sslContext); + LOGGER.info("RabbitMQ enabled SSL WITH certificate verification using custom Truststore"); + } else { + connectionFactory.useSslProtocol(); + LOGGER.info("RabbitMQ enabled SSL WITHOUT certificate verification"); + } } //Does not close the channel if an unhandled exception occurred //Can happen in QueueConsumer and it's safe to not handle the execution, it will be picked up diff --git a/metis-core/metis-core-rest/src/main/resources/default_transformation.xslt b/metis-core/metis-core-rest/src/main/resources/default_transformation.xslt index 0d5db0bd2..9e470d9b4 100644 --- a/metis-core/metis-core-rest/src/main/resources/default_transformation.xslt +++ b/metis-core/metis-core-rest/src/main/resources/default_transformation.xslt @@ -19,38 +19,42 @@ xmlns:skos="http://www.w3.org/2004/02/skos/core#" xmlns:svcs="http://rdfs.org/sioc/services#" xmlns:wgs84_pos="http://www.w3.org/2003/01/geo/wgs84_pos#" - xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:xsl="http://www.w3.org/1999/XSL/Transform" + xmlns:xs="http://www.w3.org/2001/XMLSchema" + xmlns:f="functions-namespace" - exclude-result-prefixes="xsi xsl"> - + exclude-result-prefixes="xsi xsl f"> - - - - - - + + - - - - - - + + + + + + + + + + + + + + - - @@ -61,7 +65,6 @@ - @@ -69,58 +72,68 @@ - - + - - + - + - + + + + + - - + + + + - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + - - - - + - - - + - + + + + + + + @@ -132,73 +145,113 @@ - + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + false - + - + + + + + + + + + - + + + + + + + + + + true + + + + + + + + + + - @@ -239,7 +292,6 @@ - @@ -267,7 +319,6 @@ - @@ -283,7 +334,6 @@ - @@ -298,7 +348,6 @@ - @@ -317,7 +366,6 @@ - @@ -325,7 +373,6 @@ - @@ -340,7 +387,6 @@ - @@ -358,7 +404,6 @@ - @@ -366,15 +411,52 @@ - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/metis-core/metis-core-rest/src/main/resources/metis.properties.example b/metis-core/metis-core-rest/src/main/resources/metis.properties.example index 6cf4b828e..f7f7050da 100644 --- a/metis-core/metis-core-rest/src/main/resources/metis.properties.example +++ b/metis-core/metis-core-rest/src/main/resources/metis.properties.example @@ -6,7 +6,6 @@ dps.read.timeout.in.millisecs= failsafe.margin.of.inactivity.in.secs= periodic.failsafe.check.in.millisecs= periodic.scheduler.check.in.millisecs= -redisson.lock.watchdog.timeout.in.secs= polling.timeout.for.cleaning.completion.service.in.millisecs= #If a task passed this cap the task will be cancelled period.of.no.processed.records.change.in.minutes= @@ -31,6 +30,8 @@ rabbitmq.virtual.host= rabbitmq.queue.name= rabbitmq.highest.priority= rabbitmq.enableSSL= +#True if a custom certificate is used in the truststore defined above +rabbitmq.enable.custom.truststore= #Mongo mongo.hosts= @@ -40,14 +41,23 @@ mongo.username= mongo.password= mongo.enableSSL= mongo.db= +mongo.application.name= #Redis redis.host= redis.port=0 +redis.username= redis.password= redis.enableSSL= #True if a custom certificate is used in the truststore defined above redis.enable.custom.truststore= +redisson.connection.pool.size= +redisson.connect.timeout.in.millisecs= +redisson.lock.watchdog.timeout.in.secs= +#Setting to -1 disables DNS monitoring +redisson.dns.monitor.interval.in.millisecs= +redisson.idle.connection.timeout.in.millisecs= +redisson.retry.attempts= solr.commit.period.in.mins= diff --git a/metis-core/metis-core-service/pom.xml b/metis-core/metis-core-service/pom.xml index 912757719..67f75d2b5 100644 --- a/metis-core/metis-core-service/pom.xml +++ b/metis-core/metis-core-service/pom.xml @@ -4,7 +4,7 @@ metis-core eu.europeana.metis - 3 + 4 metis-core-service @@ -40,11 +40,6 @@     - - org.mongodb - mongodb-driver-core - ${version.mongodb.driver.core} - org.apache.commons commons-lang3 @@ -62,18 +57,14 @@ org.mockito mockito-core - ${version.mockito.core} - test org.junit.jupiter junit-jupiter-api - test org.junit.jupiter junit-jupiter-engine - test org.springframework diff --git a/metis-core/metis-core-service/src/main/java/eu/europeana/metis/core/dao/WorkflowUtils.java b/metis-core/metis-core-service/src/main/java/eu/europeana/metis/core/dao/DataEvolutionUtils.java similarity index 57% rename from metis-core/metis-core-service/src/main/java/eu/europeana/metis/core/dao/WorkflowUtils.java rename to metis-core/metis-core-service/src/main/java/eu/europeana/metis/core/dao/DataEvolutionUtils.java index 936963eaa..d9d768283 100644 --- a/metis-core/metis-core-service/src/main/java/eu/europeana/metis/core/dao/WorkflowUtils.java +++ b/metis-core/metis-core-service/src/main/java/eu/europeana/metis/core/dao/DataEvolutionUtils.java @@ -1,37 +1,31 @@ package eu.europeana.metis.core.dao; -import eu.europeana.metis.utils.CommonStringValues; -import eu.europeana.metis.core.dataset.DepublishRecordId.DepublicationStatus; +import eu.europeana.metis.core.dao.WorkflowExecutionDao.ExecutionDatasetPair; +import eu.europeana.metis.core.dao.WorkflowExecutionDao.Pagination; +import eu.europeana.metis.core.dao.WorkflowExecutionDao.ResultList; import eu.europeana.metis.core.exceptions.PluginExecutionNotAllowed; -import eu.europeana.metis.core.util.DepublishRecordIdSortField; -import eu.europeana.metis.core.util.SortDirection; -import eu.europeana.metis.core.workflow.Workflow; import eu.europeana.metis.core.workflow.WorkflowExecution; import eu.europeana.metis.core.workflow.plugins.AbstractExecutablePlugin; -import eu.europeana.metis.core.workflow.plugins.AbstractExecutablePluginMetadata; import eu.europeana.metis.core.workflow.plugins.AbstractMetisPlugin; -import eu.europeana.metis.core.workflow.plugins.DepublishPluginMetadata; +import eu.europeana.metis.core.workflow.plugins.DataStatus; import eu.europeana.metis.core.workflow.plugins.ExecutablePlugin; import eu.europeana.metis.core.workflow.plugins.ExecutablePluginType; import eu.europeana.metis.core.workflow.plugins.ExecutionProgress; -import eu.europeana.metis.core.workflow.plugins.HTTPHarvestPluginMetadata; +import eu.europeana.metis.core.workflow.plugins.IndexToPublishPlugin; import eu.europeana.metis.core.workflow.plugins.MetisPlugin; -import eu.europeana.metis.core.workflow.plugins.OaipmhHarvestPluginMetadata; +import eu.europeana.metis.core.workflow.plugins.OaipmhHarvestPlugin; import eu.europeana.metis.core.workflow.plugins.PluginStatus; import eu.europeana.metis.core.workflow.plugins.PluginType; -import eu.europeana.metis.exception.BadContentException; -import eu.europeana.metis.exception.GenericMetisException; -import java.net.MalformedURLException; -import java.net.URI; -import java.net.URISyntaxException; -import java.net.URL; +import eu.europeana.metis.utils.CommonStringValues; import java.util.ArrayDeque; import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; import java.util.Date; import java.util.EnumSet; +import java.util.LinkedHashMap; import java.util.List; +import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.Set; @@ -40,13 +34,12 @@ import java.util.stream.Stream; import org.apache.commons.lang3.tuple.ImmutablePair; import org.apache.commons.lang3.tuple.Pair; -import org.apache.http.client.utils.URIBuilder; -import org.springframework.util.CollectionUtils; /** - * This class is a utility class that can answer questions related to the validation of workflows. + * This class is a utility class that can answer questions related to the history and evolution of + * the data. */ -public class WorkflowUtils { +public class DataEvolutionUtils { private static final Set HARVEST_PLUGIN_GROUP = Collections.unmodifiableSet( EnumSet.of(ExecutablePluginType.OAIPMH_HARVEST, ExecutablePluginType.HTTP_HARVEST)); @@ -61,167 +54,14 @@ public class WorkflowUtils { .unmodifiableSet(EnumSet.complementOf(EnumSet.of(ExecutablePluginType.LINK_CHECKING))); private final WorkflowExecutionDao workflowExecutionDao; - private final DepublishRecordIdDao depublishRecordIdDao; /** * Constructor. * * @param workflowExecutionDao the workflow execution dao - * @param depublishRecordIdDao the depublication record id dao */ - public WorkflowUtils(WorkflowExecutionDao workflowExecutionDao, - DepublishRecordIdDao depublishRecordIdDao) { + public DataEvolutionUtils(WorkflowExecutionDao workflowExecutionDao) { this.workflowExecutionDao = workflowExecutionDao; - this.depublishRecordIdDao = depublishRecordIdDao; - } - - /** - * This method validates the workflow plugin sequence. In particular, it checks: - *
      - *
    1. That the workflow is not empty and contains plugins with valid types,
      2. - *
    2. That the first plugin is not link checking (except when it is the only plugin),
      3. - *
    3. That no two plugins of the same type occur in the workflow,
      4. - *
    4. That if depublish is enabled no other plugins are allowed in the workflow,
      5. - *
    5. That the first plugin has a valid predecessor plugin in the dataset's history (as defined by - * {@link #getPredecessorTypes(ExecutablePluginType)}), the type of which can be overridden by the - * enforced predecessor type, and the root plugin (i.e. harvest) of which is equal to the latest - * successful harvest (i.e. no old data should be processed after new data has been introduced),
      6. - *
    6. That all subsequent plugins have a valid predecessor within the workflow (as defined by - * {@link #getPredecessorTypes(ExecutablePluginType)}),
      7. - *
    7. That harvesting plugins have valid URL settings.
      8. - *
    - * - * @param workflow The workflow to validate. - * @param enforcedPredecessorType If not null, overrides the predecessor type of the first - * plugin. - * @return The predecessor of the first plugin. Or null if no predecessor is required. - * @throws GenericMetisException which can be one of: - *
      - *
    • {@link BadContentException} In case the workflow is empty, or contains plugins with - * invalid types.
      • - *
    • {@link PluginExecutionNotAllowed} in case the plugin sequence as provided is not - * allowed.
      • - *
    - */ - public PluginWithExecutionId validateWorkflowPlugins(Workflow workflow, - ExecutablePluginType enforcedPredecessorType) throws GenericMetisException { - - // Workflow should have a plugin list. - if (workflow.getMetisPluginsMetadata() == null) { - throw new BadContentException("Workflow should not be empty."); - } - - // Compile the list of enabled plugins. - final List enabledPlugins = workflow.getMetisPluginsMetadata() - .stream().filter(AbstractExecutablePluginMetadata::isEnabled).collect(Collectors.toList()); - - // Workflow should not be empty and all should have a type. - if (enabledPlugins.isEmpty()) { - throw new BadContentException("Workflow should not be empty."); - } - if (enabledPlugins.stream().map(AbstractExecutablePluginMetadata::getExecutablePluginType) - .anyMatch(Objects::isNull)) { - throw new BadContentException( - "There are enabled plugins of which the type could not be determined."); - } - - // Validate dataset/record depublication - validateDepublishPlugin(workflow.getDatasetId(), enabledPlugins); - - // Validate and normalize the harvest parameters of harvest plugins (even if not enabled) - validateAndTrimHarvestParameters(enabledPlugins); - - // Check that first plugin is not link checking (except if it is the only plugin) - if (enabledPlugins.size() > 1 - && enabledPlugins.get(0).getPluginType() == PluginType.LINK_CHECKING) { - throw new PluginExecutionNotAllowed(CommonStringValues.PLUGIN_EXECUTION_NOT_ALLOWED); - } - - // Make sure that all enabled plugins (except the first) have a predecessor within the workflow. - final EnumSet previousTypesInWorkflow = EnumSet - .of(enabledPlugins.get(0).getExecutablePluginType()); - for (int i = 1; i < enabledPlugins.size(); i++) { - - // Find the permissible predecessors - final ExecutablePluginType pluginType = enabledPlugins.get(i).getExecutablePluginType(); - final Set permissiblePredecessors = getPredecessorTypes(pluginType); - - // Check if we have the right predecessor plugin types in the workflow - final boolean hasNoPredecessor = !permissiblePredecessors.isEmpty() && - permissiblePredecessors.stream().noneMatch(previousTypesInWorkflow::contains); - if (hasNoPredecessor) { - throw new PluginExecutionNotAllowed(CommonStringValues.PLUGIN_EXECUTION_NOT_ALLOWED); - } - - // Add the plugin type to those we have seen - previousTypesInWorkflow.add(pluginType); - } - - // We should now have seen all types. Make sure that there are no duplicates - if (previousTypesInWorkflow.size() != enabledPlugins.size()) { - throw new PluginExecutionNotAllowed(CommonStringValues.PLUGIN_EXECUTION_NOT_ALLOWED); - } - - // Check the presence of the predecessor and return it. - return computePredecessorPlugin(enabledPlugins.get(0).getExecutablePluginType(), - enforcedPredecessorType, workflow.getDatasetId()); - } - - private static void validateAndTrimHarvestParameters( - Iterable enabledPlugins) - throws BadContentException { - for (AbstractExecutablePluginMetadata pluginMetadata : enabledPlugins) { - if (pluginMetadata instanceof OaipmhHarvestPluginMetadata) { - final OaipmhHarvestPluginMetadata oaipmhMetadata = (OaipmhHarvestPluginMetadata) pluginMetadata; - final URI validatedUri = validateUrl(oaipmhMetadata.getUrl()); - oaipmhMetadata - .setUrl(new URIBuilder(validatedUri).removeQuery().setFragment(null).toString()); - oaipmhMetadata.setMetadataFormat(oaipmhMetadata.getMetadataFormat() == null ? null - : oaipmhMetadata.getMetadataFormat().trim()); - oaipmhMetadata.setSetSpec( - oaipmhMetadata.getSetSpec() == null ? null : oaipmhMetadata.getSetSpec().trim()); - } - if (pluginMetadata instanceof HTTPHarvestPluginMetadata) { - final HTTPHarvestPluginMetadata httpMetadata = (HTTPHarvestPluginMetadata) pluginMetadata; - httpMetadata.setUrl(validateUrl(httpMetadata.getUrl()).toString()); - } - } - } - - private void validateDepublishPlugin(String datasetId, - List enabledPlugins) throws BadContentException { - // If depublish requested, make sure it's the only plugin in the workflow - final Optional depublishPluginMetadata = enabledPlugins.stream() - .filter(plugin -> plugin.getExecutablePluginType().toPluginType() == PluginType.DEPUBLISH) - .map(plugin -> (DepublishPluginMetadata) plugin).findFirst(); - if (enabledPlugins.size() > 1 && depublishPluginMetadata.isPresent()) { - throw new BadContentException( - "If DEPUBLISH plugin enabled, no other enabled plugins are allowed."); - } - - // If record depublication requested, check if there are pending record ids in the db - if (depublishPluginMetadata.isPresent() && !depublishPluginMetadata.get() - .isDatasetDepublish()) { - final Set pendingDepublicationIds = depublishRecordIdDao - .getAllDepublishRecordIdsWithStatus(datasetId, - DepublishRecordIdSortField.DEPUBLICATION_STATE, SortDirection.ASCENDING, - DepublicationStatus.PENDING_DEPUBLICATION); - if (CollectionUtils.isEmpty(pendingDepublicationIds)) { - throw new BadContentException( - "Record depublication requested but there are no pending depublication record ids in the db"); - } - } - } - - private static URI validateUrl(String urlString) throws BadContentException { - if (urlString == null) { - throw new BadContentException("Harvesting parameters are missing"); - } - try { - return new URL(urlString.trim()).toURI(); - } catch (MalformedURLException | URISyntaxException e) { - throw new BadContentException("Harvesting parameters are invalid", e); - } } /** @@ -257,8 +97,7 @@ private static URI validateUrl(String urlString) throws BadContentException { * Can be null in case the given type does not require a predecessor. */ public static AbstractExecutablePlugin computePredecessorPlugin(ExecutablePluginType - pluginType, - WorkflowExecution workflowExecution) { + pluginType, WorkflowExecution workflowExecution) { // If the plugin type does not need a predecessor we are done. final Set predecessorTypes = getPredecessorTypes(pluginType); @@ -323,21 +162,21 @@ public static AbstractExecutablePlugin computePredecessorPlugin(ExecutablePlugin * null in case the given type does not require a predecessor. * @throws PluginExecutionNotAllowed In case a valid predecessor is required, but not found. */ - public PluginWithExecutionId computePredecessorPlugin( + public PluginWithExecutionId> computePredecessorPlugin( ExecutablePluginType pluginType, ExecutablePluginType enforcedPredecessorType, String datasetId) throws PluginExecutionNotAllowed { - final PluginWithExecutionId predecessorPlugin; + final PluginWithExecutionId> predecessorPlugin; final Set defaultPredecessorTypes = getPredecessorTypes(pluginType); if (pluginType == ExecutablePluginType.DEPUBLISH) { // If a DEPUBLISH Operation is requested we don't link to the predecessor plugin. However, // make sure there at least one successful exists (possibly invalidated by later reindex). - final PluginWithExecutionId successfulPublish = workflowExecutionDao + final PluginWithExecutionId> successfulPublish = workflowExecutionDao .getLatestSuccessfulExecutablePlugin(datasetId, EnumSet.of(ExecutablePluginType.PUBLISH), false); final boolean hasAtLeastOneSuccessfulPlugin = Optional.ofNullable(successfulPublish) - .filter(WorkflowUtils::pluginHasSuccessfulRecords).isPresent(); + .filter(DataEvolutionUtils::pluginHasSuccessfulRecords).isPresent(); if (!hasAtLeastOneSuccessfulPlugin) { throw new PluginExecutionNotAllowed(CommonStringValues.PLUGIN_EXECUTION_NOT_ALLOWED); } @@ -353,27 +192,27 @@ public PluginWithExecutionId computePredecessorPlugin( .>map(EnumSet::of).orElse(defaultPredecessorTypes); // Find the latest successful harvest to compare with. If none exist, throw exception. - final PluginWithExecutionId latestHarvest = Optional + final PluginWithExecutionId> latestHarvest = Optional .ofNullable(workflowExecutionDao.getLatestSuccessfulExecutablePlugin(datasetId, HARVEST_PLUGIN_GROUP, true)).orElseThrow( () -> new PluginExecutionNotAllowed(CommonStringValues.PLUGIN_EXECUTION_NOT_ALLOWED)); // Find the latest successful plugin of each type and filter on existence of successful records. - final Stream> latestSuccessfulPlugins = predecessorTypes + final Stream>> latestSuccessfulPlugins = predecessorTypes .stream().map(Collections::singleton).map( type -> workflowExecutionDao .getLatestSuccessfulExecutablePlugin(datasetId, type, true)) - .filter(Objects::nonNull).filter(WorkflowUtils::pluginHasSuccessfulRecords); + .filter(Objects::nonNull).filter(DataEvolutionUtils::pluginHasSuccessfulRecords); // Sort on finished state, so that the root check occurs as little as possible. - final Stream> sortedSuccessfulPlugins = + final Stream>> sortedSuccessfulPlugins = latestSuccessfulPlugins.sorted(Comparator.comparing( plugin -> Optional.ofNullable(plugin.getPlugin().getFinishedDate()) .orElseGet(() -> new Date(Long.MIN_VALUE)), Comparator.reverseOrder())); // Find the first plugin that satisfies the root check. If none found, throw exception. - final Predicate> rootCheck = plugin -> getRootAncestor( + final Predicate>> rootCheck = plugin -> getRootAncestor( plugin).equals(latestHarvest); predecessorPlugin = sortedSuccessfulPlugins .filter(rootCheck).findFirst().orElseThrow( @@ -383,7 +222,7 @@ public PluginWithExecutionId computePredecessorPlugin( } private static boolean pluginHasSuccessfulRecords( - PluginWithExecutionId plugin) { + PluginWithExecutionId> plugin) { final ExecutionProgress progress = plugin.getPlugin().getExecutionProgress(); return progress != null && progress.getProcessedRecords() > progress.getErrors(); } @@ -395,14 +234,15 @@ private static boolean pluginHasSuccessfulRecords( * @param plugin The plugin for which to find the root ancestor. * @return The root ancestor. Is not null. */ - public PluginWithExecutionId getRootAncestor( - PluginWithExecutionId plugin) { + public PluginWithExecutionId> getRootAncestor( + PluginWithExecutionId> plugin) { final WorkflowExecution execution = workflowExecutionDao.getById(plugin.getExecutionId()); - final List> evolution = + final List, WorkflowExecution>> evolution = compileVersionEvolution(plugin.getPlugin(), execution); - return evolution.stream().findFirst() - .map(pair -> new PluginWithExecutionId<>(pair.getRight(), pair.getLeft())) - .orElse(plugin); + if (evolution.isEmpty()) { + return plugin; + } + return new PluginWithExecutionId<>(evolution.get(0).getRight(), evolution.get(0).getLeft()); } /** @@ -414,7 +254,7 @@ public PluginWithExecutionId getRootAncestor( * type can be based on. Cannot be null, but can be the empty set in case the plugin type requires * no predecessor. */ - private static Set getPredecessorTypes(ExecutablePluginType pluginType) { + static Set getPredecessorTypes(ExecutablePluginType pluginType) { final Set predecessorTypes; switch (pluginType) { case VALIDATION_EXTERNAL: @@ -460,17 +300,17 @@ private static Set getPredecessorTypes(ExecutablePluginTyp * target version. * * @param targetPlugin The target for compiling the evolution: the result will lead to, but not - * inclide, this plugin. + * include, this plugin. * @param targetPluginExecution The execution in which this target plugin may be found. * @return The evolution. */ - public List> compileVersionEvolution( - MetisPlugin targetPlugin, WorkflowExecution targetPluginExecution) { + public List, WorkflowExecution>> compileVersionEvolution( + MetisPlugin targetPlugin, WorkflowExecution targetPluginExecution) { // Loop backwards to find the plugin. Don't add the first plugin to the result list. - Pair currentExecutionAndPlugin = new ImmutablePair<>( + Pair, WorkflowExecution> currentExecutionAndPlugin = new ImmutablePair<>( targetPlugin, targetPluginExecution); - final ArrayDeque> evolutionSteps = new ArrayDeque<>(); + final ArrayDeque, WorkflowExecution>> evolutionSteps = new ArrayDeque<>(); while (true) { // Move to the previous execution: stop when we have none or it is not executable. @@ -492,23 +332,21 @@ public List> compileVersionEvolution( return new ArrayList<>(evolutionSteps); } - Pair getPreviousExecutionAndPlugin(MetisPlugin plugin, + // Not private because of unit tests. + Pair, WorkflowExecution> getPreviousExecutionAndPlugin(MetisPlugin plugin, String datasetId) { // Check whether we are at the end of the chain. - final Date previousPluginTimestamp = plugin.getPluginMetadata() - .getRevisionTimestampPreviousPlugin(); - final PluginType previousPluginType = PluginType.getPluginTypeFromEnumName( - plugin.getPluginMetadata().getRevisionNamePreviousPlugin()); - if (previousPluginTimestamp == null || previousPluginType == null) { + final ExecutedMetisPluginId previousPluginId = ExecutedMetisPluginId.forPredecessor(plugin); + if (previousPluginId == null) { return null; } // Obtain the previous execution and plugin. final WorkflowExecution previousExecution = workflowExecutionDao - .getByTaskExecution(previousPluginTimestamp, previousPluginType, datasetId); - final AbstractMetisPlugin previousPlugin = previousExecution == null ? null - : previousExecution.getMetisPluginWithType(previousPluginType).orElse(null); + .getByTaskExecution(previousPluginId, datasetId); + final AbstractMetisPlugin previousPlugin = previousExecution == null ? null + : previousExecution.getMetisPluginWithType(previousPluginId.getPluginType()).orElse(null); if (previousExecution == null || previousPlugin == null) { return null; } @@ -517,6 +355,98 @@ Pair getPreviousExecutionAndPlugin(MetisPlugin p return new ImmutablePair<>(previousPlugin, previousExecution); } + List> getPublishOperationsSortedInversely(String datasetId) { + + // Get all workflows with finished publish plugins (theoretically we can't quite rely on order). + final Pagination pagination = workflowExecutionDao.createPagination(0, null, true); + final ResultList executionsWithPublishOperations = workflowExecutionDao + .getWorkflowExecutionsOverview(Set.of(datasetId), Set.of(PluginStatus.FINISHED), + Set.of(PluginType.PUBLISH), null, null, pagination); + + // Extract all (finished) publish plugins inversely sorted by started date (most recent first). + final List> publishOperations = new ArrayList<>(); + executionsWithPublishOperations.getResults().stream().map(ExecutionDatasetPair::getExecution) + .forEach(execution -> execution.getMetisPlugins().stream() + .filter(plugin -> plugin instanceof IndexToPublishPlugin) + .map(plugin -> (IndexToPublishPlugin) plugin) + .map(plugin -> new PluginWithExecutionId<>(execution.getId().toString(), plugin)) + .forEach(publishOperations::add)); + final Comparator> comparator = Comparator + .comparing(pair -> pair.getPlugin().getStartedDate()); + publishOperations.sort(comparator.reversed()); + + // Done + return publishOperations; + } + + /** + *

    + * This method returns a sequence of published harvest increments. This is a list of successive + * published harvests that must start with the most recent full published harvest and then + * includes all incremental published harvests (if any) that came after that. + *

    + *

    + * Note that a published harvest is a harvest that resulted in a index to publish (i.e. any + * harvest that is the root predecessor of an index to publish). + *

    + * + * @param datasetId The dataset ID for which to obtain the chain. + * @return The chain, in the form of plugin-execution pairs that are ordered chronologically. Is + * never null, but can be empty if no such chain exists (i.e. the dataset does not have a + * published harvest or we find an index after the last full harvest that is invalid or did + * somehow not originate from a harvest). + */ + public List>> getPublishedHarvestIncrements( + String datasetId) { + + // Get all publish operations sorted inversely + final var allPublishOperations = getPublishOperationsSortedInversely(datasetId); + + // Compile a list of all associated harvests (that led to one of these publish operations). + // Note: we assume that workflows don't cross each other (i.e. an earlier publish cannot have a + // later harvest). We stop when we find a full harvest (the latest full harvest). + boolean fullHarvestFound = false; + final Map>> resultHarvests = new LinkedHashMap<>(); + for (PluginWithExecutionId publishOperation : allPublishOperations) { + + // If the publish is not available, we have detected an anomaly. We are done. + if (MetisPlugin.getDataStatus(publishOperation.getPlugin()) != DataStatus.VALID && + MetisPlugin.getDataStatus(publishOperation.getPlugin()) != DataStatus.DEPRECATED) { + return Collections.emptyList(); + } + + // Get the root harvest and add it to the map. + final PluginWithExecutionId> rootHarvest = getRootAncestor( + publishOperation); + + // If the root harvest is not a harvest, we have detected an anomaly. We are done. + if (!HARVEST_PLUGIN_GROUP + .contains(rootHarvest.getPlugin().getPluginMetadata().getExecutablePluginType())) { + return Collections.emptyList(); + } + + // Add the root harvest to the result (overwrite if already there). + resultHarvests.put(ExecutedMetisPluginId.forPlugin(rootHarvest.getPlugin()), rootHarvest); + + // If the root harvest is a full harvest, we are done. + fullHarvestFound = !((rootHarvest.getPlugin() instanceof OaipmhHarvestPlugin) + && ((OaipmhHarvestPlugin) rootHarvest.getPlugin()).getPluginMetadata() + .isIncrementalHarvest()); + if (fullHarvestFound) { + break; + } + } + + // Done. Sort and return the harvests in the right order (just reversing the result). + if (!fullHarvestFound) { + return Collections.emptyList(); + } + final List>> result = new ArrayList<>( + resultHarvests.values()); + Collections.reverse(result); + return result; + } + /** * @return The plugin types that are of the 'harvesting' kind: they can occur at the beginning of * workflows and don't need another plugin type as base. diff --git a/metis-core/metis-core-service/src/main/java/eu/europeana/metis/core/dao/ExecutedMetisPluginId.java b/metis-core/metis-core-service/src/main/java/eu/europeana/metis/core/dao/ExecutedMetisPluginId.java new file mode 100644 index 000000000..0a1a80cbc --- /dev/null +++ b/metis-core/metis-core-service/src/main/java/eu/europeana/metis/core/dao/ExecutedMetisPluginId.java @@ -0,0 +1,91 @@ +package eu.europeana.metis.core.dao; + +import eu.europeana.metis.core.workflow.plugins.MetisPlugin; +import eu.europeana.metis.core.workflow.plugins.MetisPluginMetadata; +import eu.europeana.metis.core.workflow.plugins.PluginType; +import java.util.Date; +import java.util.Objects; + +/** + * Instances of this class uniquely define a plugin that has been completed. It can be used to test + * equality of executed plugins. + */ +public class ExecutedMetisPluginId { + + private final Date pluginStartedDate; + private final PluginType pluginType; + + ExecutedMetisPluginId(Date pluginStartedDate, PluginType pluginType) { + this.pluginStartedDate = pluginStartedDate; + this.pluginType = pluginType; + if (this.pluginStartedDate == null || this.pluginType == null) { + throw new IllegalArgumentException(); + } + } + + /** + * Creates the ID of this plugin. + * + * @param plugin The pluign for which to create the ID. + * @return The ID of this plugin, or null if this plugin has not been started yet. + */ + public static ExecutedMetisPluginId forPlugin(MetisPlugin plugin) { + final Date startedDate = plugin.getStartedDate(); + if (startedDate == null) { + return null; + } + return new ExecutedMetisPluginId(startedDate, plugin.getPluginType()); + } + + /** + * Extracts the ID of the predecessor plugin of this plugin. + * + * @param plugin The plugin for which to extract the predecessor ID. + * @return The ID of the predecessor, or null if no predecessor defined. + */ + public static ExecutedMetisPluginId forPredecessor(MetisPlugin plugin) { + return forPredecessor(plugin.getPluginMetadata()); + } + + /** + * Extracts the ID of the predecessor plugin of this plugin. + * + * @param metadata The metadata of the plugin for which to extract the predecessor ID. + * @return The ID of the predecessor, or null if no predecessor defined. + */ + public static ExecutedMetisPluginId forPredecessor(MetisPluginMetadata metadata) { + final Date previousPluginTimestamp = metadata.getRevisionTimestampPreviousPlugin(); + final PluginType previousPluginType = PluginType.getPluginTypeFromEnumName( + metadata.getRevisionNamePreviousPlugin()); + if (previousPluginTimestamp == null || previousPluginType == null) { + return null; + } + return new ExecutedMetisPluginId(previousPluginTimestamp, previousPluginType); + } + + public Date getPluginStartedDate() { + return pluginStartedDate; + } + + public PluginType getPluginType() { + return pluginType; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + final ExecutedMetisPluginId that = (ExecutedMetisPluginId) o; + return Objects.equals(getPluginStartedDate(), that.getPluginStartedDate()) && + getPluginType() == that.getPluginType(); + } + + @Override + public int hashCode() { + return Objects.hash(getPluginStartedDate(), getPluginType()); + } +} diff --git a/metis-core/metis-core-service/src/main/java/eu/europeana/metis/core/dao/WorkflowExecutionDao.java b/metis-core/metis-core-service/src/main/java/eu/europeana/metis/core/dao/WorkflowExecutionDao.java index a1a8f1efe..1d3359c5a 100644 --- a/metis-core/metis-core-service/src/main/java/eu/europeana/metis/core/dao/WorkflowExecutionDao.java +++ b/metis-core/metis-core-service/src/main/java/eu/europeana/metis/core/dao/WorkflowExecutionDao.java @@ -284,7 +284,7 @@ private Query runningOrInqueueQuery(String datasetId) { * values. * @return the first plugin found */ - public PluginWithExecutionId getFirstSuccessfulPlugin(String datasetId, + public PluginWithExecutionId> getFirstSuccessfulPlugin(String datasetId, Set pluginTypes) { return Optional.ofNullable(getFirstOrLastFinishedPlugin(datasetId, pluginTypes, true)) .orElse(null); @@ -299,7 +299,7 @@ public PluginWithExecutionId getFirstSuccessfulPlugin(String datase * values. * @return the last plugin found */ - public PluginWithExecutionId getLatestSuccessfulPlugin(String datasetId, + public PluginWithExecutionId> getLatestSuccessfulPlugin(String datasetId, Set pluginTypes) { return Optional.ofNullable(getFirstOrLastFinishedPlugin(datasetId, pluginTypes, false)) .orElse(null); @@ -315,7 +315,7 @@ public PluginWithExecutionId getLatestSuccessfulPlugin(String datas * @param limitToValidData Only return the result if it has valid data (see {@link DataStatus}). * @return the last plugin found */ - public PluginWithExecutionId getLatestSuccessfulExecutablePlugin( + public PluginWithExecutionId> getLatestSuccessfulExecutablePlugin( String datasetId, Set pluginTypes, boolean limitToValidData) { @@ -325,9 +325,9 @@ public PluginWithExecutionId getLatestSuccessfulExecutablePlug // Perform the database query. If nothing found, we are done. final Set convertedPluginTypes = pluginTypes.stream() .map(ExecutablePluginType::toPluginType).collect(Collectors.toSet()); - final PluginWithExecutionId uncastResultWrapper = + final PluginWithExecutionId> uncastResultWrapper = getFirstOrLastFinishedPlugin(datasetId, convertedPluginTypes, false); - final MetisPlugin uncastResult = Optional.ofNullable(uncastResultWrapper) + final MetisPlugin uncastResult = Optional.ofNullable(uncastResultWrapper) .map(PluginWithExecutionId::getPlugin).orElse(null); if (uncastResult == null) { return null; @@ -339,10 +339,10 @@ public PluginWithExecutionId getLatestSuccessfulExecutablePlug uncastResult.getId(), uncastResult.getPluginType()); return null; } - final ExecutablePlugin castResult = (ExecutablePlugin) uncastResult; + final ExecutablePlugin castResult = (ExecutablePlugin) uncastResult; // if necessary, check for the data validity. - final PluginWithExecutionId result; + final PluginWithExecutionId> result; if (limitToValidData && MetisPlugin.getDataStatus(castResult) != DataStatus.VALID) { result = null; } else { @@ -351,7 +351,7 @@ public PluginWithExecutionId getLatestSuccessfulExecutablePlug return result; } - PluginWithExecutionId getFirstOrLastFinishedPlugin(String datasetId, + PluginWithExecutionId> getFirstOrLastFinishedPlugin(String datasetId, Set pluginTypes, boolean firstFinished) { // Verify the plugin types @@ -395,11 +395,11 @@ PluginWithExecutionId getFirstOrLastFinishedPlugin(String datasetId // Because of the unwind, we know that the plugin we need is always the first one. return Optional.ofNullable(metisPluginsIterator).stream().flatMap(Collection::stream) .filter(execution -> !execution.getMetisPlugins().isEmpty()) - .map(execution -> new PluginWithExecutionId(execution, + .map(execution -> new PluginWithExecutionId>(execution, execution.getMetisPlugins().get(0))).findFirst().orElse(null); } - private void verifyEnumSetIsValidAndNotEmpty(Set set) { + private void verifyEnumSetIsValidAndNotEmpty(Set> set) { if (set == null || set.isEmpty() || set.stream().anyMatch(Objects::isNull)) { throw new IllegalArgumentException(); } @@ -464,7 +464,10 @@ public ResultList getAllWorkflowExecutions(Set datase /** * Get an overview of all WorkflowExecutions. This returns a list of executions ordered to display * an overview. First the ones in queue, then those in progress and then those that are finalized. - * They will be sorted by creation date. This method does support pagination. TODO when we migrate + * Within these categories they will be sorted by creation date (most recent first). This method + * does support pagination. + * + * TODO when we migrate * to mongo 3.4 or later, we can do this easier with new aggregation pipeline stages and * operators. The main improvements are 1) to try to map the root to the 'execution' variable so * that we don't have to look it up afterwards, and 2) to use $addFields with $switch to add the @@ -477,16 +480,22 @@ public ResultList getAllWorkflowExecutions(Set datase * @param toDate the date to where the results should end. Can be null. * @param nextPage the nextPage token * @param pageCount the number of pages that are requested - * @return a list of all the WorkflowExecutions found + * @return a list of all the WorkflowExecutions found. Is not null. */ public ResultList getWorkflowExecutionsOverview(Set datasetIds, Set pluginStatuses, Set pluginTypes, Date fromDate, Date toDate, int nextPage, int pageCount) { + return getWorkflowExecutionsOverview(datasetIds, pluginStatuses, pluginTypes, fromDate, toDate, + createPagination(nextPage, pageCount, false)); + } + + ResultList getWorkflowExecutionsOverview(Set datasetIds, + Set pluginStatuses, Set pluginTypes, Date fromDate, Date toDate, + Pagination pagination) { - return retryableExternalRequestForNetworkExceptions(() -> { + return retryableExternalRequestForNetworkExceptions(() -> { // Prepare pagination and check that there is something to query - final Pagination pagination = createPagination(nextPage, pageCount, false); if (pagination.getLimit() < 1) { return createResultList(Collections.emptyList(), pagination); } @@ -757,18 +766,16 @@ public WorkflowExecution getByExternalTaskId(long externalTaskId) { * This method retrieves the workflow execution that contains a subtask satisfying the given * parameters. * - * @param startedDate The started date of the subtask. - * @param pluginType The plugin type of the subtask. + * @param plugin The plugin ID representing the subtask. * @param datasetId The dataset ID of the workflow execution. * @return The workflow execution. */ - public WorkflowExecution getByTaskExecution(Date startedDate, PluginType pluginType, - String datasetId) { + public WorkflowExecution getByTaskExecution(ExecutedMetisPluginId plugin, String datasetId) { // Create subquery to find the correct plugin. List elemMatchFilters = new ArrayList<>(); - elemMatchFilters.add(Filters.eq(STARTED_DATE.getFieldName(), startedDate)); - elemMatchFilters.add(Filters.eq(PLUGIN_TYPE.getFieldName(), pluginType)); + elemMatchFilters.add(Filters.eq(STARTED_DATE.getFieldName(), plugin.getPluginStartedDate())); + elemMatchFilters.add(Filters.eq(PLUGIN_TYPE.getFieldName(), plugin.getPluginType())); // Create query to find workflow execution final Query query = @@ -795,7 +802,7 @@ public WorkflowExecution getAnyByXsltId(String xsltId) { return retryableExternalRequestForNetworkExceptions(query::first); } - private Pagination createPagination(int firstPage, Integer pageCount, + Pagination createPagination(int firstPage, Integer pageCount, boolean ignoreMaxServedExecutionsLimit) { // Compute the total number (including skipped pages) @@ -814,13 +821,13 @@ private Pagination createPagination(int firstPage, Integer pageCount, return new Pagination(skip, limit, maxRequested); } - private static class Pagination { + static class Pagination { private final int skip; private final int limit; private final boolean maxRequested; - Pagination(int skip, int limit, boolean maxRequested) { + private Pagination(int skip, int limit, boolean maxRequested) { this.skip = skip; this.limit = limit; this.maxRequested = maxRequested; diff --git a/metis-core/metis-core-service/src/main/java/eu/europeana/metis/core/dao/WorkflowValidationUtils.java b/metis-core/metis-core-service/src/main/java/eu/europeana/metis/core/dao/WorkflowValidationUtils.java new file mode 100644 index 000000000..a4f6a44d5 --- /dev/null +++ b/metis-core/metis-core-service/src/main/java/eu/europeana/metis/core/dao/WorkflowValidationUtils.java @@ -0,0 +1,216 @@ +package eu.europeana.metis.core.dao; + +import eu.europeana.metis.core.dataset.DepublishRecordId.DepublicationStatus; +import eu.europeana.metis.core.exceptions.PluginExecutionNotAllowed; +import eu.europeana.metis.core.util.DepublishRecordIdSortField; +import eu.europeana.metis.core.util.SortDirection; +import eu.europeana.metis.core.workflow.Workflow; +import eu.europeana.metis.core.workflow.plugins.AbstractExecutablePluginMetadata; +import eu.europeana.metis.core.workflow.plugins.DepublishPluginMetadata; +import eu.europeana.metis.core.workflow.plugins.ExecutablePlugin; +import eu.europeana.metis.core.workflow.plugins.ExecutablePluginType; +import eu.europeana.metis.core.workflow.plugins.HTTPHarvestPluginMetadata; +import eu.europeana.metis.core.workflow.plugins.OaipmhHarvestPluginMetadata; +import eu.europeana.metis.core.workflow.plugins.PluginType; +import eu.europeana.metis.exception.BadContentException; +import eu.europeana.metis.exception.GenericMetisException; +import eu.europeana.metis.utils.CommonStringValues; +import java.net.MalformedURLException; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.URL; +import java.util.EnumSet; +import java.util.List; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; +import org.apache.http.client.utils.URIBuilder; +import org.springframework.util.CollectionUtils; + +/** + * This class is a utility class that can answer questions related to the validation of workflows. + */ +public class WorkflowValidationUtils { + + private final DepublishRecordIdDao depublishRecordIdDao; + private final DataEvolutionUtils dataEvolutionUtils; + + /** + * Constructor. + * + * @param depublishRecordIdDao the depublication record id dao + * @param dataEvolutionUtils The utilities class for sorting out data evolution + */ + public WorkflowValidationUtils(DepublishRecordIdDao depublishRecordIdDao, + DataEvolutionUtils dataEvolutionUtils) { + this.depublishRecordIdDao = depublishRecordIdDao; + this.dataEvolutionUtils = dataEvolutionUtils; + } + + /** + * This method validates the workflow plugin sequence. In particular, it checks: + *
      + *
    1. That the workflow is not empty and contains plugins with valid types,
      2. + *
    2. That the first plugin is not link checking (except when it is the only plugin),
      3. + *
    3. That no two plugins of the same type occur in the workflow,
      4. + *
    4. That if depublish is enabled no other plugins are allowed in the workflow,
      5. + *
    5. That the first plugin has a valid predecessor plugin in the dataset's history (as defined by + * {@link DataEvolutionUtils#getPredecessorTypes(ExecutablePluginType)}), the type of which can be + * overridden by the enforced predecessor type, and the root plugin (i.e. harvest) of which is + * equal to the latest successful harvest (i.e. no old data should be processed after new data has + * been introduced),
      6. + *
    6. That all subsequent plugins have a valid predecessor within the workflow (as defined by + * {@link DataEvolutionUtils#getPredecessorTypes(ExecutablePluginType)}),
      7. + *
    7. That harvesting plugins have valid URL settings.
      8. + *
    + * + * @param workflow The workflow to validate. + * @param enforcedPredecessorType If not null, overrides the predecessor type of the first + * plugin. + * @return The predecessor of the first plugin. Or null if no predecessor is required. + * @throws GenericMetisException which can be one of: + *
      + *
    • {@link BadContentException} In case the workflow is empty, or contains plugins with + * invalid types.
      • + *
    • {@link PluginExecutionNotAllowed} in case the plugin sequence as provided is not + * allowed.
      • + *
    + */ + public PluginWithExecutionId> validateWorkflowPlugins(Workflow workflow, + ExecutablePluginType enforcedPredecessorType) throws GenericMetisException { + + // Workflow should have a plugin list. + if (workflow.getMetisPluginsMetadata() == null) { + throw new BadContentException("Workflow should not be empty."); + } + + // Compile the list of enabled plugins. + final List enabledPlugins = workflow.getMetisPluginsMetadata() + .stream().filter(AbstractExecutablePluginMetadata::isEnabled).collect(Collectors.toList()); + + // Workflow should not be empty and all should have a type. + if (enabledPlugins.isEmpty()) { + throw new BadContentException("Workflow should not be empty."); + } + if (enabledPlugins.stream().map(AbstractExecutablePluginMetadata::getExecutablePluginType) + .anyMatch(Objects::isNull)) { + throw new BadContentException( + "There are enabled plugins of which the type could not be determined."); + } + + // Validate dataset/record depublication + validateDepublishPlugin(workflow.getDatasetId(), enabledPlugins); + + // Validate and normalize the harvest parameters of harvest plugins (even if not enabled) + validateAndTrimHarvestParameters(workflow.getDatasetId(), enabledPlugins); + + // Check that first plugin is not link checking (except if it is the only plugin) + if (enabledPlugins.size() > 1 + && enabledPlugins.get(0).getPluginType() == PluginType.LINK_CHECKING) { + throw new PluginExecutionNotAllowed(CommonStringValues.PLUGIN_EXECUTION_NOT_ALLOWED); + } + + // Make sure that all enabled plugins (except the first) have a predecessor within the workflow. + final EnumSet previousTypesInWorkflow = EnumSet + .of(enabledPlugins.get(0).getExecutablePluginType()); + for (int i = 1; i < enabledPlugins.size(); i++) { + + // Find the permissible predecessors + final ExecutablePluginType pluginType = enabledPlugins.get(i).getExecutablePluginType(); + final Set permissiblePredecessors = DataEvolutionUtils + .getPredecessorTypes(pluginType); + + // Check if we have the right predecessor plugin types in the workflow + final boolean hasNoPredecessor = !permissiblePredecessors.isEmpty() && + permissiblePredecessors.stream().noneMatch(previousTypesInWorkflow::contains); + if (hasNoPredecessor) { + throw new PluginExecutionNotAllowed(CommonStringValues.PLUGIN_EXECUTION_NOT_ALLOWED); + } + + // Add the plugin type to those we have seen + previousTypesInWorkflow.add(pluginType); + } + + // We should now have seen all types. Make sure that there are no duplicates + if (previousTypesInWorkflow.size() != enabledPlugins.size()) { + throw new PluginExecutionNotAllowed(CommonStringValues.PLUGIN_EXECUTION_NOT_ALLOWED); + } + + // Check the presence of the predecessor and return it. + return dataEvolutionUtils + .computePredecessorPlugin(enabledPlugins.get(0).getExecutablePluginType(), + enforcedPredecessorType, workflow.getDatasetId()); + } + + private void validateAndTrimHarvestParameters(String datasetId, + Iterable enabledPlugins) throws BadContentException { + for (AbstractExecutablePluginMetadata pluginMetadata : enabledPlugins) { + if (pluginMetadata instanceof OaipmhHarvestPluginMetadata) { + final OaipmhHarvestPluginMetadata oaipmhMetadata = (OaipmhHarvestPluginMetadata) pluginMetadata; + final URI validatedUri = validateUrl(oaipmhMetadata.getUrl()); + oaipmhMetadata + .setUrl(new URIBuilder(validatedUri).removeQuery().setFragment(null).toString()); + oaipmhMetadata.setMetadataFormat(oaipmhMetadata.getMetadataFormat() == null ? null + : oaipmhMetadata.getMetadataFormat().trim()); + oaipmhMetadata.setSetSpec( + oaipmhMetadata.getSetSpec() == null ? null : oaipmhMetadata.getSetSpec().trim()); + if (oaipmhMetadata.isIncrementalHarvest() && !isIncrementalHarvestingAllowed(datasetId)) { + throw new BadContentException("Can't perform incremental harvesting for this dataset."); + } + } + if (pluginMetadata instanceof HTTPHarvestPluginMetadata) { + final HTTPHarvestPluginMetadata httpMetadata = (HTTPHarvestPluginMetadata) pluginMetadata; + httpMetadata.setUrl(validateUrl(httpMetadata.getUrl()).toString()); + } + } + } + + /** + * This method returns whether currently it is permitted/possible to perform incremental + * harvesting for the given dataset. + * + * @param datasetId The ID of the dataset for which to check. + * @return Whether we can perform incremental harvesting for the dataset. + */ + public boolean isIncrementalHarvestingAllowed(String datasetId) { + // We need to do the entire analysis to make sure that all publish actions are consistent. + return !CollectionUtils.isEmpty(dataEvolutionUtils.getPublishedHarvestIncrements(datasetId)); + } + + private void validateDepublishPlugin(String datasetId, + List enabledPlugins) throws BadContentException { + // If depublish requested, make sure it's the only plugin in the workflow + final Optional depublishPluginMetadata = enabledPlugins.stream() + .filter(plugin -> plugin.getExecutablePluginType().toPluginType() == PluginType.DEPUBLISH) + .map(plugin -> (DepublishPluginMetadata) plugin).findFirst(); + if (enabledPlugins.size() > 1 && depublishPluginMetadata.isPresent()) { + throw new BadContentException( + "If DEPUBLISH plugin enabled, no other enabled plugins are allowed."); + } + + // If record depublication requested, check if there are pending record ids in the db + if (depublishPluginMetadata.isPresent() && !depublishPluginMetadata.get() + .isDatasetDepublish()) { + final Set pendingDepublicationIds = depublishRecordIdDao + .getAllDepublishRecordIdsWithStatus(datasetId, + DepublishRecordIdSortField.DEPUBLICATION_STATE, SortDirection.ASCENDING, + DepublicationStatus.PENDING_DEPUBLICATION); + if (CollectionUtils.isEmpty(pendingDepublicationIds)) { + throw new BadContentException( + "Record depublication requested but there are no pending depublication record ids in the db"); + } + } + } + + private static URI validateUrl(String urlString) throws BadContentException { + if (urlString == null) { + throw new BadContentException("Harvesting parameters are missing"); + } + try { + return new URL(urlString.trim()).toURI(); + } catch (MalformedURLException | URISyntaxException e) { + throw new BadContentException("Harvesting parameters are invalid", e); + } + } +} diff --git a/metis-core/metis-core-service/src/main/java/eu/europeana/metis/core/execution/SchedulerExecutor.java b/metis-core/metis-core-service/src/main/java/eu/europeana/metis/core/execution/SchedulerExecutor.java index 2bfeb915b..ba669de8f 100644 --- a/metis-core/metis-core-service/src/main/java/eu/europeana/metis/core/execution/SchedulerExecutor.java +++ b/metis-core/metis-core-service/src/main/java/eu/europeana/metis/core/execution/SchedulerExecutor.java @@ -27,9 +27,9 @@ public class SchedulerExecutor { private static final Logger LOGGER = LoggerFactory.getLogger(SchedulerExecutor.class); - private final RLock lock; private final OrchestratorService orchestratorService; private final ScheduleWorkflowService scheduleWorkflowService; + private final RedissonClient redissonClient; private static final String SCHEDULER_LOCK = "schedulerLock"; private LocalDateTime lastExecutionTime = LocalDateTime.now(); @@ -44,7 +44,7 @@ public SchedulerExecutor(OrchestratorService orchestratorService, ScheduleWorkflowService scheduleWorkflowService, RedissonClient redissonClient) { this.orchestratorService = orchestratorService; this.scheduleWorkflowService = scheduleWorkflowService; - this.lock = redissonClient.getFairLock(SCHEDULER_LOCK); + this.redissonClient = redissonClient; } /** @@ -53,6 +53,7 @@ public SchedulerExecutor(OrchestratorService orchestratorService, * periodically. */ public void performScheduling() { + RLock lock = redissonClient.getFairLock(SCHEDULER_LOCK); try { lock.lock(); final LocalDateTime thisExecutionTime = LocalDateTime.now(); diff --git a/metis-core/metis-core-service/src/main/java/eu/europeana/metis/core/execution/WorkflowExecutionMonitor.java b/metis-core/metis-core-service/src/main/java/eu/europeana/metis/core/execution/WorkflowExecutionMonitor.java index f553f7fc5..40040980f 100644 --- a/metis-core/metis-core-service/src/main/java/eu/europeana/metis/core/execution/WorkflowExecutionMonitor.java +++ b/metis-core/metis-core-service/src/main/java/eu/europeana/metis/core/execution/WorkflowExecutionMonitor.java @@ -41,7 +41,7 @@ public class WorkflowExecutionMonitor { private final WorkflowExecutionDao workflowExecutionDao; private final WorkflowExecutorManager workflowExecutorManager; private final Duration failsafeLeniency; - private final RLock lock; + private final RedissonClient redissonClient; /** * The currently running executions. @@ -62,7 +62,7 @@ public WorkflowExecutionMonitor(WorkflowExecutorManager workflowExecutorManager, this.failsafeLeniency = failsafeLeniency; this.workflowExecutionDao = workflowExecutionDao; this.workflowExecutorManager = workflowExecutorManager; - this.lock = redissonClient.getFairLock(FAILSAFE_LOCK); + this.redissonClient = redissonClient; } /* DO NOT CALL THIS METHOD WITHOUT POSSESSING THE LOCK */ @@ -102,11 +102,10 @@ List updateCurrentRunningExecutions() { * to run periodically. */ public void performFailsafe() { + RLock lock = redissonClient.getFairLock(FAILSAFE_LOCK); try { - // Lock for the duration of this scheduled task lock.lock(); - // Update the execution times. This way we always have the latest values. final List allRunningWorkflowExecutions = updateCurrentRunningExecutions(); @@ -187,10 +186,10 @@ public Pair claimExecution(String workflowExecutionI WorkflowExecution workflowExecution; boolean claimed = false; + RLock lock = redissonClient.getFairLock(FAILSAFE_LOCK); + // Lock for the duration of this request + lock.lock(); try { - // Lock for the duration of this request - lock.lock(); - // Retrieve the most current version of the execution. workflowExecution = workflowExecutionDao.getById(workflowExecutionId); diff --git a/metis-core/metis-core-service/src/main/java/eu/europeana/metis/core/execution/WorkflowExecutor.java b/metis-core/metis-core-service/src/main/java/eu/europeana/metis/core/execution/WorkflowExecutor.java index ad9d4d89a..8fc51c1b8 100644 --- a/metis-core/metis-core-service/src/main/java/eu/europeana/metis/core/execution/WorkflowExecutor.java +++ b/metis-core/metis-core-service/src/main/java/eu/europeana/metis/core/execution/WorkflowExecutor.java @@ -6,8 +6,9 @@ import eu.europeana.cloud.client.dps.rest.DpsClient; import eu.europeana.cloud.common.model.dps.TaskState; import eu.europeana.cloud.service.dps.exception.DpsException; +import eu.europeana.metis.core.dao.ExecutedMetisPluginId; import eu.europeana.metis.core.dao.WorkflowExecutionDao; -import eu.europeana.metis.core.dao.WorkflowUtils; +import eu.europeana.metis.core.dao.DataEvolutionUtils; import eu.europeana.metis.core.workflow.WorkflowExecution; import eu.europeana.metis.core.workflow.WorkflowStatus; import eu.europeana.metis.core.workflow.plugins.AbstractExecutablePlugin; @@ -17,7 +18,6 @@ import eu.europeana.metis.core.workflow.plugins.ExecutablePlugin.MonitorResult; import eu.europeana.metis.core.workflow.plugins.ExecutablePluginType; import eu.europeana.metis.core.workflow.plugins.PluginStatus; -import eu.europeana.metis.core.workflow.plugins.PluginType; import eu.europeana.metis.exception.ExternalTaskException; import eu.europeana.metis.network.ExternalRequestUtil; import java.util.Date; @@ -207,7 +207,6 @@ private int getFirstPluginPositionToStart() { * @param i the index of the plugin in the list of plugins inside the workflow execution * @param plugin the provided plugin to be ran * @return true if plugin ran, false if plugin did not run - * @throws InterruptedException if an interruption occurred */ private boolean runMetisPluginWithSemaphoreAllocation(int i, AbstractMetisPlugin plugin) { // Sanity check @@ -249,20 +248,21 @@ private boolean runMetisPluginWithSemaphoreAllocation(int i, AbstractMetisPlugin * It will prepare the plugin, request the external execution and will periodically monitor, * update the plugin's progress and at the end finalize the plugin's status and finished date. * - * @param executablePlugin the plugin to run + * @param plugin the plugin to run * @param startDateToUse The date that should be used as start date (if the plugin is not already * running). * @param datasetId The dataset ID. */ - private void runMetisPlugin(AbstractExecutablePlugin executablePlugin, Date startDateToUse, + private void runMetisPlugin(AbstractExecutablePlugin plugin, Date startDateToUse, String datasetId) { try { // Compute previous plugin revision information. Only need to look within the workflow: when // scheduling the workflow, the previous plugin information is set for the first plugin. - final AbstractExecutablePluginMetadata metadata = executablePlugin.getPluginMetadata(); - if (metadata.getRevisionTimestampPreviousPlugin() == null - || metadata.getRevisionNamePreviousPlugin() == null) { - final AbstractExecutablePlugin predecessor = WorkflowUtils + final AbstractExecutablePluginMetadata metadata = plugin.getPluginMetadata(); + final ExecutedMetisPluginId executedMetisPluginId = ExecutedMetisPluginId + .forPredecessor(plugin); + if (executedMetisPluginId == null) { + final AbstractExecutablePlugin predecessor = DataEvolutionUtils .computePredecessorPlugin(metadata.getExecutablePluginType(), workflowExecution); if (predecessor != null) { metadata.setPreviousRevisionInformation(predecessor); @@ -270,23 +270,22 @@ private void runMetisPlugin(AbstractExecutablePlugin executablePlugin, Date s } // Start execution if it has not already started - if (StringUtils.isEmpty(executablePlugin.getExternalTaskId())) { - if (executablePlugin.getPluginStatus() == PluginStatus.INQUEUE) { - executablePlugin.setStartedDate(startDateToUse); + if (StringUtils.isEmpty(plugin.getExternalTaskId())) { + if (plugin.getPluginStatus() == PluginStatus.INQUEUE) { + plugin.setStartedDate(startDateToUse); } final EcloudBasePluginParameters ecloudBasePluginParameters = new EcloudBasePluginParameters( ecloudBaseUrl, ecloudProvider, workflowExecution.getEcloudDatasetId(), getExternalTaskIdOfPreviousPlugin(metadata), metisCoreBaseUrl); - executablePlugin + plugin .execute(workflowExecution.getDatasetId(), dpsClient, ecloudBasePluginParameters); } } catch (ExternalTaskException | RuntimeException e) { - LOGGER.warn(String - .format("workflowExecutionId: %s, pluginType: %s - Execution of plugin " + "failed", - workflowExecution.getId(), executablePlugin.getPluginType()), e); - executablePlugin.setFinishedDate(null); - executablePlugin.setPluginStatusAndResetFailMessage(PluginStatus.FAILED); - executablePlugin.setFailMessage(String.format(DETAILED_EXCEPTION_FORMAT, TRIGGER_ERROR_PREFIX, + LOGGER.warn(String.format("workflowExecutionId: %s, pluginType: %s - Execution of plugin " + + "failed", workflowExecution.getId(), plugin.getPluginType()), e); + plugin.setFinishedDate(null); + plugin.setPluginStatusAndResetFailMessage(PluginStatus.FAILED); + plugin.setFailMessage(String.format(DETAILED_EXCEPTION_FORMAT, TRIGGER_ERROR_PREFIX, ExceptionUtils.getStackTrace(e))); return; } finally { @@ -295,26 +294,22 @@ private void runMetisPlugin(AbstractExecutablePlugin executablePlugin, Date s // Start periodical check and wait for plugin to be done long sleepTime = TimeUnit.SECONDS.toMillis(monitorCheckIntervalInSecs); - periodicCheckingLoop(sleepTime, executablePlugin, datasetId); + periodicCheckingLoop(sleepTime, plugin, datasetId); } - private String getExternalTaskIdOfPreviousPlugin( - AbstractExecutablePluginMetadata pluginMetadata) { + private String getExternalTaskIdOfPreviousPlugin(AbstractExecutablePluginMetadata metadata) { // Get the previous plugin parameters from the plugin - if there is none, we are done. - final PluginType previousPluginType = PluginType - .getPluginTypeFromEnumName(pluginMetadata.getRevisionNamePreviousPlugin()); - final Date previousPluginStartDate = pluginMetadata.getRevisionTimestampPreviousPlugin(); - if (previousPluginType == null || previousPluginStartDate == null) { + final ExecutedMetisPluginId predecessorPlugin = ExecutedMetisPluginId.forPredecessor(metadata); + if (predecessorPlugin == null) { return null; } // Get the previous plugin based on the parameters. final WorkflowExecution previousExecution = workflowExecutionDao - .getByTaskExecution(previousPluginStartDate, previousPluginType, - workflowExecution.getDatasetId()); + .getByTaskExecution(predecessorPlugin, workflowExecution.getDatasetId()); return Optional.ofNullable(previousExecution) - .flatMap(execution -> execution.getMetisPluginWithType(previousPluginType)) + .flatMap(execution -> execution.getMetisPluginWithType(predecessorPlugin.getPluginType())) .map(this::expectExecutablePlugin).map(AbstractExecutablePlugin::getExternalTaskId) .orElse(null); } diff --git a/metis-core/metis-core-service/src/main/java/eu/europeana/metis/core/execution/WorkflowPostProcessor.java b/metis-core/metis-core-service/src/main/java/eu/europeana/metis/core/execution/WorkflowPostProcessor.java index f1b0c58b7..7b9035449 100644 --- a/metis-core/metis-core-service/src/main/java/eu/europeana/metis/core/execution/WorkflowPostProcessor.java +++ b/metis-core/metis-core-service/src/main/java/eu/europeana/metis/core/execution/WorkflowPostProcessor.java @@ -48,7 +48,7 @@ public class WorkflowPostProcessor { /** * Constructor. - * @param depublishRecordIdDao The DAO for depublished records. + * @param depublishRecordIdDao The DAO for depublished records. * @param datasetDao The DAO for datasets * @param workflowExecutionDao The DAO for workflow executions. * @param dpsClient the dps client @@ -101,7 +101,7 @@ private void depublishDatasetPostProcess(String datasetId){ DepublicationStatus.PENDING_DEPUBLICATION, null); // Find latest PUBLISH Type Plugin and set dataStatus to DELETED. - final PluginWithExecutionId latestSuccessfulPlugin = workflowExecutionDao + final PluginWithExecutionId> latestSuccessfulPlugin = workflowExecutionDao .getLatestSuccessfulPlugin(datasetId, OrchestratorService.PUBLISH_TYPES); if (Objects.nonNull(latestSuccessfulPlugin) && Objects .nonNull(latestSuccessfulPlugin.getPlugin())) { diff --git a/metis-core/metis-core-service/src/main/java/eu/europeana/metis/core/service/DatasetService.java b/metis-core/metis-core-service/src/main/java/eu/europeana/metis/core/service/DatasetService.java index 2915974bc..bd312d245 100644 --- a/metis-core/metis-core-service/src/main/java/eu/europeana/metis/core/service/DatasetService.java +++ b/metis-core/metis-core-service/src/main/java/eu/europeana/metis/core/service/DatasetService.java @@ -1,5 +1,6 @@ package eu.europeana.metis.core.service; +import eu.europeana.metis.core.common.TransformationParameters; import eu.europeana.metis.utils.CommonStringValues; import eu.europeana.metis.utils.RestEndpoints; import eu.europeana.metis.authentication.user.MetisUser; @@ -518,8 +519,10 @@ private List transformRecords(Dataset dataset, Collection record final XsltTransformer transformer; final EuropeanaIdCreator europeanIdCreator; try { - transformer = new XsltTransformer(xsltUrl, dataset.getDatasetName(), - dataset.getCountry().getName(), dataset.getLanguage().name()); + final TransformationParameters transformationParameters = new TransformationParameters( + dataset); + transformer = new XsltTransformer(xsltUrl, transformationParameters.getDatasetName(), + transformationParameters.getEdmCountry(), transformationParameters.getEdmLanguage()); europeanIdCreator = new EuropeanaIdCreator(); } catch (TransformationException e) { throw new XsltSetupException("Could not setup XSL transformation.", e); @@ -680,7 +683,7 @@ public List searchDatasetsBasedOnSearchString(MetisUser metis } return datasets.stream().map(dataset -> { - final PluginWithExecutionId latestSuccessfulExecutablePlugin = workflowExecutionDao + final PluginWithExecutionId> latestSuccessfulExecutablePlugin = workflowExecutionDao .getLatestSuccessfulExecutablePlugin(dataset.getDatasetId(), EnumSet.allOf(ExecutablePluginType.class), false); final DatasetSearchView datasetSearchView = new DatasetSearchView(); diff --git a/metis-core/metis-core-service/src/main/java/eu/europeana/metis/core/service/OrchestratorService.java b/metis-core/metis-core-service/src/main/java/eu/europeana/metis/core/service/OrchestratorService.java index 9281acbf5..6996422ad 100644 --- a/metis-core/metis-core-service/src/main/java/eu/europeana/metis/core/service/OrchestratorService.java +++ b/metis-core/metis-core-service/src/main/java/eu/europeana/metis/core/service/OrchestratorService.java @@ -11,7 +11,8 @@ import eu.europeana.metis.core.dao.WorkflowExecutionDao; import eu.europeana.metis.core.dao.WorkflowExecutionDao.ExecutionDatasetPair; import eu.europeana.metis.core.dao.WorkflowExecutionDao.ResultList; -import eu.europeana.metis.core.dao.WorkflowUtils; +import eu.europeana.metis.core.dao.DataEvolutionUtils; +import eu.europeana.metis.core.dao.WorkflowValidationUtils; import eu.europeana.metis.core.dataset.Dataset; import eu.europeana.metis.core.dataset.DatasetExecutionInformation; import eu.europeana.metis.core.dataset.DatasetExecutionInformation.PublicationStatus; @@ -98,7 +99,8 @@ public class OrchestratorService { .immutableEnumSet(ExecutablePluginType.LINK_CHECKING, ExecutablePluginType.DEPUBLISH); private final WorkflowExecutionDao workflowExecutionDao; - private final WorkflowUtils workflowUtils; + private final WorkflowValidationUtils workflowValidationUtils; + private final DataEvolutionUtils dataEvolutionUtils; private final WorkflowDao workflowDao; private final DatasetDao datasetDao; private final WorkflowExecutorManager workflowExecutorManager; @@ -114,7 +116,8 @@ public class OrchestratorService { * @param workflowExecutionFactory the orchestratorHelper instance * @param workflowDao the Dao instance to access the Workflow database * @param workflowExecutionDao the Dao instance to access the WorkflowExecution database - * @param workflowUtils The utilities class providing more functionality on top of DAOs. + * @param workflowValidationUtils A utilities class providing more functionality on top of DAOs. + * @param dataEvolutionUtils A utilities class providing more functionality on top of DAOs. * @param datasetDao the Dao instance to access the Dataset database * @param workflowExecutorManager the instance that handles the production and consumption of * workflowExecutions @@ -124,14 +127,16 @@ public class OrchestratorService { */ @Autowired public OrchestratorService(WorkflowExecutionFactory workflowExecutionFactory, - WorkflowDao workflowDao, WorkflowExecutionDao workflowExecutionDao, - WorkflowUtils workflowUtils, DatasetDao datasetDao, - WorkflowExecutorManager workflowExecutorManager, RedissonClient redissonClient, - Authorizer authorizer, DepublishRecordIdDao depublishRecordIdDao) { + WorkflowDao workflowDao, WorkflowExecutionDao workflowExecutionDao, + WorkflowValidationUtils workflowValidationUtils, DataEvolutionUtils dataEvolutionUtils, + DatasetDao datasetDao, WorkflowExecutorManager workflowExecutorManager, + RedissonClient redissonClient, Authorizer authorizer, + DepublishRecordIdDao depublishRecordIdDao) { this.workflowExecutionFactory = workflowExecutionFactory; this.workflowDao = workflowDao; this.workflowExecutionDao = workflowExecutionDao; - this.workflowUtils = workflowUtils; + this.workflowValidationUtils = workflowValidationUtils; + this.dataEvolutionUtils = dataEvolutionUtils; this.datasetDao = datasetDao; this.workflowExecutorManager = workflowExecutorManager; this.redissonClient = redissonClient; @@ -170,7 +175,7 @@ public void createWorkflow(MetisUser metisUser, String datasetId, Workflow workf } // Validate the new workflow. - workflowUtils.validateWorkflowPlugins(workflow, enforcedPredecessorType); + workflowValidationUtils.validateWorkflowPlugins(workflow, enforcedPredecessorType); // Save the workflow. workflowDao.create(workflow); @@ -209,7 +214,7 @@ public void updateWorkflow(MetisUser metisUser, String datasetId, Workflow workf } // Validate the new workflow. - workflowUtils.validateWorkflowPlugins(workflow, enforcedPredecessorType); + workflowValidationUtils.validateWorkflowPlugins(workflow, enforcedPredecessorType); // Overwrite the workflow. workflow.setId(storedWorkflow.getId()); @@ -371,7 +376,7 @@ private WorkflowExecution addWorkflowInQueueOfWorkflowExecutions(Dataset dataset } // Validate the workflow and obtain the predecessor. - final PluginWithExecutionId predecessor = workflowUtils + final PluginWithExecutionId> predecessor = workflowValidationUtils .validateWorkflowPlugins(workflow, enforcedPredecessorType); // Make sure that eCloud knows tmetisUserhis dataset (needs to happen before we create the workflow). @@ -479,12 +484,12 @@ public int getWorkflowExecutionsPerRequest() { *
  • {@link UserUnauthorizedException} if the user is not authorized to perform this task
    • * */ - public ExecutablePlugin getLatestFinishedPluginByDatasetIdIfPluginTypeAllowedForExecution( + public ExecutablePlugin getLatestFinishedPluginByDatasetIdIfPluginTypeAllowedForExecution( MetisUser metisUser, String datasetId, ExecutablePluginType pluginType, ExecutablePluginType enforcedPredecessorType) throws GenericMetisException { authorizer.authorizeReadExistingDatasetById(metisUser, datasetId); return Optional.ofNullable( - workflowUtils.computePredecessorPlugin(pluginType, enforcedPredecessorType, datasetId)) + dataEvolutionUtils.computePredecessorPlugin(pluginType, enforcedPredecessorType, datasetId)) .map(PluginWithExecutionId::getPlugin).orElse(null); } @@ -627,28 +632,28 @@ public DatasetExecutionInformation getDatasetExecutionInformation(MetisUser meti DatasetExecutionInformation getDatasetExecutionInformation(String datasetId) { // Obtain the relevant parts of the execution history - final ExecutablePlugin lastHarvestPlugin = Optional.ofNullable( + final ExecutablePlugin lastHarvestPlugin = Optional.ofNullable( workflowExecutionDao.getLatestSuccessfulExecutablePlugin(datasetId, HARVEST_TYPES, false)) .map(PluginWithExecutionId::getPlugin).orElse(null); - final PluginWithExecutionId firstPublishPluginWithExecutionId = workflowExecutionDao + final PluginWithExecutionId> firstPublishPluginWithExecutionId = workflowExecutionDao .getFirstSuccessfulPlugin(datasetId, PUBLISH_TYPES); - final MetisPlugin firstPublishPlugin = firstPublishPluginWithExecutionId == null ? null + final MetisPlugin firstPublishPlugin = firstPublishPluginWithExecutionId == null ? null : firstPublishPluginWithExecutionId.getPlugin(); - final ExecutablePlugin lastExecutablePreviewPlugin = Optional.ofNullable(workflowExecutionDao + final ExecutablePlugin lastExecutablePreviewPlugin = Optional.ofNullable(workflowExecutionDao .getLatestSuccessfulExecutablePlugin(datasetId, EXECUTABLE_PREVIEW_TYPES, false)) .map(PluginWithExecutionId::getPlugin).orElse(null); - final ExecutablePlugin lastExecutablePublishPlugin = Optional.ofNullable(workflowExecutionDao + final ExecutablePlugin lastExecutablePublishPlugin = Optional.ofNullable(workflowExecutionDao .getLatestSuccessfulExecutablePlugin(datasetId, EXECUTABLE_PUBLISH_TYPES, false)) .map(PluginWithExecutionId::getPlugin).orElse(null); - final PluginWithExecutionId latestPreviewPluginWithExecutionId = workflowExecutionDao + final PluginWithExecutionId> latestPreviewPluginWithExecutionId = workflowExecutionDao .getLatestSuccessfulPlugin(datasetId, PREVIEW_TYPES); - final PluginWithExecutionId latestPublishPluginWithExecutionId = workflowExecutionDao + final PluginWithExecutionId> latestPublishPluginWithExecutionId = workflowExecutionDao .getLatestSuccessfulPlugin(datasetId, PUBLISH_TYPES); - final MetisPlugin lastPreviewPlugin = latestPreviewPluginWithExecutionId == null ? null + final MetisPlugin lastPreviewPlugin = latestPreviewPluginWithExecutionId == null ? null : latestPreviewPluginWithExecutionId.getPlugin(); - final MetisPlugin lastPublishPlugin = latestPublishPluginWithExecutionId == null ? null + final MetisPlugin lastPublishPlugin = latestPublishPluginWithExecutionId == null ? null : latestPublishPluginWithExecutionId.getPlugin(); - final ExecutablePlugin lastExecutableDepublishPlugin = Optional.ofNullable(workflowExecutionDao + final ExecutablePlugin lastExecutableDepublishPlugin = Optional.ofNullable(workflowExecutionDao .getLatestSuccessfulExecutablePlugin(datasetId, EXECUTABLE_DEPUBLISH_TYPES, false)) .map(PluginWithExecutionId::getPlugin).orElse(null); @@ -910,13 +915,13 @@ public VersionEvolution getRecordEvolutionForVersion(MetisUser metisUser, String } // Find the plugin (workflow step) in question. - final AbstractMetisPlugin targetPlugin = execution.getMetisPluginWithType(pluginType) + final AbstractMetisPlugin targetPlugin = execution.getMetisPluginWithType(pluginType) .orElseThrow(() -> new NoWorkflowExecutionFoundException(String .format("No plugin of type %s found for workflowExecution with id: %s", pluginType.name(), execution))); // Compile the version evolution. - final Collection> evolutionSteps = workflowUtils + final Collection, WorkflowExecution>> evolutionSteps = dataEvolutionUtils .compileVersionEvolution(targetPlugin, execution); final VersionEvolution versionEvolution = new VersionEvolution(); versionEvolution.setEvolutionSteps(evolutionSteps.stream().map(step -> { @@ -930,6 +935,29 @@ public VersionEvolution getRecordEvolutionForVersion(MetisUser metisUser, String return versionEvolution; } + /** + * This method returns whether currently it is permitted/possible to perform incremental + * harvesting for the given dataset. + * + * @param metisUser the user wishing to perform this operation + * @param datasetId The ID of the dataset for which to check. + * @return Whether we can perform incremental harvesting for the dataset. + * @throws GenericMetisException which can be one of: + *
      + *
    • {@link NoDatasetFoundException} if the dataset identifier provided does not exist
      • + *
    • {@link UserUnauthorizedException} if the user is not authorized to perform this task
      • + *
    + */ + public boolean isIncrementalHarvestingAllowed(MetisUser metisUser, String datasetId) + throws GenericMetisException { + + // Check that the user is authorized + authorizer.authorizeReadExistingDatasetById(metisUser, datasetId); + + // Do the check. + return workflowValidationUtils.isIncrementalHarvestingAllowed(datasetId); + } + public int getSolrCommitPeriodInMins() { synchronized (this) { return solrCommitPeriodInMins; diff --git a/metis-core/metis-core-service/src/main/java/eu/europeana/metis/core/service/WorkflowExecutionFactory.java b/metis-core/metis-core-service/src/main/java/eu/europeana/metis/core/service/WorkflowExecutionFactory.java index 37915e65d..6fc2d3fcd 100644 --- a/metis-core/metis-core-service/src/main/java/eu/europeana/metis/core/service/WorkflowExecutionFactory.java +++ b/metis-core/metis-core-service/src/main/java/eu/europeana/metis/core/service/WorkflowExecutionFactory.java @@ -1,10 +1,11 @@ package eu.europeana.metis.core.service; +import eu.europeana.metis.core.common.TransformationParameters; +import eu.europeana.metis.core.dao.DataEvolutionUtils; import eu.europeana.metis.core.dao.DatasetXsltDao; import eu.europeana.metis.core.dao.DepublishRecordIdDao; import eu.europeana.metis.core.dao.PluginWithExecutionId; import eu.europeana.metis.core.dao.WorkflowExecutionDao; -import eu.europeana.metis.core.dao.WorkflowUtils; import eu.europeana.metis.core.dataset.Dataset; import eu.europeana.metis.core.dataset.DatasetXslt; import eu.europeana.metis.core.dataset.DepublishRecordId.DepublicationStatus; @@ -29,7 +30,6 @@ import java.util.ArrayList; import java.util.EnumSet; import java.util.List; -import java.util.Locale; import java.util.Set; import java.util.function.BooleanSupplier; import org.apache.commons.lang3.StringUtils; @@ -46,7 +46,7 @@ public class WorkflowExecutionFactory { private final DatasetXsltDao datasetXsltDao; private final DepublishRecordIdDao depublishRecordIdDao; private final WorkflowExecutionDao workflowExecutionDao; - private final WorkflowUtils workflowUtils; + private final DataEvolutionUtils dataEvolutionUtils; private ValidationProperties validationExternalProperties; // Use getter and setter! private ValidationProperties validationInternalProperties; // Use getter and setter! @@ -59,25 +59,25 @@ public class WorkflowExecutionFactory { * @param datasetXsltDao the Dao instance to access the dataset xslts * @param depublishRecordIdDao The Dao instance to access depublish records. * @param workflowExecutionDao the Dao instance to access the workflow executions - * @param workflowUtils the utilities class for workflow operations + * @param dataEvolutionUtils the utilities class for workflow operations */ public WorkflowExecutionFactory(DatasetXsltDao datasetXsltDao, DepublishRecordIdDao depublishRecordIdDao, WorkflowExecutionDao workflowExecutionDao, - WorkflowUtils workflowUtils) { + DataEvolutionUtils dataEvolutionUtils) { this.datasetXsltDao = datasetXsltDao; this.depublishRecordIdDao = depublishRecordIdDao; this.workflowExecutionDao = workflowExecutionDao; - this.workflowUtils = workflowUtils; + this.dataEvolutionUtils = dataEvolutionUtils; } // Expect the dataset to be synced with eCloud. // Does not save the workflow execution. WorkflowExecution createWorkflowExecution(Workflow workflow, Dataset dataset, - PluginWithExecutionId predecessor, int priority) + PluginWithExecutionId> predecessor, int priority) throws BadContentException { // Create the plugins - final List workflowPlugins = new ArrayList<>(); + final List> workflowPlugins = new ArrayList<>(); final List typesInWorkflow = new ArrayList<>(); for (AbstractExecutablePluginMetadata pluginMetadata : workflow.getMetisPluginsMetadata()) { if (pluginMetadata.isEnabled()) { @@ -97,8 +97,8 @@ WorkflowExecution createWorkflowExecution(Workflow workflow, Dataset dataset, return new WorkflowExecution(dataset, workflowPlugins, priority); } - private AbstractExecutablePlugin createWorkflowExecutionPlugin(Dataset dataset, - PluginWithExecutionId workflowPredecessor, + private AbstractExecutablePlugin createWorkflowExecutionPlugin(Dataset dataset, + PluginWithExecutionId> workflowPredecessor, AbstractExecutablePluginMetadata pluginMetadata, List typesInWorkflowBeforeThisPlugin) throws BadContentException { @@ -177,19 +177,19 @@ private AbstractExecutablePlugin createWorkflowExecutionPlugin(Dataset dataset, * @return Whether to apply redirection as part of this plugin. */ private boolean shouldRedirectsBePerformed(Dataset dataset, - PluginWithExecutionId workflowPredecessor, + PluginWithExecutionId> workflowPredecessor, ExecutablePluginType executablePluginType, List typesInWorkflowBeforeThisPlugin) { // Get some history from the database: find the latest successful plugin of the same type. - final PluginWithExecutionId latestSuccessfulPlugin = workflowExecutionDao + final PluginWithExecutionId> latestSuccessfulPlugin = workflowExecutionDao .getLatestSuccessfulExecutablePlugin(dataset.getDatasetId(), EnumSet.of(executablePluginType), true); // Check if we can find the answer in the workflow itself. Iterate backwards and see what we find. for (int i = typesInWorkflowBeforeThisPlugin.size() - 1; i >= 0; i--) { final ExecutablePluginType type = typesInWorkflowBeforeThisPlugin.get(i); - if (WorkflowUtils.getHarvestPluginGroup().contains(type)) { + if (DataEvolutionUtils.getHarvestPluginGroup().contains(type)) { // If we find a harvest (occurring after any plugin of this type), // we know we need to perform redirects only if there is a non null latest successful plugin or there are datasets to redirect from. return latestSuccessfulPlugin != null || !CollectionUtils @@ -217,8 +217,8 @@ private boolean shouldRedirectsBePerformed(Dataset dataset, // If this plugin's harvest cannot be determined, assume it is not the same (this shouldn't // happen as we checked the workflow already). This is a lambda: we wish to evaluate on demand. final BooleanSupplier rootDiffersForLatestPlugin = () -> workflowPredecessor == null - || !workflowUtils.getRootAncestor(latestSuccessfulPlugin) - .equals(workflowUtils.getRootAncestor(workflowPredecessor)); + || !dataEvolutionUtils.getRootAncestor(latestSuccessfulPlugin) + .equals(dataEvolutionUtils.getRootAncestor(workflowPredecessor)); // In either of these situations, we perform a redirect. performRedirect = @@ -254,10 +254,10 @@ private void setupXsltIdForPluginMetadata(Dataset dataset, if (xsltObject != null && StringUtils.isNotEmpty(xsltObject.getXslt())) { pluginMetadata.setXsltId(xsltObject.getId().toString()); } - //DatasetName in Transformation should be a concatenation datasetId_datasetName - pluginMetadata.setDatasetName(dataset.getDatasetId() + "_" + dataset.getDatasetName()); - pluginMetadata.setCountry(dataset.getCountry().getName()); - pluginMetadata.setLanguage(dataset.getLanguage().name().toLowerCase(Locale.US)); + final TransformationParameters transformationParameters = new TransformationParameters(dataset); + pluginMetadata.setDatasetName(transformationParameters.getDatasetName()); + pluginMetadata.setCountry(transformationParameters.getEdmCountry()); + pluginMetadata.setLanguage(transformationParameters.getEdmLanguage()); } private void setupDepublishPluginMetadata(Dataset dataset, DepublishPluginMetadata pluginMetadata) diff --git a/metis-core/metis-core-service/src/test/java/eu/europeana/metis/core/dao/TestWorkflowUtils.java b/metis-core/metis-core-service/src/test/java/eu/europeana/metis/core/dao/TestDataEvolutionUtils.java similarity index 55% rename from metis-core/metis-core-service/src/test/java/eu/europeana/metis/core/dao/TestWorkflowUtils.java rename to metis-core/metis-core-service/src/test/java/eu/europeana/metis/core/dao/TestDataEvolutionUtils.java index f9d152224..fcb6b3ef3 100644 --- a/metis-core/metis-core-service/src/test/java/eu/europeana/metis/core/dao/TestWorkflowUtils.java +++ b/metis-core/metis-core-service/src/test/java/eu/europeana/metis/core/dao/TestDataEvolutionUtils.java @@ -10,24 +10,26 @@ import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.ArgumentMatchers.isNull; +import static org.mockito.ArgumentMatchers.same; import static org.mockito.Mockito.doReturn; -import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.reset; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.when; -import eu.europeana.metis.core.dataset.DepublishRecordId.DepublicationStatus; +import eu.europeana.metis.core.dao.WorkflowExecutionDao.ExecutionDatasetPair; +import eu.europeana.metis.core.dao.WorkflowExecutionDao.Pagination; +import eu.europeana.metis.core.dao.WorkflowExecutionDao.ResultList; +import eu.europeana.metis.core.dataset.Dataset; import eu.europeana.metis.core.exceptions.PluginExecutionNotAllowed; -import eu.europeana.metis.core.util.DepublishRecordIdSortField; -import eu.europeana.metis.core.util.SortDirection; import eu.europeana.metis.core.utils.TestObjectFactory; -import eu.europeana.metis.core.workflow.Workflow; import eu.europeana.metis.core.workflow.WorkflowExecution; import eu.europeana.metis.core.workflow.plugins.AbstractExecutablePlugin; import eu.europeana.metis.core.workflow.plugins.AbstractExecutablePluginMetadata; import eu.europeana.metis.core.workflow.plugins.AbstractMetisPlugin; import eu.europeana.metis.core.workflow.plugins.AbstractMetisPluginMetadata; +import eu.europeana.metis.core.workflow.plugins.DataStatus; import eu.europeana.metis.core.workflow.plugins.DepublishPluginMetadata; import eu.europeana.metis.core.workflow.plugins.EnrichmentPluginMetadata; import eu.europeana.metis.core.workflow.plugins.ExecutablePlugin; @@ -35,13 +37,16 @@ import eu.europeana.metis.core.workflow.plugins.ExecutablePluginMetadata; import eu.europeana.metis.core.workflow.plugins.ExecutablePluginType; import eu.europeana.metis.core.workflow.plugins.ExecutionProgress; +import eu.europeana.metis.core.workflow.plugins.HTTPHarvestPlugin; import eu.europeana.metis.core.workflow.plugins.HTTPHarvestPluginMetadata; import eu.europeana.metis.core.workflow.plugins.IndexToPreviewPluginMetadata; +import eu.europeana.metis.core.workflow.plugins.IndexToPublishPlugin; import eu.europeana.metis.core.workflow.plugins.IndexToPublishPluginMetadata; import eu.europeana.metis.core.workflow.plugins.LinkCheckingPluginMetadata; import eu.europeana.metis.core.workflow.plugins.MediaProcessPluginMetadata; import eu.europeana.metis.core.workflow.plugins.MetisPlugin; import eu.europeana.metis.core.workflow.plugins.NormalizationPluginMetadata; +import eu.europeana.metis.core.workflow.plugins.OaipmhHarvestPlugin; import eu.europeana.metis.core.workflow.plugins.OaipmhHarvestPluginMetadata; import eu.europeana.metis.core.workflow.plugins.PluginStatus; import eu.europeana.metis.core.workflow.plugins.PluginType; @@ -50,8 +55,6 @@ import eu.europeana.metis.core.workflow.plugins.TransformationPluginMetadata; import eu.europeana.metis.core.workflow.plugins.ValidationExternalPluginMetadata; import eu.europeana.metis.core.workflow.plugins.ValidationInternalPluginMetadata; -import eu.europeana.metis.exception.BadContentException; -import eu.europeana.metis.exception.GenericMetisException; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -60,7 +63,7 @@ import java.util.List; import java.util.Optional; import java.util.Set; -import java.util.stream.Collectors; +import java.util.function.Function; import org.apache.commons.lang3.tuple.ImmutablePair; import org.apache.commons.lang3.tuple.Pair; import org.bson.types.ObjectId; @@ -73,36 +76,34 @@ * @author Simon Tzanakis (Simon.Tzanakis@europeana.eu) * @since 2018-02-01 */ -class TestWorkflowUtils { +class TestDataEvolutionUtils { private static final String DATASET_ID = Integer.toString(TestObjectFactory.DATASETID); - private static WorkflowUtils workflowUtils; + private static DataEvolutionUtils dataEvolutionUtils; private static WorkflowExecutionDao workflowExecutionDao; - private static DepublishRecordIdDao depublishRecordIdDao; @BeforeAll static void prepare() { workflowExecutionDao = mock(WorkflowExecutionDao.class); - depublishRecordIdDao = mock(DepublishRecordIdDao.class); - workflowUtils = spy(new WorkflowUtils(workflowExecutionDao, depublishRecordIdDao)); + dataEvolutionUtils = spy(new DataEvolutionUtils(workflowExecutionDao)); } @AfterEach void cleanUp() { - reset(workflowUtils, workflowExecutionDao, depublishRecordIdDao); + reset(dataEvolutionUtils, workflowExecutionDao); } @Test void testComputePredecessorPlugin_HarvestPlugin() throws PluginExecutionNotAllowed { assertNull( - workflowUtils + dataEvolutionUtils .computePredecessorPlugin(ExecutablePluginType.OAIPMH_HARVEST, null, DATASET_ID)); assertNull( - workflowUtils + dataEvolutionUtils .computePredecessorPlugin(ExecutablePluginType.HTTP_HARVEST, null, DATASET_ID)); - assertNull(workflowUtils.computePredecessorPlugin(ExecutablePluginType.OAIPMH_HARVEST, + assertNull(dataEvolutionUtils.computePredecessorPlugin(ExecutablePluginType.OAIPMH_HARVEST, ExecutablePluginType.TRANSFORMATION, DATASET_ID)); - assertNull(workflowUtils.computePredecessorPlugin(ExecutablePluginType.HTTP_HARVEST, + assertNull(dataEvolutionUtils.computePredecessorPlugin(ExecutablePluginType.HTTP_HARVEST, ExecutablePluginType.TRANSFORMATION, DATASET_ID)); Mockito.verify(workflowExecutionDao, Mockito.never()) .getLatestSuccessfulExecutablePlugin(anyString(), any(), anyBoolean()); @@ -115,7 +116,7 @@ void testComputePredecessorPlugin() throws PluginExecutionNotAllowed { testComputePredecessorPlugin(new OaipmhHarvestPluginMetadata(), Collections.emptySet(), null); testComputePredecessorPlugin(new HTTPHarvestPluginMetadata(), Collections.emptySet(), null); testComputePredecessorPlugin(new ValidationExternalPluginMetadata(), - WorkflowUtils.getHarvestPluginGroup(), null); + DataEvolutionUtils.getHarvestPluginGroup(), null); testComputePredecessorPlugin(new TransformationPluginMetadata(), EnumSet.of(ExecutablePluginType.VALIDATION_EXTERNAL), null); testComputePredecessorPlugin(new ValidationInternalPluginMetadata(), @@ -133,7 +134,7 @@ void testComputePredecessorPlugin() throws PluginExecutionNotAllowed { testComputePredecessorPlugin(new DepublishPluginMetadata(), EnumSet.of(ExecutablePluginType.PUBLISH), null); testComputePredecessorPlugin(new LinkCheckingPluginMetadata(), - WorkflowUtils.getAllExceptLinkGroup(), null); + DataEvolutionUtils.getAllExceptLinkGroup(), null); // Test enforcing a predecessor type. testComputePredecessorPlugin(new OaipmhHarvestPluginMetadata(), Collections.emptySet(), @@ -178,25 +179,25 @@ private void testComputePredecessorPlugin(ExecutablePluginMetadata metadata, } if (predecessorTypes.isEmpty() || metadata.getExecutablePluginType() == ExecutablePluginType.DEPUBLISH) { - assertNull(workflowUtils + assertNull(dataEvolutionUtils .computePredecessorPlugin(metadata.getExecutablePluginType(), enforcedPluginType, DATASET_ID)); } else { assertNotNull(recentPredecessorPlugin); when(workflowExecutionDao.getLatestSuccessfulExecutablePlugin(DATASET_ID, - WorkflowUtils.getHarvestPluginGroup(), true)) + DataEvolutionUtils.getHarvestPluginGroup(), true)) .thenReturn(new PluginWithExecutionId<>(rootExecutionId.toString(), rootPlugin)); when(workflowExecutionDao.getById(predecessorExecutionId.toString())) .thenReturn(predecessorExecution); - final List> evolution = + final List, WorkflowExecution>> evolution = Arrays.asList(ImmutablePair.of(rootPlugin, rootExecution), ImmutablePair.of(mock(AbstractExecutablePlugin.class), rootExecution), ImmutablePair.of(mock(AbstractExecutablePlugin.class), rootExecution)); - when(workflowUtils.compileVersionEvolution(recentPredecessorPlugin, predecessorExecution)) + when(dataEvolutionUtils.compileVersionEvolution(recentPredecessorPlugin, predecessorExecution)) .thenReturn(evolution); // Test without errors - final PluginWithExecutionId withoutErrorsResult = workflowUtils + final PluginWithExecutionId> withoutErrorsResult = dataEvolutionUtils .computePredecessorPlugin(metadata.getExecutablePluginType(), enforcedPluginType, DATASET_ID); assertSame(recentPredecessorPlugin, withoutErrorsResult.getPlugin()); @@ -206,25 +207,25 @@ private void testComputePredecessorPlugin(ExecutablePluginMetadata metadata, final AbstractExecutablePlugin otherRootPlugin = mock(AbstractExecutablePlugin.class); final String otherRootPluginId = "other root plugin ID"; when(otherRootPlugin.getId()).thenReturn(otherRootPluginId); - when(workflowUtils.compileVersionEvolution(recentPredecessorPlugin, predecessorExecution)) + when(dataEvolutionUtils.compileVersionEvolution(recentPredecessorPlugin, predecessorExecution)) .thenReturn( Collections.singletonList(ImmutablePair.of(otherRootPlugin, rootExecution))); assertThrows(PluginExecutionNotAllowed.class, - () -> workflowUtils.computePredecessorPlugin(metadata.getExecutablePluginType(), + () -> dataEvolutionUtils.computePredecessorPlugin(metadata.getExecutablePluginType(), enforcedPluginType, DATASET_ID)); - when(workflowUtils.compileVersionEvolution(recentPredecessorPlugin, predecessorExecution)) + when(dataEvolutionUtils.compileVersionEvolution(recentPredecessorPlugin, predecessorExecution)) .thenReturn(Collections.singletonList(ImmutablePair.of(rootPlugin, rootExecution))); - assertSame(recentPredecessorPlugin, workflowUtils.computePredecessorPlugin( + assertSame(recentPredecessorPlugin, dataEvolutionUtils.computePredecessorPlugin( metadata.getExecutablePluginType(), enforcedPluginType, DATASET_ID).getPlugin()); // Test with errors recentPredecessorPlugin.getExecutionProgress().setErrors(1); - assertThrows(PluginExecutionNotAllowed.class, () -> workflowUtils.computePredecessorPlugin( + assertThrows(PluginExecutionNotAllowed.class, () -> dataEvolutionUtils.computePredecessorPlugin( metadata.getExecutablePluginType(), enforcedPluginType, DATASET_ID)); // Test without progress information recentPredecessorPlugin.setExecutionProgress(null); - assertThrows(PluginExecutionNotAllowed.class, () -> workflowUtils.computePredecessorPlugin( + assertThrows(PluginExecutionNotAllowed.class, () -> dataEvolutionUtils.computePredecessorPlugin( metadata.getExecutablePluginType(), enforcedPluginType, DATASET_ID)); } } @@ -264,198 +265,20 @@ void testComputePredecessorPluginForWorkflowExecution() { // Execute the call expecting a successful result. assertSame(lastCandidate, - WorkflowUtils + DataEvolutionUtils .computePredecessorPlugin(ExecutablePluginType.MEDIA_PROCESS, workflowExecution)); // Execute the call for plugin type not requiring predecessor assertNull( - WorkflowUtils + DataEvolutionUtils .computePredecessorPlugin(ExecutablePluginType.HTTP_HARVEST, workflowExecution)); // Execute the call for failed result assertThrows(IllegalArgumentException.class, - () -> WorkflowUtils + () -> DataEvolutionUtils .computePredecessorPlugin(ExecutablePluginType.PUBLISH, workflowExecution)); } - @Test - void testValidateWorkflowPlugins_testWorkflowComposition() throws GenericMetisException { - - // Create successful predecessor - final ExecutablePluginType predecessorType = ExecutablePluginType.OAIPMH_HARVEST; - final AbstractExecutablePlugin predecessor = - ExecutablePluginFactory.createPlugin(new OaipmhHarvestPluginMetadata()); - predecessor.setExecutionProgress(new ExecutionProgress()); - predecessor.getExecutionProgress().setProcessedRecords(1); - predecessor.getExecutionProgress().setErrors(0); - doReturn(new PluginWithExecutionId<>("", predecessor)).when(workflowUtils) - .computePredecessorPlugin(any(), eq(predecessorType), eq(DATASET_ID)); - - // Test allowed workflow - assertSame(predecessor, workflowUtils.validateWorkflowPlugins(createWorkflow( - ExecutablePluginType.OAIPMH_HARVEST), predecessorType).getPlugin()); - assertSame(predecessor, workflowUtils.validateWorkflowPlugins(createWorkflow( - ExecutablePluginType.NORMALIZATION, ExecutablePluginType.ENRICHMENT, - ExecutablePluginType.LINK_CHECKING), predecessorType).getPlugin()); - assertSame(predecessor, workflowUtils.validateWorkflowPlugins(createWorkflow( - ExecutablePluginType.ENRICHMENT, ExecutablePluginType.OAIPMH_HARVEST), predecessorType) - .getPlugin()); - - // Test workflow with empty list - assertThrows(BadContentException.class, () -> workflowUtils - .validateWorkflowPlugins(createWorkflow(), predecessorType)); - - // Test workflow with null list - final Workflow workflowWithNullList = new Workflow(); - workflowWithNullList.setMetisPluginsMetadata(null); - assertThrows(BadContentException.class, () -> workflowUtils - .validateWorkflowPlugins(workflowWithNullList, predecessorType)); - - // Test workflow with plugin with invalid type - assertThrows(BadContentException.class, () -> workflowUtils.validateWorkflowPlugins( - createWorkflow(ExecutablePluginType.NORMALIZATION, null, - ExecutablePluginType.LINK_CHECKING), predecessorType)); - - // Test workflow with two plugins, one of which is depublish - Workflow workflowDepublishAndOai = new Workflow(); - workflowDepublishAndOai.setDatasetId(Integer.toString(TestObjectFactory.DATASETID)); - OaipmhHarvestPluginMetadata oaipmhHarvestPluginMetadata = new OaipmhHarvestPluginMetadata(); - oaipmhHarvestPluginMetadata.setEnabled(true); - DepublishPluginMetadata depublishPluginMetadata = new DepublishPluginMetadata(); - depublishPluginMetadata.setEnabled(true); - depublishPluginMetadata.setDatasetDepublish(true); - List abstractMetisPluginMetadata = new ArrayList<>(2); - abstractMetisPluginMetadata.add(oaipmhHarvestPluginMetadata); - abstractMetisPluginMetadata.add(depublishPluginMetadata); - workflowDepublishAndOai.setMetisPluginsMetadata(abstractMetisPluginMetadata); - assertThrows(BadContentException.class, - () -> workflowUtils.validateWorkflowPlugins(workflowDepublishAndOai, null)); - - // Test if workflow contains record depublish that record ids exist - Workflow workflowDepublish = new Workflow(); - workflowDepublish.setDatasetId(Integer.toString(TestObjectFactory.DATASETID)); - depublishPluginMetadata.setDatasetDepublish(false); - abstractMetisPluginMetadata.clear(); - abstractMetisPluginMetadata.add(depublishPluginMetadata); - workflowDepublish.setMetisPluginsMetadata(abstractMetisPluginMetadata); - when(depublishRecordIdDao - .getAllDepublishRecordIdsWithStatus(workflowDepublish.getDatasetId(), - DepublishRecordIdSortField.DEPUBLICATION_STATE, SortDirection.ASCENDING, - DepublicationStatus.PENDING_DEPUBLICATION)).thenReturn(Collections.emptySet()); - assertThrows(BadContentException.class, () -> workflowUtils - .validateWorkflowPlugins(workflowDepublish, null)); - - // Test workflow starting with link checking. - assertSame(predecessor, workflowUtils.validateWorkflowPlugins( - createWorkflow(ExecutablePluginType.LINK_CHECKING), predecessorType).getPlugin()); - assertThrows(PluginExecutionNotAllowed.class, () -> workflowUtils.validateWorkflowPlugins( - createWorkflow(ExecutablePluginType.LINK_CHECKING, ExecutablePluginType.TRANSFORMATION), - predecessorType)); - - // Test workflow with gaps - assertThrows(PluginExecutionNotAllowed.class, () -> workflowUtils.validateWorkflowPlugins( - createWorkflow(ExecutablePluginType.TRANSFORMATION, ExecutablePluginType.ENRICHMENT), - predecessorType)); - - // Test workflow with duplicate types - assertThrows(PluginExecutionNotAllowed.class, () -> workflowUtils.validateWorkflowPlugins( - createWorkflow(ExecutablePluginType.TRANSFORMATION, ExecutablePluginType.ENRICHMENT, - ExecutablePluginType.ENRICHMENT), predecessorType)); - assertThrows(PluginExecutionNotAllowed.class, () -> workflowUtils.validateWorkflowPlugins( - createWorkflow(ExecutablePluginType.TRANSFORMATION, ExecutablePluginType.LINK_CHECKING, - ExecutablePluginType.LINK_CHECKING), predecessorType)); - - // Test workflow with disabled plugins: valid before disabling, but invalid after. - final Workflow workflowWithDisabledPlugins = createWorkflow( - ExecutablePluginType.NORMALIZATION, - ExecutablePluginType.ENRICHMENT, ExecutablePluginType.MEDIA_PROCESS); - assertSame(predecessor, workflowUtils.validateWorkflowPlugins(workflowWithDisabledPlugins, - predecessorType).getPlugin()); - when(workflowWithDisabledPlugins.getMetisPluginsMetadata().get(1).isEnabled()) - .thenReturn(false); - assertThrows(PluginExecutionNotAllowed.class, () -> workflowUtils.validateWorkflowPlugins( - workflowWithDisabledPlugins, predecessorType)); - - // Test workflow with bad predecessor - doThrow(PluginExecutionNotAllowed.class).when(workflowUtils) - .computePredecessorPlugin(any(), eq(predecessorType), eq(DATASET_ID)); - assertThrows(PluginExecutionNotAllowed.class, () -> workflowUtils.validateWorkflowPlugins( - createWorkflow(ExecutablePluginType.ENRICHMENT, ExecutablePluginType.OAIPMH_HARVEST), - predecessorType)); - } - - private Workflow createWorkflow(ExecutablePluginType... pluginTypes) { - final Workflow workflow = new Workflow(); - workflow.setDatasetId(DATASET_ID); - workflow.setMetisPluginsMetadata(Arrays.stream(pluginTypes).map(type -> { - final AbstractExecutablePluginMetadata plugin = mock( - AbstractExecutablePluginMetadata.class); - when(plugin.isEnabled()).thenReturn(true); - doReturn(type).when(plugin).getExecutablePluginType(); - return plugin; - }).collect(Collectors.toList())); - return workflow; - } - - @Test - void testValidateWorkflowPlugins_testHarvestingParameters() throws GenericMetisException { - - // Prepare correct url variables - final String simpleUrl = "http://test.com/path"; - final String urlWithFragmentAndQuery = simpleUrl + "#fragment?query=1"; - final String metadataFormat = "metadataFormatParameter"; - final String setSpec = "setSpecParameter"; - - // Create oai harvesting with all parameters - final OaipmhHarvestPluginMetadata oai = new OaipmhHarvestPluginMetadata(); - oai.setEnabled(true); - oai.setUrl(" " + urlWithFragmentAndQuery + " "); - oai.setMetadataFormat(" " + metadataFormat + " "); - oai.setSetSpec(" " + setSpec + " "); - - // Create http harvesting - final HTTPHarvestPluginMetadata http = new HTTPHarvestPluginMetadata(); - http.setEnabled(true); - http.setUrl(" " + urlWithFragmentAndQuery + " "); - - // Create the workflow and execute the method - final Workflow workflow = new Workflow(); - workflow.setMetisPluginsMetadata(Arrays.asList(oai, http)); - workflowUtils.validateWorkflowPlugins(workflow, null); - - // Test output - assertEquals(simpleUrl, oai.getUrl()); - assertEquals(metadataFormat, oai.getMetadataFormat()); - assertEquals(setSpec, oai.getSetSpec()); - assertEquals(urlWithFragmentAndQuery, http.getUrl()); - - // Create oai harvesting with only url - oai.setUrl(urlWithFragmentAndQuery); - oai.setMetadataFormat(null); - oai.setSetSpec(null); - - // Create the workflow and execute the method - workflow.setMetisPluginsMetadata(Collections.singletonList(oai)); - workflowUtils.validateWorkflowPlugins(workflow, null); - - // Test output - assertEquals(simpleUrl, oai.getUrl()); - assertNull(oai.getMetadataFormat()); - assertNull(oai.getSetSpec()); - - // Test OAI with invalid URL - oai.setUrl("invalid URL"); - workflow.setMetisPluginsMetadata(Collections.singletonList(oai)); - assertThrows(BadContentException.class, - () -> workflowUtils.validateWorkflowPlugins(workflow, null)); - - // Test HTTP with missing URL - http.setUrl(null); - workflow.setMetisPluginsMetadata(Collections.singletonList(http)); - assertThrows(BadContentException.class, - () -> workflowUtils.validateWorkflowPlugins(workflow, null)); - } - @Test void testGetRecordEvolutionForVersionHappyFlow() { @@ -466,14 +289,14 @@ void testGetRecordEvolutionForVersionHappyFlow() { final AbstractExecutablePlugin plugin3 = mock(AbstractExecutablePlugin.class); final WorkflowExecution execution1 = createWorkflowExecution(datasetId, plugin1); final WorkflowExecution execution2 = createWorkflowExecution(datasetId, plugin2, plugin3); - doReturn(null).when(workflowUtils).getPreviousExecutionAndPlugin(plugin1, datasetId); - doReturn(new ImmutablePair<>(plugin1, execution1)).when(workflowUtils) + doReturn(null).when(dataEvolutionUtils).getPreviousExecutionAndPlugin(plugin1, datasetId); + doReturn(new ImmutablePair<>(plugin1, execution1)).when(dataEvolutionUtils) .getPreviousExecutionAndPlugin(plugin2, datasetId); - doReturn(new ImmutablePair<>(plugin2, execution2)).when(workflowUtils) + doReturn(new ImmutablePair<>(plugin2, execution2)).when(dataEvolutionUtils) .getPreviousExecutionAndPlugin(plugin3, datasetId); // Execute the call to examine all three - final List> resultForThree = workflowUtils + final List, WorkflowExecution>> resultForThree = dataEvolutionUtils .compileVersionEvolution(plugin3, execution2); assertNotNull(resultForThree); assertEquals(2, resultForThree.size()); @@ -483,7 +306,7 @@ void testGetRecordEvolutionForVersionHappyFlow() { assertSame(execution2, resultForThree.get(1).getRight()); // Execute the call to examine just two - final List> resultForTwo = workflowUtils + final List, WorkflowExecution>> resultForTwo = dataEvolutionUtils .compileVersionEvolution(plugin2, execution2); assertNotNull(resultForTwo); assertEquals(1, resultForTwo.size()); @@ -491,13 +314,13 @@ void testGetRecordEvolutionForVersionHappyFlow() { assertSame(execution1, resultForThree.get(0).getRight()); // Execute the call to examine just one - final List> resultForOne = workflowUtils + final List, WorkflowExecution>> resultForOne = dataEvolutionUtils .compileVersionEvolution(plugin1, execution1); assertNotNull(resultForOne); assertTrue(resultForOne.isEmpty()); } - private WorkflowExecution createWorkflowExecution(String datasetId, + private static WorkflowExecution createWorkflowExecution(String datasetId, AbstractMetisPlugin... plugins) { final WorkflowExecution result = new WorkflowExecution(); result.setId(new ObjectId()); @@ -520,38 +343,38 @@ void testGetPreviousExecutionAndPlugin() { previousPluginTime); // Test the absence of one or both of the pointers to a previous execution. - assertNull(workflowUtils.getPreviousExecutionAndPlugin(createMetisPlugin( + assertNull(dataEvolutionUtils.getPreviousExecutionAndPlugin(createMetisPlugin( pluginType, null, null), datasetId)); - assertNull(workflowUtils.getPreviousExecutionAndPlugin(createMetisPlugin( + assertNull(dataEvolutionUtils.getPreviousExecutionAndPlugin(createMetisPlugin( pluginType, previousPluginType, null), datasetId)); - assertNull(workflowUtils.getPreviousExecutionAndPlugin(createMetisPlugin( + assertNull(dataEvolutionUtils.getPreviousExecutionAndPlugin(createMetisPlugin( pluginType, null, previousPluginTime), datasetId)); // Test the absence of the execution despite the presence of the pointers. when(workflowExecutionDao - .getByTaskExecution(eq(previousPluginTime), eq(previousPluginType), eq(datasetId))) + .getByTaskExecution(eq(new ExecutedMetisPluginId(previousPluginTime, previousPluginType)), eq(datasetId))) .thenReturn(null); - assertNull(workflowUtils.getPreviousExecutionAndPlugin(plugin, datasetId)); + assertNull(dataEvolutionUtils.getPreviousExecutionAndPlugin(plugin, datasetId)); when(workflowExecutionDao - .getByTaskExecution(eq(previousPluginTime), eq(previousPluginType), eq(datasetId))) + .getByTaskExecution(eq(new ExecutedMetisPluginId(previousPluginTime, previousPluginType)), eq(datasetId))) .thenReturn(previousExecution); // Test the absence of the plugin despite the presence of the pointers. when(previousExecution.getMetisPluginWithType(eq(previousPluginType))).thenReturn( Optional.empty()); - assertNull(workflowUtils.getPreviousExecutionAndPlugin(plugin, datasetId)); + assertNull(dataEvolutionUtils.getPreviousExecutionAndPlugin(plugin, datasetId)); when(previousExecution.getMetisPluginWithType(eq(previousPluginType))) .thenReturn(Optional.of(previousPlugin)); // Test the happy flow - final Pair result = workflowUtils + final Pair, WorkflowExecution> result = dataEvolutionUtils .getPreviousExecutionAndPlugin(plugin, datasetId); assertNotNull(result); assertSame(previousExecution, result.getRight()); assertSame(previousPlugin, result.getLeft()); } - private AbstractMetisPlugin createMetisPlugin(PluginType type, PluginType previousType, + private static AbstractMetisPlugin createMetisPlugin(PluginType type, PluginType previousType, Date previousDate) { AbstractMetisPluginMetadata metadata = mock(AbstractMetisPluginMetadata.class); when(metadata.getPluginType()).thenReturn(type); @@ -563,4 +386,162 @@ private AbstractMetisPlugin createMetisPlugin(PluginType type, PluginType previo when(result.getPluginMetadata()).thenReturn(metadata); return result; } + + @Test + void testGetPublishedHarvestIncrements() { + + // Create a bunch of harvest and index plugins and link them + final var fullOaiHarvest1 = createOaiHarvestPlugin(new Date(10), false, "A"); + final var incrementalOaiHarvest2 = createOaiHarvestPlugin(new Date(20), true, "B"); + final var httpHarvest3 = createExecutableMetisPlugin(ExecutablePluginType.HTTP_HARVEST, + new Date(30), HTTPHarvestPlugin.class, HTTPHarvestPluginMetadata.class, "C"); + final var fullOaiHarvest4 = createOaiHarvestPlugin(new Date(40), false, "D"); + final var incrementalOaiHarvest5 = createOaiHarvestPlugin(new Date(50), true, "E"); + final var indexPlugin1 = createIndexToPublish(new Date(11), "F"); + final var indexPlugin2a = createIndexToPublish(new Date(21), "G"); + final var indexPlugin2b = createIndexToPublish(new Date(22), "H"); + final var indexPlugin3a = createIndexToPublish(new Date(31), "I"); + final var indexPlugin3b = createIndexToPublish(new Date(32), "J"); + final var indexPlugin4a = createIndexToPublish(new Date(41), "K"); + final var indexPlugin4b = createIndexToPublish(new Date(42), "L"); + final var indexPlugin5a = createIndexToPublish(new Date(51), "M"); + final var indexPlugin5b = createIndexToPublish(new Date(52), "N"); + doReturn(fullOaiHarvest1).when(dataEvolutionUtils).getRootAncestor(same(indexPlugin1)); + doReturn(incrementalOaiHarvest2).when(dataEvolutionUtils).getRootAncestor(same(indexPlugin2a)); + doReturn(incrementalOaiHarvest2).when(dataEvolutionUtils).getRootAncestor(same(indexPlugin2b)); + doReturn(httpHarvest3).when(dataEvolutionUtils).getRootAncestor(same(indexPlugin3a)); + doReturn(httpHarvest3).when(dataEvolutionUtils).getRootAncestor(same(indexPlugin3b)); + doReturn(fullOaiHarvest4).when(dataEvolutionUtils).getRootAncestor(same(indexPlugin4a)); + doReturn(fullOaiHarvest4).when(dataEvolutionUtils).getRootAncestor(same(indexPlugin4b)); + doReturn(incrementalOaiHarvest5).when(dataEvolutionUtils).getRootAncestor(same(indexPlugin5a)); + doReturn(incrementalOaiHarvest5).when(dataEvolutionUtils).getRootAncestor(same(indexPlugin5b)); + + // Test happy flow with two OAI harvests. Only last full and last incremented to be returned. + final var listOfAllOaiIndex = List.of(indexPlugin5b, indexPlugin5a, indexPlugin4b, + indexPlugin4a, indexPlugin2b, indexPlugin2a, indexPlugin1); + doReturn(listOfAllOaiIndex).when(dataEvolutionUtils) + .getPublishOperationsSortedInversely(DATASET_ID); + final var result1 = dataEvolutionUtils.getPublishedHarvestIncrements(DATASET_ID); + assertListSameItems(List.of(fullOaiHarvest4, incrementalOaiHarvest5), result1); + + // Test happy flow with an http harvest + final var listOfHttpAndOaiIndex = List.of(indexPlugin5b, indexPlugin5a, indexPlugin3b, + indexPlugin3a, indexPlugin2b, indexPlugin2a, indexPlugin1); + doReturn(listOfHttpAndOaiIndex).when(dataEvolutionUtils) + .getPublishOperationsSortedInversely(DATASET_ID); + final var result2 = dataEvolutionUtils.getPublishedHarvestIncrements(DATASET_ID); + assertListSameItems(List.of(httpHarvest3, incrementalOaiHarvest5), result2); + + // Test happy flow with just one full harvest + doReturn(List.of(indexPlugin1)).when(dataEvolutionUtils) + .getPublishOperationsSortedInversely(DATASET_ID); + final var result3 = dataEvolutionUtils.getPublishedHarvestIncrements(DATASET_ID); + assertListSameItems(List.of(fullOaiHarvest1), result3); + + // Test flow with no harvest + doReturn(Collections.emptyList()).when(dataEvolutionUtils) + .getPublishOperationsSortedInversely(DATASET_ID); + assertTrue(dataEvolutionUtils.getPublishedHarvestIncrements(DATASET_ID).isEmpty()); + + // Test flow with only an incremental harvest + doReturn(List.of(indexPlugin5b)).when(dataEvolutionUtils) + .getPublishOperationsSortedInversely(DATASET_ID); + assertTrue(dataEvolutionUtils.getPublishedHarvestIncrements(DATASET_ID).isEmpty()); + + // Test flow with invalid harvests or non-harvests + doReturn(List.of(indexPlugin5a, indexPlugin4a, indexPlugin1)).when(dataEvolutionUtils) + .getPublishOperationsSortedInversely(DATASET_ID); + doReturn(DataStatus.DELETED).when(indexPlugin4a.getPlugin()).getDataStatus(); + assertTrue(dataEvolutionUtils.getPublishedHarvestIncrements(DATASET_ID).isEmpty()); + doReturn(DataStatus.DEPRECATED).when(indexPlugin4a.getPlugin()).getDataStatus(); + assertListSameItems(List.of(fullOaiHarvest4, incrementalOaiHarvest5), + dataEvolutionUtils.getPublishedHarvestIncrements(DATASET_ID)); + doReturn(DataStatus.VALID).when(indexPlugin4a.getPlugin()).getDataStatus(); + assertListSameItems(List.of(fullOaiHarvest4, incrementalOaiHarvest5), + dataEvolutionUtils.getPublishedHarvestIncrements(DATASET_ID)); + doReturn(indexPlugin4a).when(dataEvolutionUtils).getRootAncestor(same(indexPlugin4a)); + assertTrue(dataEvolutionUtils.getPublishedHarvestIncrements(DATASET_ID).isEmpty()); + } + + private void assertListSameItems(List expected, List actual) { + assertListSameItems(expected, actual, Function.identity()); + } + + private void assertListSameItems(List expected, List actual, + Function extractor) { + assertNotNull(expected); + assertEquals(expected.size(), actual.size()); + for (int i = 0; i < expected.size(); i++) { + assertSame(expected.get(i), extractor.apply(actual.get(i))); + } + } + + private static > + PluginWithExecutionId createExecutableMetisPlugin(ExecutablePluginType type, Date startedDate, + Class pluginClass, Class metadataClass, String executionId) { + M metadata = mock(metadataClass); + doReturn(type).when(metadata).getExecutablePluginType(); + T result = mock(pluginClass); + when(result.getPluginType()).thenReturn(type.toPluginType()); + when(result.getPluginMetadata()).thenReturn(metadata); + when(result.getStartedDate()).thenReturn(startedDate); + return new PluginWithExecutionId<>(executionId, result); + } + + private static PluginWithExecutionId createIndexToPublish(Date startedDate, + String executionId) { + return createExecutableMetisPlugin(ExecutablePluginType.PUBLISH, startedDate, + IndexToPublishPlugin.class, IndexToPublishPluginMetadata.class, executionId); + } + + private static PluginWithExecutionId createOaiHarvestPlugin(Date startedDate, + boolean incremental, String executionId) { + final PluginWithExecutionId result = createExecutableMetisPlugin( + ExecutablePluginType.OAIPMH_HARVEST, startedDate, OaipmhHarvestPlugin.class, + OaipmhHarvestPluginMetadata.class, executionId); + when(result.getPlugin().getPluginMetadata().isIncrementalHarvest()).thenReturn(incremental); + return result; + } + + @Test + void testGetPublishOperationsSortedInversely(){ + + // Create some objects + final var otherPluginA = createOaiHarvestPlugin(new Date(0), false, null).getPlugin(); + final var indexPluginA = createIndexToPublish(new Date(1), null).getPlugin(); + final var indexPluginB1 = createIndexToPublish(new Date(2), null).getPlugin(); + final var indexPluginB2 = createIndexToPublish(new Date(3), null).getPlugin(); + final var executionA = createWorkflowExecution(DATASET_ID, otherPluginA, indexPluginA); + final var executionB = createWorkflowExecution(DATASET_ID, indexPluginB1, indexPluginB2); + final var pagination = mock(Pagination.class); + + // Test happy flow + final var input = new ResultList<>(List.of(new ExecutionDatasetPair(new Dataset(), executionA), + new ExecutionDatasetPair(new Dataset(), executionB)), true); + doReturn(pagination).when(workflowExecutionDao).createPagination(0, null, true); + doReturn(input).when(workflowExecutionDao) + .getWorkflowExecutionsOverview(eq(Set.of(DATASET_ID)), eq(Set.of(PluginStatus.FINISHED)), + eq(Set.of(PluginType.PUBLISH)), isNull(), isNull(), same(pagination)); + final List> result1 = dataEvolutionUtils + .getPublishOperationsSortedInversely(DATASET_ID); + assertListSameItems(List.of(indexPluginB2, indexPluginB1, indexPluginA), result1, + PluginWithExecutionId::getPlugin); + + // Test happy flow with different order + doReturn(new Date(13)).when(indexPluginA).getStartedDate(); + doReturn(new Date(12)).when(indexPluginB1).getStartedDate(); + doReturn(new Date(11)).when(indexPluginB2).getStartedDate(); + final List> result2 = dataEvolutionUtils + .getPublishOperationsSortedInversely(DATASET_ID); + assertListSameItems(List.of(indexPluginA, indexPluginB1, indexPluginB2), result2, + PluginWithExecutionId::getPlugin); + + // Test for no results + doReturn(new ResultList<>(Collections.emptyList(), true)).when(workflowExecutionDao) + .getWorkflowExecutionsOverview(eq(Set.of(DATASET_ID)), eq(Set.of(PluginStatus.FINISHED)), + eq(Set.of(PluginType.PUBLISH)), isNull(), isNull(), same(pagination)); + final List> result3 = dataEvolutionUtils + .getPublishOperationsSortedInversely(DATASET_ID); + assertListSameItems(Collections.emptyList(), result3, PluginWithExecutionId::getPlugin); + } } diff --git a/metis-core/metis-core-service/src/test/java/eu/europeana/metis/core/dao/TestWorkflowExecutionDao.java b/metis-core/metis-core-service/src/test/java/eu/europeana/metis/core/dao/TestWorkflowExecutionDao.java index 935e913a8..dedc44292 100644 --- a/metis-core/metis-core-service/src/test/java/eu/europeana/metis/core/dao/TestWorkflowExecutionDao.java +++ b/metis-core/metis-core-service/src/test/java/eu/europeana/metis/core/dao/TestWorkflowExecutionDao.java @@ -265,14 +265,14 @@ void getFirstOrLastFinishedPlugin_CheckFirstAndLast() { final String executionFirstId = workflowExecutionDao.create(workflowExecutionFirst); final String executionSecondId = workflowExecutionDao.create(workflowExecutionSecond); - PluginWithExecutionId latestFinishedWorkflowExecution = workflowExecutionDao + PluginWithExecutionId> latestFinishedWorkflowExecution = workflowExecutionDao .getFirstOrLastFinishedPlugin(Integer.toString(TestObjectFactory.DATASETID), EnumSet.of(PluginType.OAIPMH_HARVEST), false); assertEquals(latestFinishedWorkflowExecution.getPlugin().getFinishedDate(), workflowExecutionSecond.getMetisPlugins().get(0).getFinishedDate()); assertEquals(executionSecondId, latestFinishedWorkflowExecution.getExecutionId()); - PluginWithExecutionId firstFinishedWorkflowExecution = workflowExecutionDao + PluginWithExecutionId> firstFinishedWorkflowExecution = workflowExecutionDao .getFirstOrLastFinishedPlugin(Integer.toString(TestObjectFactory.DATASETID), EnumSet.of(PluginType.OAIPMH_HARVEST), true); assertEquals(firstFinishedWorkflowExecution.getPlugin().getFinishedDate(), @@ -282,7 +282,7 @@ void getFirstOrLastFinishedPlugin_CheckFirstAndLast() { @Test void getFirstOrLastFinishedPlugin_isNull() { - PluginWithExecutionId latestFinishedWorkflowExecution = workflowExecutionDao + PluginWithExecutionId> latestFinishedWorkflowExecution = workflowExecutionDao .getFirstOrLastFinishedPlugin(Integer.toString(TestObjectFactory.DATASETID), EnumSet.of(PluginType.OAIPMH_HARVEST), false); assertNull(latestFinishedWorkflowExecution); diff --git a/metis-core/metis-core-service/src/test/java/eu/europeana/metis/core/dao/TestWorkflowValidationUtils.java b/metis-core/metis-core-service/src/test/java/eu/europeana/metis/core/dao/TestWorkflowValidationUtils.java new file mode 100644 index 000000000..78fab6220 --- /dev/null +++ b/metis-core/metis-core-service/src/test/java/eu/europeana/metis/core/dao/TestWorkflowValidationUtils.java @@ -0,0 +1,262 @@ +package eu.europeana.metis.core.dao; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.reset; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.when; + +import eu.europeana.metis.core.dataset.DepublishRecordId.DepublicationStatus; +import eu.europeana.metis.core.exceptions.PluginExecutionNotAllowed; +import eu.europeana.metis.core.util.DepublishRecordIdSortField; +import eu.europeana.metis.core.util.SortDirection; +import eu.europeana.metis.core.utils.TestObjectFactory; +import eu.europeana.metis.core.workflow.Workflow; +import eu.europeana.metis.core.workflow.plugins.AbstractExecutablePlugin; +import eu.europeana.metis.core.workflow.plugins.AbstractExecutablePluginMetadata; +import eu.europeana.metis.core.workflow.plugins.DepublishPluginMetadata; +import eu.europeana.metis.core.workflow.plugins.ExecutablePluginFactory; +import eu.europeana.metis.core.workflow.plugins.ExecutablePluginType; +import eu.europeana.metis.core.workflow.plugins.ExecutionProgress; +import eu.europeana.metis.core.workflow.plugins.HTTPHarvestPluginMetadata; +import eu.europeana.metis.core.workflow.plugins.OaipmhHarvestPluginMetadata; +import eu.europeana.metis.exception.BadContentException; +import eu.europeana.metis.exception.GenericMetisException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +public class TestWorkflowValidationUtils { + + private static final String DATASET_ID = Integer.toString(TestObjectFactory.DATASETID); + private static WorkflowValidationUtils validationUtils; + private static DepublishRecordIdDao depublishRecordIdDao; + private static DataEvolutionUtils dataEvolutionUtils; + + @BeforeAll + static void prepare() { + depublishRecordIdDao = mock(DepublishRecordIdDao.class); + dataEvolutionUtils = mock(DataEvolutionUtils.class); + validationUtils = spy(new WorkflowValidationUtils(depublishRecordIdDao, dataEvolutionUtils)); + } + + @AfterEach + void cleanUp() { + reset(validationUtils, depublishRecordIdDao, dataEvolutionUtils); + } + + @Test + void testValidateWorkflowPlugins_testWorkflowComposition() throws GenericMetisException { + + // Create successful predecessor + final ExecutablePluginType predecessorType = ExecutablePluginType.OAIPMH_HARVEST; + final AbstractExecutablePlugin predecessor = + ExecutablePluginFactory.createPlugin(new OaipmhHarvestPluginMetadata()); + predecessor.setExecutionProgress(new ExecutionProgress()); + predecessor.getExecutionProgress().setProcessedRecords(1); + predecessor.getExecutionProgress().setErrors(0); + doReturn(new PluginWithExecutionId<>("", predecessor)).when(dataEvolutionUtils) + .computePredecessorPlugin(any(), eq(predecessorType), eq(DATASET_ID)); + + // Test allowed workflow + assertSame(predecessor, validationUtils.validateWorkflowPlugins(createWorkflow( + ExecutablePluginType.OAIPMH_HARVEST), predecessorType).getPlugin()); + assertSame(predecessor, validationUtils.validateWorkflowPlugins(createWorkflow( + ExecutablePluginType.NORMALIZATION, ExecutablePluginType.ENRICHMENT, + ExecutablePluginType.LINK_CHECKING), predecessorType).getPlugin()); + assertSame(predecessor, validationUtils.validateWorkflowPlugins(createWorkflow( + ExecutablePluginType.ENRICHMENT, ExecutablePluginType.OAIPMH_HARVEST), predecessorType) + .getPlugin()); + + // Test workflow with empty list + assertThrows(BadContentException.class, () -> validationUtils + .validateWorkflowPlugins(createWorkflow(), predecessorType)); + + // Test workflow with null list + final Workflow workflowWithNullList = new Workflow(); + workflowWithNullList.setMetisPluginsMetadata(null); + assertThrows(BadContentException.class, () -> validationUtils + .validateWorkflowPlugins(workflowWithNullList, predecessorType)); + + // Test workflow with plugin with invalid type + assertThrows(BadContentException.class, () -> validationUtils.validateWorkflowPlugins( + createWorkflow(ExecutablePluginType.NORMALIZATION, null, + ExecutablePluginType.LINK_CHECKING), predecessorType)); + + // Test workflow with two plugins, one of which is depublish + Workflow workflowDepublishAndOai = new Workflow(); + workflowDepublishAndOai.setDatasetId(Integer.toString(TestObjectFactory.DATASETID)); + OaipmhHarvestPluginMetadata oaipmhHarvestPluginMetadata = new OaipmhHarvestPluginMetadata(); + oaipmhHarvestPluginMetadata.setEnabled(true); + DepublishPluginMetadata depublishPluginMetadata = new DepublishPluginMetadata(); + depublishPluginMetadata.setEnabled(true); + depublishPluginMetadata.setDatasetDepublish(true); + List abstractMetisPluginMetadata = new ArrayList<>(2); + abstractMetisPluginMetadata.add(oaipmhHarvestPluginMetadata); + abstractMetisPluginMetadata.add(depublishPluginMetadata); + workflowDepublishAndOai.setMetisPluginsMetadata(abstractMetisPluginMetadata); + assertThrows(BadContentException.class, + () -> validationUtils.validateWorkflowPlugins(workflowDepublishAndOai, null)); + + // Test if workflow contains record depublish that record ids exist + Workflow workflowDepublish = new Workflow(); + workflowDepublish.setDatasetId(Integer.toString(TestObjectFactory.DATASETID)); + depublishPluginMetadata.setDatasetDepublish(false); + abstractMetisPluginMetadata.clear(); + abstractMetisPluginMetadata.add(depublishPluginMetadata); + workflowDepublish.setMetisPluginsMetadata(abstractMetisPluginMetadata); + when(depublishRecordIdDao + .getAllDepublishRecordIdsWithStatus(workflowDepublish.getDatasetId(), + DepublishRecordIdSortField.DEPUBLICATION_STATE, SortDirection.ASCENDING, + DepublicationStatus.PENDING_DEPUBLICATION)).thenReturn(Collections.emptySet()); + assertThrows(BadContentException.class, () -> validationUtils + .validateWorkflowPlugins(workflowDepublish, null)); + + // Test workflow starting with link checking. + assertSame(predecessor, validationUtils.validateWorkflowPlugins( + createWorkflow(ExecutablePluginType.LINK_CHECKING), predecessorType).getPlugin()); + assertThrows(PluginExecutionNotAllowed.class, () -> validationUtils.validateWorkflowPlugins( + createWorkflow(ExecutablePluginType.LINK_CHECKING, ExecutablePluginType.TRANSFORMATION), + predecessorType)); + + // Test workflow with gaps + assertThrows(PluginExecutionNotAllowed.class, () -> validationUtils.validateWorkflowPlugins( + createWorkflow(ExecutablePluginType.TRANSFORMATION, ExecutablePluginType.ENRICHMENT), + predecessorType)); + + // Test workflow with duplicate types + assertThrows(PluginExecutionNotAllowed.class, () -> validationUtils.validateWorkflowPlugins( + createWorkflow(ExecutablePluginType.TRANSFORMATION, ExecutablePluginType.ENRICHMENT, + ExecutablePluginType.ENRICHMENT), predecessorType)); + assertThrows(PluginExecutionNotAllowed.class, () -> validationUtils.validateWorkflowPlugins( + createWorkflow(ExecutablePluginType.TRANSFORMATION, ExecutablePluginType.LINK_CHECKING, + ExecutablePluginType.LINK_CHECKING), predecessorType)); + + // Test workflow with disabled plugins: valid before disabling, but invalid after. + final Workflow workflowWithDisabledPlugins = createWorkflow( + ExecutablePluginType.NORMALIZATION, + ExecutablePluginType.ENRICHMENT, ExecutablePluginType.MEDIA_PROCESS); + assertSame(predecessor, validationUtils.validateWorkflowPlugins(workflowWithDisabledPlugins, + predecessorType).getPlugin()); + when(workflowWithDisabledPlugins.getMetisPluginsMetadata().get(1).isEnabled()) + .thenReturn(false); + assertThrows(PluginExecutionNotAllowed.class, () -> validationUtils.validateWorkflowPlugins( + workflowWithDisabledPlugins, predecessorType)); + + // Test workflow with bad predecessor + doThrow(PluginExecutionNotAllowed.class).when(dataEvolutionUtils) + .computePredecessorPlugin(any(), eq(predecessorType), eq(DATASET_ID)); + assertThrows(PluginExecutionNotAllowed.class, () -> validationUtils.validateWorkflowPlugins( + createWorkflow(ExecutablePluginType.ENRICHMENT, ExecutablePluginType.OAIPMH_HARVEST), + predecessorType)); + } + + private Workflow createWorkflow(ExecutablePluginType... pluginTypes) { + final Workflow workflow = new Workflow(); + workflow.setDatasetId(DATASET_ID); + workflow.setMetisPluginsMetadata(Arrays.stream(pluginTypes).map(type -> { + final AbstractExecutablePluginMetadata plugin = mock( + AbstractExecutablePluginMetadata.class); + when(plugin.isEnabled()).thenReturn(true); + doReturn(type).when(plugin).getExecutablePluginType(); + return plugin; + }).collect(Collectors.toList())); + return workflow; + } + + @Test + void testValidateWorkflowPlugins_testHarvestingParameters() throws GenericMetisException { + + // Prepare correct url variables + final String simpleUrl = "http://test.com/path"; + final String urlWithFragmentAndQuery = simpleUrl + "#fragment?query=1"; + final String metadataFormat = "metadataFormatParameter"; + final String setSpec = "setSpecParameter"; + + // Create oai harvesting with all parameters + final OaipmhHarvestPluginMetadata oai = new OaipmhHarvestPluginMetadata(); + oai.setEnabled(true); + oai.setUrl(" " + urlWithFragmentAndQuery + " "); + oai.setMetadataFormat(" " + metadataFormat + " "); + oai.setSetSpec(" " + setSpec + " "); + + // Create http harvesting + final HTTPHarvestPluginMetadata http = new HTTPHarvestPluginMetadata(); + http.setEnabled(true); + http.setUrl(" " + urlWithFragmentAndQuery + " "); + + // Create the workflow and execute the method + final Workflow workflow = new Workflow(); + workflow.setDatasetId(DATASET_ID); + workflow.setMetisPluginsMetadata(Arrays.asList(oai, http)); + validationUtils.validateWorkflowPlugins(workflow, null); + + // Test output + assertEquals(simpleUrl, oai.getUrl()); + assertEquals(metadataFormat, oai.getMetadataFormat()); + assertEquals(setSpec, oai.getSetSpec()); + assertEquals(urlWithFragmentAndQuery, http.getUrl()); + + // Create oai harvesting with only url + oai.setUrl(urlWithFragmentAndQuery); + oai.setMetadataFormat(null); + oai.setSetSpec(null); + + // Create the workflow and execute the method + workflow.setMetisPluginsMetadata(Collections.singletonList(oai)); + validationUtils.validateWorkflowPlugins(workflow, null); + + // Test output + assertEquals(simpleUrl, oai.getUrl()); + assertNull(oai.getMetadataFormat()); + assertNull(oai.getSetSpec()); + + // Test OAI with invalid URL + oai.setUrl("invalid URL"); + workflow.setMetisPluginsMetadata(Collections.singletonList(oai)); + assertThrows(BadContentException.class, + () -> validationUtils.validateWorkflowPlugins(workflow, null)); + + // Test HTTP with missing URL + http.setUrl(null); + workflow.setMetisPluginsMetadata(Collections.singletonList(http)); + assertThrows(BadContentException.class, + () -> validationUtils.validateWorkflowPlugins(workflow, null)); + + // Test incremental OAI + oai.setUrl(urlWithFragmentAndQuery); + oai.setIncrementalHarvest(true); + workflow.setMetisPluginsMetadata(Collections.singletonList(oai)); + doReturn(true).when(validationUtils).isIncrementalHarvestingAllowed(DATASET_ID); + validationUtils.validateWorkflowPlugins(workflow, null); + doReturn(false).when(validationUtils).isIncrementalHarvestingAllowed(DATASET_ID); + assertThrows(BadContentException.class, + () -> validationUtils.validateWorkflowPlugins(workflow, null)); + } + + @Test + void testIsIncrementalHarvestingAllowed() { + doReturn(List.of(new PluginWithExecutionId<>((String) null, null))) + .when(dataEvolutionUtils).getPublishedHarvestIncrements(DATASET_ID); + assertTrue(validationUtils.isIncrementalHarvestingAllowed(DATASET_ID)); + doReturn(Collections.emptyList()).when(dataEvolutionUtils) + .getPublishedHarvestIncrements(DATASET_ID); + assertFalse(validationUtils.isIncrementalHarvestingAllowed(DATASET_ID)); + doReturn(null).when(dataEvolutionUtils).getPublishedHarvestIncrements(DATASET_ID); + assertFalse(validationUtils.isIncrementalHarvestingAllowed(DATASET_ID)); + } +} diff --git a/metis-core/metis-core-service/src/test/java/eu/europeana/metis/core/service/TestOrchestratorService.java b/metis-core/metis-core-service/src/test/java/eu/europeana/metis/core/service/TestOrchestratorService.java index 9b77a9592..a862ebb04 100644 --- a/metis-core/metis-core-service/src/test/java/eu/europeana/metis/core/service/TestOrchestratorService.java +++ b/metis-core/metis-core-service/src/test/java/eu/europeana/metis/core/service/TestOrchestratorService.java @@ -34,7 +34,8 @@ import eu.europeana.metis.core.dao.WorkflowExecutionDao; import eu.europeana.metis.core.dao.WorkflowExecutionDao.ExecutionDatasetPair; import eu.europeana.metis.core.dao.WorkflowExecutionDao.ResultList; -import eu.europeana.metis.core.dao.WorkflowUtils; +import eu.europeana.metis.core.dao.DataEvolutionUtils; +import eu.europeana.metis.core.dao.WorkflowValidationUtils; import eu.europeana.metis.core.dataset.Dataset; import eu.europeana.metis.core.dataset.DatasetExecutionInformation; import eu.europeana.metis.core.dataset.DatasetXslt; @@ -114,7 +115,8 @@ class TestOrchestratorService { private static final int SOLR_COMMIT_PERIOD_IN_MINS = 15; private static WorkflowExecutionDao workflowExecutionDao; - private static WorkflowUtils workflowUtils; + private static DataEvolutionUtils dataEvolutionUtils; + private static WorkflowValidationUtils validationUtils; private static WorkflowDao workflowDao; private static DatasetDao datasetDao; private static DatasetXsltDao datasetXsltDao; @@ -128,7 +130,8 @@ class TestOrchestratorService { @BeforeAll static void prepare() { workflowExecutionDao = mock(WorkflowExecutionDao.class); - workflowUtils = mock(WorkflowUtils.class); + dataEvolutionUtils = mock(DataEvolutionUtils.class); + validationUtils = mock(WorkflowValidationUtils.class); workflowDao = mock(WorkflowDao.class); datasetDao = mock(DatasetDao.class); datasetXsltDao = mock(DatasetXsltDao.class); @@ -138,22 +141,22 @@ static void prepare() { authorizer = mock(Authorizer.class); workflowExecutionFactory = spy(new WorkflowExecutionFactory(datasetXsltDao, - depublishRecordIdDao, workflowExecutionDao, workflowUtils)); + depublishRecordIdDao, workflowExecutionDao, dataEvolutionUtils)); workflowExecutionFactory.setValidationExternalProperties( new ValidationProperties("url-ext", "schema-ext", "schematron-ext")); workflowExecutionFactory.setValidationInternalProperties( new ValidationProperties("url-int", "schema-int", "schematron-int")); orchestratorService = spy(new OrchestratorService(workflowExecutionFactory, workflowDao, - workflowExecutionDao, workflowUtils, datasetDao, workflowExecutorManager, redissonClient, - authorizer, depublishRecordIdDao)); + workflowExecutionDao, validationUtils, dataEvolutionUtils, datasetDao, + workflowExecutorManager, redissonClient, authorizer, depublishRecordIdDao)); orchestratorService.setSolrCommitPeriodInMins(SOLR_COMMIT_PERIOD_IN_MINS); } @AfterEach void cleanUp() { Mockito.reset(workflowExecutionDao); - Mockito.reset(workflowUtils); + Mockito.reset(validationUtils); Mockito.reset(workflowDao); Mockito.reset(datasetDao); Mockito.reset(workflowExecutorManager); @@ -188,7 +191,7 @@ void createWorkflowOrderOfPluginsNotAllowed() throws Exception { Dataset dataset = TestObjectFactory.createDataset("datasetName"); workflow.setDatasetId(dataset.getDatasetId()); when(datasetDao.getDatasetByDatasetId(dataset.getDatasetId())).thenReturn(dataset); - doThrow(PluginExecutionNotAllowed.class).when(workflowUtils) + doThrow(PluginExecutionNotAllowed.class).when(validationUtils) .validateWorkflowPlugins(workflow, null); assertThrows(PluginExecutionNotAllowed.class, () -> orchestratorService @@ -458,7 +461,7 @@ void addWorkflowInQueueOfWorkflowExecutions_NoHarvestPlugin() throws Exception { ExecutionProgress executionProgress = new ExecutionProgress(); executionProgress.setProcessedRecords(5); oaipmhHarvestPlugin.setExecutionProgress(executionProgress); - when(workflowUtils.validateWorkflowPlugins(workflow, null)) + when(validationUtils.validateWorkflowPlugins(workflow, null)) .thenReturn(new PluginWithExecutionId<>("execution id", oaipmhHarvestPlugin)); RLock rlock = mock(RLock.class); when(redissonClient.getFairLock(anyString())).thenReturn(rlock); @@ -482,7 +485,7 @@ void addWorkflowInQueueOfWorkflowExecutions_NoHarvestPlugin_NoProcessPlugin() .thenReturn(dataset); when(workflowDao.getWorkflow(workflow.getDatasetId())).thenReturn(workflow); when(redissonClient.getFairLock(anyString())).thenReturn(Mockito.mock(RLock.class)); - when(workflowUtils.validateWorkflowPlugins(workflow, null)) + when(validationUtils.validateWorkflowPlugins(workflow, null)) .thenThrow(new PluginExecutionNotAllowed("")); assertThrows(PluginExecutionNotAllowed.class, () -> orchestratorService .addWorkflowInQueueOfWorkflowExecutions(metisUser, dataset.getDatasetId(), null, null, 0)); @@ -590,7 +593,7 @@ void addWorkflowInQueueOfWorkflowExecutions_WorkflowIsEmpty() throws Exception { when(authorizer.authorizeWriteExistingDatasetById(metisUser, dataset.getDatasetId())) .thenReturn(dataset); when(workflowDao.getWorkflow(dataset.getDatasetId())).thenReturn(workflow); - when(workflowUtils.validateWorkflowPlugins(workflow, null)) + when(validationUtils.validateWorkflowPlugins(workflow, null)) .thenThrow(new BadContentException("")); assertThrows(BadContentException.class, () -> orchestratorService .addWorkflowInQueueOfWorkflowExecutions(metisUser, dataset.getDatasetId(), null, null, 0)); @@ -647,8 +650,9 @@ void getLatestSuccessfulFinishedPluginByDatasetIdIfPluginTypeAllowedForExecution final AbstractExecutablePlugin oaipmhHarvestPlugin = ExecutablePluginFactory .createPlugin(new OaipmhHarvestPluginMetadata()); when(authorizer.authorizeReadExistingDatasetById(metisUser, datasetId)).thenReturn(null); - when(workflowUtils.computePredecessorPlugin(ExecutablePluginType.VALIDATION_EXTERNAL, null, - datasetId)).thenReturn(new PluginWithExecutionId<>("execution ID", oaipmhHarvestPlugin)); + doReturn(new PluginWithExecutionId<>("execution ID", oaipmhHarvestPlugin)) + .when(dataEvolutionUtils) + .computePredecessorPlugin(ExecutablePluginType.VALIDATION_EXTERNAL, null, datasetId); assertSame(oaipmhHarvestPlugin, orchestratorService .getLatestFinishedPluginByDatasetIdIfPluginTypeAllowedForExecution(metisUser, datasetId, ExecutablePluginType.VALIDATION_EXTERNAL, null)); @@ -662,7 +666,7 @@ void getLatestFinishedPluginByDatasetIdIfPluginTypeAllowedForExecution_PluginExe final MetisUser metisUser = TestObjectFactory.createMetisUser(TestObjectFactory.EMAIL); final String datasetId = Integer.toString(TestObjectFactory.DATASETID); when(authorizer.authorizeReadExistingDatasetById(metisUser, datasetId)).thenReturn(null); - when(workflowUtils.computePredecessorPlugin(ExecutablePluginType.VALIDATION_EXTERNAL, null, + when(dataEvolutionUtils.computePredecessorPlugin(ExecutablePluginType.VALIDATION_EXTERNAL, null, datasetId)).thenThrow(new PluginExecutionNotAllowed("")); assertThrows(PluginExecutionNotAllowed.class, () -> orchestratorService .getLatestFinishedPluginByDatasetIdIfPluginTypeAllowedForExecution(metisUser, @@ -1120,8 +1124,8 @@ void testGetRecordEvolutionForVersionHappyFlow() throws GenericMetisException { final List> evolutionWithContent = Arrays .asList( ImmutablePair.of(plugin1, execution1), ImmutablePair.of(plugin2, execution2)); - doReturn(evolutionWithContent).when(workflowUtils).compileVersionEvolution(plugin3, execution2); - doReturn(new ArrayList<>()).when(workflowUtils).compileVersionEvolution(plugin1, execution1); + doReturn(evolutionWithContent).when(dataEvolutionUtils).compileVersionEvolution(plugin3, execution2); + doReturn(new ArrayList<>()).when(dataEvolutionUtils).compileVersionEvolution(plugin1, execution1); // Execute the call and expect an evolution with content. final VersionEvolution resultForThree = orchestratorService.getRecordEvolutionForVersion( diff --git a/metis-core/pom.xml b/metis-core/pom.xml index c5b3e53fe..687814f89 100644 --- a/metis-core/pom.xml +++ b/metis-core/pom.xml @@ -4,7 +4,7 @@ metis-framework eu.europeana.metis - 3 + 4 metis-core pom diff --git a/metis-data-checker/metis-data-checker-common/pom.xml b/metis-data-checker/metis-data-checker-common/pom.xml index 70756e1ad..ddfeb5674 100644 --- a/metis-data-checker/metis-data-checker-common/pom.xml +++ b/metis-data-checker/metis-data-checker-common/pom.xml @@ -4,7 +4,7 @@ metis-data-checker eu.europeana.metis - 3 + 4 metis-data-checker-common diff --git a/metis-data-checker/metis-data-checker-rest/pom.xml b/metis-data-checker/metis-data-checker-rest/pom.xml index e254e7272..b950a3f43 100644 --- a/metis-data-checker/metis-data-checker-rest/pom.xml +++ b/metis-data-checker/metis-data-checker-rest/pom.xml @@ -4,7 +4,7 @@ metis-data-checker eu.europeana.metis - 3 + 4 metis-data-checker-rest war @@ -56,24 +56,19 @@ com.jayway.jsonpath json-path-assert - ${version.json.path} test org.junit.jupiter junit-jupiter-api - test org.junit.jupiter junit-jupiter-engine - test org.mockito mockito-core - ${version.mockito.core} - test org.springframework diff --git a/metis-data-checker/metis-data-checker-rest/src/main/java/eu/europeana/metis/data/checker/config/Application.java b/metis-data-checker/metis-data-checker-rest/src/main/java/eu/europeana/metis/data/checker/config/Application.java index 063045caa..9998dbf51 100644 --- a/metis-data-checker/metis-data-checker-rest/src/main/java/eu/europeana/metis/data/checker/config/Application.java +++ b/metis-data-checker/metis-data-checker-rest/src/main/java/eu/europeana/metis/data/checker/config/Application.java @@ -88,6 +88,8 @@ public class Application implements WebMvcConfigurer, InitializingBean { private String mongoDb; @Value("${mongo.enableSSL}") private boolean mongoEnableSSL; + @Value("${mongo.application.name}") + private String mongoApplicationName; @Value("${solr.hosts}") private String[] solrHosts; @@ -133,7 +135,8 @@ public void afterPropertiesSet() throws Exception { // Set the Mongo properties settings.getMongoProperties().setAllProperties(mongoHosts, mongoPorts, - mongoAuthenticationDb, mongoUsername, mongoPassword, mongoEnableSSL, null); + mongoAuthenticationDb, mongoUsername, mongoPassword, mongoEnableSSL, null, + mongoApplicationName); settings.setMongoDatabaseName(mongoDb); // Set Solr properties diff --git a/metis-data-checker/metis-data-checker-rest/src/main/resources/data.checker.properties.example b/metis-data-checker/metis-data-checker-rest/src/main/resources/data.checker.properties.example index 53c07d345..ce147f8e6 100644 --- a/metis-data-checker/metis-data-checker-rest/src/main/resources/data.checker.properties.example +++ b/metis-data-checker/metis-data-checker-rest/src/main/resources/data.checker.properties.example @@ -12,6 +12,7 @@ mongo.username= mongo.password= mongo.db= mongo.enableSSL= +mongo.application.name= solr.hosts= diff --git a/metis-data-checker/metis-data-checker-service/pom.xml b/metis-data-checker/metis-data-checker-service/pom.xml index 86a2c0ff1..7dc0162cf 100644 --- a/metis-data-checker/metis-data-checker-service/pom.xml +++ b/metis-data-checker/metis-data-checker-service/pom.xml @@ -4,7 +4,7 @@ metis-data-checker eu.europeana.metis - 3 + 4 metis-data-checker-service @@ -26,12 +26,6 @@ eu.europeana.metis metis-data-checker-common ${project.version} - - - org.mongodb - mongodb-driver-core - - eu.europeana.metis @@ -51,18 +45,14 @@ org.junit.jupiter junit-jupiter-api - test org.junit.jupiter junit-jupiter-engine - test org.mockito mockito-core - ${version.mockito.core} - test org.springframework diff --git a/metis-data-checker/metis-data-checker-service/src/main/java/eu/europeana/metis/data/checker/service/DataCheckerService.java b/metis-data-checker/metis-data-checker-service/src/main/java/eu/europeana/metis/data/checker/service/DataCheckerService.java index a71b40f97..f50592344 100644 --- a/metis-data-checker/metis-data-checker-service/src/main/java/eu/europeana/metis/data/checker/service/DataCheckerService.java +++ b/metis-data-checker/metis-data-checker-service/src/main/java/eu/europeana/metis/data/checker/service/DataCheckerService.java @@ -1,6 +1,5 @@ package eu.europeana.metis.data.checker.service; -import com.google.common.base.Strings; import eu.europeana.indexing.exception.IndexingException; import eu.europeana.metis.data.checker.common.exception.DataCheckerServiceException; import eu.europeana.metis.data.checker.common.model.DatasetProperties; @@ -25,6 +24,7 @@ import java.util.function.Function; import java.util.stream.Collectors; import javax.annotation.PreDestroy; +import org.apache.commons.lang3.StringUtils; import org.apache.solr.client.solrj.SolrServerException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -139,7 +139,7 @@ private ExtendedValidationResult compileResult(final List final List succeededResults; if (includeRecordIds) { succeededResults = taskResults.stream().filter(ValidationTaskResult::isSuccess) - .map(ValidationTaskResult::getRecordId).filter(record -> !Strings.isNullOrEmpty(record)) + .map(ValidationTaskResult::getRecordId).filter(StringUtils::isNotBlank) .collect(Collectors.toList()); } else { succeededResults = null; diff --git a/metis-data-checker/metis-data-checker-service/src/main/java/eu/europeana/metis/data/checker/service/persistence/RecordIndexingService.java b/metis-data-checker/metis-data-checker-service/src/main/java/eu/europeana/metis/data/checker/service/persistence/RecordIndexingService.java index 9b2be598d..7137e1ff0 100644 --- a/metis-data-checker/metis-data-checker-service/src/main/java/eu/europeana/metis/data/checker/service/persistence/RecordIndexingService.java +++ b/metis-data-checker/metis-data-checker-service/src/main/java/eu/europeana/metis/data/checker/service/persistence/RecordIndexingService.java @@ -18,6 +18,7 @@ import eu.europeana.indexing.AbstractConnectionProvider; import eu.europeana.indexing.Indexer; import eu.europeana.indexing.IndexerFactory; +import eu.europeana.indexing.IndexingProperties; import eu.europeana.indexing.exception.IndexingException; import eu.europeana.metis.mongo.dao.RecordDao; import eu.europeana.metis.schema.jibx.RDF; @@ -64,7 +65,7 @@ public RecordIndexingService(AbstractConnectionProvider connectionProvider) thro * @throws IndexingException In case indexing failed. */ public void createRecord(RDF rdf, Date recordDate) throws IndexingException { - indexer.indexRdf(rdf, recordDate, false, null, false); + indexer.indexRdf(rdf, new IndexingProperties(recordDate, false, null, false, true)); } /** diff --git a/metis-data-checker/metis-data-checker-service/src/test/java/eu/europeana/metis/data/checker/service/persistence/TestRecordIndexingService.java b/metis-data-checker/metis-data-checker-service/src/test/java/eu/europeana/metis/data/checker/service/persistence/TestRecordIndexingService.java index 5d940bda4..f8b31cfff 100644 --- a/metis-data-checker/metis-data-checker-service/src/test/java/eu/europeana/metis/data/checker/service/persistence/TestRecordIndexingService.java +++ b/metis-data-checker/metis-data-checker-service/src/test/java/eu/europeana/metis/data/checker/service/persistence/TestRecordIndexingService.java @@ -21,12 +21,8 @@ void test() throws Exception { Date recordDate = new java.util.Date(); recordIndexingService.createRecord(rdf, recordDate); - Mockito.verify(indexer, Mockito.times(1)).indexRdf(Mockito.any(), Mockito.any(Date.class), - Mockito.anyBoolean(), Mockito.any(), Mockito.anyBoolean() - ); - Mockito.verify(indexer, Mockito.times(1)).indexRdf(Mockito.eq(rdf), Mockito.any(Date.class), - Mockito.anyBoolean(), Mockito.any(), Mockito.anyBoolean() - ); + Mockito.verify(indexer, Mockito.times(1)).indexRdf(Mockito.any(), Mockito.any()); + Mockito.verify(indexer, Mockito.times(1)).indexRdf(Mockito.eq(rdf), Mockito.any()); } } diff --git a/metis-data-checker/pom.xml b/metis-data-checker/pom.xml index fc55970aa..b26a8e502 100644 --- a/metis-data-checker/pom.xml +++ b/metis-data-checker/pom.xml @@ -4,7 +4,7 @@ metis-framework eu.europeana.metis - 3 + 4 metis-data-checker pom diff --git a/metis-dereference/metis-dereference-common/pom.xml b/metis-dereference/metis-dereference-common/pom.xml index 0b6e62ca7..7da1d6cf3 100644 --- a/metis-dereference/metis-dereference-common/pom.xml +++ b/metis-dereference/metis-dereference-common/pom.xml @@ -4,7 +4,7 @@ metis-dereference eu.europeana.metis - 3 + 4 metis-dereference-common @@ -38,12 +38,10 @@ org.junit.jupiter junit-jupiter-api - test org.junit.jupiter junit-jupiter-engine - test \ No newline at end of file diff --git a/metis-dereference/metis-dereference-import/pom.xml b/metis-dereference/metis-dereference-import/pom.xml index 331efade0..bbaf2326b 100644 --- a/metis-dereference/metis-dereference-import/pom.xml +++ b/metis-dereference/metis-dereference-import/pom.xml @@ -3,35 +3,31 @@ metis-dereference eu.europeana.metis - 3 + 4 4.0.0 metis-dereference-import 3.0.0-M3 - 2.0.9 + 2.2.1 org.junit.jupiter junit-jupiter-api - test org.junit.jupiter junit-jupiter-engine - test org.mockito mockito-core - ${version.mockito.core} - test eu.europeana.metis metis-dereference-common - 3 + ${project.version} com.fasterxml.jackson.dataformat diff --git a/metis-dereference/metis-dereference-rest/pom.xml b/metis-dereference/metis-dereference-rest/pom.xml index 4bfdee410..b659939b6 100644 --- a/metis-dereference/metis-dereference-rest/pom.xml +++ b/metis-dereference/metis-dereference-rest/pom.xml @@ -4,7 +4,7 @@ metis-dereference eu.europeana.metis - 3 + 4 metis-dereference-rest war @@ -77,7 +77,6 @@ com.jayway.jsonpath json-path-assert - ${version.json.path} test @@ -88,18 +87,14 @@ org.junit.jupiter junit-jupiter-api - test org.junit.jupiter junit-jupiter-engine - test org.mockito mockito-core - ${version.mockito.core} - test org.springframework diff --git a/metis-dereference/metis-dereference-rest/src/main/java/eu/europeana/metis/dereference/rest/config/Application.java b/metis-dereference/metis-dereference-rest/src/main/java/eu/europeana/metis/dereference/rest/config/Application.java index 8f28d9fd4..d3ae9a83b 100644 --- a/metis-dereference/metis-dereference-rest/src/main/java/eu/europeana/metis/dereference/rest/config/Application.java +++ b/metis-dereference/metis-dereference-rest/src/main/java/eu/europeana/metis/dereference/rest/config/Application.java @@ -58,6 +58,8 @@ public class Application implements WebMvcConfigurer, InitializingBean { private String mongoUsername; @Value("${mongo.password}") private String mongoPassword; + @Value("${mongo.application.name}") + private String mongoApplicationName; @Value("${entity.db}") private String entityDb; @Value("${vocabulary.db}") @@ -83,6 +85,7 @@ public void afterPropertiesSet() { final MongoProperties mongoProperties = new MongoProperties<>( IllegalArgumentException::new); mongoProperties.setMongoHosts(mongoHosts, mongoPorts); + mongoProperties.setApplicationName(mongoApplicationName); mongoClientEntity = new MongoClientProvider<>(mongoProperties).createMongoClient(); mongoClientVocabulary = new MongoClientProvider<>(mongoProperties).createMongoClient(); } diff --git a/metis-dereference/metis-dereference-rest/src/main/resources/dereferencing.properties.example b/metis-dereference/metis-dereference-rest/src/main/resources/dereferencing.properties.example index 72dbcde87..3ee045635 100644 --- a/metis-dereference/metis-dereference-rest/src/main/resources/dereferencing.properties.example +++ b/metis-dereference/metis-dereference-rest/src/main/resources/dereferencing.properties.example @@ -10,5 +10,6 @@ mongo.hosts= mongo.port= mongo.username= mongo.password= +mongo.application.name= entity.db= vocabulary.db= \ No newline at end of file diff --git a/metis-dereference/metis-dereference-service/pom.xml b/metis-dereference/metis-dereference-service/pom.xml index c5d8d77c5..d6b01e92e 100644 --- a/metis-dereference/metis-dereference-service/pom.xml +++ b/metis-dereference/metis-dereference-service/pom.xml @@ -4,7 +4,7 @@ metis-dereference eu.europeana.metis - 3 + 4 metis-dereference-service @@ -37,11 +37,6 @@ jackson-databind ${version.jackson} - - org.mongodb - mongodb-driver-core - ${version.mongodb.driver.core} - org.springframework spring-context @@ -62,18 +57,14 @@ org.junit.jupiter junit-jupiter-api - test org.junit.jupiter junit-jupiter-engine - test org.mockito mockito-core - ${version.mockito.core} - test eu.europeana.metis diff --git a/metis-dereference/pom.xml b/metis-dereference/pom.xml index 52dc4863a..959566961 100644 --- a/metis-dereference/pom.xml +++ b/metis-dereference/pom.xml @@ -3,7 +3,7 @@ 4.0.0 eu.europeana.metis - 3 + 4 metis-framework diff --git a/metis-enrichment/metis-enrichment-client/pom.xml b/metis-enrichment/metis-enrichment-client/pom.xml index b9d11c1ea..697bf6514 100644 --- a/metis-enrichment/metis-enrichment-client/pom.xml +++ b/metis-enrichment/metis-enrichment-client/pom.xml @@ -4,7 +4,7 @@ metis-enrichment eu.europeana.metis - 3 + 4 metis-enrichment-client jar @@ -22,8 +22,6 @@ org.mockito mockito-core - ${version.mockito.core} - test eu.europeana.metis diff --git a/metis-enrichment/metis-enrichment-client/src/main/java/eu/europeana/enrichment/rest/client/enrichment/MetisRecordParser.java b/metis-enrichment/metis-enrichment-client/src/main/java/eu/europeana/enrichment/rest/client/enrichment/MetisRecordParser.java index ff2348454..6fcc7be3e 100644 --- a/metis-enrichment/metis-enrichment-client/src/main/java/eu/europeana/enrichment/rest/client/enrichment/MetisRecordParser.java +++ b/metis-enrichment/metis-enrichment-client/src/main/java/eu/europeana/enrichment/rest/client/enrichment/MetisRecordParser.java @@ -43,10 +43,11 @@ public class MetisRecordParser implements RecordParser { @Override public Set parseSearchTerms(RDF rdf) { - final ProxyType providerProxy = RdfProxyUtils.getProviderProxy(rdf); + final List providerProxies = RdfProxyUtils.getProviderProxies(rdf); final Map> result = new HashMap<>(); for (FieldType field : FieldType.values()) { - field.extractFieldValuesForEnrichment(providerProxy) + providerProxies.stream().map(field::extractFieldValuesForEnrichment) + .flatMap(Collection::stream) .forEach(value -> result.computeIfAbsent(value, key -> new HashSet<>()).add(field)); } return result.entrySet().stream().map(entry -> new SearchTermContext(entry.getKey().getValue(), diff --git a/metis-enrichment/metis-enrichment-client/src/main/java/eu/europeana/enrichment/utils/DereferenceUtils.java b/metis-enrichment/metis-enrichment-client/src/main/java/eu/europeana/enrichment/utils/DereferenceUtils.java index 8103ca8d3..06192d394 100644 --- a/metis-enrichment/metis-enrichment-client/src/main/java/eu/europeana/enrichment/utils/DereferenceUtils.java +++ b/metis-enrichment/metis-enrichment-client/src/main/java/eu/europeana/enrichment/utils/DereferenceUtils.java @@ -11,7 +11,6 @@ import eu.europeana.metis.schema.jibx.ResourceType; import eu.europeana.metis.schema.jibx.TimeSpanType; import eu.europeana.metis.schema.jibx.WebResourceType; -import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Optional; @@ -47,8 +46,7 @@ public static Set extractReferencesForDereferencing(RDF rdf) { extractValues(rdf.getPlaceList(), item -> dereferencePlace(item, result)); extractValues(rdf.getTimeSpanList(), item -> dereferenceTimespan(item, result)); extractValues(rdf.getWebResourceList(), item -> dereferenceWebResource(item, result)); - extractValues(Collections.singletonList(RdfProxyUtils.getProviderProxy(rdf)), - item -> dereferenceProxy(item, result)); + extractValues(RdfProxyUtils.getProviderProxies(rdf), item -> dereferenceProxy(item, result)); // Clean up the result: no null values and no objects that we already have. result.remove(null); diff --git a/metis-enrichment/metis-enrichment-client/src/main/java/eu/europeana/enrichment/utils/EnrichmentUtils.java b/metis-enrichment/metis-enrichment-client/src/main/java/eu/europeana/enrichment/utils/EnrichmentUtils.java index b4646ac03..e220f3a23 100644 --- a/metis-enrichment/metis-enrichment-client/src/main/java/eu/europeana/enrichment/utils/EnrichmentUtils.java +++ b/metis-enrichment/metis-enrichment-client/src/main/java/eu/europeana/enrichment/utils/EnrichmentUtils.java @@ -3,6 +3,7 @@ import eu.europeana.metis.schema.jibx.Aggregation; import eu.europeana.metis.schema.jibx.Completeness; import eu.europeana.metis.schema.jibx.EuropeanaAggregationType; +import eu.europeana.metis.schema.jibx.EuropeanaType; import eu.europeana.metis.schema.jibx.EuropeanaType.Choice; import eu.europeana.metis.schema.jibx.LiteralType; import eu.europeana.metis.schema.jibx.ProxyType; @@ -52,27 +53,25 @@ private EnrichmentUtils() { public static void setAdditionalData(RDF rdf) { // Get the provider and europeana proxy - final ProxyType providerProxy = rdf.getProxyList().stream() - .filter(proxy -> !isEuropeanaProxy(proxy)).findAny().orElse(null); - final ProxyType europeanaProxy = rdf.getProxyList().stream() - .filter(EnrichmentUtils::isEuropeanaProxy).findAny().orElse(null); - if (providerProxy == null || europeanaProxy == null) { + final List providerProxies = RdfProxyUtils.getProviderProxies(rdf); + if (providerProxies.isEmpty()) { return; } + final ProxyType europeanaProxy = RdfProxyUtils.getEuropeanaProxy(rdf); // Calculate completeness first EuropeanaAggregationType europeanaAggregation = rdf.getEuropeanaAggregationList().stream() .findAny().orElse(null); if (europeanaAggregation != null) { Completeness completeness = new Completeness(); - completeness.setString(Integer - .toString(computeEuropeanaCompleteness(providerProxy, rdf.getAggregationList().get(0)))); + completeness.setString(Integer.toString( + computeEuropeanaCompleteness(providerProxies, rdf.getAggregationList()))); europeanaAggregation.setCompleteness(completeness); } // Obtain the date strings from the various proxy fields. - final List dateStrings = - providerProxy.getChoiceList().stream().map(EnrichmentUtils::getDateFromChoice) + final List dateStrings = providerProxies.stream().map(EuropeanaType::getChoiceList) + .flatMap(Collection::stream).map(EnrichmentUtils::getDateFromChoice) .filter(Objects::nonNull).collect(Collectors.toList()); // Parse them and set them in the europeana proxy. @@ -103,10 +102,6 @@ private static String getDateFromChoice(Choice choice) { return result == null ? null : result.getString(); } - private static boolean isEuropeanaProxy(ProxyType proxy) { - return proxy.getEuropeanaProxy() != null && proxy.getEuropeanaProxy().isEuropeanaProxy(); - } - /** * Calculates the Europeana Completeness. * @@ -124,25 +119,27 @@ private static boolean isEuropeanaProxy(ProxyType proxy) { * value of points is 5 * * - * @param providerProxy the provider proxy - * @param aggregation the provider aggregation + * @param providerProxies the provider proxies + * @param providerAggregations the provider aggregations * @return the points awarded to the record */ - private static int computeEuropeanaCompleteness(final ProxyType providerProxy, - final Aggregation aggregation) { + private static int computeEuropeanaCompleteness(final List providerProxies, + final List providerAggregations) { + List tags = new ArrayList<>(); List descriptions = new ArrayList<>(); List titles = new ArrayList<>(); - List choices = providerProxy.getChoiceList(); - if (choices != null) { - Map, String> uniqueResourceOrLiteralTypeClassesMap = createCollectionsForResourceOrLiteralType( - choices, descriptions, titles); - addResourceOrLiteralTypeFromMapsToList(uniqueResourceOrLiteralTypeClassesMap, tags); - } - String thumbnailUrl = Optional.ofNullable(aggregation.getObject()) - .map(ResourceType::getResource) - .orElse(null); + List choices = providerProxies.stream().map(EuropeanaType::getChoiceList) + .filter(Objects::nonNull).flatMap(Collection::stream).collect(Collectors.toList()); + Map, String> uniqueResourceOrLiteralTypeClassesMap = createCollectionsForResourceOrLiteralType( + choices, descriptions, titles); + addResourceOrLiteralTypeFromMapsToList(uniqueResourceOrLiteralTypeClassesMap, tags); + + final String thumbnailUrl = Optional.ofNullable(providerAggregations).stream() + .flatMap(Collection::stream).filter(Objects::nonNull).map(Aggregation::getObject) + .filter(Objects::nonNull).map(ResourceType::getResource).filter(Objects::nonNull) + .findAny().orElse(null); return completenessCalculation(thumbnailUrl, titles, descriptions, tags); } @@ -226,8 +223,7 @@ private static void addChoiceToList(BooleanSupplier booleanSupplier, Supplie } private static void addResourceOrLiteralTypeFromMapsToList( - final Map, String> uniqueResourceOrLiteralTypeClassesMap, - List tags) { + final Map, String> uniqueResourceOrLiteralTypeClassesMap, List tags) { uniqueResourceOrLiteralTypeClassesMap.values().stream().filter(StringUtils::isNotBlank) .forEach(tags::add); } diff --git a/metis-enrichment/metis-enrichment-client/src/main/java/eu/europeana/enrichment/utils/RdfProxyUtils.java b/metis-enrichment/metis-enrichment-client/src/main/java/eu/europeana/enrichment/utils/RdfProxyUtils.java index 1887ef473..6aa67a83f 100644 --- a/metis-enrichment/metis-enrichment-client/src/main/java/eu/europeana/enrichment/utils/RdfProxyUtils.java +++ b/metis-enrichment/metis-enrichment-client/src/main/java/eu/europeana/enrichment/utils/RdfProxyUtils.java @@ -6,6 +6,7 @@ import eu.europeana.metis.schema.jibx.ProxyType; import eu.europeana.metis.schema.jibx.RDF; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.EnumMap; import java.util.List; @@ -14,7 +15,6 @@ import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; -import java.util.stream.Stream; import org.apache.commons.lang3.StringUtils; /** @@ -50,8 +50,8 @@ public static void appendLinkToEuropeanaProxy(RDF rdf, String link, } private static Map> getAllProxyLinksPerType(RDF rdf) { - final List allChoices = Optional.ofNullable(rdf.getProxyList()) - .map(List::stream).orElseGet(Stream::empty).filter(Objects::nonNull) + final List allChoices = Optional.ofNullable(rdf.getProxyList()).stream() + .flatMap(Collection::stream).filter(Objects::nonNull) .map(ProxyType::getChoiceList).filter(Objects::nonNull).flatMap(List::stream) .filter(Objects::nonNull).collect(Collectors.toList()); final Map> result = new EnumMap<>(FieldType.class); @@ -66,29 +66,32 @@ private static Map> getAllProxyLinksPerType(RDF rdf) { } /** - * Retrieve the Provider proxy from the proxy list in the {@link RDF} + * Retrieve all Provider proxy from the proxy list in the {@link RDF} * * @param rdf the rdf used to search for the proxy - * @return the Provider proxy + * @return the Provider proxy list. Could be empty, but is not null. */ - public static ProxyType getProviderProxy(RDF rdf) { - for (ProxyType proxyType : rdf.getProxyList()) { - if (proxyType.getEuropeanaProxy() == null - || !proxyType.getEuropeanaProxy().isEuropeanaProxy()) { - return proxyType; - } - } - throw new IllegalArgumentException("Could not find provider proxy."); + public static List getProviderProxies(RDF rdf) { + return Optional.ofNullable(rdf.getProxyList()).stream().flatMap(Collection::stream) + .filter(Objects::nonNull).filter(proxy -> !isEuropeanaProxy(proxy)) + .collect(Collectors.toList()); } - private static ProxyType getEuropeanaProxy(RDF rdf) { - for (ProxyType proxyType : rdf.getProxyList()) { - if (proxyType.getEuropeanaProxy() != null - && proxyType.getEuropeanaProxy().isEuropeanaProxy()) { - return proxyType; - } - } - throw new IllegalArgumentException("Could not find Europeana proxy."); + private static boolean isEuropeanaProxy(ProxyType proxy) { + return proxy.getEuropeanaProxy() != null && proxy.getEuropeanaProxy().isEuropeanaProxy(); + } + + /** + * Retrieve the Europeana proxy from the proxy list in the {@link RDF} + * + * @param rdf the rdf used to search for the proxy + * @return the Europeana proxy. Is not null. + * @throws IllegalArgumentException in case the RDF does not have a Europeana proxy. + */ + public static ProxyType getEuropeanaProxy(RDF rdf) { + return Optional.ofNullable(rdf.getProxyList()).stream().flatMap(Collection::stream) + .filter(Objects::nonNull).filter(RdfProxyUtils::isEuropeanaProxy).findAny() + .orElseThrow(() -> new IllegalArgumentException("Could not find Europeana proxy.")); } private static void replaceProxy(RDF rdf, ProxyType europeanaProxy) { diff --git a/metis-enrichment/metis-enrichment-client/src/test/java/eu/europeana/enrichment/utils/EnrichmentUtilsTest.java b/metis-enrichment/metis-enrichment-client/src/test/java/eu/europeana/enrichment/utils/EnrichmentUtilsTest.java new file mode 100644 index 000000000..1d2b8e1fe --- /dev/null +++ b/metis-enrichment/metis-enrichment-client/src/test/java/eu/europeana/enrichment/utils/EnrichmentUtilsTest.java @@ -0,0 +1,222 @@ +package eu.europeana.enrichment.utils; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.spy; + +import eu.europeana.metis.schema.jibx.Aggregation; +import eu.europeana.metis.schema.jibx.Created; +import eu.europeana.metis.schema.jibx.Date; +import eu.europeana.metis.schema.jibx.EuropeanaAggregationType; +import eu.europeana.metis.schema.jibx.EuropeanaProxy; +import eu.europeana.metis.schema.jibx.EuropeanaType.Choice; +import eu.europeana.metis.schema.jibx.Issued; +import eu.europeana.metis.schema.jibx.ProxyType; +import eu.europeana.metis.schema.jibx.RDF; +import eu.europeana.metis.schema.jibx.Temporal; +import eu.europeana.metis.schema.jibx.Title; +import eu.europeana.metis.schema.jibx._Object; +import java.util.ArrayList; +import java.util.Collections; + +import java.util.List; +import java.util.stream.Collectors; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +public class EnrichmentUtilsTest { + + private final static RDF TEST_RDF = spy(new RDF()); + private final static ProxyType PROXY_EUROPEANA = new ProxyType(); + private final static ProxyType PROXY_PROVIDER = new ProxyType(); + private final static EuropeanaAggregationType EUROPEANA_AGGREGATION_TYPE = new EuropeanaAggregationType(); + + @BeforeEach + void setUp() { + + EuropeanaProxy europeanaProxy = new EuropeanaProxy(); + EuropeanaProxy providerProxy = new EuropeanaProxy(); + europeanaProxy.setEuropeanaProxy(true); + providerProxy.setEuropeanaProxy(false); + + PROXY_EUROPEANA.setAbout("/proxy/europeana/260/_kmo_av_sid_45006"); + PROXY_EUROPEANA.setEuropeanaProxy(europeanaProxy); + + PROXY_PROVIDER.setAbout("/proxy/provider/260/_kmo_av_sid_45006"); + PROXY_PROVIDER.setEuropeanaProxy(providerProxy); + + EUROPEANA_AGGREGATION_TYPE.setAbout("/aggregation/europeana/260/_kmo_av_sid_45006"); + + TEST_RDF.setEuropeanaAggregationList(Collections.singletonList(EUROPEANA_AGGREGATION_TYPE)); + + doReturn(Collections.singletonList(new Aggregation())).when(TEST_RDF).getAggregationList(); + } + + @Test + void testSetAdditionalDataAllYearFieldValues(){ + + Choice dateChoice = new Choice(); + Date date = new Date(); + Choice temporalChoice = new Choice(); + Temporal temporal = new Temporal(); + Choice createdChoice = new Choice(); + Created created = new Created(); + Choice issuedChoice = new Choice(); + Issued issued = new Issued(); + + date.setString("1990"); + dateChoice.setDate(date); + + temporal.setString("1991"); + temporalChoice.setTemporal(temporal); + + created.setString("1992"); + createdChoice.setCreated(created); + + issued.setString("1993"); + issuedChoice.setIssued(issued); + + ArrayList choices = new ArrayList<>(); + choices.add(dateChoice); + choices.add(temporalChoice); + choices.add(createdChoice); + choices.add(issuedChoice); + + PROXY_PROVIDER.setChoiceList(choices); + + ArrayList proxyList = new ArrayList<>(); + proxyList.add(PROXY_EUROPEANA); + proxyList.add(PROXY_PROVIDER); + + TEST_RDF.setProxyList(proxyList); + + EnrichmentUtils.setAdditionalData(TEST_RDF); + + ProxyType proxyResult = TEST_RDF.getProxyList().stream() + .filter(x -> x.getEuropeanaProxy().isEuropeanaProxy()) + .collect(Collectors.toList()).get(0); + + assertEquals(4, proxyResult.getYearList().size()); + assertEquals("1990", proxyResult.getYearList().get(0).getString()); + assertEquals("1991", proxyResult.getYearList().get(1).getString()); + assertEquals("1992", proxyResult.getYearList().get(2).getString()); + assertEquals("1993", proxyResult.getYearList().get(3).getString()); + + } + + @Test + void testSetAdditionalDataYearFieldWithoutDuplicates(){ + Choice dateChoice = new Choice(); + Date date = new Date(); + Choice temporalChoice = new Choice(); + Temporal temporal = new Temporal(); + + date.setString("1990"); + dateChoice.setDate(date); + + temporal.setString("1990"); + temporalChoice.setTemporal(temporal); + + ArrayList choices = new ArrayList<>(); + choices.add(dateChoice); + choices.add(temporalChoice); + + PROXY_PROVIDER.setChoiceList(choices); + + ArrayList proxyList = new ArrayList<>(); + proxyList.add(PROXY_EUROPEANA); + proxyList.add(PROXY_PROVIDER); + + TEST_RDF.setProxyList(proxyList); + + EnrichmentUtils.setAdditionalData(TEST_RDF); + + ProxyType proxyResult = TEST_RDF.getProxyList().stream() + .filter(x -> x.getEuropeanaProxy().isEuropeanaProxy()) + .collect(Collectors.toList()).get(0); + + assertEquals(1, proxyResult.getYearList().size()); + assertEquals("1990", proxyResult.getYearList().get(0).getString()); + + } + + @Test + void testSetAdditionalDataCompletenessNone(){ + + ArrayList proxyList = new ArrayList<>(); + proxyList.add(PROXY_EUROPEANA); + proxyList.add(PROXY_PROVIDER); + + TEST_RDF.setProxyList(proxyList); + + EnrichmentUtils.setAdditionalData(TEST_RDF); + + EuropeanaAggregationType aggregationTypeResult = TEST_RDF.getEuropeanaAggregationList().get(0); + + assertEquals("0", aggregationTypeResult.getCompleteness().getString()); + } + + @Test + void testSetAdditionalDataCompletenessMoreThanZero() { + Aggregation aggregation = spy(new Aggregation()); + + Choice dateChoice = new Choice(); + Date date = new Date(); + Choice titleChoice = new Choice(); + Title title = new Title(); + + date.setString("1990"); + dateChoice.setDate(date); + title.setString( + "The Sudbury Neutrino Observatory: Observation of Flavor Change for Solar Neutrinos"); + titleChoice.setTitle(title); + + ArrayList choices = new ArrayList<>(); + choices.add(dateChoice); + choices.add(titleChoice); + + PROXY_PROVIDER.setChoiceList(choices); + + ArrayList proxyList = new ArrayList<>(); + proxyList.add(PROXY_EUROPEANA); + proxyList.add(PROXY_PROVIDER); + + TEST_RDF.setProxyList(proxyList); + + _Object object = new _Object(); + object.setResource("/260/_kmo_av_sid_45006"); + doReturn(object).when(aggregation).getObject(); + doReturn(Collections.singletonList(aggregation)).when(TEST_RDF).getAggregationList(); + + EnrichmentUtils.setAdditionalData(TEST_RDF); + + EuropeanaAggregationType aggregationTypeResult = TEST_RDF.getEuropeanaAggregationList().get(0); + + assertTrue(Integer.parseInt(aggregationTypeResult.getCompleteness().getString()) > 0); + } + + @Test + void testSetAdditionalDataEmptyProxies() { + RDF newRdf = new RDF(); + RDF toCompare = new RDF(); + ProxyType emptyEuropeanaProxy = new ProxyType(); + emptyEuropeanaProxy.setEuropeanaProxy(new EuropeanaProxy()); + emptyEuropeanaProxy.getEuropeanaProxy().setEuropeanaProxy(true); + ProxyType emptyProviderProxy = new ProxyType(); + EuropeanaAggregationType emptyAggregation = new EuropeanaAggregationType(); + + List proxyList = List.of(emptyEuropeanaProxy, emptyProviderProxy); + + newRdf.setEuropeanaAggregationList(Collections.singletonList(emptyAggregation)); + newRdf.setProxyList(proxyList); + toCompare.setEuropeanaAggregationList(Collections.singletonList(emptyAggregation)); + toCompare.setProxyList(proxyList); + + EnrichmentUtils.setAdditionalData(newRdf); + + assertArrayEquals(toCompare.getProxyList().toArray(), newRdf.getProxyList().toArray()); + assertEquals(toCompare.getEuropeanaAggregationList().get(0), newRdf.getEuropeanaAggregationList().get(0)); + } +} diff --git a/metis-enrichment/metis-enrichment-client/src/test/java/eu/europeana/enrichment/utils/ItemExtractorUtilsTest.java b/metis-enrichment/metis-enrichment-client/src/test/java/eu/europeana/enrichment/utils/ItemExtractorUtilsTest.java new file mode 100644 index 000000000..2b5477623 --- /dev/null +++ b/metis-enrichment/metis-enrichment-client/src/test/java/eu/europeana/enrichment/utils/ItemExtractorUtilsTest.java @@ -0,0 +1,488 @@ +package eu.europeana.enrichment.utils; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import eu.europeana.enrichment.api.external.model.Agent; +import eu.europeana.enrichment.api.external.model.EnrichmentBase; +import eu.europeana.enrichment.api.external.model.Label; +import eu.europeana.enrichment.api.external.model.LabelResource; +import eu.europeana.enrichment.api.external.model.Part; +import eu.europeana.enrichment.api.external.model.Resource; +import eu.europeana.metis.schema.jibx.AboutType; +import eu.europeana.metis.schema.jibx.AltLabel; +import eu.europeana.metis.schema.jibx.Concept.Choice; +import eu.europeana.metis.schema.jibx.LiteralType; +import eu.europeana.metis.schema.jibx.LiteralType.Lang; +import eu.europeana.metis.schema.jibx.ResourceOrLiteralType; +import eu.europeana.metis.schema.jibx.ResourceType; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; +import org.junit.jupiter.api.Test; +import org.junit.platform.commons.util.StringUtils; + +public class ItemExtractorUtilsTest { + + + @Test + void testSetAbout(){ + + EnrichmentBase enrichmentBase = new Agent(); + AboutType aboutType = new AboutType(); + + enrichmentBase.setAbout("About"); + + ItemExtractorUtils.setAbout(enrichmentBase, aboutType); + + assertEquals("About", aboutType.getAbout()); + + } + + @Test + void testExtractItems(){ + List inputList = new ArrayList<>(); + Integer integer1 = 1; + Integer integer2 = 2; + Integer integer3 = 3; + inputList.add(integer1); + inputList.add(integer2); + inputList.add(integer3); + + List output = ItemExtractorUtils.extractItems(inputList, Double::valueOf); + + for(Integer elem: inputList){ + assertTrue(output.contains(Double.valueOf(elem))); + } + + } + + @Test + void testExtractItemsReturnEmpty(){ + List output = ItemExtractorUtils.extractItems(null, String::valueOf); + assertTrue(output.isEmpty()); + + } + + @Test + void testExtractLabels(){ + + List - - org.mongodb - mongodb-driver-core - ${version.mongodb.driver.core} - compile - com.fasterxml.jackson.module jackson-module-jaxb-annotations @@ -66,11 +60,6 @@ swagger-annotations ${version.swagger.annotations} - - redis.clients - jedis - ${version.jedis} - diff --git a/metis-enrichment/metis-enrichment-common/src/main/java/eu/europeana/enrichment/api/external/ReferenceValue.java b/metis-enrichment/metis-enrichment-common/src/main/java/eu/europeana/enrichment/api/external/ReferenceValue.java index e4244d655..f9df8f083 100644 --- a/metis-enrichment/metis-enrichment-common/src/main/java/eu/europeana/enrichment/api/external/ReferenceValue.java +++ b/metis-enrichment/metis-enrichment-common/src/main/java/eu/europeana/enrichment/api/external/ReferenceValue.java @@ -6,6 +6,7 @@ import java.util.Collections; import java.util.HashSet; import java.util.List; +import java.util.Objects; import java.util.Set; import javax.xml.bind.Unmarshaller; import javax.xml.bind.annotation.XmlAccessType; @@ -84,4 +85,22 @@ public void afterUnmarshal(Unmarshaller unmarshaller, Object parent) { //Remove duplicates from list after unmarshal entityTypes = new ArrayList<>(new HashSet<>(entityTypes)); } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + ReferenceValue that = (ReferenceValue) o; + return Objects.equals(reference, that.reference) && Objects + .equals(entityTypes, that.entityTypes); + } + + @Override + public int hashCode() { + return Objects.hash(reference, entityTypes); + } } diff --git a/metis-enrichment/metis-enrichment-common/src/main/java/eu/europeana/enrichment/api/external/SearchValue.java b/metis-enrichment/metis-enrichment-common/src/main/java/eu/europeana/enrichment/api/external/SearchValue.java index 9c77d6600..541f17f11 100644 --- a/metis-enrichment/metis-enrichment-common/src/main/java/eu/europeana/enrichment/api/external/SearchValue.java +++ b/metis-enrichment/metis-enrichment-common/src/main/java/eu/europeana/enrichment/api/external/SearchValue.java @@ -6,6 +6,7 @@ import java.util.Collections; import java.util.HashSet; import java.util.List; +import java.util.Objects; import java.util.Set; import javax.xml.bind.Unmarshaller; import javax.xml.bind.annotation.XmlAccessType; @@ -35,9 +36,11 @@ public class SearchValue { /** * Internally duplicates are removed when value is set or unmarshalled. */ + //TODO: 26-02-2021 We initialized the List to fix the bug MET-3245. A error message would show + //TODO: when this field was missing. This is not the best fix and we should figure out another solution @XmlElement(name = "entityType") @JsonProperty("entityType") - private List entityTypes; + private List entityTypes = new ArrayList<>(); public SearchValue() { // Required for XML (un)marshalling. @@ -86,8 +89,8 @@ public void setEntityTypes(List entityTypes) { } /** - * This method is REQUIRED so that after unmarshalling the list contents of {@code entityTypes} - * are cleaned to remove any duplicates. + * This method is REQUIRED so that after unmarshalling the list contents of {@code + * entityTypes} are cleaned to remove any duplicates. * * @param unmarshaller the unmarshaller * @param parent the parent @@ -96,4 +99,22 @@ public void afterUnmarshal(Unmarshaller unmarshaller, Object parent) { //Remove duplicates from list after unmarshal entityTypes = new ArrayList<>(new HashSet<>(entityTypes)); } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + SearchValue that = (SearchValue) o; + return Objects.equals(value, that.value) && Objects.equals(language, that.language) && Objects + .equals(entityTypes, that.entityTypes); + } + + @Override + public int hashCode() { + return Objects.hash(value, language, entityTypes); + } } diff --git a/metis-enrichment/metis-enrichment-common/src/main/java/eu/europeana/enrichment/api/external/model/LabelInfo.java b/metis-enrichment/metis-enrichment-common/src/main/java/eu/europeana/enrichment/api/external/model/LabelInfo.java index d0f2ef598..e777a25bd 100644 --- a/metis-enrichment/metis-enrichment-common/src/main/java/eu/europeana/enrichment/api/external/model/LabelInfo.java +++ b/metis-enrichment/metis-enrichment-common/src/main/java/eu/europeana/enrichment/api/external/model/LabelInfo.java @@ -4,6 +4,7 @@ import eu.europeana.enrichment.internal.model.AbstractEnrichmentEntity; import java.util.ArrayList; import java.util.List; +import java.util.Objects; /** * Contains the labels that come from the preLabel fields of a contextual class. @@ -26,6 +27,7 @@ public LabelInfo() { /** * Constructor with all parameters + * * @param lowerCaseLabel the lower case label list * @param lang the language of the labels */ @@ -51,4 +53,21 @@ public void setLowerCaseLabel(List lowerCaseLabel) { this.lowerCaseLabel = lowerCaseLabel == null ? new ArrayList<>() : new ArrayList<>(lowerCaseLabel); } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + LabelInfo labelInfo = (LabelInfo) o; + return Objects.equals(lang, labelInfo.lang) && lowerCaseLabel.equals(labelInfo.lowerCaseLabel); + } + + @Override + public int hashCode() { + return Objects.hash(lang, lowerCaseLabel); + } } diff --git a/metis-enrichment/metis-enrichment-common/src/main/java/eu/europeana/enrichment/internal/model/Address.java b/metis-enrichment/metis-enrichment-common/src/main/java/eu/europeana/enrichment/internal/model/Address.java index 3d7377ccd..8ed374ef5 100644 --- a/metis-enrichment/metis-enrichment-common/src/main/java/eu/europeana/enrichment/internal/model/Address.java +++ b/metis-enrichment/metis-enrichment-common/src/main/java/eu/europeana/enrichment/internal/model/Address.java @@ -1,6 +1,7 @@ package eu.europeana.enrichment.internal.model; import dev.morphia.annotations.Embedded; +import java.util.Objects; @Embedded public class Address { @@ -68,4 +69,28 @@ public String getVcardHasGeo() { public void setVcardHasGeo(String vcardHasGeo) { this.vcardHasGeo = vcardHasGeo; } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + Address address = (Address) o; + return Objects.equals(about, address.about) && Objects + .equals(vcardStreetAddress, address.vcardStreetAddress) && Objects + .equals(vcardLocality, address.vcardLocality) && Objects + .equals(vcardPostalCode, address.vcardPostalCode) && Objects + .equals(vcardCountryName, address.vcardCountryName) && Objects + .equals(vcardPostOfficeBox, address.vcardPostOfficeBox) && Objects + .equals(vcardHasGeo, address.vcardHasGeo); + } + + @Override + public int hashCode() { + return Objects.hash(about, vcardStreetAddress, vcardLocality, vcardPostalCode, vcardCountryName, + vcardPostOfficeBox, vcardHasGeo); + } } diff --git a/metis-enrichment/metis-enrichment-rest/pom.xml b/metis-enrichment/metis-enrichment-rest/pom.xml index 47e11917c..dbfdb4c10 100644 --- a/metis-enrichment/metis-enrichment-rest/pom.xml +++ b/metis-enrichment/metis-enrichment-rest/pom.xml @@ -4,7 +4,7 @@ metis-enrichment eu.europeana.metis - 3 + 4 metis-enrichment-rest war @@ -42,11 +42,6 @@ - - org.mongodb - mongodb-driver-core - ${version.mongodb.driver.core} - eu.europeana.corelib corelib-web @@ -87,7 +82,6 @@ com.jayway.jsonpath json-path-assert - ${version.json.path} test @@ -98,10 +92,7 @@ org.mockito mockito-core - ${version.mockito.core} - test - javax.servlet javax.servlet-api diff --git a/metis-enrichment/metis-enrichment-rest/src/main/java/eu/europeana/enrichment/rest/EnrichmentController.java b/metis-enrichment/metis-enrichment-rest/src/main/java/eu/europeana/enrichment/rest/EnrichmentController.java index 86c8ae8c4..e9490095f 100644 --- a/metis-enrichment/metis-enrichment-rest/src/main/java/eu/europeana/enrichment/rest/EnrichmentController.java +++ b/metis-enrichment/metis-enrichment-rest/src/main/java/eu/europeana/enrichment/rest/EnrichmentController.java @@ -60,7 +60,7 @@ public EnrichmentController(EnrichmentService enrichmentService) { public EnrichmentResultList search( @ApiParam("SearchTerms") @RequestBody EnrichmentSearch enrichmentSearch) { return new EnrichmentResultList(enrichmentService - .enrichByEnrichmentSearchValues(enrichmentSearch.getSearchValues())); + .enrichByEnrichmentSearchValues(enrichmentSearch.getSearchValues())); } /** @@ -76,8 +76,8 @@ public EnrichmentResultList search( @ResponseBody @ApiResponses(value = {@ApiResponse(code = 400, message = "Error processing the result")}) public EnrichmentBase equivalence(@ApiParam("uri") @RequestParam("uri") String uri) { - List result = enrichmentService - .enrichByEquivalenceValues(new ReferenceValue(uri, Collections.emptySet())); + List result = enrichmentService + .enrichByEquivalenceValues(new ReferenceValue(uri, Collections.emptySet())); return result.stream().findFirst().orElse(null); //TODO 07-12-2020: For this case it is expected only one as a result, but we should handle this better in the future } @@ -118,7 +118,8 @@ public EnrichmentResultList equivalence(@RequestBody EnrichmentReference enrichm @ApiResponses(value = {@ApiResponse(code = 400, message = "Error processing the result")}) public EnrichmentResultList entityId(@RequestBody List uris) { // TODO: 17/11/2020 Support for xml input. Requires a wrapper class with a field of the list of uris - final List enrichmentBaseWrappers = uris.stream() + final List enrichmentBaseWrappers = + uris.stream() .map(enrichmentService::enrichById) .map(result -> Optional.ofNullable(result).map(List::of).orElseGet(Collections::emptyList)) .map(EnrichmentResultBaseWrapper::new) diff --git a/metis-enrichment/metis-enrichment-rest/src/main/java/eu/europeana/enrichment/rest/config/Application.java b/metis-enrichment/metis-enrichment-rest/src/main/java/eu/europeana/enrichment/rest/config/Application.java index 634db3f46..454487a6f 100644 --- a/metis-enrichment/metis-enrichment-rest/src/main/java/eu/europeana/enrichment/rest/config/Application.java +++ b/metis-enrichment/metis-enrichment-rest/src/main/java/eu/europeana/enrichment/rest/config/Application.java @@ -55,6 +55,8 @@ public class Application implements WebMvcConfigurer, InitializingBean { private int enrichmentMongoPort; @Value("${enrichment.mongo.database}") private String enrichmentMongoDatabase; + @Value("${enrichment.mongo.application.name}") + private String enrichmentMongoApplicationName; private MongoClient mongoClient; @@ -92,6 +94,7 @@ MongoClient getMongoClient() { IllegalArgumentException::new); mongoProperties .setMongoHosts(new String[]{enrichmentMongoHost}, new int[]{enrichmentMongoPort}); + mongoProperties.setApplicationName(enrichmentMongoApplicationName); mongoClient = new MongoClientProvider<>(mongoProperties).createMongoClient(); return mongoClient; } diff --git a/metis-enrichment/metis-enrichment-rest/src/main/java/eu/europeana/enrichment/rest/exception/EnrichmentException.java b/metis-enrichment/metis-enrichment-rest/src/main/java/eu/europeana/enrichment/rest/exception/EnrichmentException.java deleted file mode 100644 index 643ab8e68..000000000 --- a/metis-enrichment/metis-enrichment-rest/src/main/java/eu/europeana/enrichment/rest/exception/EnrichmentException.java +++ /dev/null @@ -1,30 +0,0 @@ -package eu.europeana.enrichment.rest.exception; - -/** - * Exception used during enrichment functionality. - */ -public class EnrichmentException extends Exception { - - /** - * Constructs a new exception with the specified detail message. - * - * @param message the detail message. The detail message is saved for later retrieval by the {@link #getMessage()} method. - */ - public EnrichmentException(String message) { - super(message); - } - - /** - * Constructs a new exception with the specified detail message and - * cause. - * - * @param message the detail message (which is saved for later retrieval by the {@link #getMessage()} method). - * @param cause the cause (which is saved for later retrieval by the - * {@link #getCause()} method). (A null value is - * permitted, and indicates that the cause is nonexistent or - * unknown.) - */ - public EnrichmentException(String message, Throwable cause) { - super(message, cause); - } -} diff --git a/metis-enrichment/metis-enrichment-rest/src/main/java/eu/europeana/enrichment/rest/exception/RestResponseExceptionHandler.java b/metis-enrichment/metis-enrichment-rest/src/main/java/eu/europeana/enrichment/rest/exception/RestResponseExceptionHandler.java index c6816b5f1..f26a97c91 100644 --- a/metis-enrichment/metis-enrichment-rest/src/main/java/eu/europeana/enrichment/rest/exception/RestResponseExceptionHandler.java +++ b/metis-enrichment/metis-enrichment-rest/src/main/java/eu/europeana/enrichment/rest/exception/RestResponseExceptionHandler.java @@ -14,16 +14,17 @@ public class RestResponseExceptionHandler { /** - * Handle {@link EnrichmentException} + * Handle {@link Exception} * * @param response the response that should be updated * @param exception the exception thrown - * @return {@link StructuredExceptionWrapper} a json friendly class that contains the error message for the client + * @return {@link StructuredExceptionWrapper} a json friendly class that contains the error + * message for the client */ @ResponseBody - @ExceptionHandler(EnrichmentException.class) + @ExceptionHandler(Exception.class) public StructuredExceptionWrapper handleResponse(HttpServletResponse response, - EnrichmentException exception) { + Exception exception) { response.setStatus(HttpStatus.BAD_REQUEST.value()); return new StructuredExceptionWrapper(exception.getMessage()); } diff --git a/metis-enrichment/metis-enrichment-rest/src/main/resources/enrichment.properties.example b/metis-enrichment/metis-enrichment-rest/src/main/resources/enrichment.properties.example index 07fa7a2b1..68ad705a3 100644 --- a/metis-enrichment/metis-enrichment-rest/src/main/resources/enrichment.properties.example +++ b/metis-enrichment/metis-enrichment-rest/src/main/resources/enrichment.properties.example @@ -8,4 +8,5 @@ socks.proxy.password= #Mongo enrichment.mongo.host= enrichment.mongo.database= -enrichment.mongo.port= \ No newline at end of file +enrichment.mongo.port= +enrichment.mongo.application.name= \ No newline at end of file diff --git a/metis-enrichment/metis-enrichment-rest/src/test/java/eu/europeana/enrichment/rest/EnrichmentControllerTest.java b/metis-enrichment/metis-enrichment-rest/src/test/java/eu/europeana/enrichment/rest/EnrichmentControllerTest.java index 194761680..b0ee68347 100644 --- a/metis-enrichment/metis-enrichment-rest/src/test/java/eu/europeana/enrichment/rest/EnrichmentControllerTest.java +++ b/metis-enrichment/metis-enrichment-rest/src/test/java/eu/europeana/enrichment/rest/EnrichmentControllerTest.java @@ -2,8 +2,6 @@ import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.core.Is.is; -import static org.mockito.Matchers.anyListOf; -import static org.mockito.Matchers.anySet; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; @@ -12,21 +10,36 @@ import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.xpath; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.ObjectWriter; +import com.fasterxml.jackson.databind.SerializationFeature; +import eu.europeana.enrichment.api.external.EnrichmentReference; +import eu.europeana.enrichment.api.external.EnrichmentSearch; import eu.europeana.enrichment.api.external.ReferenceValue; import eu.europeana.enrichment.api.external.SearchValue; import eu.europeana.enrichment.api.external.model.Agent; +import eu.europeana.enrichment.api.external.model.Concept; import eu.europeana.enrichment.api.external.model.EnrichmentResultBaseWrapper; import eu.europeana.enrichment.api.external.model.Label; +import eu.europeana.enrichment.api.external.model.Part; +import eu.europeana.enrichment.api.external.model.Place; +import eu.europeana.enrichment.api.external.model.Resource; +import eu.europeana.enrichment.api.external.model.Timespan; import eu.europeana.enrichment.rest.exception.RestResponseExceptionHandler; import eu.europeana.enrichment.service.EnrichmentService; +import eu.europeana.enrichment.utils.EntityType; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Set; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import org.mockito.Mockito; +import org.springframework.http.HttpStatus; import org.springframework.http.MediaType; import org.springframework.test.web.servlet.MockMvc; import org.springframework.test.web.servlet.setup.MockMvcBuilders; @@ -35,15 +48,16 @@ public class EnrichmentControllerTest { private static MockMvc enrichmentControllerMock; private static EnrichmentService enrichmentServiceMock; + private static Map namespaceMap; @BeforeAll public static void setUp() { + namespaceMap = getNamespaceMap(); enrichmentServiceMock = mock(EnrichmentService.class); - EnrichmentController oldEnrichmentController = new EnrichmentController(enrichmentServiceMock); - enrichmentControllerMock = MockMvcBuilders.standaloneSetup(oldEnrichmentController) - .setControllerAdvice(new RestResponseExceptionHandler()) - .build(); + EnrichmentController enrichmentController = new EnrichmentController(enrichmentServiceMock); + enrichmentControllerMock = MockMvcBuilders.standaloneSetup(enrichmentController) + .setControllerAdvice(new RestResponseExceptionHandler()).build(); } @AfterEach @@ -52,91 +66,255 @@ public void tearDown() { } @Test - public void getByUri_JSON() throws Exception { - String uri = "http://www.example.com"; - Agent agent = getAgent(uri); - ReferenceValue reference = new ReferenceValue(uri, anySet()); - when(enrichmentServiceMock.enrichByEquivalenceValues(reference)).thenReturn(List.of(agent)); - enrichmentControllerMock.perform(get("/enrich/entity/equivalence") - .param("uri", "http://www.example.com") - .accept(MediaType.APPLICATION_JSON)) - .andExpect(status().is(200)) - .andExpect(jsonPath("$.about", is("http://www.example.com"))) + public void testEquivalenceInputURIInputJSON() throws Exception { + final Agent agent = getAgent(); + final EnrichmentReference enrichmentReference = new EnrichmentReference(); + final ReferenceValue referenceValue = new ReferenceValue(agent.getAbout(), + Collections.emptySet()); + enrichmentReference.setReferenceValues(List.of(referenceValue)); + when(enrichmentServiceMock.enrichByEquivalenceValues(referenceValue)) + .thenReturn(List.of(agent)); + enrichmentControllerMock.perform( + get("/enrich/entity/equivalence").param("uri", agent.getAbout()) + .accept(MediaType.APPLICATION_JSON)).andExpect(status().is(200)) + .andExpect(jsonPath("$.about", is(agent.getAbout()))) .andExpect(jsonPath("$.altLabelList[?(@.lang=='en')].value", containsInAnyOrder("labelEn"))) .andExpect( jsonPath("$.altLabelList[?(@.lang=='nl')].value", containsInAnyOrder("labelNl"))); } @Test - public void getByUri_XML() throws Exception { - - String uri = "http://www.example.com"; - Agent agent = getAgent(uri); - ReferenceValue reference = new ReferenceValue(uri, anySet()); - when(enrichmentServiceMock.enrichByEquivalenceValues(reference)).thenReturn(List.of(agent)); - Map namespaceMap = getNamespaceMap(); - enrichmentControllerMock.perform(get("/enrich/entity/equivalence") - .param("uri", "http://www.example.com") - .accept(MediaType.APPLICATION_XML_VALUE)) - .andExpect(status().is(200)) - .andExpect(xpath("edm:Agent/@rdf:about", namespaceMap).string("http://www.example.com")) + public void testEquivalenceInputURIInputXML() throws Exception { + final Agent agent = getAgent(); + final EnrichmentReference enrichmentReference = new EnrichmentReference(); + final ReferenceValue referenceValue = new ReferenceValue(agent.getAbout(), + Collections.emptySet()); + enrichmentReference.setReferenceValues(List.of(referenceValue)); + + when(enrichmentServiceMock.enrichByEquivalenceValues(referenceValue)) + .thenReturn(List.of(agent)); + enrichmentControllerMock.perform( + get("/enrich/entity/equivalence").param("uri", agent.getAbout()) + .accept(MediaType.APPLICATION_XML_VALUE)).andExpect(status().is(200)) + .andExpect(xpath("edm:Agent/@rdf:about", namespaceMap).string(agent.getAbout())) .andExpect(xpath("edm:Agent/skos:altLabel[@xml:lang='en']", namespaceMap).string("labelEn")) .andExpect( xpath("edm:Agent/skos:altLabel[@xml:lang='nl']", namespaceMap).string("labelNl")); } @Test - public void enrich_XML() throws Exception { - String uri = "http://www.example.com"; - String body = - "{\n" - + " \"inputValueList\": [\n" - + " {\n" - + " \"value\": \"AgentName\",\n" - + " \"vocabularies\": [\n" - + " \"AGENT\"\n" - + " ]\n" - + " }\n" - + " ]\n" - + "}"; - - Map namespaceMap = getNamespaceMap(); - Agent agent = getAgent(uri); - - when(enrichmentServiceMock.enrichByEnrichmentSearchValues(anyListOf(SearchValue.class))) + public void testEquivalenceWithEnrichmentReference() throws Exception { + final Agent agent = getAgent(); + final EnrichmentReference enrichmentReference = new EnrichmentReference(); + final ReferenceValue referenceValue = new ReferenceValue(agent.getAbout(), + Set.of(EntityType.AGENT)); + enrichmentReference.setReferenceValues(List.of(referenceValue)); + + String requestJson = convertToJson(enrichmentReference); + + when(enrichmentServiceMock.enrichByEquivalenceValues(referenceValue)) + .thenReturn(List.of(agent)); + enrichmentControllerMock.perform(post("/enrich/entity/equivalence").content(requestJson) + .accept(MediaType.APPLICATION_XML_VALUE).contentType(MediaType.APPLICATION_JSON)) + .andExpect(status().is(200)).andExpect( + xpath("metis:results/metis:result/edm:Agent/@rdf:about", namespaceMap) + .string(agent.getAbout())).andExpect( + xpath("metis:results/metis:result/edm:Agent/skos:altLabel[@xml:lang='en']", namespaceMap) + .string("labelEn")).andExpect( + xpath("metis:results/metis:result/edm:Agent/skos:altLabel[@xml:lang='nl']", namespaceMap) + .string("labelNl")).andExpect( + xpath("metis:results/metis:result/edm:Agent/rdaGr2:dateOfBirth[@xml:lang='en']", + namespaceMap).string("10-10-10")); + } + + @Test + public void testEntityId() throws Exception { + final Agent agent = getAgent(); + String body = "[\"" + agent.getAbout() + "\"]"; + + when(enrichmentServiceMock.enrichById(agent.getAbout())).thenReturn(agent); + enrichmentControllerMock.perform( + post("/enrich/entity/id").content(body).accept(MediaType.APPLICATION_XML_VALUE) + .contentType(MediaType.APPLICATION_JSON)).andExpect(status().is(200)).andExpect( + xpath("metis:results/metis:result/edm:Agent/@rdf:about", namespaceMap) + .string(agent.getAbout())).andExpect( + xpath("metis:results/metis:result/edm:Agent/skos:altLabel[@xml:lang='en']", namespaceMap) + .string("labelEn")).andExpect( + xpath("metis:results/metis:result/edm:Agent/skos:altLabel[@xml:lang='nl']", namespaceMap) + .string("labelNl")).andExpect( + xpath("metis:results/metis:result/edm:Agent/rdaGr2:dateOfBirth[@xml:lang='en']", + namespaceMap).string("10-10-10")); + } + + @Test + public void testSearchInputXML_Agent() throws Exception { + final Agent agent = getAgent(); + final EnrichmentSearch enrichmentSearch = new EnrichmentSearch(); + final SearchValue searchValue = new SearchValue("value", null, EntityType.AGENT); + enrichmentSearch.setSearchValues(List.of(searchValue)); + String requestJson = convertToJson(enrichmentSearch); + + when(enrichmentServiceMock.enrichByEnrichmentSearchValues(List.of(searchValue))) .thenReturn(List.of(new EnrichmentResultBaseWrapper(List.of(agent)))); - enrichmentControllerMock.perform(post("/enrich/entity/search") - .content(body) - .accept(MediaType.APPLICATION_XML_VALUE) - .contentType(MediaType.APPLICATION_JSON)) - .andExpect(status().is(200)) - .andExpect(xpath("metis:results/metis:result/edm:Agent/@rdf:about", namespaceMap) - .string("http://www.example.com")) - .andExpect(xpath("metis:results/metis:result/edm:Agent/skos:altLabel[@xml:lang='en']", namespaceMap) - .string("labelEn")) - .andExpect(xpath("metis:results/metis:result/edm:Agent/skos:altLabel[@xml:lang='nl']", namespaceMap) - .string("labelNl")) - .andExpect(xpath("metis:results/metis:result/edm:Agent/rdaGr2:dateOfBirth[@xml:lang='en']", namespaceMap) - .string("10-10-10")); - } - - private Agent getAgent(String uri) { + enrichmentControllerMock.perform( + post("/enrich/entity/search").content(requestJson).accept(MediaType.APPLICATION_XML_VALUE) + .contentType(MediaType.APPLICATION_JSON)).andExpect(status().is(200)).andExpect( + xpath("metis:results/metis:result/edm:Agent/@rdf:about", namespaceMap) + .string(agent.getAbout())).andExpect( + xpath("metis:results/metis:result/edm:Agent/skos:altLabel[@xml:lang='en']", namespaceMap) + .string("labelEn")).andExpect( + xpath("metis:results/metis:result/edm:Agent/skos:altLabel[@xml:lang='nl']", namespaceMap) + .string("labelNl")).andExpect( + xpath("metis:results/metis:result/edm:Agent/rdaGr2:dateOfBirth[@xml:lang='en']", + namespaceMap).string("10-10-10")); + } + + @Test + public void testSearchInputXML_Concept() throws Exception { + final Concept concept = getConcept(); + final EnrichmentSearch enrichmentSearch = new EnrichmentSearch(); + final SearchValue searchValue = new SearchValue("value", null, EntityType.CONCEPT); + enrichmentSearch.setSearchValues(List.of(searchValue)); + String requestJson = convertToJson(enrichmentSearch); + + when(enrichmentServiceMock.enrichByEnrichmentSearchValues(List.of(searchValue))) + .thenReturn(List.of(new EnrichmentResultBaseWrapper(List.of(concept)))); + + enrichmentControllerMock.perform( + post("/enrich/entity/search").content(requestJson).accept(MediaType.APPLICATION_XML_VALUE) + .contentType(MediaType.APPLICATION_JSON)).andExpect(status().is(200)).andExpect( + xpath("metis:results/metis:result/skos:Concept/@rdf:about", namespaceMap) + .string(concept.getAbout())).andExpect( + xpath("metis:results/metis:result/skos:Concept/skos:altLabel[@xml:lang='en']", namespaceMap) + .string("labelEn")).andExpect( + xpath("metis:results/metis:result/skos:Concept/skos:altLabel[@xml:lang='nl']", namespaceMap) + .string("labelNl")).andExpect( + xpath("metis:results/metis:result/skos:Concept/skos:broader/@rdf:resource", namespaceMap) + .string(concept.getBroader().get(0).getResource())); + } + + @Test + public void testSearchInputXML_Timespan() throws Exception { + final Timespan timespan = getTimespan(); + final EnrichmentSearch enrichmentSearch = new EnrichmentSearch(); + final SearchValue searchValue = new SearchValue("value", null, EntityType.TIMESPAN); + enrichmentSearch.setSearchValues(List.of(searchValue)); + String requestJson = convertToJson(enrichmentSearch); + + when(enrichmentServiceMock.enrichByEnrichmentSearchValues(List.of(searchValue))) + .thenReturn(List.of(new EnrichmentResultBaseWrapper(List.of(timespan)))); + + enrichmentControllerMock.perform( + post("/enrich/entity/search").content(requestJson).accept(MediaType.APPLICATION_XML_VALUE) + .contentType(MediaType.APPLICATION_JSON)).andExpect(status().is(200)).andExpect( + xpath("metis:results/metis:result/edm:Timespan/@rdf:about", namespaceMap) + .string(timespan.getAbout())).andExpect( + xpath("metis:results/metis:result/edm:Timespan/skos:altLabel[@xml:lang='en']", namespaceMap) + .string("labelEn")).andExpect( + xpath("metis:results/metis:result/edm:Timespan/skos:altLabel[@xml:lang='nl']", namespaceMap) + .string("labelNl")).andExpect( + xpath("metis:results/metis:result/edm:Timespan/dcterms:isPartOf/@rdf:resource", + namespaceMap).string(timespan.getIsPartOf().getResource())); + } + + @Test + public void testSearchInputXML_Place() throws Exception { + final Place place = getPlace(); + final EnrichmentSearch enrichmentSearch = new EnrichmentSearch(); + final SearchValue searchValue = new SearchValue("value", null, EntityType.PLACE); + enrichmentSearch.setSearchValues(List.of(searchValue)); + String requestJson = convertToJson(enrichmentSearch); + + when(enrichmentServiceMock.enrichByEnrichmentSearchValues(List.of(searchValue))) + .thenReturn(List.of(new EnrichmentResultBaseWrapper(List.of(place)))); + + enrichmentControllerMock.perform( + post("/enrich/entity/search").content(requestJson).accept(MediaType.APPLICATION_XML_VALUE) + .contentType(MediaType.APPLICATION_JSON)).andExpect(status().is(200)).andExpect( + xpath("metis:results/metis:result/edm:Place/@rdf:about", namespaceMap) + .string(place.getAbout())).andExpect( + xpath("metis:results/metis:result/edm:Place/skos:altLabel[@xml:lang='en']", namespaceMap) + .string("labelEn")).andExpect( + xpath("metis:results/metis:result/edm:Place/skos:altLabel[@xml:lang='nl']", namespaceMap) + .string("labelNl")).andExpect( + xpath("metis:results/metis:result/edm:Place/dcterms:isPartOf/@rdf:resource", namespaceMap) + .string(place.getIsPartOf().getResource())); + } + + @Test + public void testSearchInputXML_Exception() throws Exception { + String invalidJson = "{{}"; + enrichmentControllerMock.perform( + post("/enrich/entity/search").content(invalidJson).accept(MediaType.APPLICATION_XML_VALUE) + .contentType(MediaType.APPLICATION_JSON)) + .andExpect(status().is(HttpStatus.BAD_REQUEST.value())) + .andExpect(xpath("error").exists()); + } + + private String convertToJson(Object object) throws JsonProcessingException { + ObjectMapper mapper = new ObjectMapper(); + mapper.configure(SerializationFeature.WRAP_ROOT_VALUE, false); + ObjectWriter ow = mapper.writer().withDefaultPrettyPrinter(); + return ow.writeValueAsString(object); + } + + private Agent getAgent() { Agent agent = new Agent(); - agent.setAbout(uri); + agent.setAbout("http://agent.org"); List diff --git a/metis-enrichment/metis-enrichment-service/src/main/java/eu/europeana/enrichment/service/Converter.java b/metis-enrichment/metis-enrichment-service/src/main/java/eu/europeana/enrichment/service/Converter.java index df418673a..0a739c1a7 100644 --- a/metis-enrichment/metis-enrichment-service/src/main/java/eu/europeana/enrichment/service/Converter.java +++ b/metis-enrichment/metis-enrichment-service/src/main/java/eu/europeana/enrichment/service/Converter.java @@ -12,13 +12,16 @@ import eu.europeana.enrichment.internal.model.AgentEnrichmentEntity; import eu.europeana.enrichment.internal.model.ConceptEnrichmentEntity; import eu.europeana.enrichment.internal.model.EnrichmentTerm; +import eu.europeana.enrichment.internal.model.OrganizationEnrichmentEntity; import eu.europeana.enrichment.internal.model.PlaceEnrichmentEntity; import eu.europeana.enrichment.internal.model.TimespanEnrichmentEntity; import eu.europeana.enrichment.utils.EntityType; import java.util.ArrayList; import java.util.Arrays; +import java.util.Date; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.stream.Collectors; import org.apache.commons.lang.StringUtils; @@ -172,6 +175,17 @@ private static Agent convertAgent(AgentEnrichmentEntity agentEntityEnrichment) { return output; } + static EnrichmentTerm organizationImplToEnrichmentTerm( + OrganizationEnrichmentEntity organizationEnrichmentEntity, Date created, Date updated) { + final EnrichmentTerm enrichmentTerm = new EnrichmentTerm(); + enrichmentTerm.setEnrichmentEntity(organizationEnrichmentEntity); + enrichmentTerm.setEntityType(EntityType.ORGANIZATION); + enrichmentTerm.setCreated(Objects.requireNonNullElseGet(created, Date::new)); + enrichmentTerm.setUpdated(updated); + + return enrichmentTerm; + } + private static List