Merge pull request #11 from rjd1/1.3.0

1.3.0

.fixtures.yml

@@ -5,7 +5,7 @@ fixtures:
   forge_modules:
     kafka_connect:
        repo: "rjd1/kafka_connect"
-       ref: "1.2.0"
+       ref: "1.3.0"
     stdlib: "puppetlabs/stdlib"
     apt: "puppetlabs/apt"
     java: "puppetlabs/java"

CHANGELOG.md

@@ -2,6 +2,15 @@
 
 Release notes for the rjd1-kafka_connect module.
 
+## Release 1.3.0
+
+2024-02-19 - Improved logging support
+
+ - Added support for time-based log rotation
+ - Added ability to enable sending logs to stdout/console
+ - Added support for setting custom log4j config lines
+ - Some various class updates
+
 ## Release 1.2.0
 
 2024-01-29 - Parameter updates & class enhancements

README.md

@@ -20,7 +20,7 @@ Welcome to the kafka_connect Puppet module!
     * * [Remove a connector](#remove-a-connector)
     * * [Pause a connector](#pause-a-connector)
     * * [Managing secrets config data](#managing-secrets-config-data)
-    * [Managing connectors directly through the provider](#managing-connectors-directly-through-the-provider)
+    * [Managing connectors directly through the resource type](#managing-connectors-directly-through-the-resource-type)
     * * [Examples](#examples)
 4. [Reference - An under-the-hood peek at what the module is doing and how](REFERENCE.md)
 5. [Limitations - OS compatibility, etc.](#limitations)
@@ -179,7 +179,7 @@ kafka_connect::secrets:
     ensure: 'absent'
 ```
 
-### Managing connectors directly through the provider
+### Managing connectors directly through the resource type
 
 #### Examples
 
@@ -200,6 +200,7 @@ To pause:
     connector_state_ensure => 'PAUSED',
   }
 ```
+
 To remove:
 
 ```puppet
@@ -209,9 +210,15 @@ To remove:
   }
 ```
 
+Command to remove through the Puppet RAL:
+
+```bash
+$ puppet resource manage_connector some-kc-connector ensure=absent enable_delete=true
+```
+
 ## Limitations
 
-Tested with Confluent 7.1.1 & 7.5.1 on Amazon Linux 2. Should also work on other Redhat as well as Debian-based systems.
+Tested with Confluent 7.x on Amazon Linux 2. Should also work on other RedHat as well as Debian-based systems.
 
 Each secrets file should contain only one key-value pair.
 

REFERENCE.md

@@ -24,6 +24,8 @@
 
 ### Data types
 
+* [`Kafka_connect::HubPlugins`](#kafka_connecthubplugins): Validate the Confluent Hub plugins list.
+* [`Kafka_connect::LogAppender`](#kafka_connectlogappender): Validate the log4j file appender.
 * [`Kafka_connect::Loglevel`](#kafka_connectloglevel): Matches all valid log4j loglevels.
 
 ## Classes
@@ -55,6 +57,16 @@ class { 'kafka_connect':
 
 ##### 
 
+```puppet
+class { 'kafka_connect':
+  log4j_enable_stdout       => true,
+  log4j_custom_config_lines => [ 'log4j.logger.io.confluent.connect.elasticsearch=DEBUG' ],
+  confluent_hub_plugins     => [ 'confluentinc/kafka-connect-elasticsearch:latest' ],
+}
+```
+
+##### 
+
 ```puppet
 class { 'kafka_connect':
   manage_connectors_only => true,
@@ -90,9 +102,13 @@ The following parameters are available in the `kafka_connect` class:
 * [`key_converter`](#key_converter)
 * [`key_converter_schemas_enable`](#key_converter_schemas_enable)
 * [`listeners`](#listeners)
+* [`log4j_file_appender`](#log4j_file_appender)
 * [`log4j_appender_file_path`](#log4j_appender_file_path)
 * [`log4j_appender_max_file_size`](#log4j_appender_max_file_size)
 * [`log4j_appender_max_backup_index`](#log4j_appender_max_backup_index)
+* [`log4j_appender_date_pattern`](#log4j_appender_date_pattern)
+* [`log4j_enable_stdout`](#log4j_enable_stdout)
+* [`log4j_custom_config_lines`](#log4j_custom_config_lines)
 * [`log4j_loglevel_rootlogger`](#log4j_loglevel_rootlogger)
 * [`offset_flush_interval_ms`](#offset_flush_interval_ms)
 * [`offset_storage_topic`](#offset_storage_topic)
@@ -212,10 +228,11 @@ Default value: `'/usr/share/confluent-hub-components'`
 
 ##### <a name="confluent_hub_plugins"></a>`confluent_hub_plugins`
 
-Data type: `Array[String]`
+Data type: `Kafka_connect::HubPlugins`
 
 List of Confluent Hub plugins to install.
 Each should be in the format author/name:semantic-version, e.g. 'acme/fancy-plugin:0.1.0'
+Also accepts 'latest' in place of a specific version.
 
 Default value: `[]`
 
@@ -299,6 +316,14 @@ Config value to set for 'listeners'.
 
 Default value: `'HTTP://:8083'`
 
+##### <a name="log4j_file_appender"></a>`log4j_file_appender`
+
+Data type: `Kafka_connect::LogAppender`
+
+Log4j file appender type to use (RollingFileAppender or DailyRollingFileAppender).
+
+Default value: `'RollingFileAppender'`
+
 ##### <a name="log4j_appender_file_path"></a>`log4j_appender_file_path`
 
 Data type: `Stdlib::Absolutepath`
@@ -312,6 +337,7 @@ Default value: `'/var/log/confluent/connect.log'`
 Data type: `String[1]`
 
 Config value to set for 'log4j.appender.file.MaxFileSize'.
+Only used if log4j_file_appender = 'RollingFileAppender'.
 
 Default value: `'100MB'`
 
@@ -320,9 +346,37 @@ Default value: `'100MB'`
 Data type: `Integer`
 
 Config value to set for 'log4j.appender.file.MaxBackupIndex'.
+Only used if log4j_file_appender = 'RollingFileAppender'.
 
 Default value: `10`
 
+##### <a name="log4j_appender_date_pattern"></a>`log4j_appender_date_pattern`
+
+Data type: `String[1]`
+
+Config value to set for 'log4j.appender.file.DatePattern'.
+Only used if log4j_file_appender = 'DailyRollingFileAppender'.
+
+Default value: `'\'.\'yyyy-MM-dd-HH'`
+
+##### <a name="log4j_enable_stdout"></a>`log4j_enable_stdout`
+
+Data type: `Boolean`
+
+Option to enable logging to stdout/console.
+
+Default value: ``false``
+
+##### <a name="log4j_custom_config_lines"></a>`log4j_custom_config_lines`
+
+Data type: `Optional[Array[String[1]]]`
+
+Option to provide additional custom logging configuration.
+Can be used, for example, to adjust the log level for a specific connector type.
+See: https://docs.confluent.io/platform/current/connect/logging.html#use-the-kconnect-log4j-properties-file
+
+Default value: ``undef``
+
 ##### <a name="log4j_loglevel_rootlogger"></a>`log4j_loglevel_rootlogger`
 
 Data type: `Kafka_connect::Loglevel`
@@ -469,15 +523,15 @@ Default value: `'/etc/kafka-connect'`
 
 ##### <a name="owner"></a>`owner`
 
-Data type: `String[1]`
+Data type: `Variant[String[1], Integer]`
 
 Owner to set on config files.
 
 Default value: `'cp-kafka-connect'`
 
 ##### <a name="group"></a>`group`
 
-Data type: `String[1]`
+Data type: `Variant[String[1], Integer]`
 
 Group to set on config files.
 
@@ -631,6 +685,26 @@ Default value: ``false``
 
 ## Data types
 
+### <a name="kafka_connecthubplugins"></a>`Kafka_connect::HubPlugins`
+
+Validate the Confluent Hub plugins list.
+
+Alias of
+
+```puppet
+Array[Optional[Pattern[/^\w+\/[a-zA-z0-9]{1,}[a-zA-z0-9\-]{0,}:(\d+\.\d+\.\d+|latest)$/]]]
+```
+
+### <a name="kafka_connectlogappender"></a>`Kafka_connect::LogAppender`
+
+Validate the log4j file appender.
+
+Alias of
+
+```puppet
+Enum['DailyRollingFileAppender', 'RollingFileAppender']
+```
+
 ### <a name="kafka_connectloglevel"></a>`Kafka_connect::Loglevel`
 
 Matches all valid log4j loglevels.

manifests/confluent_repo.pp

@@ -19,6 +19,8 @@
     }
 
   } elsif $facts['os']['family'] == 'Debian' {
+    include apt
+
     apt::source { 'confluent':
       ensure   => $kafka_connect::repo_ensure,
       comment  => 'Confluent repository',

manifests/init.pp

@@ -37,6 +37,7 @@
 # @param confluent_hub_plugins
 #   List of Confluent Hub plugins to install.
 #   Each should be in the format author/name:semantic-version, e.g. 'acme/fancy-plugin:0.1.0'
+#   Also accepts 'latest' in place of a specific version.
 #
 # @param confluent_hub_client_package_name
 #   Name of the Confluent Hub Client package.
@@ -68,14 +69,31 @@
 # @param listeners
 #   Config value to set for 'listeners'.
 #
+# @param log4j_file_appender
+#   Log4j file appender type to use (RollingFileAppender or DailyRollingFileAppender).
+#
 # @param log4j_appender_file_path
 #   Config value to set for 'log4j.appender.file.File'.
 #
 # @param log4j_appender_max_file_size
 #   Config value to set for 'log4j.appender.file.MaxFileSize'.
+#   Only used if log4j_file_appender = 'RollingFileAppender'.
 #
 # @param log4j_appender_max_backup_index
 #   Config value to set for 'log4j.appender.file.MaxBackupIndex'.
+#   Only used if log4j_file_appender = 'RollingFileAppender'.
+#
+# @param log4j_appender_date_pattern
+#   Config value to set for 'log4j.appender.file.DatePattern'.
+#   Only used if log4j_file_appender = 'DailyRollingFileAppender'.
+#
+# @param log4j_enable_stdout
+#   Option to enable logging to stdout/console.
+#
+# @param log4j_custom_config_lines
+#   Option to provide additional custom logging configuration.
+#   Can be used, for example, to adjust the log level for a specific connector type.
+#   See: https://docs.confluent.io/platform/current/connect/logging.html#use-the-kconnect-log4j-properties-file
 #
 # @param log4j_loglevel_rootlogger
 #   Config value to set for 'log4j.rootLogger'.
@@ -171,6 +189,13 @@
 #
 # @example
 #   class { 'kafka_connect':
+#     log4j_enable_stdout       => true,
+#     log4j_custom_config_lines => [ 'log4j.logger.io.confluent.connect.elasticsearch=DEBUG' ],
+#     confluent_hub_plugins     => [ 'confluentinc/kafka-connect-elasticsearch:latest' ],
+#   }
+#
+# @example
+#   class { 'kafka_connect':
 #     manage_connectors_only => true,
 #     connector_config_dir   => '/opt/kafka-connect/etc',
 #     rest_port              => 8084,
@@ -197,7 +222,7 @@
   String[1]                   $schema_registry_package_name        = 'confluent-schema-registry',
   String[1]                   $confluent_rest_utils_package_name   = 'confluent-rest-utils',
   Stdlib::Absolutepath        $confluent_hub_plugin_path           = '/usr/share/confluent-hub-components',
-  Array[String]               $confluent_hub_plugins               = [],
+  Kafka_connect::HubPlugins   $confluent_hub_plugins               = [],
   String[1]                   $confluent_hub_client_package_name   = 'confluent-hub-client',
   String[1]                   $confluent_common_package_name       = 'confluent-common',
 
@@ -210,9 +235,13 @@
   String[1]                   $key_converter                       = 'org.apache.kafka.connect.json.JsonConverter',
   Boolean                     $key_converter_schemas_enable        = true,
   Stdlib::HTTPUrl             $listeners                           = 'HTTP://:8083',
+  Kafka_connect::LogAppender  $log4j_file_appender                 = 'RollingFileAppender',
   Stdlib::Absolutepath        $log4j_appender_file_path            = '/var/log/confluent/connect.log',
   String[1]                   $log4j_appender_max_file_size        = '100MB',
   Integer                     $log4j_appender_max_backup_index     = 10,
+  String[1]                   $log4j_appender_date_pattern         = '\'.\'yyyy-MM-dd-HH',
+  Boolean                     $log4j_enable_stdout                 = false,
+  Optional[Array[String[1]]]  $log4j_custom_config_lines           = undef,
   Kafka_connect::Loglevel     $log4j_loglevel_rootlogger           = 'INFO',
   Integer                     $offset_flush_interval_ms            = 10000,
   String[1]                   $offset_storage_topic                = 'connect-offsets',

manifests/install.pp

@@ -32,11 +32,6 @@
   }
 
   $kafka_connect::confluent_hub_plugins.each |$plugin| {
-    unless $plugin =~ /^\w+\/[a-zA-z0-9]{1,}[a-zA-z0-9\-]{0,}:\d+\.\d+\.\d+$/ {
-      fail("Unexpected plugin value encountered: ${plugin} \
-        \n This should be in the format author/name:semantic-version, e.g. acme/fancy-plugin:0.1.0 \n")
-    }
-
     $author = regsubst($plugin,'^(\w+)\/.+:.+$','\1')
     $name   = regsubst($plugin,'^(\w+)\/([a-zA-z0-9]{1,}[a-zA-z0-9\-]{0,}):.+$','\2')
 

metadata.json

@@ -1,6 +1,6 @@
 {
   "name": "rjd1-kafka_connect",
-  "version": "1.2.0",
+  "version": "1.3.0",
   "author": "rjd1",
   "summary": "Puppet module to manage Kafka Connect.",
   "license": "Apache-2.0",

templates/connect-distributed.properties.erb

@@ -1,4 +1,5 @@
-##
+#### MANAGED BY PUPPET ##
+#
 # Licensed to the Apache Software Foundation (ASF) under one or more
 # contributor license agreements.  See the NOTICE file distributed with
 # this work for additional information regarding copyright ownership.