From 7d5f6b6255be326c564df8a4d2919b7ec55865a3 Mon Sep 17 00:00:00 2001 From: admi zakaryae Date: Thu, 18 Jul 2024 11:18:36 +0100 Subject: [PATCH 1/2] adding timezone field,header opc_meta_data property, and workload identity auth method fixing issues related to the previous commit fixing issue related to previous commit 2 fixing last commit issue fixing last commit issue fixing last commit issue deleting GemFile.lock fixing collection source issue --- Gemfile | 1 - examples/apache.conf | 28 ++++---- fluent-plugin-oci-logging-analytics.gemspec | 3 +- lib/fluent/dto/logEvents.rb | 7 +- lib/fluent/enums/source.rb | 4 ++ lib/fluent/metrics/metricsLabels.rb | 3 +- .../plugin/out_oci-logging-analytics.rb | 71 +++++++++++++++++-- lib/fluent/version/version.rb | 5 ++ 8 files changed, 95 insertions(+), 27 deletions(-) create mode 100644 lib/fluent/enums/source.rb create mode 100644 lib/fluent/version/version.rb diff --git a/Gemfile b/Gemfile index 784e23a..92128fa 100755 --- a/Gemfile +++ b/Gemfile @@ -2,6 +2,5 @@ ## The Universal Permissive License (UPL), Version 1.0 as shown at https://oss.oracle.com/licenses/upl/ source "https://rubygems.org" - gemspec diff --git a/examples/apache.conf b/examples/apache.conf index 756e386..b0fc62e 100644 --- a/examples/apache.conf +++ b/examples/apache.conf @@ -9,8 +9,8 @@ @type tail @id in_tail_apacheError - path /var/log/apacheError.log - pos_file /var/log/fluentd-apacheError.log.pos + path /var/log/apacheError.log + pos_file /var/log/fluentd-apacheError.log.pos read_from_head true path_key tailed_path tag oci.apacheError @@ -20,20 +20,20 @@ # Add below prometheus config block only when you need output plugin metrics. - - @type prometheus - bind 0.0.0.0 - port 24231 - metrics_path /metrics - +# +# @type prometheus +# bind 0.0.0.0 +# port 24231 +# metrics_path /metrics +# @type record_transformer enable_ruby true - oci_la_global_metadata ${{: , :}} + #oci_la_global_metadata ${{: , :}} oci_la_entity_id # If same across sources. Else keep this in individual filters - oci_la_entity_type # If same across sources. Else keep this in individual filters + oci_la_entity_type # If same across sources. Else keep this in individual filters @@ -41,24 +41,24 @@ @type record_transformer enable_ruby true - oci_la_metadata ${{: , :}} + oci_la_metadata ${{'Error ID': '1', Environment: 'preprod', 'Client Host Region': 'PST'}} oci_la_log_source_name oci_la_log_group_id oci_la_log_path "${record['tailed_path']}" + oci_la_timezone "UTC" tag ${tag} @type oci-logging-analytics - namespace - # Auth config file details + namespace config_file_location ~/.oci/config profile_name DEFAULT # Buffer Configuration @type file - path /var/log + path /var/log retry_forever true disable_chunk_backup true diff --git a/fluent-plugin-oci-logging-analytics.gemspec b/fluent-plugin-oci-logging-analytics.gemspec index 47e3843..c65cb13 100755 --- a/fluent-plugin-oci-logging-analytics.gemspec +++ b/fluent-plugin-oci-logging-analytics.gemspec @@ -1,12 +1,13 @@ ## Copyright (c) 2021, 2022 Oracle and/or its affiliates. ## The Universal Permissive License (UPL), Version 1.0 as shown at https://oss.oracle.com/licenses/upl/ +require_relative './lib/fluent/version/version' lib = File.expand_path("../lib", __FILE__) $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib) Gem::Specification.new do |spec| spec.name = "fluent-plugin-oci-logging-analytics" - spec.version = "2.0.6" + spec.version = Version::VERSION spec.authors = ["Oracle","OCI Observability: Logging Analytics"] spec.email = ["oci_la_plugins_grp@oracle.com"] diff --git a/lib/fluent/dto/logEvents.rb b/lib/fluent/dto/logEvents.rb index 959f1b9..e59d1f6 100755 --- a/lib/fluent/dto/logEvents.rb +++ b/lib/fluent/dto/logEvents.rb @@ -2,9 +2,9 @@ ## The Universal Permissive License (UPL), Version 1.0 as shown at https://oss.oracle.com/licenses/upl/ class LogEvents - attr_accessor :entityId, :entityType, :logSourceName, :logPath, :logRecords , :metadata + attr_accessor :entityId, :entityType, :logSourceName, :logPath, :logRecords , :metadata, :timezone def initialize(lrpe_key, fluentd_records) - @metadata, @entityId, @entityType, @logSourceName, @logPath = lrpe_key + @metadata, @entityId, @entityType, @logSourceName, @logPath, @timezone = lrpe_key @logRecords = fluentd_records.map{ |record| record['message'] } @@ -17,7 +17,8 @@ def to_hash entityType: @entityType, logSourceName: @logSourceName, logPath: @logPath, - logRecords: @logRecords + logRecords: @logRecords, + timezone:@timezone }.compact end diff --git a/lib/fluent/enums/source.rb b/lib/fluent/enums/source.rb new file mode 100644 index 0000000..776c2f5 --- /dev/null +++ b/lib/fluent/enums/source.rb @@ -0,0 +1,4 @@ +module Source + FLUENTD = :fluentd + KUBERNETES_SOLUTION = :kubernetes_solution +end diff --git a/lib/fluent/metrics/metricsLabels.rb b/lib/fluent/metrics/metricsLabels.rb index 4201a42..35099ec 100644 --- a/lib/fluent/metrics/metricsLabels.rb +++ b/lib/fluent/metrics/metricsLabels.rb @@ -1,5 +1,5 @@ class MetricsLabels - attr_accessor :worker_id, :tag, :logGroupId, :logSourceName, :logSet, :invalid_reason, :records_valid, :records_per_tag, :latency + attr_accessor :worker_id, :tag, :logGroupId, :logSourceName, :logSet, :invalid_reason, :records_valid, :records_per_tag, :latency,:timezone def initialize @worker_id = nil @tag = nil @@ -10,5 +10,6 @@ def initialize @records_valid = 0 @records_per_tag = 0 @latency = 0 + @timezone = nil end end \ No newline at end of file diff --git a/lib/fluent/plugin/out_oci-logging-analytics.rb b/lib/fluent/plugin/out_oci-logging-analytics.rb index 47b1a21..7b2e9bb 100755 --- a/lib/fluent/plugin/out_oci-logging-analytics.rb +++ b/lib/fluent/plugin/out_oci-logging-analytics.rb @@ -1,17 +1,18 @@ ## Copyright (c) 2021, 2022 Oracle and/or its affiliates. ## The Universal Permissive License (UPL), Version 1.0 as shown at https://oss.oracle.com/licenses/upl/ - require 'fluent/plugin/output' require "benchmark" require 'zip' require 'yajl' require 'yajl/json_gem' +# require 'tzinfo' require 'logger' require_relative '../dto/logEventsJson' require_relative '../dto/logEvents' require_relative '../metrics/prometheusMetrics' require_relative '../metrics/metricsLabels' +require_relative '../enums/source' # Import only specific OCI modules to improve load times and reduce the memory requirements. require 'oci/auth/auth' @@ -36,7 +37,6 @@ require 'oci/waiter' require 'oci/retry/retry' require 'oci/object_storage/object_storage' - module OCI class << self attr_accessor :sdk_name @@ -97,7 +97,8 @@ class OutOracleOCILogAnalytics < Output config_param :zip_file_location, :string, :default => nil desc 'The kubernetes_metadata_keys_mapping.' config_param :kubernetes_metadata_keys_mapping, :hash, :default => {"container_name":"Container","namespace_name":"Namespace","pod_name":"Pod","container_image":"Container Image Name","host":"Node"} - + desc 'opc-meta-properties' + config_param :collection_source, :string, :default => Source::FLUENTD #**************************************************************** desc 'The http proxy to be used.' @@ -256,6 +257,14 @@ def initialize_loganalytics_client() else @@loganalytics_client = OCI::LogAnalytics::LogAnalyticsClient.new(config: OCI::Config.new, signer: instance_principals_signer) end + when "WorkloadIdentity" + workload_identity_signer = OCI::Auth::Signers::oke_workload_resource_principal_signer + if is_valid(@endpoint) + @@loganalytics_client = OCI::LogAnalytics::LogAnalyticsClient.new(config: OCI::Config.new, endpoint: @endpoint, signer: workload_identity_signer) + @@logger.info {"loganalytics_client initialised with endpoint: #{@endpoint}"} + else + @@loganalytics_client = OCI::LogAnalytics::LogAnalyticsClient.new(config: OCI::Config.new, signer: workload_identity_signer) + end when "ConfigFile" my_config = OCI::ConfigFileLoader.load_config(config_file_location: @config_file_location, profile_name: @profile_name) if is_valid(@endpoint) @@ -628,6 +637,8 @@ def group_by_logGroupId(chunk) latency = 0 records_per_tag = 0 + + tag_metrics_set = Hash.new logGroup_labels_set = Hash.new @@ -637,8 +648,8 @@ def group_by_logGroupId(chunk) tags_per_logGroupId = Hash.new tag_logSet_map = Hash.new tag_metadata_map = Hash.new + timezoneValuesByTag = Hash.new incoming_records = 0 - chunk.each do |time, record| incoming_records += 1 metricsLabels = MetricsLabels.new @@ -722,6 +733,8 @@ def group_by_logGroupId(chunk) end next end + + # metricsLabels.timezone = record["oci_la_timezone"] metricsLabels.logGroupId = record["oci_la_log_group_id"] metricsLabels.logSourceName = record["oci_la_log_source_name"] if record["oci_la_log_set"] != nil @@ -770,6 +783,25 @@ def group_by_logGroupId(chunk) tags_per_logGroupId[record["oci_la_log_group_id"]] = record["tag"] end end + # validating the timezone field + if !timezoneValuesByTag.has_key?(record["tag"]) + begin + timezoneIdentifier = record["oci_la_timezone"] + unless is_valid(timezoneIdentifier) + record["oci_la_timezone"] = nil + else + isTimezoneExist = timezone_exist? timezoneIdentifier + unless isTimezoneExist + @@logger.warn { "Invalid timezone '#{timezoneIdentifier}', using default UTC." } + record["oci_la_timezone"] = "UTC" + end + + end + timezoneValuesByTag[record["tag"]] = record["oci_la_timezone"] + end + else + record["oci_la_timezone"] = timezoneValuesByTag[record["tag"]] + end records << record ensure @@ -916,6 +948,14 @@ def write(chunk) end end end + def timezone_exist?(tz) + begin + TZInfo::Timezone.get(tz) + return true + rescue TZInfo::InvalidTimezoneIdentifier + return false + end + end # Each oci_la_log_set will correspond to a separate file in the zip # Only MAX_FILES_PER_ZIP files are allowed per zip. @@ -958,6 +998,21 @@ def get_logSets_map_per_logGroupId(oci_la_log_group_id,records_per_logGroupId) # takes a fluentD chunk and converts it to an in-memory zipfile, populating metrics hash provided # Any exception raised is passed into the metrics hash, to be re-thrown from write() + def getCollectionSource(input) + collections_src = [] + if !is_valid input + collections_src.unshift("source:#{Source::FLUENTD}") + else + if input == Source::FLUENTD.to_s or input == Source::KUBERNETES_SOLUTION.to_s + collections_src.unshift("source:#{input}") + else + # source not define ! using default source 'fluentd' + collections_src.unshift("source:#{Source::FLUENTD}") + end + end + collections_src + end + def get_zipped_stream(oci_la_log_group_id,oci_la_global_metadata,records_per_logSet_map) begin current, = Time.now @@ -970,8 +1025,9 @@ def get_zipped_stream(oci_la_log_group_id,oci_la_global_metadata,records_per_log record['oci_la_metadata'], record['oci_la_entity_id'], record['oci_la_entity_type'], - record['oci_la_log_source_name'] , - record['oci_la_log_path'] + record['oci_la_log_source_name'], + record['oci_la_log_path'], + record['oci_la_timezone'] ]}.map { |lrpe_key, records_per_lrpe| number_of_records += records_per_lrpe.length LogEvents.new(lrpe_key, records_per_lrpe) @@ -1021,9 +1077,10 @@ def save_zip_to_local(oci_la_log_group_id, zippedstream, current_s) # upload zipped stream to oci def upload_to_oci(oci_la_log_group_id, number_of_records, zippedstream, metricsLabels_array) begin + collection_src_prop = getCollectionSource @collection_source error_reason = nil error_code = nil - opts = {payload_type: "ZIP"} + opts = { payload_type: "ZIP", opc_meta_properties:collection_src_prop} response = @@loganalytics_client.upload_log_events_file(namespace_name=@namespace, logGroupId=oci_la_log_group_id , diff --git a/lib/fluent/version/version.rb b/lib/fluent/version/version.rb new file mode 100644 index 0000000..be03b69 --- /dev/null +++ b/lib/fluent/version/version.rb @@ -0,0 +1,5 @@ +# frozen_string_literal: true + +module Version + VERSION = "2.0.7".freeze +end From 92696d88d7182ab7d33f7ed962bb2d8bbe4e41f0 Mon Sep 17 00:00:00 2001 From: admi ZAKARYAE Date: Sat, 5 Oct 2024 15:45:06 +0100 Subject: [PATCH 2/2] revert apache.conf file changes --- examples/apache.conf | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/examples/apache.conf b/examples/apache.conf index b0fc62e..10c76c4 100644 --- a/examples/apache.conf +++ b/examples/apache.conf @@ -9,8 +9,8 @@ @type tail @id in_tail_apacheError - path /var/log/apacheError.log - pos_file /var/log/fluentd-apacheError.log.pos + path /var/log/apacheError.log + pos_file /var/log/fluentd-apacheError.log.pos read_from_head true path_key tailed_path tag oci.apacheError @@ -20,18 +20,18 @@ # Add below prometheus config block only when you need output plugin metrics. -# -# @type prometheus -# bind 0.0.0.0 -# port 24231 -# metrics_path /metrics -# + + @type prometheus + bind 0.0.0.0 + port 24231 + metrics_path /metrics + @type record_transformer enable_ruby true - #oci_la_global_metadata ${{: , :}} + oci_la_global_metadata ${{: , :}} oci_la_entity_id # If same across sources. Else keep this in individual filters oci_la_entity_type # If same across sources. Else keep this in individual filters @@ -41,24 +41,24 @@ @type record_transformer enable_ruby true - oci_la_metadata ${{'Error ID': '1', Environment: 'preprod', 'Client Host Region': 'PST'}} + oci_la_metadata ${{: , : oci_la_log_group_id oci_la_log_path "${record['tailed_path']}" - oci_la_timezone "UTC" + oci_la_timezone tag ${tag} @type oci-logging-analytics - namespace + namespace config_file_location ~/.oci/config profile_name DEFAULT # Buffer Configuration @type file - path /var/log + path /var/log retry_forever true disable_chunk_backup true