From 593089e3cc04222773f0d5275ca3d628da8ebc8f Mon Sep 17 00:00:00 2001 From: Andreas Braun Date: Wed, 9 Sep 2020 15:24:19 +0200 Subject: [PATCH 001/139] WIP: first stab at structured logging --- src/libmongoc/CMakeLists.txt | 2 + src/libmongoc/src/mongoc/mongoc-cluster.c | 4 + .../mongoc/mongoc-structured-log-private.h | 40 ++++++ .../src/mongoc/mongoc-structured-log.c | 120 ++++++++++++++++++ .../src/mongoc/mongoc-structured-log.h | 66 ++++++++++ src/libmongoc/src/mongoc/mongoc.h | 1 + 6 files changed, 233 insertions(+) create mode 100644 src/libmongoc/src/mongoc/mongoc-structured-log-private.h create mode 100644 src/libmongoc/src/mongoc/mongoc-structured-log.c create mode 100644 src/libmongoc/src/mongoc/mongoc-structured-log.h diff --git a/src/libmongoc/CMakeLists.txt b/src/libmongoc/CMakeLists.txt index fed07d51882..cfabf2901b1 100644 --- a/src/libmongoc/CMakeLists.txt +++ b/src/libmongoc/CMakeLists.txt @@ -647,6 +647,7 @@ set (MONGOC_SOURCES ${PROJECT_SOURCE_DIR}/src/mongoc/mongoc-stream-gridfs-download.c ${PROJECT_SOURCE_DIR}/src/mongoc/mongoc-stream-gridfs-upload.c ${PROJECT_SOURCE_DIR}/src/mongoc/mongoc-stream-socket.c + ${PROJECT_SOURCE_DIR}/src/mongoc/mongoc-structured-log.c ${PROJECT_SOURCE_DIR}/src/mongoc/mongoc-timeout.c ${PROJECT_SOURCE_DIR}/src/mongoc/mongoc-topology.c ${PROJECT_SOURCE_DIR}/src/mongoc/mongoc-topology-background-monitoring.c @@ -761,6 +762,7 @@ set (HEADERS ${PROJECT_SOURCE_DIR}/src/mongoc/mongoc-stream-file.h ${PROJECT_SOURCE_DIR}/src/mongoc/mongoc-stream-gridfs.h ${PROJECT_SOURCE_DIR}/src/mongoc/mongoc-stream-socket.h + ${PROJECT_SOURCE_DIR}/src/mongoc/mongoc-structured-log.h ${PROJECT_SOURCE_DIR}/src/mongoc/mongoc-topology-description.h ${PROJECT_SOURCE_DIR}/src/mongoc/mongoc-uri.h ${PROJECT_SOURCE_DIR}/src/mongoc/mongoc-version-functions.h diff --git a/src/libmongoc/src/mongoc/mongoc-cluster.c b/src/libmongoc/src/mongoc/mongoc-cluster.c index fab09a2635b..f901fa9a063 100644 --- a/src/libmongoc/src/mongoc/mongoc-cluster.c +++ b/src/libmongoc/src/mongoc/mongoc-cluster.c @@ -56,6 +56,7 @@ #include "mongoc-handshake-private.h" #include "mongoc-cluster-aws-private.h" #include "mongoc-error-private.h" +#include "mongoc-structured-log-private.h" #include #include @@ -532,6 +533,9 @@ mongoc_cluster_run_command_monitored (mongoc_cluster_t *cluster, mongoc_cmd_t *c } } + // @todo Provide explicit session + mongoc_structured_log_command_started(cmd, request_id, false); + if (callbacks->started) { mongoc_apm_command_started_init_with_cmd ( &started_event, cmd, request_id, &is_redacted, cluster->client->apm_context); diff --git a/src/libmongoc/src/mongoc/mongoc-structured-log-private.h b/src/libmongoc/src/mongoc/mongoc-structured-log-private.h new file mode 100644 index 00000000000..535ada3e79a --- /dev/null +++ b/src/libmongoc/src/mongoc/mongoc-structured-log-private.h @@ -0,0 +1,40 @@ +/* + * Copyright 2015 MongoDB, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "mongoc-prelude.h" +#include "mongoc-structured-log.h" +#include "mongoc-cmd-private.h" + +#ifndef MONGOC_STRUCTRURED_LOG_PRIVATE_H +#define MONGOC_STRUCTRURED_LOG_PRIVATE_H + +// @todo Decide if we want a log entry structure +struct _mongoc_structured_log_entry_t { + mongoc_structured_log_level_t level; + mongoc_structured_log_component_t component; + char* message; + bson_t* context; +}; + +void mongoc_structured_log_command_started(mongoc_cmd_t *cmd, + uint32_t request_id, + // Driver connection ID + // Server connection Id + bool explicit_session); + +#define MONGOC_STRUCTURED_LOG_COMMAND_STARTED() + +#endif /* MONGOC_STRUCTURED_LOG_PRIVATE_H */ diff --git a/src/libmongoc/src/mongoc/mongoc-structured-log.c b/src/libmongoc/src/mongoc/mongoc-structured-log.c new file mode 100644 index 00000000000..db7d12c2dfa --- /dev/null +++ b/src/libmongoc/src/mongoc/mongoc-structured-log.c @@ -0,0 +1,120 @@ +/* + * Copyright 2013 MongoDB, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +#if defined(__linux__) +#include +#elif defined(_WIN32) +#include +#elif defined(__FreeBSD__) +#include +#else +#include +#endif +#include +#include + +#include "mongoc-structured-log.h" +#include "mongoc-structured-log-private.h" +#include "mongoc-thread-private.h" + +static bson_once_t once = BSON_ONCE_INIT; +static bson_mutex_t gStructuredLogMutex; +static mongoc_structured_log_func_t gStructuredLogger = NULL; +static void *gStructuredLoggerData; + +static BSON_ONCE_FUN (_mongoc_ensure_mutex_once) +{ + bson_mutex_init (&gStructuredLogMutex); + + BSON_ONCE_RETURN; +} + +static bson_t* mongoc_log_structured_create_context(const char *message, va_list *ap) +{ + bson_t *context; + bcon_append_ctx_t ctx; + + context = BCON_NEW ("message", *message); + + bcon_append_ctx_init (&ctx); + bcon_append_ctx_va (context, &ctx, ap); + + return context; +} + +void +mongoc_structured_log_set_handler (mongoc_structured_log_func_t log_func, void *user_data) +{ + bson_once (&once, &_mongoc_ensure_mutex_once); + + bson_mutex_lock (&gStructuredLogMutex); + gStructuredLogger = log_func; + gStructuredLoggerData = user_data; + bson_mutex_unlock (&gStructuredLogMutex); +} + +void mongoc_log_structured (mongoc_structured_log_level_t level, + mongoc_structured_log_component_t component, + const char *message, + ...) +{ + va_list ap; + bson_t *context; + + if (!gStructuredLogger) { + return; + } + + va_start (ap, message); + context = mongoc_log_structured_create_context(message, &ap); + va_end (ap); + + bson_mutex_lock (&gStructuredLogMutex); + gStructuredLogger (level, component, message, context, gStructuredLoggerData); + bson_mutex_unlock (&gStructuredLogMutex); + + bson_destroy(context); +} + +void mongoc_structured_log_command_started(mongoc_cmd_t *cmd, + uint32_t request_id, + // Driver connection ID + // Server connection Id + bool explicit_session) +{ + mongoc_log_structured( + MONGOC_STRUCTURED_LOG_LEVEL_INFO, + MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND, + "Command started", + "command", + BCON_DOCUMENT (cmd->command), // @todo Convert to canonical extJson here? + "databaseName", + cmd->db_name, + "commandName", + cmd->command_name, + "requestId", + BCON_INT32 (request_id), + "operationId", + cmd->operation_id, + "driverConnectionId", + BCON_INT32 (0), // @todo Provide driverConnectionId + "serverConnectionId", + BCON_INT32 (0), // @todo Provide serverConnectionId + "explicitSession", + explicit_session + ); +} diff --git a/src/libmongoc/src/mongoc/mongoc-structured-log.h b/src/libmongoc/src/mongoc/mongoc-structured-log.h new file mode 100644 index 00000000000..6df2c08ed28 --- /dev/null +++ b/src/libmongoc/src/mongoc/mongoc-structured-log.h @@ -0,0 +1,66 @@ +/* + * Copyright 2013 MongoDB, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "mongoc-prelude.h" + +#ifndef MONGOC_STRUCTURED_LOG_H +#define MONGOC_STRUCTURED_LOG_H + +#include + +#include "mongoc-macros.h" + +BSON_BEGIN_DECLS + +typedef enum { + MONGOC_STRUCTURED_LOG_LEVEL_EMERGENCY = 0, + MONGOC_STRUCTURED_LOG_LEVEL_ALERT = 1, + MONGOC_STRUCTURED_LOG_LEVEL_CRITICAL = 2, + MONGOC_STRUCTURED_LOG_LEVEL_ERROR = 3, + MONGOC_STRUCTURED_LOG_LEVEL_WARNING = 4, + MONGOC_STRUCTURED_LOG_LEVEL_NOTICE = 5, + MONGOC_STRUCTURED_LOG_LEVEL_INFO = 6, + MONGOC_STRUCTURED_LOG_LEVEL_DEBUG = 7, + MONGOC_STRUCTURED_LOG_LEVEL_TRACE = 8, +} mongoc_structured_log_level_t; + +typedef enum { + MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND, + MONGOC_STRUCTURED_LOG_COMPONENT_SDAM, + MONGOC_STRUCTURED_LOG_COMPONENT_SERVER_SELECTION, + MONGOC_STRUCTURED_LOG_COMPONENT_CONNECTION, +} mongoc_structured_log_component_t; + +typedef struct _mongoc_structured_log_entry_t mongoc_structured_log_entry_t; + +typedef void (*mongoc_structured_log_func_t) (mongoc_structured_log_level_t level, + mongoc_structured_log_component_t component, + const char *message, + bson_t *context, + void *user_data); +MONGOC_EXPORT (void) +mongoc_structured_log_set_handler (mongoc_structured_log_func_t log_func, void *user_data); + +MONGOC_EXPORT (void) +mongoc_log_structured (mongoc_structured_log_level_t level, + mongoc_structured_log_component_t component, + const char *message, + ...); + +BSON_END_DECLS + + +#endif /* MONGOC_STRUCTURED_LOG_H */ diff --git a/src/libmongoc/src/mongoc/mongoc.h b/src/libmongoc/src/mongoc/mongoc.h index 6dab2fc3dda..cccedc54286 100644 --- a/src/libmongoc/src/mongoc/mongoc.h +++ b/src/libmongoc/src/mongoc/mongoc.h @@ -56,6 +56,7 @@ #include "mongoc-stream-file.h" #include "mongoc-stream-gridfs.h" #include "mongoc-stream-socket.h" +#include "mongoc-structured-log.h" #include "mongoc-uri.h" #include "mongoc-write-concern.h" #include "mongoc-version.h" From dfee17ac3cb3236d8d97b3a1c21cde4e760572a1 Mon Sep 17 00:00:00 2001 From: Andreas Braun Date: Fri, 11 Sep 2020 14:04:12 +0200 Subject: [PATCH 002/139] Make context generation lazy to save on resources --- src/libmongoc/src/mongoc/mongoc-cluster.c | 4 +- .../mongoc/mongoc-structured-log-private.h | 27 ++- .../src/mongoc/mongoc-structured-log.c | 185 ++++++++++++++---- .../src/mongoc/mongoc-structured-log.h | 20 +- 4 files changed, 182 insertions(+), 54 deletions(-) diff --git a/src/libmongoc/src/mongoc/mongoc-cluster.c b/src/libmongoc/src/mongoc/mongoc-cluster.c index f901fa9a063..52b0c67ce15 100644 --- a/src/libmongoc/src/mongoc/mongoc-cluster.c +++ b/src/libmongoc/src/mongoc/mongoc-cluster.c @@ -533,8 +533,8 @@ mongoc_cluster_run_command_monitored (mongoc_cluster_t *cluster, mongoc_cmd_t *c } } - // @todo Provide explicit session - mongoc_structured_log_command_started(cmd, request_id, false); + // @todo Provide explicit session and connection IDs + mongoc_structured_log_command_started (cmd, request_id, 0, 0, false); if (callbacks->started) { mongoc_apm_command_started_init_with_cmd ( diff --git a/src/libmongoc/src/mongoc/mongoc-structured-log-private.h b/src/libmongoc/src/mongoc/mongoc-structured-log-private.h index 535ada3e79a..c0a16de9905 100644 --- a/src/libmongoc/src/mongoc/mongoc-structured-log-private.h +++ b/src/libmongoc/src/mongoc/mongoc-structured-log-private.h @@ -21,19 +21,30 @@ #ifndef MONGOC_STRUCTRURED_LOG_PRIVATE_H #define MONGOC_STRUCTRURED_LOG_PRIVATE_H -// @todo Decide if we want a log entry structure +typedef void (*mongoc_structured_log_build_context_t) (bson_t *context, va_list *context_data); + struct _mongoc_structured_log_entry_t { mongoc_structured_log_level_t level; mongoc_structured_log_component_t component; - char* message; - bson_t* context; + const char* message; + mongoc_structured_log_build_context_t build_context; + va_list *context_data; + bson_t *context; }; -void mongoc_structured_log_command_started(mongoc_cmd_t *cmd, - uint32_t request_id, - // Driver connection ID - // Server connection Id - bool explicit_session); +void +mongoc_structured_log (mongoc_structured_log_level_t level, + mongoc_structured_log_component_t component, + const char *message, + mongoc_structured_log_build_context_t build_context, + ...); + +void +mongoc_structured_log_command_started (mongoc_cmd_t *cmd, + uint32_t request_id, + uint32_t driver_connection_id, + uint32_t server_connection_id, + bool explicit_session); #define MONGOC_STRUCTURED_LOG_COMMAND_STARTED() diff --git a/src/libmongoc/src/mongoc/mongoc-structured-log.c b/src/libmongoc/src/mongoc/mongoc-structured-log.c index db7d12c2dfa..5b2ca3e2b75 100644 --- a/src/libmongoc/src/mongoc/mongoc-structured-log.c +++ b/src/libmongoc/src/mongoc/mongoc-structured-log.c @@ -26,6 +26,7 @@ #endif #include #include +#include #include "mongoc-structured-log.h" #include "mongoc-structured-log-private.h" @@ -33,7 +34,7 @@ static bson_once_t once = BSON_ONCE_INIT; static bson_mutex_t gStructuredLogMutex; -static mongoc_structured_log_func_t gStructuredLogger = NULL; +static mongoc_structured_log_func_t gStructuredLogger = mongoc_structured_log_default_handler; static void *gStructuredLoggerData; static BSON_ONCE_FUN (_mongoc_ensure_mutex_once) @@ -43,17 +44,61 @@ static BSON_ONCE_FUN (_mongoc_ensure_mutex_once) BSON_ONCE_RETURN; } -static bson_t* mongoc_log_structured_create_context(const char *message, va_list *ap) +static void +mongoc_structured_log_entry_init (mongoc_structured_log_entry_t *entry, + mongoc_structured_log_level_t level, + mongoc_structured_log_component_t component, + const char *message, + mongoc_structured_log_build_context_t build_context, + va_list *context_data) { - bson_t *context; - bcon_append_ctx_t ctx; + entry->level = level; + entry->component = component; + entry->message = message; + entry->build_context = build_context; + entry->context_data = context_data; + entry->context = NULL; +} - context = BCON_NEW ("message", *message); +static void +mongoc_structured_log_entry_destroy (mongoc_structured_log_entry_t *entry) +{ + if (entry->context) { + bson_free (entry->context); + } +} - bcon_append_ctx_init (&ctx); - bcon_append_ctx_va (context, &ctx, ap); +bson_t* +mongoc_structured_log_entry_get_context (mongoc_structured_log_entry_t *entry) +{ + if (entry->context) { + return entry->context; + } - return context; + entry->context = BCON_NEW( + "message", + BCON_UTF8(entry->message) + ); + + if (entry->build_context) { + entry->build_context(entry->context, entry->context_data); + } + + entry->context_data = NULL; + + return entry->context; +} + +mongoc_structured_log_level_t +mongoc_structured_log_entry_get_level (mongoc_structured_log_entry_t *entry) +{ + return entry->level; +} + +mongoc_structured_log_component_t +mongoc_structured_log_entry_get_component (mongoc_structured_log_entry_t *entry) +{ + return entry->component; } void @@ -67,54 +112,126 @@ mongoc_structured_log_set_handler (mongoc_structured_log_func_t log_func, void * bson_mutex_unlock (&gStructuredLogMutex); } -void mongoc_log_structured (mongoc_structured_log_level_t level, - mongoc_structured_log_component_t component, - const char *message, - ...) +void +mongoc_structured_log (mongoc_structured_log_level_t level, + mongoc_structured_log_component_t component, + const char *message, + mongoc_structured_log_build_context_t build_context, + ...) { - va_list ap; - bson_t *context; + va_list context_data; + mongoc_structured_log_entry_t entry; if (!gStructuredLogger) { return; } - va_start (ap, message); - context = mongoc_log_structured_create_context(message, &ap); - va_end (ap); + va_start (context_data, build_context); + mongoc_structured_log_entry_init (&entry, level, component, message, build_context, &context_data); bson_mutex_lock (&gStructuredLogMutex); - gStructuredLogger (level, component, message, context, gStructuredLoggerData); + gStructuredLogger (&entry, gStructuredLoggerData); bson_mutex_unlock (&gStructuredLogMutex); - bson_destroy(context); + mongoc_structured_log_entry_destroy (&entry); + va_end (context_data); } -void mongoc_structured_log_command_started(mongoc_cmd_t *cmd, - uint32_t request_id, - // Driver connection ID - // Server connection Id - bool explicit_session) +static void +mongoc_log_structured_build_command_context(bson_t *context, va_list *context_data) { - mongoc_log_structured( - MONGOC_STRUCTURED_LOG_LEVEL_INFO, - MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND, - "Command started", + mongoc_cmd_t *cmd = va_arg (*context_data, mongoc_cmd_t*); + uint32_t request_id = va_arg (*context_data, uint32_t); + uint32_t driver_connection_id = va_arg (*context_data, uint32_t); + uint32_t server_connection_id = va_arg (*context_data, uint32_t); + bool explicit_session = !!va_arg (*context_data, int); + + char* cmd_json = bson_as_canonical_extended_json(cmd->command, NULL); + + BCON_APPEND( + context, "command", - BCON_DOCUMENT (cmd->command), // @todo Convert to canonical extJson here? + BCON_UTF8(cmd_json), "databaseName", - cmd->db_name, + BCON_UTF8(cmd->db_name), "commandName", - cmd->command_name, + BCON_UTF8(cmd->command_name), "requestId", BCON_INT32 (request_id), "operationId", - cmd->operation_id, + BCON_INT64(cmd->operation_id), "driverConnectionId", - BCON_INT32 (0), // @todo Provide driverConnectionId + BCON_INT32 (driver_connection_id), "serverConnectionId", - BCON_INT32 (0), // @todo Provide serverConnectionId + BCON_INT32 (server_connection_id), "explicitSession", + BCON_BOOL(explicit_session) + ); + + bson_free (cmd_json); +} + +void +mongoc_structured_log_command_started (mongoc_cmd_t *cmd, + uint32_t request_id, + uint32_t driver_connection_id, + uint32_t server_connection_id, + bool explicit_session) +{ + mongoc_structured_log ( + MONGOC_STRUCTURED_LOG_LEVEL_INFO, + MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND, + "Command started", + mongoc_log_structured_build_command_context, + cmd, + request_id, + driver_connection_id, + server_connection_id, explicit_session ); } + +static int +_mongoc_structured_log_convert_level (mongoc_structured_log_level_t level) +{ + switch (level) + { + case MONGOC_STRUCTURED_LOG_LEVEL_TRACE: + case MONGOC_STRUCTURED_LOG_LEVEL_DEBUG: + return LOG_DEBUG; + + case MONGOC_STRUCTURED_LOG_LEVEL_INFO: + return LOG_INFO; + + case MONGOC_STRUCTURED_LOG_LEVEL_NOTICE: + return LOG_NOTICE; + + case MONGOC_STRUCTURED_LOG_LEVEL_WARNING: + return LOG_WARNING; + + case MONGOC_STRUCTURED_LOG_LEVEL_ERROR: + return LOG_ERR; + + case MONGOC_STRUCTURED_LOG_LEVEL_CRITICAL: + return LOG_CRIT; + + case MONGOC_STRUCTURED_LOG_LEVEL_ALERT: + return LOG_ALERT; + + case MONGOC_STRUCTURED_LOG_LEVEL_EMERGENCY: + return LOG_EMERG; + + default: + return LOG_ERR; + } +} + +void +mongoc_structured_log_default_handler (mongoc_structured_log_entry_t *entry, void *user_data) +{ + char *message = bson_as_json (mongoc_structured_log_entry_get_context (entry), NULL); + + syslog (_mongoc_structured_log_convert_level (mongoc_structured_log_entry_get_level (entry)), "%s", message); + + bson_free (message); +} diff --git a/src/libmongoc/src/mongoc/mongoc-structured-log.h b/src/libmongoc/src/mongoc/mongoc-structured-log.h index 6df2c08ed28..eff0eae7920 100644 --- a/src/libmongoc/src/mongoc/mongoc-structured-log.h +++ b/src/libmongoc/src/mongoc/mongoc-structured-log.h @@ -46,21 +46,21 @@ typedef enum { typedef struct _mongoc_structured_log_entry_t mongoc_structured_log_entry_t; -typedef void (*mongoc_structured_log_func_t) (mongoc_structured_log_level_t level, - mongoc_structured_log_component_t component, - const char *message, - bson_t *context, - void *user_data); +typedef void (*mongoc_structured_log_func_t) (mongoc_structured_log_entry_t *entry, + void *user_data); + MONGOC_EXPORT (void) mongoc_structured_log_set_handler (mongoc_structured_log_func_t log_func, void *user_data); +MONGOC_EXPORT (mongoc_structured_log_level_t) +mongoc_structured_log_entry_get_level (mongoc_structured_log_entry_t *entry); + +MONGOC_EXPORT (mongoc_structured_log_component_t) +mongoc_structured_log_entry_get_component (mongoc_structured_log_entry_t *entry); + MONGOC_EXPORT (void) -mongoc_log_structured (mongoc_structured_log_level_t level, - mongoc_structured_log_component_t component, - const char *message, - ...); +mongoc_structured_log_default_handler (mongoc_structured_log_entry_t *entry, void *user_data); BSON_END_DECLS - #endif /* MONGOC_STRUCTURED_LOG_H */ From 826707f1d0a808c5aa874b767da164d302dc7d62 Mon Sep 17 00:00:00 2001 From: Andreas Braun Date: Fri, 11 Sep 2020 14:16:47 +0200 Subject: [PATCH 003/139] Remove syslog and log to stderr --- .../src/mongoc/mongoc-structured-log.c | 42 +++---------------- 1 file changed, 5 insertions(+), 37 deletions(-) diff --git a/src/libmongoc/src/mongoc/mongoc-structured-log.c b/src/libmongoc/src/mongoc/mongoc-structured-log.c index 5b2ca3e2b75..f3228dd8f07 100644 --- a/src/libmongoc/src/mongoc/mongoc-structured-log.c +++ b/src/libmongoc/src/mongoc/mongoc-structured-log.c @@ -26,7 +26,6 @@ #endif #include #include -#include #include "mongoc-structured-log.h" #include "mongoc-structured-log-private.h" @@ -191,47 +190,16 @@ mongoc_structured_log_command_started (mongoc_cmd_t *cmd, ); } -static int -_mongoc_structured_log_convert_level (mongoc_structured_log_level_t level) -{ - switch (level) - { - case MONGOC_STRUCTURED_LOG_LEVEL_TRACE: - case MONGOC_STRUCTURED_LOG_LEVEL_DEBUG: - return LOG_DEBUG; - - case MONGOC_STRUCTURED_LOG_LEVEL_INFO: - return LOG_INFO; - - case MONGOC_STRUCTURED_LOG_LEVEL_NOTICE: - return LOG_NOTICE; - - case MONGOC_STRUCTURED_LOG_LEVEL_WARNING: - return LOG_WARNING; - - case MONGOC_STRUCTURED_LOG_LEVEL_ERROR: - return LOG_ERR; - - case MONGOC_STRUCTURED_LOG_LEVEL_CRITICAL: - return LOG_CRIT; - - case MONGOC_STRUCTURED_LOG_LEVEL_ALERT: - return LOG_ALERT; - - case MONGOC_STRUCTURED_LOG_LEVEL_EMERGENCY: - return LOG_EMERG; - - default: - return LOG_ERR; - } -} - void mongoc_structured_log_default_handler (mongoc_structured_log_entry_t *entry, void *user_data) { char *message = bson_as_json (mongoc_structured_log_entry_get_context (entry), NULL); - syslog (_mongoc_structured_log_convert_level (mongoc_structured_log_entry_get_level (entry)), "%s", message); + fprintf (stderr, + "Structured log: %d, %d, %s", + mongoc_structured_log_entry_get_level (entry), + mongoc_structured_log_entry_get_component (entry), + message); bson_free (message); } From 689671436e9f355b262210db1207852577182263 Mon Sep 17 00:00:00 2001 From: Andreas Braun Date: Fri, 11 Sep 2020 16:46:19 +0200 Subject: [PATCH 004/139] Don't clear context_data --- src/libmongoc/src/mongoc/mongoc-structured-log.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/libmongoc/src/mongoc/mongoc-structured-log.c b/src/libmongoc/src/mongoc/mongoc-structured-log.c index f3228dd8f07..dfc849df2a0 100644 --- a/src/libmongoc/src/mongoc/mongoc-structured-log.c +++ b/src/libmongoc/src/mongoc/mongoc-structured-log.c @@ -83,8 +83,6 @@ mongoc_structured_log_entry_get_context (mongoc_structured_log_entry_t *entry) entry->build_context(entry->context, entry->context_data); } - entry->context_data = NULL; - return entry->context; } From 1518a7a2f9ab12a903265e889f98f8153ab31935 Mon Sep 17 00:00:00 2001 From: Andreas Braun Date: Tue, 15 Sep 2020 13:54:15 +0200 Subject: [PATCH 005/139] Update signatures --- src/libmongoc/src/mongoc/mongoc-structured-log.c | 6 +++--- src/libmongoc/src/mongoc/mongoc-structured-log.h | 7 +++++-- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/src/libmongoc/src/mongoc/mongoc-structured-log.c b/src/libmongoc/src/mongoc/mongoc-structured-log.c index dfc849df2a0..e96824c71da 100644 --- a/src/libmongoc/src/mongoc/mongoc-structured-log.c +++ b/src/libmongoc/src/mongoc/mongoc-structured-log.c @@ -67,7 +67,7 @@ mongoc_structured_log_entry_destroy (mongoc_structured_log_entry_t *entry) } } -bson_t* +const bson_t* mongoc_structured_log_entry_get_context (mongoc_structured_log_entry_t *entry) { if (entry->context) { @@ -87,13 +87,13 @@ mongoc_structured_log_entry_get_context (mongoc_structured_log_entry_t *entry) } mongoc_structured_log_level_t -mongoc_structured_log_entry_get_level (mongoc_structured_log_entry_t *entry) +mongoc_structured_log_entry_get_level (const mongoc_structured_log_entry_t *entry) { return entry->level; } mongoc_structured_log_component_t -mongoc_structured_log_entry_get_component (mongoc_structured_log_entry_t *entry) +mongoc_structured_log_entry_get_component (const mongoc_structured_log_entry_t *entry) { return entry->component; } diff --git a/src/libmongoc/src/mongoc/mongoc-structured-log.h b/src/libmongoc/src/mongoc/mongoc-structured-log.h index eff0eae7920..156f345a0c1 100644 --- a/src/libmongoc/src/mongoc/mongoc-structured-log.h +++ b/src/libmongoc/src/mongoc/mongoc-structured-log.h @@ -52,11 +52,14 @@ typedef void (*mongoc_structured_log_func_t) (mongoc_structured_log_entry_t *ent MONGOC_EXPORT (void) mongoc_structured_log_set_handler (mongoc_structured_log_func_t log_func, void *user_data); +MONGOC_EXPORT (const bson_t*) +mongoc_structured_log_entry_get_context (mongoc_structured_log_entry_t *entry); + MONGOC_EXPORT (mongoc_structured_log_level_t) -mongoc_structured_log_entry_get_level (mongoc_structured_log_entry_t *entry); +mongoc_structured_log_entry_get_level (const mongoc_structured_log_entry_t *entry); MONGOC_EXPORT (mongoc_structured_log_component_t) -mongoc_structured_log_entry_get_component (mongoc_structured_log_entry_t *entry); +mongoc_structured_log_entry_get_component (const mongoc_structured_log_entry_t *entry); MONGOC_EXPORT (void) mongoc_structured_log_default_handler (mongoc_structured_log_entry_t *entry, void *user_data); From a9456950aba8529436a6e3f3a316199ca0e30282 Mon Sep 17 00:00:00 2001 From: Andreas Braun Date: Tue, 15 Sep 2020 13:54:24 +0200 Subject: [PATCH 006/139] Explicitly track whether context was already built --- .../mongoc/mongoc-structured-log-private.h | 1 + .../src/mongoc/mongoc-structured-log.c | 19 +++++-------------- 2 files changed, 6 insertions(+), 14 deletions(-) diff --git a/src/libmongoc/src/mongoc/mongoc-structured-log-private.h b/src/libmongoc/src/mongoc/mongoc-structured-log-private.h index c0a16de9905..bc5fd84c090 100644 --- a/src/libmongoc/src/mongoc/mongoc-structured-log-private.h +++ b/src/libmongoc/src/mongoc/mongoc-structured-log-private.h @@ -30,6 +30,7 @@ struct _mongoc_structured_log_entry_t { mongoc_structured_log_build_context_t build_context; va_list *context_data; bson_t *context; + bool context_built; }; void diff --git a/src/libmongoc/src/mongoc/mongoc-structured-log.c b/src/libmongoc/src/mongoc/mongoc-structured-log.c index e96824c71da..a271a9e1a99 100644 --- a/src/libmongoc/src/mongoc/mongoc-structured-log.c +++ b/src/libmongoc/src/mongoc/mongoc-structured-log.c @@ -56,31 +56,22 @@ mongoc_structured_log_entry_init (mongoc_structured_log_entry_t *entry, entry->message = message; entry->build_context = build_context; entry->context_data = context_data; - entry->context = NULL; + entry->context = BCON_NEW("message", BCON_UTF8 (message)); + entry->context_built = false; } static void mongoc_structured_log_entry_destroy (mongoc_structured_log_entry_t *entry) { - if (entry->context) { - bson_free (entry->context); - } + bson_free (entry->context); } const bson_t* mongoc_structured_log_entry_get_context (mongoc_structured_log_entry_t *entry) { - if (entry->context) { - return entry->context; - } - - entry->context = BCON_NEW( - "message", - BCON_UTF8(entry->message) - ); - - if (entry->build_context) { + if (!entry->context_built && entry->build_context) { entry->build_context(entry->context, entry->context_data); + entry->context_built = true; } return entry->context; From 370b757986ad32cf7c512c200006d0c2ee7ef196 Mon Sep 17 00:00:00 2001 From: Andreas Braun Date: Tue, 15 Sep 2020 14:10:48 +0200 Subject: [PATCH 007/139] Fix coding style --- .../src/mongoc/mongoc-structured-log.c | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/libmongoc/src/mongoc/mongoc-structured-log.c b/src/libmongoc/src/mongoc/mongoc-structured-log.c index a271a9e1a99..c560a1f8344 100644 --- a/src/libmongoc/src/mongoc/mongoc-structured-log.c +++ b/src/libmongoc/src/mongoc/mongoc-structured-log.c @@ -56,7 +56,7 @@ mongoc_structured_log_entry_init (mongoc_structured_log_entry_t *entry, entry->message = message; entry->build_context = build_context; entry->context_data = context_data; - entry->context = BCON_NEW("message", BCON_UTF8 (message)); + entry->context = BCON_NEW ("message", BCON_UTF8 (message)); entry->context_built = false; } @@ -70,7 +70,7 @@ const bson_t* mongoc_structured_log_entry_get_context (mongoc_structured_log_entry_t *entry) { if (!entry->context_built && entry->build_context) { - entry->build_context(entry->context, entry->context_data); + entry->build_context (entry->context, entry->context_data); entry->context_built = true; } @@ -126,7 +126,7 @@ mongoc_structured_log (mongoc_structured_log_level_t level, } static void -mongoc_log_structured_build_command_context(bson_t *context, va_list *context_data) +mongoc_log_structured_build_command_context (bson_t *context, va_list *context_data) { mongoc_cmd_t *cmd = va_arg (*context_data, mongoc_cmd_t*); uint32_t request_id = va_arg (*context_data, uint32_t); @@ -134,26 +134,26 @@ mongoc_log_structured_build_command_context(bson_t *context, va_list *context_da uint32_t server_connection_id = va_arg (*context_data, uint32_t); bool explicit_session = !!va_arg (*context_data, int); - char* cmd_json = bson_as_canonical_extended_json(cmd->command, NULL); + char* cmd_json = bson_as_canonical_extended_json (cmd->command, NULL); - BCON_APPEND( + BCON_APPEND ( context, "command", - BCON_UTF8(cmd_json), + BCON_UTF8 (cmd_json), "databaseName", - BCON_UTF8(cmd->db_name), + BCON_UTF8 (cmd->db_name), "commandName", - BCON_UTF8(cmd->command_name), + BCON_UTF8 (cmd->command_name), "requestId", BCON_INT32 (request_id), "operationId", - BCON_INT64(cmd->operation_id), + BCON_INT64 (cmd->operation_id), "driverConnectionId", BCON_INT32 (driver_connection_id), "serverConnectionId", BCON_INT32 (server_connection_id), "explicitSession", - BCON_BOOL(explicit_session) + BCON_BOOL (explicit_session) ); bson_free (cmd_json); From 79e52cbbdb59269bea5c53cc9eb3cb056bf02109 Mon Sep 17 00:00:00 2001 From: Andreas Braun Date: Thu, 17 Sep 2020 11:02:58 +0200 Subject: [PATCH 008/139] Extract command logging and add structs --- src/libmongoc/CMakeLists.txt | 1 + src/libmongoc/src/mongoc/mongoc-cluster.c | 2 +- .../mongoc-structured-log-command-private.h | 51 +++++ .../mongoc/mongoc-structured-log-command.c | 179 ++++++++++++++++++ .../mongoc/mongoc-structured-log-private.h | 25 +-- .../src/mongoc/mongoc-structured-log.c | 104 ++-------- 6 files changed, 261 insertions(+), 101 deletions(-) create mode 100644 src/libmongoc/src/mongoc/mongoc-structured-log-command-private.h create mode 100644 src/libmongoc/src/mongoc/mongoc-structured-log-command.c diff --git a/src/libmongoc/CMakeLists.txt b/src/libmongoc/CMakeLists.txt index cfabf2901b1..a9b23b12a39 100644 --- a/src/libmongoc/CMakeLists.txt +++ b/src/libmongoc/CMakeLists.txt @@ -648,6 +648,7 @@ set (MONGOC_SOURCES ${PROJECT_SOURCE_DIR}/src/mongoc/mongoc-stream-gridfs-upload.c ${PROJECT_SOURCE_DIR}/src/mongoc/mongoc-stream-socket.c ${PROJECT_SOURCE_DIR}/src/mongoc/mongoc-structured-log.c + ${PROJECT_SOURCE_DIR}/src/mongoc/mongoc-structured-log-command.c ${PROJECT_SOURCE_DIR}/src/mongoc/mongoc-timeout.c ${PROJECT_SOURCE_DIR}/src/mongoc/mongoc-topology.c ${PROJECT_SOURCE_DIR}/src/mongoc/mongoc-topology-background-monitoring.c diff --git a/src/libmongoc/src/mongoc/mongoc-cluster.c b/src/libmongoc/src/mongoc/mongoc-cluster.c index 52b0c67ce15..97abf5c41f9 100644 --- a/src/libmongoc/src/mongoc/mongoc-cluster.c +++ b/src/libmongoc/src/mongoc/mongoc-cluster.c @@ -56,7 +56,7 @@ #include "mongoc-handshake-private.h" #include "mongoc-cluster-aws-private.h" #include "mongoc-error-private.h" -#include "mongoc-structured-log-private.h" +#include "mongoc-structured-log-command-private.h" #include #include diff --git a/src/libmongoc/src/mongoc/mongoc-structured-log-command-private.h b/src/libmongoc/src/mongoc/mongoc-structured-log-command-private.h new file mode 100644 index 00000000000..387f2ce7ad8 --- /dev/null +++ b/src/libmongoc/src/mongoc/mongoc-structured-log-command-private.h @@ -0,0 +1,51 @@ +/* + * Copyright 2015 MongoDB, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "mongoc-prelude.h" +#include "mongoc-structured-log.h" +#include "mongoc-cmd-private.h" + +#ifndef MONGOC_STRUCTRURED_LOG_COMMAND_PRIVATE_H +#define MONGOC_STRUCTRURED_LOG_COMMAND_PRIVATE_H + +typedef struct { + const mongoc_cmd_t *cmd; + const bson_t *reply; + const bson_error_t *error; + int64_t duration; + uint32_t request_id; + uint32_t driver_connection_id; + uint32_t server_connection_id; + bool explicit_session; +} _mongoc_structured_log_command_t; + +void +mongoc_structured_log_command_started (const mongoc_cmd_t *cmd, + uint32_t request_id, + uint32_t driver_connection_id, + uint32_t server_connection_id, + bool explicit_session); + +void +mongoc_structured_log_command_success (const mongoc_cmd_t *cmd, + const bson_t *reply, + uint64_t duration, + uint32_t request_id, + uint32_t driver_connection_id, + uint32_t server_connection_id, + bool explicit_session); + +#endif /* MONGOC_STRUCTURED_LOG_COMMAND_PRIVATE_H */ diff --git a/src/libmongoc/src/mongoc/mongoc-structured-log-command.c b/src/libmongoc/src/mongoc/mongoc-structured-log-command.c new file mode 100644 index 00000000000..9512ff87cf8 --- /dev/null +++ b/src/libmongoc/src/mongoc/mongoc-structured-log-command.c @@ -0,0 +1,179 @@ +/* + * Copyright 2013 MongoDB, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +#if defined(__linux__) +#include +#elif defined(_WIN32) +#include +#elif defined(__FreeBSD__) +#include +#else +#include +#endif +#include + +#include "mongoc-structured-log.h" +#include "mongoc-structured-log-private.h" +#include "mongoc-structured-log-command-private.h" + +static void +mongoc_log_structured_build_command_started_message (mongoc_structured_log_entry_t *entry) +{ + char* cmd_json; + _mongoc_structured_log_command_t *log_command = entry->command; + + BSON_ASSERT (entry->component == MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND); + + cmd_json = bson_as_canonical_extended_json (log_command->cmd->command, NULL); + + BCON_APPEND ( + entry->structured_message, + "command", + BCON_UTF8 (cmd_json), + "databaseName", + BCON_UTF8 (log_command->cmd->db_name), + "commandName", + BCON_UTF8 (log_command->cmd->command_name), + "requestId", + BCON_INT32 (log_command->request_id), + "operationId", + BCON_INT64 (log_command->cmd->operation_id), + "driverConnectionId", + BCON_INT32 (log_command->driver_connection_id), + "serverConnectionId", + BCON_INT32 (log_command->server_connection_id), + "explicitSession", + BCON_BOOL (log_command->explicit_session) + ); + + bson_free (cmd_json); +} + +static void +mongoc_log_structured_build_command_succeeded_message (mongoc_structured_log_entry_t *entry) +{ + char* reply_json; + _mongoc_structured_log_command_t *log_command = entry->command; + + BSON_ASSERT (entry->component == MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND); + + reply_json = bson_as_canonical_extended_json (log_command->reply, NULL); + + BCON_APPEND ( + entry->structured_message, + "reply", + BCON_UTF8 (reply_json), + "duration", + BCON_INT64 (log_command->duration), + "databaseName", + BCON_UTF8 (log_command->cmd->db_name), + "commandName", + BCON_UTF8 (log_command->cmd->command_name), + "requestId", + BCON_INT32 (log_command->request_id), + "operationId", + BCON_INT64 (log_command->cmd->operation_id), + "driverConnectionId", + BCON_INT32 (log_command->driver_connection_id), + "serverConnectionId", + BCON_INT32 (log_command->server_connection_id), + "explicitSession", + BCON_BOOL (log_command->explicit_session) + ); + + bson_free (reply_json); +} + +void +mongoc_structured_log_command_started (const mongoc_cmd_t *cmd, + uint32_t request_id, + uint32_t driver_connection_id, + uint32_t server_connection_id, + bool explicit_session) +{ + _mongoc_structured_log_command_t command_log = { + .cmd = cmd, + .request_id = request_id, + .driver_connection_id = driver_connection_id, + .server_connection_id = server_connection_id, + .explicit_session = explicit_session, + }; + + mongoc_structured_log ( + MONGOC_STRUCTURED_LOG_LEVEL_INFO, + MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND, + "Command started", + mongoc_log_structured_build_command_started_message, + &command_log + ); +} + +void +mongoc_structured_log_command_success (const mongoc_cmd_t *cmd, + const bson_t *reply, + uint64_t duration, + uint32_t request_id, + uint32_t driver_connection_id, + uint32_t server_connection_id, + bool explicit_session) +{ + _mongoc_structured_log_command_t command_log = { + .cmd = cmd, + .reply = reply, + .duration = duration, + .request_id = request_id, + .driver_connection_id = driver_connection_id, + .server_connection_id = server_connection_id, + .explicit_session = explicit_session, + }; + + mongoc_structured_log ( + MONGOC_STRUCTURED_LOG_LEVEL_INFO, + MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND, + "Command succeeded", + mongoc_log_structured_build_command_succeeded_message, + &command_log + ); +} + +void +mongoc_structured_log_command_failure (const mongoc_cmd_t *cmd, + const bson_t *reply, + const bson_error_t *error, + uint32_t request_id, + uint32_t driver_connection_id, + uint32_t server_connection_id, + bool explicit_session) +{ + _mongoc_structured_log_command_t command_log = { + .cmd = cmd, + .reply = reply, + .error = error, + .request_id = request_id, + .driver_connection_id = driver_connection_id, + .server_connection_id = server_connection_id, + .explicit_session = explicit_session, + }; + + mongoc_structured_log ( + MONGOC_STRUCTURED_LOG_LEVEL_INFO, + MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND, + "Command succeeded", + mongoc_log_structured_build_command_succeeded_message, + &command_log + ); +} diff --git a/src/libmongoc/src/mongoc/mongoc-structured-log-private.h b/src/libmongoc/src/mongoc/mongoc-structured-log-private.h index bc5fd84c090..3a292be4638 100644 --- a/src/libmongoc/src/mongoc/mongoc-structured-log-private.h +++ b/src/libmongoc/src/mongoc/mongoc-structured-log-private.h @@ -17,36 +17,29 @@ #include "mongoc-prelude.h" #include "mongoc-structured-log.h" #include "mongoc-cmd-private.h" +#include "mongoc-structured-log-command-private.h" #ifndef MONGOC_STRUCTRURED_LOG_PRIVATE_H #define MONGOC_STRUCTRURED_LOG_PRIVATE_H -typedef void (*mongoc_structured_log_build_context_t) (bson_t *context, va_list *context_data); +typedef void (*mongoc_structured_log_build_message_t) (mongoc_structured_log_entry_t *entry); struct _mongoc_structured_log_entry_t { mongoc_structured_log_level_t level; mongoc_structured_log_component_t component; const char* message; - mongoc_structured_log_build_context_t build_context; - va_list *context_data; - bson_t *context; - bool context_built; + bson_t *structured_message; + mongoc_structured_log_build_message_t build_message_func; + union { + _mongoc_structured_log_command_t *command; + }; }; void mongoc_structured_log (mongoc_structured_log_level_t level, mongoc_structured_log_component_t component, const char *message, - mongoc_structured_log_build_context_t build_context, - ...); - -void -mongoc_structured_log_command_started (mongoc_cmd_t *cmd, - uint32_t request_id, - uint32_t driver_connection_id, - uint32_t server_connection_id, - bool explicit_session); - -#define MONGOC_STRUCTURED_LOG_COMMAND_STARTED() + mongoc_structured_log_build_message_t build_message_func, + void *structured_message_data); #endif /* MONGOC_STRUCTURED_LOG_PRIVATE_H */ diff --git a/src/libmongoc/src/mongoc/mongoc-structured-log.c b/src/libmongoc/src/mongoc/mongoc-structured-log.c index c560a1f8344..ca7c7d99a8e 100644 --- a/src/libmongoc/src/mongoc/mongoc-structured-log.c +++ b/src/libmongoc/src/mongoc/mongoc-structured-log.c @@ -43,38 +43,26 @@ static BSON_ONCE_FUN (_mongoc_ensure_mutex_once) BSON_ONCE_RETURN; } -static void -mongoc_structured_log_entry_init (mongoc_structured_log_entry_t *entry, - mongoc_structured_log_level_t level, - mongoc_structured_log_component_t component, - const char *message, - mongoc_structured_log_build_context_t build_context, - va_list *context_data) -{ - entry->level = level; - entry->component = component; - entry->message = message; - entry->build_context = build_context; - entry->context_data = context_data; - entry->context = BCON_NEW ("message", BCON_UTF8 (message)); - entry->context_built = false; -} - static void mongoc_structured_log_entry_destroy (mongoc_structured_log_entry_t *entry) { - bson_free (entry->context); + if (entry->structured_message) { + bson_free (entry->structured_message); + } } const bson_t* mongoc_structured_log_entry_get_context (mongoc_structured_log_entry_t *entry) { - if (!entry->context_built && entry->build_context) { - entry->build_context (entry->context, entry->context_data); - entry->context_built = true; + if (!entry->structured_message) { + entry->structured_message = BCON_NEW ("message", BCON_UTF8 (entry->message)); + + if (entry->build_message_func) { + entry->build_message_func (entry); + } } - return entry->context; + return entry->structured_message; } mongoc_structured_log_level_t @@ -104,79 +92,27 @@ void mongoc_structured_log (mongoc_structured_log_level_t level, mongoc_structured_log_component_t component, const char *message, - mongoc_structured_log_build_context_t build_context, - ...) + mongoc_structured_log_build_message_t build_message_func, + void *structured_message_data) { - va_list context_data; - mongoc_structured_log_entry_t entry; + mongoc_structured_log_entry_t entry = { + level, + component, + message, + NULL, + build_message_func, + structured_message_data, + }; if (!gStructuredLogger) { return; } - va_start (context_data, build_context); - mongoc_structured_log_entry_init (&entry, level, component, message, build_context, &context_data); - bson_mutex_lock (&gStructuredLogMutex); gStructuredLogger (&entry, gStructuredLoggerData); bson_mutex_unlock (&gStructuredLogMutex); mongoc_structured_log_entry_destroy (&entry); - va_end (context_data); -} - -static void -mongoc_log_structured_build_command_context (bson_t *context, va_list *context_data) -{ - mongoc_cmd_t *cmd = va_arg (*context_data, mongoc_cmd_t*); - uint32_t request_id = va_arg (*context_data, uint32_t); - uint32_t driver_connection_id = va_arg (*context_data, uint32_t); - uint32_t server_connection_id = va_arg (*context_data, uint32_t); - bool explicit_session = !!va_arg (*context_data, int); - - char* cmd_json = bson_as_canonical_extended_json (cmd->command, NULL); - - BCON_APPEND ( - context, - "command", - BCON_UTF8 (cmd_json), - "databaseName", - BCON_UTF8 (cmd->db_name), - "commandName", - BCON_UTF8 (cmd->command_name), - "requestId", - BCON_INT32 (request_id), - "operationId", - BCON_INT64 (cmd->operation_id), - "driverConnectionId", - BCON_INT32 (driver_connection_id), - "serverConnectionId", - BCON_INT32 (server_connection_id), - "explicitSession", - BCON_BOOL (explicit_session) - ); - - bson_free (cmd_json); -} - -void -mongoc_structured_log_command_started (mongoc_cmd_t *cmd, - uint32_t request_id, - uint32_t driver_connection_id, - uint32_t server_connection_id, - bool explicit_session) -{ - mongoc_structured_log ( - MONGOC_STRUCTURED_LOG_LEVEL_INFO, - MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND, - "Command started", - mongoc_log_structured_build_command_context, - cmd, - request_id, - driver_connection_id, - server_connection_id, - explicit_session - ); } void From de7b41e7e873f504d872f909736a27d9adc53cd2 Mon Sep 17 00:00:00 2001 From: Andreas Braun Date: Thu, 17 Sep 2020 14:53:23 +0200 Subject: [PATCH 009/139] Refactor signatures and log all command results --- src/libmongoc/src/mongoc/mongoc-client.c | 50 ++++++-- src/libmongoc/src/mongoc/mongoc-cluster.c | 87 +++++++++----- .../src/mongoc/mongoc-cursor-legacy.c | 16 +-- src/libmongoc/src/mongoc/mongoc-cursor.c | 49 ++++++-- .../mongoc-structured-log-command-private.h | 23 +++- .../mongoc/mongoc-structured-log-command.c | 109 ++++++++++++------ .../src/mongoc/mongoc-structured-log.c | 2 +- 7 files changed, 241 insertions(+), 95 deletions(-) diff --git a/src/libmongoc/src/mongoc/mongoc-client.c b/src/libmongoc/src/mongoc/mongoc-client.c index 80f15f83055..e6dc1e09a81 100644 --- a/src/libmongoc/src/mongoc/mongoc-client.c +++ b/src/libmongoc/src/mongoc/mongoc-client.c @@ -63,6 +63,7 @@ #include "mongoc-ssl-private.h" #include "mongoc-cmd-private.h" #include "mongoc-opts-private.h" +#include "mongoc-structured-log-command-private.h" #endif #if defined(MONGOC_ENABLE_SSL_OPENSSL) && OPENSSL_VERSION_NUMBER >= 0x10100000L @@ -2159,12 +2160,17 @@ _mongoc_client_monitor_op_killcursors (mongoc_cluster_t *cluster, client = cluster->client; + bson_init (&doc); + _mongoc_client_prepare_killcursors_command (cursor_id, collection, &doc); + + // @todo Provide missing arguments + mongoc_structured_log_command_started (&doc, "killCursors", db, operation_id, cluster->request_id, 0, 0, false); + if (!client->apm_callbacks.started) { + bson_destroy (&doc); return; } - bson_init (&doc); - _mongoc_client_prepare_killcursors_command (cursor_id, collection, &doc); mongoc_apm_command_started_init (&event, &doc, db, @@ -2203,10 +2209,6 @@ _mongoc_client_monitor_op_killcursors_succeeded (mongoc_cluster_t *cluster, client = cluster->client; - if (!client->apm_callbacks.succeeded) { - EXIT; - } - /* fake server reply to killCursors command: {ok: 1, cursorsUnknown: [42]} */ bson_init (&doc); bson_append_int32 (&doc, "ok", 2, 1); @@ -2214,6 +2216,22 @@ _mongoc_client_monitor_op_killcursors_succeeded (mongoc_cluster_t *cluster, bson_array_builder_append_int64 (cursors_unknown, cursor_id); bson_append_array_builder_end (&doc, cursors_unknown); + // @todo Provide missing arguments + mongoc_structured_log_command_success ( + "killCursors", + operation_id, + &doc, + duration, + cluster->request_id, + 0, + 0, + false); + + if (!client->apm_callbacks.succeeded) { + bson_destroy (&doc); + EXIT; + } + mongoc_apm_command_succeeded_init (&event, duration, &doc, @@ -2251,14 +2269,26 @@ _mongoc_client_monitor_op_killcursors_failed (mongoc_cluster_t *cluster, client = cluster->client; - if (!client->apm_callbacks.failed) { - EXIT; - } - /* fake server reply to killCursors command: {ok: 0} */ bson_init (&doc); bson_append_int32 (&doc, "ok", 2, 0); + // @todo Provide missing arguments + mongoc_structured_log_command_failure ( + "killCursors", + operation_id, + &doc, + error, + cluster->request_id, + 0, + 0, + false); + + if (!client->apm_callbacks.failed) { + bson_destroy (&doc); + EXIT; + } + mongoc_apm_command_failed_init (&event, duration, "killCursors", diff --git a/src/libmongoc/src/mongoc/mongoc-cluster.c b/src/libmongoc/src/mongoc/mongoc-cluster.c index 97abf5c41f9..c9ae79dffe3 100644 --- a/src/libmongoc/src/mongoc/mongoc-cluster.c +++ b/src/libmongoc/src/mongoc/mongoc-cluster.c @@ -533,8 +533,8 @@ mongoc_cluster_run_command_monitored (mongoc_cluster_t *cluster, mongoc_cmd_t *c } } - // @todo Provide explicit session and connection IDs - mongoc_structured_log_command_started (cmd, request_id, 0, 0, false); + // @todo Provide missing arguments + mongoc_structured_log_command_started (cmd->command, cmd->command_name, cmd->db_name, cmd->operation_id, request_id, 0, 0, false); if (callbacks->started) { mongoc_apm_command_started_init_with_cmd ( @@ -546,8 +546,10 @@ mongoc_cluster_run_command_monitored (mongoc_cluster_t *cluster, mongoc_cmd_t *c retval = mongoc_cluster_run_opmsg (cluster, cmd, reply, error); - if (retval && callbacks->succeeded) { + if (retval) { bson_t fake_reply = BSON_INITIALIZER; + int64_t duration = bson_get_monotonic_time() - started; + /* * Unacknowledged writes must provide a CommandSucceededEvent with an * {ok: 1} reply. @@ -556,42 +558,71 @@ mongoc_cluster_run_command_monitored (mongoc_cluster_t *cluster, mongoc_cmd_t *c if (!cmd->is_acknowledged) { bson_append_int32 (&fake_reply, "ok", 2, 1); } - mongoc_apm_command_succeeded_init (&succeeded_event, - bson_get_monotonic_time () - started, - cmd->is_acknowledged ? reply : &fake_reply, + + // @todo Provide missing arguments + mongoc_structured_log_command_success ( + cmd->command_name, + cmd->operation_id, + cmd->is_acknowledged ? reply : &fake_reply, + duration, + request_id, + 0, + 0, + false); + + if (callbacks->succeeded) { + mongoc_apm_command_succeeded_init (&succeeded_event, + duration, + cmd->is_acknowledged ? reply : &fake_reply, + cmd->command_name, + cmd->db_name, + request_id, + cmd->operation_id, + &server_stream->sd->host, + server_id, + &server_stream->sd->service_id, + server_stream->sd->server_connection_id, + is_redacted, + cluster->client->apm_context); + + callbacks->succeeded (&succeeded_event); + mongoc_apm_command_succeeded_cleanup (&succeeded_event); + } + + bson_destroy (&fake_reply); + } else { + int64_t duration = bson_get_monotonic_time() - started; + + // @todo Provide missing arguments + mongoc_structured_log_command_failure ( + cmd->command_name, + cmd->operation_id, + reply, + error, + cluster->request_id, + 0, + 0, + false); + + if (callbacks->failed) { + mongoc_apm_command_failed_init (&failed_event, + duration, cmd->command_name, cmd->db_name, + error, + reply, request_id, cmd->operation_id, &server_stream->sd->host, server_id, &server_stream->sd->service_id, - server_stream->sd->server_connection_id, - is_redacted, - cluster->client->apm_context); - - callbacks->succeeded (&succeeded_event); - mongoc_apm_command_succeeded_cleanup (&succeeded_event); - bson_destroy (&fake_reply); - } - if (!retval && callbacks->failed) { - mongoc_apm_command_failed_init (&failed_event, - bson_get_monotonic_time () - started, - cmd->command_name, - cmd->db_name, - error, - reply, - request_id, - cmd->operation_id, - &server_stream->sd->host, - server_id, - &server_stream->sd->service_id, server_stream->sd->server_connection_id, is_redacted, cluster->client->apm_context); - callbacks->failed (&failed_event); - mongoc_apm_command_failed_cleanup (&failed_event); + callbacks->failed (&failed_event); + mongoc_apm_command_failed_cleanup (&failed_event); + } } if (retval && _mongoc_cse_is_enabled (cluster->client)) { diff --git a/src/libmongoc/src/mongoc/mongoc-cursor-legacy.c b/src/libmongoc/src/mongoc/mongoc-cursor-legacy.c index 818cb09f630..7cb3b690168 100644 --- a/src/libmongoc/src/mongoc/mongoc-cursor-legacy.c +++ b/src/libmongoc/src/mongoc/mongoc-cursor-legacy.c @@ -32,6 +32,7 @@ #include "mongoc-write-concern-private.h" #include "mongoc-read-prefs-private.h" #include "mongoc-rpc-private.h" +#include "mongoc-structured-log-command-private.h" #include @@ -46,14 +47,19 @@ _mongoc_cursor_monitor_legacy_get_more (mongoc_cursor_t *cursor, mongoc_server_s ENTRY; client = cursor->client; + _mongoc_cursor_prepare_getmore_command (cursor, &doc); + db = bson_strndup (cursor->ns, cursor->dblen); + + // @todo Provide missing arguments + mongoc_structured_log_command_started (&doc, "getMore", db, cursor->operation_id, client->cluster.request_id, 0, 0, false); + if (!client->apm_callbacks.started) { /* successful */ + bson_destroy (&doc); + bson_free (db); RETURN (true); } - _mongoc_cursor_prepare_getmore_command (cursor, &doc); - - db = bson_strndup (cursor->ns, cursor->dblen); mongoc_apm_command_started_init (&event, &doc, db, @@ -89,10 +95,6 @@ _mongoc_cursor_monitor_legacy_query (mongoc_cursor_t *cursor, ENTRY; client = cursor->client; - if (!client->apm_callbacks.started) { - /* successful */ - RETURN (true); - } bson_init (&doc); db = bson_strndup (cursor->ns, cursor->dblen); diff --git a/src/libmongoc/src/mongoc/mongoc-cursor.c b/src/libmongoc/src/mongoc/mongoc-cursor.c index 948194b50b6..7500ab1762d 100644 --- a/src/libmongoc/src/mongoc/mongoc-cursor.c +++ b/src/libmongoc/src/mongoc/mongoc-cursor.c @@ -29,6 +29,7 @@ #include "mongoc-write-concern-private.h" #include "mongoc-read-prefs-private.h" #include "mongoc-aggregate-private.h" +#include "mongoc-structured-log-command-private.h" #include #include @@ -644,13 +645,17 @@ _mongoc_cursor_monitor_command (mongoc_cursor_t *cursor, ENTRY; client = cursor->client; + db = bson_strndup (cursor->ns, cursor->dblen); + + // @todo Provide missing arguments + mongoc_structured_log_command_started (cmd, cmd_name, db, cursor->operation_id, client->cluster.request_id, 0, 0, false); + if (!client->apm_callbacks.started) { /* successful */ + bson_free (db); RETURN (true); } - db = bson_strndup (cursor->ns, cursor->dblen); - mongoc_apm_command_started_init (&event, cmd, db, @@ -710,10 +715,6 @@ _mongoc_cursor_monitor_succeeded (mongoc_cursor_t *cursor, client = cursor->client; - if (!client->apm_callbacks.succeeded) { - EXIT; - } - /* we sent OP_QUERY/OP_GETMORE, fake a reply to find/getMore command: * {ok: 1, cursor: {id: 17, ns: "...", first/nextBatch: [ ... docs ... ]}} */ @@ -730,6 +731,22 @@ _mongoc_cursor_monitor_succeeded (mongoc_cursor_t *cursor, bson_destroy (&docs_array); + // @todo Provide missing arguments + mongoc_structured_log_command_success ( + cmd_name, + cursor->operation_id, + &reply, + duration, + client->cluster.request_id, + 0, + 0, + false); + + if (!client->apm_callbacks.succeeded) { + bson_destroy (&reply); + EXIT; + } + mongoc_apm_command_succeeded_init (&event, duration, &reply, @@ -767,16 +784,28 @@ _mongoc_cursor_monitor_failed (mongoc_cursor_t *cursor, client = cursor->client; - if (!client->apm_callbacks.failed) { - EXIT; - } - /* we sent OP_QUERY/OP_GETMORE, fake a reply to find/getMore command: * {ok: 0} */ bsonBuildDecl (reply, kv ("ok", int32 (0))); char *db = bson_strndup (cursor->ns, cursor->dblen); + // @todo Provide missing arguments + mongoc_structured_log_command_failure ( + cmd_name, + cursor->operation_id, + &reply, + &cursor->error, + client->cluster.request_id, + 0, + 0, + false); + + if (!client->apm_callbacks.failed) { + bson_destroy (&reply); + EXIT; + } + mongoc_apm_command_failed_init (&event, duration, cmd_name, diff --git a/src/libmongoc/src/mongoc/mongoc-structured-log-command-private.h b/src/libmongoc/src/mongoc/mongoc-structured-log-command-private.h index 387f2ce7ad8..c47ee751519 100644 --- a/src/libmongoc/src/mongoc/mongoc-structured-log-command-private.h +++ b/src/libmongoc/src/mongoc/mongoc-structured-log-command-private.h @@ -22,10 +22,13 @@ #define MONGOC_STRUCTRURED_LOG_COMMAND_PRIVATE_H typedef struct { - const mongoc_cmd_t *cmd; + const char *command_name; + const char *db_name; + const bson_t *command; const bson_t *reply; const bson_error_t *error; int64_t duration; + int64_t operation_id; uint32_t request_id; uint32_t driver_connection_id; uint32_t server_connection_id; @@ -33,14 +36,18 @@ typedef struct { } _mongoc_structured_log_command_t; void -mongoc_structured_log_command_started (const mongoc_cmd_t *cmd, +mongoc_structured_log_command_started (const bson_t *command, + const char *command_name, + const char *db_name, + int64_t operation_id, uint32_t request_id, uint32_t driver_connection_id, uint32_t server_connection_id, bool explicit_session); void -mongoc_structured_log_command_success (const mongoc_cmd_t *cmd, +mongoc_structured_log_command_success (const char *command_name, + int64_t operation_id, const bson_t *reply, uint64_t duration, uint32_t request_id, @@ -48,4 +55,14 @@ mongoc_structured_log_command_success (const mongoc_cmd_t *cmd, uint32_t server_connection_id, bool explicit_session); +void +mongoc_structured_log_command_failure (const char *command_name, + int64_t operation_id, + const bson_t *reply, + const bson_error_t *error, + uint32_t request_id, + uint32_t driver_connection_id, + uint32_t server_connection_id, + bool explicit_session); + #endif /* MONGOC_STRUCTURED_LOG_COMMAND_PRIVATE_H */ diff --git a/src/libmongoc/src/mongoc/mongoc-structured-log-command.c b/src/libmongoc/src/mongoc/mongoc-structured-log-command.c index 9512ff87cf8..46887aab064 100644 --- a/src/libmongoc/src/mongoc/mongoc-structured-log-command.c +++ b/src/libmongoc/src/mongoc/mongoc-structured-log-command.c @@ -31,27 +31,18 @@ #include "mongoc-structured-log-command-private.h" static void -mongoc_log_structured_build_command_started_message (mongoc_structured_log_entry_t *entry) +_mongoc_log_structured_append_command_data (mongoc_structured_log_entry_t *entry) { - char* cmd_json; _mongoc_structured_log_command_t *log_command = entry->command; - BSON_ASSERT (entry->component == MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND); - - cmd_json = bson_as_canonical_extended_json (log_command->cmd->command, NULL); - BCON_APPEND ( entry->structured_message, - "command", - BCON_UTF8 (cmd_json), - "databaseName", - BCON_UTF8 (log_command->cmd->db_name), "commandName", - BCON_UTF8 (log_command->cmd->command_name), + BCON_UTF8 (log_command->command_name), "requestId", BCON_INT32 (log_command->request_id), "operationId", - BCON_INT64 (log_command->cmd->operation_id), + BCON_INT64 (log_command->operation_id), "driverConnectionId", BCON_INT32 (log_command->driver_connection_id), "serverConnectionId", @@ -59,6 +50,27 @@ mongoc_log_structured_build_command_started_message (mongoc_structured_log_entry "explicitSession", BCON_BOOL (log_command->explicit_session) ); +} + +static void +mongoc_log_structured_build_command_started_message (mongoc_structured_log_entry_t *entry) +{ + char* cmd_json; + _mongoc_structured_log_command_t *log_command = entry->command; + + BSON_ASSERT (entry->component == MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND); + + cmd_json = bson_as_canonical_extended_json (log_command->command, NULL); + + _mongoc_log_structured_append_command_data (entry); + + BCON_APPEND ( + entry->structured_message, + "databaseName", + BCON_UTF8 (log_command->db_name), + "command", + BCON_UTF8 (cmd_json) + ); bson_free (cmd_json); } @@ -73,40 +85,61 @@ mongoc_log_structured_build_command_succeeded_message (mongoc_structured_log_ent reply_json = bson_as_canonical_extended_json (log_command->reply, NULL); + _mongoc_log_structured_append_command_data (entry); + BCON_APPEND ( entry->structured_message, - "reply", - BCON_UTF8 (reply_json), "duration", BCON_INT64 (log_command->duration), - "databaseName", - BCON_UTF8 (log_command->cmd->db_name), - "commandName", - BCON_UTF8 (log_command->cmd->command_name), - "requestId", - BCON_INT32 (log_command->request_id), - "operationId", - BCON_INT64 (log_command->cmd->operation_id), - "driverConnectionId", - BCON_INT32 (log_command->driver_connection_id), - "serverConnectionId", - BCON_INT32 (log_command->server_connection_id), - "explicitSession", - BCON_BOOL (log_command->explicit_session) - ); + "reply", + BCON_UTF8 (reply_json) + ); + + bson_free (reply_json); +} + +static void +mongoc_log_structured_build_command_failed_message (mongoc_structured_log_entry_t *entry) +{ + char* reply_json; + _mongoc_structured_log_command_t *log_command = entry->command; + + BSON_ASSERT (entry->component == MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND); + + reply_json = bson_as_canonical_extended_json (log_command->reply, NULL); + + _mongoc_log_structured_append_command_data (entry); + + BCON_APPEND ( + entry->structured_message, + "reply", + BCON_UTF8 (reply_json)); + + if (log_command->error) { + BCON_APPEND ( + entry->structured_message, + "failure", + BCON_UTF8 (log_command->error->message)); + } bson_free (reply_json); } void -mongoc_structured_log_command_started (const mongoc_cmd_t *cmd, +mongoc_structured_log_command_started (const bson_t *command, + const char *command_name, + const char *db_name, + int64_t operation_id, uint32_t request_id, uint32_t driver_connection_id, uint32_t server_connection_id, bool explicit_session) { _mongoc_structured_log_command_t command_log = { - .cmd = cmd, + .command = command, + .command_name = command_name, + .db_name = db_name, + .operation_id = operation_id, .request_id = request_id, .driver_connection_id = driver_connection_id, .server_connection_id = server_connection_id, @@ -123,7 +156,8 @@ mongoc_structured_log_command_started (const mongoc_cmd_t *cmd, } void -mongoc_structured_log_command_success (const mongoc_cmd_t *cmd, +mongoc_structured_log_command_success (const char *command_name, + int64_t operation_id, const bson_t *reply, uint64_t duration, uint32_t request_id, @@ -132,7 +166,8 @@ mongoc_structured_log_command_success (const mongoc_cmd_t *cmd, bool explicit_session) { _mongoc_structured_log_command_t command_log = { - .cmd = cmd, + .command_name = command_name, + .operation_id = operation_id, .reply = reply, .duration = duration, .request_id = request_id, @@ -151,7 +186,8 @@ mongoc_structured_log_command_success (const mongoc_cmd_t *cmd, } void -mongoc_structured_log_command_failure (const mongoc_cmd_t *cmd, +mongoc_structured_log_command_failure (const char *command_name, + int64_t operation_id, const bson_t *reply, const bson_error_t *error, uint32_t request_id, @@ -160,7 +196,8 @@ mongoc_structured_log_command_failure (const mongoc_cmd_t *cmd, bool explicit_session) { _mongoc_structured_log_command_t command_log = { - .cmd = cmd, + .command_name = command_name, + .operation_id = operation_id, .reply = reply, .error = error, .request_id = request_id, @@ -173,7 +210,7 @@ mongoc_structured_log_command_failure (const mongoc_cmd_t *cmd, MONGOC_STRUCTURED_LOG_LEVEL_INFO, MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND, "Command succeeded", - mongoc_log_structured_build_command_succeeded_message, + mongoc_log_structured_build_command_failed_message, &command_log ); } diff --git a/src/libmongoc/src/mongoc/mongoc-structured-log.c b/src/libmongoc/src/mongoc/mongoc-structured-log.c index ca7c7d99a8e..c7388f74e8a 100644 --- a/src/libmongoc/src/mongoc/mongoc-structured-log.c +++ b/src/libmongoc/src/mongoc/mongoc-structured-log.c @@ -121,7 +121,7 @@ mongoc_structured_log_default_handler (mongoc_structured_log_entry_t *entry, voi char *message = bson_as_json (mongoc_structured_log_entry_get_context (entry), NULL); fprintf (stderr, - "Structured log: %d, %d, %s", + "Structured log: %d, %d, %s\n", mongoc_structured_log_entry_get_level (entry), mongoc_structured_log_entry_get_component (entry), message); From 411502aa5984b6a9fce4b345d736975af9ed8764 Mon Sep 17 00:00:00 2001 From: Andreas Braun Date: Fri, 18 Sep 2020 09:14:12 +0200 Subject: [PATCH 010/139] Keep default logger private --- src/libmongoc/src/mongoc/mongoc-structured-log.c | 5 ++++- src/libmongoc/src/mongoc/mongoc-structured-log.h | 3 --- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/libmongoc/src/mongoc/mongoc-structured-log.c b/src/libmongoc/src/mongoc/mongoc-structured-log.c index c7388f74e8a..43b663c4cc1 100644 --- a/src/libmongoc/src/mongoc/mongoc-structured-log.c +++ b/src/libmongoc/src/mongoc/mongoc-structured-log.c @@ -31,6 +31,9 @@ #include "mongoc-structured-log-private.h" #include "mongoc-thread-private.h" +static void +mongoc_structured_log_default_handler (mongoc_structured_log_entry_t *entry, void *user_data); + static bson_once_t once = BSON_ONCE_INIT; static bson_mutex_t gStructuredLogMutex; static mongoc_structured_log_func_t gStructuredLogger = mongoc_structured_log_default_handler; @@ -115,7 +118,7 @@ mongoc_structured_log (mongoc_structured_log_level_t level, mongoc_structured_log_entry_destroy (&entry); } -void +static void mongoc_structured_log_default_handler (mongoc_structured_log_entry_t *entry, void *user_data) { char *message = bson_as_json (mongoc_structured_log_entry_get_context (entry), NULL); diff --git a/src/libmongoc/src/mongoc/mongoc-structured-log.h b/src/libmongoc/src/mongoc/mongoc-structured-log.h index 156f345a0c1..3e16fc4f85e 100644 --- a/src/libmongoc/src/mongoc/mongoc-structured-log.h +++ b/src/libmongoc/src/mongoc/mongoc-structured-log.h @@ -61,9 +61,6 @@ mongoc_structured_log_entry_get_level (const mongoc_structured_log_entry_t *entr MONGOC_EXPORT (mongoc_structured_log_component_t) mongoc_structured_log_entry_get_component (const mongoc_structured_log_entry_t *entry); -MONGOC_EXPORT (void) -mongoc_structured_log_default_handler (mongoc_structured_log_entry_t *entry, void *user_data); - BSON_END_DECLS #endif /* MONGOC_STRUCTURED_LOG_H */ From c5b85e6cc1102f43ee11c319495ebe041ca38e85 Mon Sep 17 00:00:00 2001 From: Andreas Braun Date: Fri, 18 Sep 2020 09:22:12 +0200 Subject: [PATCH 011/139] Rename context to message --- src/libmongoc/src/mongoc/mongoc-structured-log.c | 4 ++-- src/libmongoc/src/mongoc/mongoc-structured-log.h | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/libmongoc/src/mongoc/mongoc-structured-log.c b/src/libmongoc/src/mongoc/mongoc-structured-log.c index 43b663c4cc1..d6f2ca210ac 100644 --- a/src/libmongoc/src/mongoc/mongoc-structured-log.c +++ b/src/libmongoc/src/mongoc/mongoc-structured-log.c @@ -55,7 +55,7 @@ mongoc_structured_log_entry_destroy (mongoc_structured_log_entry_t *entry) } const bson_t* -mongoc_structured_log_entry_get_context (mongoc_structured_log_entry_t *entry) +mongoc_structured_log_entry_get_message (mongoc_structured_log_entry_t *entry) { if (!entry->structured_message) { entry->structured_message = BCON_NEW ("message", BCON_UTF8 (entry->message)); @@ -121,7 +121,7 @@ mongoc_structured_log (mongoc_structured_log_level_t level, static void mongoc_structured_log_default_handler (mongoc_structured_log_entry_t *entry, void *user_data) { - char *message = bson_as_json (mongoc_structured_log_entry_get_context (entry), NULL); + char *message = bson_as_json (mongoc_structured_log_entry_get_message (entry), NULL); fprintf (stderr, "Structured log: %d, %d, %s\n", diff --git a/src/libmongoc/src/mongoc/mongoc-structured-log.h b/src/libmongoc/src/mongoc/mongoc-structured-log.h index 3e16fc4f85e..5907e1cb5a7 100644 --- a/src/libmongoc/src/mongoc/mongoc-structured-log.h +++ b/src/libmongoc/src/mongoc/mongoc-structured-log.h @@ -53,7 +53,7 @@ MONGOC_EXPORT (void) mongoc_structured_log_set_handler (mongoc_structured_log_func_t log_func, void *user_data); MONGOC_EXPORT (const bson_t*) -mongoc_structured_log_entry_get_context (mongoc_structured_log_entry_t *entry); +mongoc_structured_log_entry_get_message (mongoc_structured_log_entry_t *entry); MONGOC_EXPORT (mongoc_structured_log_level_t) mongoc_structured_log_entry_get_level (const mongoc_structured_log_entry_t *entry); From 06f50fe906e3d2b8d0fee09e2e4ac026d720baf1 Mon Sep 17 00:00:00 2001 From: Andreas Braun Date: Fri, 18 Sep 2020 10:12:29 +0200 Subject: [PATCH 012/139] Add tests for structured logging --- src/libmongoc/CMakeLists.txt | 1 + .../mongoc/mongoc-structured-log-private.h | 3 + .../src/mongoc/mongoc-structured-log.c | 8 ++ src/libmongoc/tests/test-libmongoc-main.c | 1 + .../tests/test-mongoc-structured-log.c | 135 ++++++++++++++++++ 5 files changed, 148 insertions(+) create mode 100644 src/libmongoc/tests/test-mongoc-structured-log.c diff --git a/src/libmongoc/CMakeLists.txt b/src/libmongoc/CMakeLists.txt index a9b23b12a39..2a850d6de84 100644 --- a/src/libmongoc/CMakeLists.txt +++ b/src/libmongoc/CMakeLists.txt @@ -1143,6 +1143,7 @@ set (test-libmongoc-sources ${PROJECT_SOURCE_DIR}/tests/test-mongoc-ssl.c ${PROJECT_SOURCE_DIR}/tests/test-mongoc-stream.c ${PROJECT_SOURCE_DIR}/tests/test-mongoc-streamable-hello.c + ${PROJECT_SOURCE_DIR}/tests/test-mongoc-structured-log.c ${PROJECT_SOURCE_DIR}/tests/test-mongoc-thread.c ${PROJECT_SOURCE_DIR}/tests/test-mongoc-timeout.c ${PROJECT_SOURCE_DIR}/tests/test-mongoc-topology-description.c diff --git a/src/libmongoc/src/mongoc/mongoc-structured-log-private.h b/src/libmongoc/src/mongoc/mongoc-structured-log-private.h index 3a292be4638..55b960c7f04 100644 --- a/src/libmongoc/src/mongoc/mongoc-structured-log-private.h +++ b/src/libmongoc/src/mongoc/mongoc-structured-log-private.h @@ -42,4 +42,7 @@ mongoc_structured_log (mongoc_structured_log_level_t level, mongoc_structured_log_build_message_t build_message_func, void *structured_message_data); +void +_mongoc_structured_log_get_handler (mongoc_structured_log_func_t *log_func, void **user_data); + #endif /* MONGOC_STRUCTURED_LOG_PRIVATE_H */ diff --git a/src/libmongoc/src/mongoc/mongoc-structured-log.c b/src/libmongoc/src/mongoc/mongoc-structured-log.c index d6f2ca210ac..a9c94a83da7 100644 --- a/src/libmongoc/src/mongoc/mongoc-structured-log.c +++ b/src/libmongoc/src/mongoc/mongoc-structured-log.c @@ -131,3 +131,11 @@ mongoc_structured_log_default_handler (mongoc_structured_log_entry_t *entry, voi bson_free (message); } + +/* just for testing */ +void +_mongoc_structured_log_get_handler (mongoc_structured_log_func_t *log_func, void **user_data) +{ + *log_func = gStructuredLogger; + *user_data = gStructuredLoggerData; +} diff --git a/src/libmongoc/tests/test-libmongoc-main.c b/src/libmongoc/tests/test-libmongoc-main.c index 7eaa348c180..921145caf01 100644 --- a/src/libmongoc/tests/test-libmongoc-main.c +++ b/src/libmongoc/tests/test-libmongoc-main.c @@ -82,6 +82,7 @@ main (int argc, char *argv[]) TEST_INSTALL (test_linux_distro_scanner_install); TEST_INSTALL (test_list_install); TEST_INSTALL (test_log_install); + TEST_INSTALL (test_structured_log_install); TEST_INSTALL (test_long_namespace_install); TEST_INSTALL (test_matcher_install); TEST_INSTALL (test_mongos_pinning_install); diff --git a/src/libmongoc/tests/test-mongoc-structured-log.c b/src/libmongoc/tests/test-mongoc-structured-log.c new file mode 100644 index 00000000000..d2d649034c9 --- /dev/null +++ b/src/libmongoc/tests/test-mongoc-structured-log.c @@ -0,0 +1,135 @@ +/* + * Copyright 2015 MongoDB, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include "mongoc/mongoc-structured-log-private.h" +#include "TestSuite.h" + +struct log_assumption { + mongoc_structured_log_entry_t expected_entry; + int expected_calls; + int calls; +}; + +struct structured_log_state { + mongoc_structured_log_func_t handler; + void *data; +}; + +static void +save_state (struct structured_log_state *state) +{ + _mongoc_structured_log_get_handler (&state->handler, &state->data); +} + +static void +restore_state (const struct structured_log_state *state) +{ + mongoc_structured_log_set_handler (state->handler, state->data); +} + +static void +structured_log_func (mongoc_structured_log_entry_t *entry, void *user_data) +{ + struct log_assumption *assumption = (struct log_assumption*) user_data; + + assumption->calls++; + + ASSERT_CMPINT (assumption->calls, <=, assumption->expected_calls); + + ASSERT_CMPINT (entry->level, ==, assumption->expected_entry.level); + ASSERT_CMPINT (entry->component, ==, assumption->expected_entry.component); + ASSERT (bson_equal (mongoc_structured_log_entry_get_message (entry), assumption->expected_entry.structured_message)); +} + +void +test_plain_log_entry () +{ + struct structured_log_state old_state; + struct log_assumption assumption = { + .expected_entry = { + .level = MONGOC_STRUCTURED_LOG_LEVEL_WARNING, + .component = MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND, + .message = "Plain log entry", + .structured_message = BCON_NEW ("message", BCON_UTF8 ("Plain log entry")), + }, + .expected_calls = 1, + .calls = 0, + }; + + save_state (&old_state); + + mongoc_structured_log_set_handler (structured_log_func, &assumption); + + mongoc_structured_log ( + MONGOC_STRUCTURED_LOG_LEVEL_WARNING, + MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND, + "Plain log entry", + NULL, + NULL); + + ASSERT_CMPINT (assumption.calls, =, 1); + + restore_state (&old_state); +} + +void +_test_append_extra_data (mongoc_structured_log_entry_t *entry) +{ + BCON_APPEND (entry->structured_message, "extra", BCON_INT32 (1)); +} + +void +test_log_entry_with_extra_data () +{ + struct structured_log_state old_state; + struct log_assumption assumption = { + .expected_entry = { + .level = MONGOC_STRUCTURED_LOG_LEVEL_WARNING, + .component = MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND, + .message = "Plain log entry", + .structured_message = BCON_NEW ( + "message", BCON_UTF8 ("Plain log entry"), + "extra", BCON_INT32 (1) + ), + }, + .expected_calls = 1, + .calls = 0, + }; + + save_state (&old_state); + + mongoc_structured_log_set_handler (structured_log_func, &assumption); + + mongoc_structured_log ( + MONGOC_STRUCTURED_LOG_LEVEL_WARNING, + MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND, + "Plain log entry", + _test_append_extra_data, + NULL); + + ASSERT_CMPINT (assumption.calls, =, 1); + + restore_state (&old_state); +} + +void +test_structured_log_install (TestSuite *suite) +{ + TestSuite_Add (suite, "/Structured_Log/plain", test_plain_log_entry); + TestSuite_Add (suite, "/Structured_Log/with_extra_data", test_log_entry_with_extra_data); +} From de6370ab3d855630f3d6cdfabfead5c12e59303a Mon Sep 17 00:00:00 2001 From: Andreas Braun Date: Fri, 18 Sep 2020 12:57:28 +0200 Subject: [PATCH 013/139] Run clang-format --- .../mongoc/mongoc-structured-log-command.c | 118 ++++++++---------- .../mongoc/mongoc-structured-log-private.h | 8 +- .../src/mongoc/mongoc-structured-log.c | 29 +++-- .../src/mongoc/mongoc-structured-log.h | 15 ++- .../tests/test-mongoc-structured-log.c | 63 +++++----- 5 files changed, 119 insertions(+), 114 deletions(-) diff --git a/src/libmongoc/src/mongoc/mongoc-structured-log-command.c b/src/libmongoc/src/mongoc/mongoc-structured-log-command.c index 46887aab064..4c5db6446f6 100644 --- a/src/libmongoc/src/mongoc/mongoc-structured-log-command.c +++ b/src/libmongoc/src/mongoc/mongoc-structured-log-command.c @@ -31,31 +31,31 @@ #include "mongoc-structured-log-command-private.h" static void -_mongoc_log_structured_append_command_data (mongoc_structured_log_entry_t *entry) +_mongoc_log_structured_append_command_data ( + mongoc_structured_log_entry_t *entry) { _mongoc_structured_log_command_t *log_command = entry->command; - BCON_APPEND ( - entry->structured_message, - "commandName", - BCON_UTF8 (log_command->command_name), - "requestId", - BCON_INT32 (log_command->request_id), - "operationId", - BCON_INT64 (log_command->operation_id), - "driverConnectionId", - BCON_INT32 (log_command->driver_connection_id), - "serverConnectionId", - BCON_INT32 (log_command->server_connection_id), - "explicitSession", - BCON_BOOL (log_command->explicit_session) - ); + BCON_APPEND (entry->structured_message, + "commandName", + BCON_UTF8 (log_command->command_name), + "requestId", + BCON_INT32 (log_command->request_id), + "operationId", + BCON_INT64 (log_command->operation_id), + "driverConnectionId", + BCON_INT32 (log_command->driver_connection_id), + "serverConnectionId", + BCON_INT32 (log_command->server_connection_id), + "explicitSession", + BCON_BOOL (log_command->explicit_session)); } static void -mongoc_log_structured_build_command_started_message (mongoc_structured_log_entry_t *entry) +mongoc_log_structured_build_command_started_message ( + mongoc_structured_log_entry_t *entry) { - char* cmd_json; + char *cmd_json; _mongoc_structured_log_command_t *log_command = entry->command; BSON_ASSERT (entry->component == MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND); @@ -64,21 +64,20 @@ mongoc_log_structured_build_command_started_message (mongoc_structured_log_entry _mongoc_log_structured_append_command_data (entry); - BCON_APPEND ( - entry->structured_message, - "databaseName", - BCON_UTF8 (log_command->db_name), - "command", - BCON_UTF8 (cmd_json) - ); + BCON_APPEND (entry->structured_message, + "databaseName", + BCON_UTF8 (log_command->db_name), + "command", + BCON_UTF8 (cmd_json)); bson_free (cmd_json); } static void -mongoc_log_structured_build_command_succeeded_message (mongoc_structured_log_entry_t *entry) +mongoc_log_structured_build_command_succeeded_message ( + mongoc_structured_log_entry_t *entry) { - char* reply_json; + char *reply_json; _mongoc_structured_log_command_t *log_command = entry->command; BSON_ASSERT (entry->component == MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND); @@ -87,21 +86,20 @@ mongoc_log_structured_build_command_succeeded_message (mongoc_structured_log_ent _mongoc_log_structured_append_command_data (entry); - BCON_APPEND ( - entry->structured_message, - "duration", - BCON_INT64 (log_command->duration), - "reply", - BCON_UTF8 (reply_json) - ); + BCON_APPEND (entry->structured_message, + "duration", + BCON_INT64 (log_command->duration), + "reply", + BCON_UTF8 (reply_json)); bson_free (reply_json); } static void -mongoc_log_structured_build_command_failed_message (mongoc_structured_log_entry_t *entry) +mongoc_log_structured_build_command_failed_message ( + mongoc_structured_log_entry_t *entry) { - char* reply_json; + char *reply_json; _mongoc_structured_log_command_t *log_command = entry->command; BSON_ASSERT (entry->component == MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND); @@ -110,16 +108,12 @@ mongoc_log_structured_build_command_failed_message (mongoc_structured_log_entry_ _mongoc_log_structured_append_command_data (entry); - BCON_APPEND ( - entry->structured_message, - "reply", - BCON_UTF8 (reply_json)); + BCON_APPEND (entry->structured_message, "reply", BCON_UTF8 (reply_json)); if (log_command->error) { - BCON_APPEND ( - entry->structured_message, - "failure", - BCON_UTF8 (log_command->error->message)); + BCON_APPEND (entry->structured_message, + "failure", + BCON_UTF8 (log_command->error->message)); } bson_free (reply_json); @@ -146,13 +140,11 @@ mongoc_structured_log_command_started (const bson_t *command, .explicit_session = explicit_session, }; - mongoc_structured_log ( - MONGOC_STRUCTURED_LOG_LEVEL_INFO, - MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND, - "Command started", - mongoc_log_structured_build_command_started_message, - &command_log - ); + mongoc_structured_log (MONGOC_STRUCTURED_LOG_LEVEL_INFO, + MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND, + "Command started", + mongoc_log_structured_build_command_started_message, + &command_log); } void @@ -176,13 +168,11 @@ mongoc_structured_log_command_success (const char *command_name, .explicit_session = explicit_session, }; - mongoc_structured_log ( - MONGOC_STRUCTURED_LOG_LEVEL_INFO, - MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND, - "Command succeeded", - mongoc_log_structured_build_command_succeeded_message, - &command_log - ); + mongoc_structured_log (MONGOC_STRUCTURED_LOG_LEVEL_INFO, + MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND, + "Command succeeded", + mongoc_log_structured_build_command_succeeded_message, + &command_log); } void @@ -206,11 +196,9 @@ mongoc_structured_log_command_failure (const char *command_name, .explicit_session = explicit_session, }; - mongoc_structured_log ( - MONGOC_STRUCTURED_LOG_LEVEL_INFO, - MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND, - "Command succeeded", - mongoc_log_structured_build_command_failed_message, - &command_log - ); + mongoc_structured_log (MONGOC_STRUCTURED_LOG_LEVEL_INFO, + MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND, + "Command succeeded", + mongoc_log_structured_build_command_failed_message, + &command_log); } diff --git a/src/libmongoc/src/mongoc/mongoc-structured-log-private.h b/src/libmongoc/src/mongoc/mongoc-structured-log-private.h index 55b960c7f04..44eee0cd0e4 100644 --- a/src/libmongoc/src/mongoc/mongoc-structured-log-private.h +++ b/src/libmongoc/src/mongoc/mongoc-structured-log-private.h @@ -22,12 +22,13 @@ #ifndef MONGOC_STRUCTRURED_LOG_PRIVATE_H #define MONGOC_STRUCTRURED_LOG_PRIVATE_H -typedef void (*mongoc_structured_log_build_message_t) (mongoc_structured_log_entry_t *entry); +typedef void (*mongoc_structured_log_build_message_t) ( + mongoc_structured_log_entry_t *entry); struct _mongoc_structured_log_entry_t { mongoc_structured_log_level_t level; mongoc_structured_log_component_t component; - const char* message; + const char *message; bson_t *structured_message; mongoc_structured_log_build_message_t build_message_func; union { @@ -43,6 +44,7 @@ mongoc_structured_log (mongoc_structured_log_level_t level, void *structured_message_data); void -_mongoc_structured_log_get_handler (mongoc_structured_log_func_t *log_func, void **user_data); +_mongoc_structured_log_get_handler (mongoc_structured_log_func_t *log_func, + void **user_data); #endif /* MONGOC_STRUCTURED_LOG_PRIVATE_H */ diff --git a/src/libmongoc/src/mongoc/mongoc-structured-log.c b/src/libmongoc/src/mongoc/mongoc-structured-log.c index a9c94a83da7..a4dc70f1c98 100644 --- a/src/libmongoc/src/mongoc/mongoc-structured-log.c +++ b/src/libmongoc/src/mongoc/mongoc-structured-log.c @@ -32,11 +32,13 @@ #include "mongoc-thread-private.h" static void -mongoc_structured_log_default_handler (mongoc_structured_log_entry_t *entry, void *user_data); +mongoc_structured_log_default_handler (mongoc_structured_log_entry_t *entry, + void *user_data); static bson_once_t once = BSON_ONCE_INIT; static bson_mutex_t gStructuredLogMutex; -static mongoc_structured_log_func_t gStructuredLogger = mongoc_structured_log_default_handler; +static mongoc_structured_log_func_t gStructuredLogger = + mongoc_structured_log_default_handler; static void *gStructuredLoggerData; static BSON_ONCE_FUN (_mongoc_ensure_mutex_once) @@ -54,11 +56,12 @@ mongoc_structured_log_entry_destroy (mongoc_structured_log_entry_t *entry) } } -const bson_t* +const bson_t * mongoc_structured_log_entry_get_message (mongoc_structured_log_entry_t *entry) { if (!entry->structured_message) { - entry->structured_message = BCON_NEW ("message", BCON_UTF8 (entry->message)); + entry->structured_message = + BCON_NEW ("message", BCON_UTF8 (entry->message)); if (entry->build_message_func) { entry->build_message_func (entry); @@ -69,19 +72,22 @@ mongoc_structured_log_entry_get_message (mongoc_structured_log_entry_t *entry) } mongoc_structured_log_level_t -mongoc_structured_log_entry_get_level (const mongoc_structured_log_entry_t *entry) +mongoc_structured_log_entry_get_level ( + const mongoc_structured_log_entry_t *entry) { return entry->level; } mongoc_structured_log_component_t -mongoc_structured_log_entry_get_component (const mongoc_structured_log_entry_t *entry) +mongoc_structured_log_entry_get_component ( + const mongoc_structured_log_entry_t *entry) { return entry->component; } void -mongoc_structured_log_set_handler (mongoc_structured_log_func_t log_func, void *user_data) +mongoc_structured_log_set_handler (mongoc_structured_log_func_t log_func, + void *user_data) { bson_once (&once, &_mongoc_ensure_mutex_once); @@ -119,9 +125,11 @@ mongoc_structured_log (mongoc_structured_log_level_t level, } static void -mongoc_structured_log_default_handler (mongoc_structured_log_entry_t *entry, void *user_data) +mongoc_structured_log_default_handler (mongoc_structured_log_entry_t *entry, + void *user_data) { - char *message = bson_as_json (mongoc_structured_log_entry_get_message (entry), NULL); + char *message = + bson_as_json (mongoc_structured_log_entry_get_message (entry), NULL); fprintf (stderr, "Structured log: %d, %d, %s\n", @@ -134,7 +142,8 @@ mongoc_structured_log_default_handler (mongoc_structured_log_entry_t *entry, voi /* just for testing */ void -_mongoc_structured_log_get_handler (mongoc_structured_log_func_t *log_func, void **user_data) +_mongoc_structured_log_get_handler (mongoc_structured_log_func_t *log_func, + void **user_data) { *log_func = gStructuredLogger; *user_data = gStructuredLoggerData; diff --git a/src/libmongoc/src/mongoc/mongoc-structured-log.h b/src/libmongoc/src/mongoc/mongoc-structured-log.h index 5907e1cb5a7..b7f8377c109 100644 --- a/src/libmongoc/src/mongoc/mongoc-structured-log.h +++ b/src/libmongoc/src/mongoc/mongoc-structured-log.h @@ -46,20 +46,23 @@ typedef enum { typedef struct _mongoc_structured_log_entry_t mongoc_structured_log_entry_t; -typedef void (*mongoc_structured_log_func_t) (mongoc_structured_log_entry_t *entry, - void *user_data); +typedef void (*mongoc_structured_log_func_t) ( + mongoc_structured_log_entry_t *entry, void *user_data); MONGOC_EXPORT (void) -mongoc_structured_log_set_handler (mongoc_structured_log_func_t log_func, void *user_data); +mongoc_structured_log_set_handler (mongoc_structured_log_func_t log_func, + void *user_data); -MONGOC_EXPORT (const bson_t*) +MONGOC_EXPORT (const bson_t *) mongoc_structured_log_entry_get_message (mongoc_structured_log_entry_t *entry); MONGOC_EXPORT (mongoc_structured_log_level_t) -mongoc_structured_log_entry_get_level (const mongoc_structured_log_entry_t *entry); +mongoc_structured_log_entry_get_level ( + const mongoc_structured_log_entry_t *entry); MONGOC_EXPORT (mongoc_structured_log_component_t) -mongoc_structured_log_entry_get_component (const mongoc_structured_log_entry_t *entry); +mongoc_structured_log_entry_get_component ( + const mongoc_structured_log_entry_t *entry); BSON_END_DECLS diff --git a/src/libmongoc/tests/test-mongoc-structured-log.c b/src/libmongoc/tests/test-mongoc-structured-log.c index d2d649034c9..9089261c10b 100644 --- a/src/libmongoc/tests/test-mongoc-structured-log.c +++ b/src/libmongoc/tests/test-mongoc-structured-log.c @@ -45,7 +45,7 @@ restore_state (const struct structured_log_state *state) static void structured_log_func (mongoc_structured_log_entry_t *entry, void *user_data) { - struct log_assumption *assumption = (struct log_assumption*) user_data; + struct log_assumption *assumption = (struct log_assumption *) user_data; assumption->calls++; @@ -53,7 +53,8 @@ structured_log_func (mongoc_structured_log_entry_t *entry, void *user_data) ASSERT_CMPINT (entry->level, ==, assumption->expected_entry.level); ASSERT_CMPINT (entry->component, ==, assumption->expected_entry.component); - ASSERT (bson_equal (mongoc_structured_log_entry_get_message (entry), assumption->expected_entry.structured_message)); + ASSERT (bson_equal (mongoc_structured_log_entry_get_message (entry), + assumption->expected_entry.structured_message)); } void @@ -61,12 +62,14 @@ test_plain_log_entry () { struct structured_log_state old_state; struct log_assumption assumption = { - .expected_entry = { - .level = MONGOC_STRUCTURED_LOG_LEVEL_WARNING, - .component = MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND, - .message = "Plain log entry", - .structured_message = BCON_NEW ("message", BCON_UTF8 ("Plain log entry")), - }, + .expected_entry = + { + .level = MONGOC_STRUCTURED_LOG_LEVEL_WARNING, + .component = MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND, + .message = "Plain log entry", + .structured_message = + BCON_NEW ("message", BCON_UTF8 ("Plain log entry")), + }, .expected_calls = 1, .calls = 0, }; @@ -75,12 +78,11 @@ test_plain_log_entry () mongoc_structured_log_set_handler (structured_log_func, &assumption); - mongoc_structured_log ( - MONGOC_STRUCTURED_LOG_LEVEL_WARNING, - MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND, - "Plain log entry", - NULL, - NULL); + mongoc_structured_log (MONGOC_STRUCTURED_LOG_LEVEL_WARNING, + MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND, + "Plain log entry", + NULL, + NULL); ASSERT_CMPINT (assumption.calls, =, 1); @@ -98,15 +100,16 @@ test_log_entry_with_extra_data () { struct structured_log_state old_state; struct log_assumption assumption = { - .expected_entry = { - .level = MONGOC_STRUCTURED_LOG_LEVEL_WARNING, - .component = MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND, - .message = "Plain log entry", - .structured_message = BCON_NEW ( - "message", BCON_UTF8 ("Plain log entry"), - "extra", BCON_INT32 (1) - ), - }, + .expected_entry = + { + .level = MONGOC_STRUCTURED_LOG_LEVEL_WARNING, + .component = MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND, + .message = "Plain log entry", + .structured_message = BCON_NEW ("message", + BCON_UTF8 ("Plain log entry"), + "extra", + BCON_INT32 (1)), + }, .expected_calls = 1, .calls = 0, }; @@ -115,12 +118,11 @@ test_log_entry_with_extra_data () mongoc_structured_log_set_handler (structured_log_func, &assumption); - mongoc_structured_log ( - MONGOC_STRUCTURED_LOG_LEVEL_WARNING, - MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND, - "Plain log entry", - _test_append_extra_data, - NULL); + mongoc_structured_log (MONGOC_STRUCTURED_LOG_LEVEL_WARNING, + MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND, + "Plain log entry", + _test_append_extra_data, + NULL); ASSERT_CMPINT (assumption.calls, =, 1); @@ -131,5 +133,6 @@ void test_structured_log_install (TestSuite *suite) { TestSuite_Add (suite, "/Structured_Log/plain", test_plain_log_entry); - TestSuite_Add (suite, "/Structured_Log/with_extra_data", test_log_entry_with_extra_data); + TestSuite_Add ( + suite, "/Structured_Log/with_extra_data", test_log_entry_with_extra_data); } From 405c4f6e7032860df20e8a401b2c7b5473b4d96f Mon Sep 17 00:00:00 2001 From: Andreas Braun Date: Fri, 18 Sep 2020 12:59:45 +0200 Subject: [PATCH 014/139] Update copyright years --- .../src/mongoc/mongoc-structured-log-command-private.h | 2 +- src/libmongoc/src/mongoc/mongoc-structured-log-command.c | 2 +- src/libmongoc/src/mongoc/mongoc-structured-log-private.h | 2 +- src/libmongoc/src/mongoc/mongoc-structured-log.c | 2 +- src/libmongoc/src/mongoc/mongoc-structured-log.h | 2 +- src/libmongoc/tests/test-mongoc-structured-log.c | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/libmongoc/src/mongoc/mongoc-structured-log-command-private.h b/src/libmongoc/src/mongoc/mongoc-structured-log-command-private.h index c47ee751519..a9560d5b8a9 100644 --- a/src/libmongoc/src/mongoc/mongoc-structured-log-command-private.h +++ b/src/libmongoc/src/mongoc/mongoc-structured-log-command-private.h @@ -1,5 +1,5 @@ /* - * Copyright 2015 MongoDB, Inc. + * Copyright 2020 MongoDB, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/src/libmongoc/src/mongoc/mongoc-structured-log-command.c b/src/libmongoc/src/mongoc/mongoc-structured-log-command.c index 4c5db6446f6..7c4ff2ef9bb 100644 --- a/src/libmongoc/src/mongoc/mongoc-structured-log-command.c +++ b/src/libmongoc/src/mongoc/mongoc-structured-log-command.c @@ -1,5 +1,5 @@ /* - * Copyright 2013 MongoDB, Inc. + * Copyright 2020 MongoDB, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/src/libmongoc/src/mongoc/mongoc-structured-log-private.h b/src/libmongoc/src/mongoc/mongoc-structured-log-private.h index 44eee0cd0e4..777dc7e0f7a 100644 --- a/src/libmongoc/src/mongoc/mongoc-structured-log-private.h +++ b/src/libmongoc/src/mongoc/mongoc-structured-log-private.h @@ -1,5 +1,5 @@ /* - * Copyright 2015 MongoDB, Inc. + * Copyright 2020 MongoDB, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/src/libmongoc/src/mongoc/mongoc-structured-log.c b/src/libmongoc/src/mongoc/mongoc-structured-log.c index a4dc70f1c98..0b79c5d25c4 100644 --- a/src/libmongoc/src/mongoc/mongoc-structured-log.c +++ b/src/libmongoc/src/mongoc/mongoc-structured-log.c @@ -1,5 +1,5 @@ /* - * Copyright 2013 MongoDB, Inc. + * Copyright 2020 MongoDB, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/src/libmongoc/src/mongoc/mongoc-structured-log.h b/src/libmongoc/src/mongoc/mongoc-structured-log.h index b7f8377c109..adaf0bcb548 100644 --- a/src/libmongoc/src/mongoc/mongoc-structured-log.h +++ b/src/libmongoc/src/mongoc/mongoc-structured-log.h @@ -1,5 +1,5 @@ /* - * Copyright 2013 MongoDB, Inc. + * Copyright 2020 MongoDB, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/src/libmongoc/tests/test-mongoc-structured-log.c b/src/libmongoc/tests/test-mongoc-structured-log.c index 9089261c10b..200f79ed99b 100644 --- a/src/libmongoc/tests/test-mongoc-structured-log.c +++ b/src/libmongoc/tests/test-mongoc-structured-log.c @@ -1,5 +1,5 @@ /* - * Copyright 2015 MongoDB, Inc. + * Copyright 2020 MongoDB, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. From cb88c6dd50d57221e817bba1172dae1801639e6a Mon Sep 17 00:00:00 2001 From: Andreas Braun Date: Fri, 18 Sep 2020 13:04:49 +0200 Subject: [PATCH 015/139] Add rudimentary documentation for new public API --- .../src/mongoc/mongoc-structured-log.h | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/src/libmongoc/src/mongoc/mongoc-structured-log.h b/src/libmongoc/src/mongoc/mongoc-structured-log.h index adaf0bcb548..3b9dae259bf 100644 --- a/src/libmongoc/src/mongoc/mongoc-structured-log.h +++ b/src/libmongoc/src/mongoc/mongoc-structured-log.h @@ -49,17 +49,47 @@ typedef struct _mongoc_structured_log_entry_t mongoc_structured_log_entry_t; typedef void (*mongoc_structured_log_func_t) ( mongoc_structured_log_entry_t *entry, void *user_data); +/** + * mongoc_structured_log_set_handler: + * @log_func: A function to handle structured messages. + * @user_data: User data for @log_func. + * + * Sets the function to be called to handle structured log messages. + */ MONGOC_EXPORT (void) mongoc_structured_log_set_handler (mongoc_structured_log_func_t log_func, void *user_data); +/** + * mongoc_structured_log_entry_get_message: + * @entry: A log entry to extract the message from + * + * Returns the structured message as a bson_t pointer. + * + * When this function is called, the message is lazily generated if it hasn't + * already been generated. Note that it is not safe to call this method outside + * of a log handler, as the data needed to assemble the message may have been + * freed already. + */ MONGOC_EXPORT (const bson_t *) mongoc_structured_log_entry_get_message (mongoc_structured_log_entry_t *entry); +/** + * mongoc_structured_log_entry_get_level: + * @entry: A log entry to read the level from + * + * Returns the severity level of the structured log entry + */ MONGOC_EXPORT (mongoc_structured_log_level_t) mongoc_structured_log_entry_get_level ( const mongoc_structured_log_entry_t *entry); +/** + * mongoc_structured_log_entry_get_component: + * @entry: A log entry to read the component from + * + * Returns the component of the structured log entry + */ MONGOC_EXPORT (mongoc_structured_log_component_t) mongoc_structured_log_entry_get_component ( const mongoc_structured_log_entry_t *entry); From e23f5ae617d2c9895ebd799d41fc6d279dc27665 Mon Sep 17 00:00:00 2001 From: Andreas Braun Date: Tue, 22 Sep 2020 11:18:14 +0200 Subject: [PATCH 016/139] Fix wrong log message for command failed log entry --- src/libmongoc/src/mongoc/mongoc-structured-log-command.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libmongoc/src/mongoc/mongoc-structured-log-command.c b/src/libmongoc/src/mongoc/mongoc-structured-log-command.c index 7c4ff2ef9bb..4d797094579 100644 --- a/src/libmongoc/src/mongoc/mongoc-structured-log-command.c +++ b/src/libmongoc/src/mongoc/mongoc-structured-log-command.c @@ -198,7 +198,7 @@ mongoc_structured_log_command_failure (const char *command_name, mongoc_structured_log (MONGOC_STRUCTURED_LOG_LEVEL_INFO, MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND, - "Command succeeded", + "Command failed", mongoc_log_structured_build_command_failed_message, &command_log); } From b463f8268a3f3d9d68979f6a92695b8d19955383 Mon Sep 17 00:00:00 2001 From: Andreas Braun Date: Tue, 22 Sep 2020 11:24:55 +0200 Subject: [PATCH 017/139] Add spec compliant default logger --- .../mongoc/mongoc-structured-log-private.h | 2 + .../src/mongoc/mongoc-structured-log.c | 81 ++++++++++++++++++- 2 files changed, 82 insertions(+), 1 deletion(-) diff --git a/src/libmongoc/src/mongoc/mongoc-structured-log-private.h b/src/libmongoc/src/mongoc/mongoc-structured-log-private.h index 777dc7e0f7a..752e5b31423 100644 --- a/src/libmongoc/src/mongoc/mongoc-structured-log-private.h +++ b/src/libmongoc/src/mongoc/mongoc-structured-log-private.h @@ -22,6 +22,8 @@ #ifndef MONGOC_STRUCTRURED_LOG_PRIVATE_H #define MONGOC_STRUCTRURED_LOG_PRIVATE_H +#define MONGOC_STRUCTURED_LOG_DEFAULT_LEVEL MONGOC_STRUCTURED_LOG_LEVEL_WARNING; + typedef void (*mongoc_structured_log_build_message_t) ( mongoc_structured_log_entry_t *entry); diff --git a/src/libmongoc/src/mongoc/mongoc-structured-log.c b/src/libmongoc/src/mongoc/mongoc-structured-log.c index 0b79c5d25c4..f1a7e989e5a 100644 --- a/src/libmongoc/src/mongoc/mongoc-structured-log.c +++ b/src/libmongoc/src/mongoc/mongoc-structured-log.c @@ -40,6 +40,7 @@ static bson_mutex_t gStructuredLogMutex; static mongoc_structured_log_func_t gStructuredLogger = mongoc_structured_log_default_handler; static void *gStructuredLoggerData; +static FILE *log_stream; static BSON_ONCE_FUN (_mongoc_ensure_mutex_once) { @@ -124,14 +125,92 @@ mongoc_structured_log (mongoc_structured_log_level_t level, mongoc_structured_log_entry_destroy (&entry); } +static mongoc_structured_log_level_t +_mongoc_structured_log_get_log_level_from_env (const char *variable) +{ + const char* level = getenv (variable); + + if (!level) { + return MONGOC_STRUCTURED_LOG_DEFAULT_LEVEL; + } else if (!strcasecmp (level, "trace")) { + return MONGOC_STRUCTURED_LOG_LEVEL_TRACE; + } else if (!strcasecmp (level, "debug")) { + return MONGOC_STRUCTURED_LOG_LEVEL_DEBUG; + } else if (!strcasecmp (level, "info")) { + return MONGOC_STRUCTURED_LOG_LEVEL_INFO; + } else if (!strcasecmp (level, "notice")) { + return MONGOC_STRUCTURED_LOG_LEVEL_NOTICE; + } else if (!strcasecmp (level, "warn")) { + return MONGOC_STRUCTURED_LOG_LEVEL_WARNING; + } else if (!strcasecmp (level, "error")) { + return MONGOC_STRUCTURED_LOG_LEVEL_ERROR; + } else if (!strcasecmp (level, "critical")) { + return MONGOC_STRUCTURED_LOG_LEVEL_CRITICAL; + } else if (!strcasecmp (level, "alert")) { + return MONGOC_STRUCTURED_LOG_LEVEL_ALERT; + } else if (!strcasecmp (level, "emergency")) { + return MONGOC_STRUCTURED_LOG_LEVEL_EMERGENCY; + } else { + MONGOC_ERROR ("Invalid log level %s read for variable %s", level, variable); + exit (EXIT_FAILURE); + } +} + +static mongoc_structured_log_level_t +_mongoc_structured_log_get_log_level (mongoc_structured_log_component_t component) +{ + switch (component) { + case MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND: + return _mongoc_structured_log_get_log_level_from_env ("MONGODB_LOGGING_COMMAND"); + case MONGOC_STRUCTURED_LOG_COMPONENT_CONNECTION: + return _mongoc_structured_log_get_log_level_from_env ("MONGODB_LOGGING_CONNECTION"); + case MONGOC_STRUCTURED_LOG_COMPONENT_SDAM: + return _mongoc_structured_log_get_log_level_from_env ("MONGODB_LOGGING_SDAM"); + case MONGOC_STRUCTURED_LOG_COMPONENT_SERVER_SELECTION: + return _mongoc_structured_log_get_log_level_from_env ("MONGODB_LOGGING_SERVER_SELECTION"); + default: + MONGOC_ERROR ("Requesting log level for unsupported component %d", component); + exit (EXIT_FAILURE); + } +} + +static void +_mongoc_structured_log_initialize_stream () +{ + const char* log_target = getenv("MONGODB_LOGGING_PATH"); + bool log_to_stderr = !log_target || !strcmp (log_target, "stderr"); + + log_stream = log_to_stderr ? stderr : fopen (log_target, "a"); + if (!log_stream) { + MONGOC_ERROR ("Cannot open log file %s for writing", log_target); + exit (EXIT_FAILURE); + } +} + +static FILE* +_mongoc_structured_log_get_stream () +{ + if (!log_stream) { + _mongoc_structured_log_initialize_stream (); + } + + return log_stream; +} + static void mongoc_structured_log_default_handler (mongoc_structured_log_entry_t *entry, void *user_data) { + mongoc_structured_log_level_t log_level = _mongoc_structured_log_get_log_level (mongoc_structured_log_entry_get_component (entry)); + + if (log_level < mongoc_structured_log_entry_get_level (entry)) { + return; + } + char *message = bson_as_json (mongoc_structured_log_entry_get_message (entry), NULL); - fprintf (stderr, + fprintf (_mongoc_structured_log_get_stream (), "Structured log: %d, %d, %s\n", mongoc_structured_log_entry_get_level (entry), mongoc_structured_log_entry_get_component (entry), From 2e49e0da97863ec7c970a828f4556b4cc1856ac4 Mon Sep 17 00:00:00 2001 From: Andreas Braun Date: Tue, 22 Sep 2020 12:50:27 +0200 Subject: [PATCH 018/139] Run clang-format --- src/libmongoc/src/mongoc/mongoc-client.c | 37 +++++++------- src/libmongoc/src/mongoc/mongoc-cluster.c | 48 +++++++++++-------- .../src/mongoc/mongoc-cursor-legacy.c | 9 +++- src/libmongoc/src/mongoc/mongoc-cursor.c | 43 +++++++++-------- .../src/mongoc/mongoc-structured-log.c | 31 +++++++----- 5 files changed, 97 insertions(+), 71 deletions(-) diff --git a/src/libmongoc/src/mongoc/mongoc-client.c b/src/libmongoc/src/mongoc/mongoc-client.c index e6dc1e09a81..0d40669c825 100644 --- a/src/libmongoc/src/mongoc/mongoc-client.c +++ b/src/libmongoc/src/mongoc/mongoc-client.c @@ -2164,7 +2164,8 @@ _mongoc_client_monitor_op_killcursors (mongoc_cluster_t *cluster, _mongoc_client_prepare_killcursors_command (cursor_id, collection, &doc); // @todo Provide missing arguments - mongoc_structured_log_command_started (&doc, "killCursors", db, operation_id, cluster->request_id, 0, 0, false); + mongoc_structured_log_command_started ( + &doc, "killCursors", db, operation_id, cluster->request_id, 0, 0, false); if (!client->apm_callbacks.started) { bson_destroy (&doc); @@ -2217,15 +2218,14 @@ _mongoc_client_monitor_op_killcursors_succeeded (mongoc_cluster_t *cluster, bson_append_array_builder_end (&doc, cursors_unknown); // @todo Provide missing arguments - mongoc_structured_log_command_success ( - "killCursors", - operation_id, - &doc, - duration, - cluster->request_id, - 0, - 0, - false); + mongoc_structured_log_command_success ("killCursors", + operation_id, + &doc, + duration, + cluster->request_id, + 0, + 0, + false); if (!client->apm_callbacks.succeeded) { bson_destroy (&doc); @@ -2274,15 +2274,14 @@ _mongoc_client_monitor_op_killcursors_failed (mongoc_cluster_t *cluster, bson_append_int32 (&doc, "ok", 2, 0); // @todo Provide missing arguments - mongoc_structured_log_command_failure ( - "killCursors", - operation_id, - &doc, - error, - cluster->request_id, - 0, - 0, - false); + mongoc_structured_log_command_failure ("killCursors", + operation_id, + &doc, + error, + cluster->request_id, + 0, + 0, + false); if (!client->apm_callbacks.failed) { bson_destroy (&doc); diff --git a/src/libmongoc/src/mongoc/mongoc-cluster.c b/src/libmongoc/src/mongoc/mongoc-cluster.c index c9ae79dffe3..17e4e175225 100644 --- a/src/libmongoc/src/mongoc/mongoc-cluster.c +++ b/src/libmongoc/src/mongoc/mongoc-cluster.c @@ -534,7 +534,14 @@ mongoc_cluster_run_command_monitored (mongoc_cluster_t *cluster, mongoc_cmd_t *c } // @todo Provide missing arguments - mongoc_structured_log_command_started (cmd->command, cmd->command_name, cmd->db_name, cmd->operation_id, request_id, 0, 0, false); + mongoc_structured_log_command_started (cmd->command, + cmd->command_name, + cmd->db_name, + cmd->operation_id, + request_id, + 0, + 0, + false); if (callbacks->started) { mongoc_apm_command_started_init_with_cmd ( @@ -548,7 +555,7 @@ mongoc_cluster_run_command_monitored (mongoc_cluster_t *cluster, mongoc_cmd_t *c if (retval) { bson_t fake_reply = BSON_INITIALIZER; - int64_t duration = bson_get_monotonic_time() - started; + int64_t duration = bson_get_monotonic_time () - started; /* * Unacknowledged writes must provide a CommandSucceededEvent with an @@ -560,15 +567,15 @@ mongoc_cluster_run_command_monitored (mongoc_cluster_t *cluster, mongoc_cmd_t *c } // @todo Provide missing arguments - mongoc_structured_log_command_success ( - cmd->command_name, - cmd->operation_id, - cmd->is_acknowledged ? reply : &fake_reply, - duration, - request_id, - 0, - 0, - false); + mongoc_structured_log_command_success (cmd->command_name, + cmd->operation_id, + cmd->is_acknowledged ? reply + : &fake_reply, + duration, + request_id, + 0, + 0, + false); if (callbacks->succeeded) { mongoc_apm_command_succeeded_init (&succeeded_event, @@ -591,18 +598,17 @@ mongoc_cluster_run_command_monitored (mongoc_cluster_t *cluster, mongoc_cmd_t *c bson_destroy (&fake_reply); } else { - int64_t duration = bson_get_monotonic_time() - started; + int64_t duration = bson_get_monotonic_time () - started; // @todo Provide missing arguments - mongoc_structured_log_command_failure ( - cmd->command_name, - cmd->operation_id, - reply, - error, - cluster->request_id, - 0, - 0, - false); + mongoc_structured_log_command_failure (cmd->command_name, + cmd->operation_id, + reply, + error, + cluster->request_id, + 0, + 0, + false); if (callbacks->failed) { mongoc_apm_command_failed_init (&failed_event, diff --git a/src/libmongoc/src/mongoc/mongoc-cursor-legacy.c b/src/libmongoc/src/mongoc/mongoc-cursor-legacy.c index 7cb3b690168..c7fae54e10a 100644 --- a/src/libmongoc/src/mongoc/mongoc-cursor-legacy.c +++ b/src/libmongoc/src/mongoc/mongoc-cursor-legacy.c @@ -51,7 +51,14 @@ _mongoc_cursor_monitor_legacy_get_more (mongoc_cursor_t *cursor, mongoc_server_s db = bson_strndup (cursor->ns, cursor->dblen); // @todo Provide missing arguments - mongoc_structured_log_command_started (&doc, "getMore", db, cursor->operation_id, client->cluster.request_id, 0, 0, false); + mongoc_structured_log_command_started (&doc, + "getMore", + db, + cursor->operation_id, + client->cluster.request_id, + 0, + 0, + false); if (!client->apm_callbacks.started) { /* successful */ diff --git a/src/libmongoc/src/mongoc/mongoc-cursor.c b/src/libmongoc/src/mongoc/mongoc-cursor.c index 7500ab1762d..47872536b0e 100644 --- a/src/libmongoc/src/mongoc/mongoc-cursor.c +++ b/src/libmongoc/src/mongoc/mongoc-cursor.c @@ -648,7 +648,14 @@ _mongoc_cursor_monitor_command (mongoc_cursor_t *cursor, db = bson_strndup (cursor->ns, cursor->dblen); // @todo Provide missing arguments - mongoc_structured_log_command_started (cmd, cmd_name, db, cursor->operation_id, client->cluster.request_id, 0, 0, false); + mongoc_structured_log_command_started (cmd, + cmd_name, + db, + cursor->operation_id, + client->cluster.request_id, + 0, + 0, + false); if (!client->apm_callbacks.started) { /* successful */ @@ -732,15 +739,14 @@ _mongoc_cursor_monitor_succeeded (mongoc_cursor_t *cursor, bson_destroy (&docs_array); // @todo Provide missing arguments - mongoc_structured_log_command_success ( - cmd_name, - cursor->operation_id, - &reply, - duration, - client->cluster.request_id, - 0, - 0, - false); + mongoc_structured_log_command_success (cmd_name, + cursor->operation_id, + &reply, + duration, + client->cluster.request_id, + 0, + 0, + false); if (!client->apm_callbacks.succeeded) { bson_destroy (&reply); @@ -791,15 +797,14 @@ _mongoc_cursor_monitor_failed (mongoc_cursor_t *cursor, char *db = bson_strndup (cursor->ns, cursor->dblen); // @todo Provide missing arguments - mongoc_structured_log_command_failure ( - cmd_name, - cursor->operation_id, - &reply, - &cursor->error, - client->cluster.request_id, - 0, - 0, - false); + mongoc_structured_log_command_failure (cmd_name, + cursor->operation_id, + &reply, + &cursor->error, + client->cluster.request_id, + 0, + 0, + false); if (!client->apm_callbacks.failed) { bson_destroy (&reply); diff --git a/src/libmongoc/src/mongoc/mongoc-structured-log.c b/src/libmongoc/src/mongoc/mongoc-structured-log.c index f1a7e989e5a..5d337c4cad9 100644 --- a/src/libmongoc/src/mongoc/mongoc-structured-log.c +++ b/src/libmongoc/src/mongoc/mongoc-structured-log.c @@ -128,7 +128,7 @@ mongoc_structured_log (mongoc_structured_log_level_t level, static mongoc_structured_log_level_t _mongoc_structured_log_get_log_level_from_env (const char *variable) { - const char* level = getenv (variable); + const char *level = getenv (variable); if (!level) { return MONGOC_STRUCTURED_LOG_DEFAULT_LEVEL; @@ -151,25 +151,32 @@ _mongoc_structured_log_get_log_level_from_env (const char *variable) } else if (!strcasecmp (level, "emergency")) { return MONGOC_STRUCTURED_LOG_LEVEL_EMERGENCY; } else { - MONGOC_ERROR ("Invalid log level %s read for variable %s", level, variable); + MONGOC_ERROR ( + "Invalid log level %s read for variable %s", level, variable); exit (EXIT_FAILURE); } } static mongoc_structured_log_level_t -_mongoc_structured_log_get_log_level (mongoc_structured_log_component_t component) +_mongoc_structured_log_get_log_level ( + mongoc_structured_log_component_t component) { switch (component) { case MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND: - return _mongoc_structured_log_get_log_level_from_env ("MONGODB_LOGGING_COMMAND"); + return _mongoc_structured_log_get_log_level_from_env ( + "MONGODB_LOGGING_COMMAND"); case MONGOC_STRUCTURED_LOG_COMPONENT_CONNECTION: - return _mongoc_structured_log_get_log_level_from_env ("MONGODB_LOGGING_CONNECTION"); + return _mongoc_structured_log_get_log_level_from_env ( + "MONGODB_LOGGING_CONNECTION"); case MONGOC_STRUCTURED_LOG_COMPONENT_SDAM: - return _mongoc_structured_log_get_log_level_from_env ("MONGODB_LOGGING_SDAM"); + return _mongoc_structured_log_get_log_level_from_env ( + "MONGODB_LOGGING_SDAM"); case MONGOC_STRUCTURED_LOG_COMPONENT_SERVER_SELECTION: - return _mongoc_structured_log_get_log_level_from_env ("MONGODB_LOGGING_SERVER_SELECTION"); + return _mongoc_structured_log_get_log_level_from_env ( + "MONGODB_LOGGING_SERVER_SELECTION"); default: - MONGOC_ERROR ("Requesting log level for unsupported component %d", component); + MONGOC_ERROR ("Requesting log level for unsupported component %d", + component); exit (EXIT_FAILURE); } } @@ -177,7 +184,7 @@ _mongoc_structured_log_get_log_level (mongoc_structured_log_component_t componen static void _mongoc_structured_log_initialize_stream () { - const char* log_target = getenv("MONGODB_LOGGING_PATH"); + const char *log_target = getenv ("MONGODB_LOGGING_PATH"); bool log_to_stderr = !log_target || !strcmp (log_target, "stderr"); log_stream = log_to_stderr ? stderr : fopen (log_target, "a"); @@ -187,7 +194,7 @@ _mongoc_structured_log_initialize_stream () } } -static FILE* +static FILE * _mongoc_structured_log_get_stream () { if (!log_stream) { @@ -201,7 +208,9 @@ static void mongoc_structured_log_default_handler (mongoc_structured_log_entry_t *entry, void *user_data) { - mongoc_structured_log_level_t log_level = _mongoc_structured_log_get_log_level (mongoc_structured_log_entry_get_component (entry)); + mongoc_structured_log_level_t log_level = + _mongoc_structured_log_get_log_level ( + mongoc_structured_log_entry_get_component (entry)); if (log_level < mongoc_structured_log_entry_get_level (entry)) { return; From 78f1970ba6cf19ea93ac0daed81cf2b75c1560fe Mon Sep 17 00:00:00 2001 From: Andreas Braun Date: Tue, 22 Sep 2020 13:10:16 +0200 Subject: [PATCH 019/139] Log client creation --- src/libmongoc/CMakeLists.txt | 1 + src/libmongoc/src/mongoc/mongoc-client.c | 3 ++ ...mongoc-structured-log-connection-private.h | 27 +++++++++++++ .../mongoc/mongoc-structured-log-connection.c | 40 +++++++++++++++++++ 4 files changed, 71 insertions(+) create mode 100644 src/libmongoc/src/mongoc/mongoc-structured-log-connection-private.h create mode 100644 src/libmongoc/src/mongoc/mongoc-structured-log-connection.c diff --git a/src/libmongoc/CMakeLists.txt b/src/libmongoc/CMakeLists.txt index 2a850d6de84..b415bcbd486 100644 --- a/src/libmongoc/CMakeLists.txt +++ b/src/libmongoc/CMakeLists.txt @@ -649,6 +649,7 @@ set (MONGOC_SOURCES ${PROJECT_SOURCE_DIR}/src/mongoc/mongoc-stream-socket.c ${PROJECT_SOURCE_DIR}/src/mongoc/mongoc-structured-log.c ${PROJECT_SOURCE_DIR}/src/mongoc/mongoc-structured-log-command.c + ${PROJECT_SOURCE_DIR}/src/mongoc/mongoc-structured-log-connection.c ${PROJECT_SOURCE_DIR}/src/mongoc/mongoc-timeout.c ${PROJECT_SOURCE_DIR}/src/mongoc/mongoc-topology.c ${PROJECT_SOURCE_DIR}/src/mongoc/mongoc-topology-background-monitoring.c diff --git a/src/libmongoc/src/mongoc/mongoc-client.c b/src/libmongoc/src/mongoc/mongoc-client.c index 0d40669c825..544f9f87880 100644 --- a/src/libmongoc/src/mongoc/mongoc-client.c +++ b/src/libmongoc/src/mongoc/mongoc-client.c @@ -64,6 +64,7 @@ #include "mongoc-cmd-private.h" #include "mongoc-opts-private.h" #include "mongoc-structured-log-command-private.h" +#include "mongoc-structured-log-connection-private.h" #endif #if defined(MONGOC_ENABLE_SSL_OPENSSL) && OPENSSL_VERSION_NUMBER >= 0x10100000L @@ -1123,6 +1124,8 @@ _mongoc_client_new_from_topology (mongoc_topology_t *topology) } #endif + mongoc_structured_log_connection_client_created (); + mongoc_counter_clients_active_inc (); return client; diff --git a/src/libmongoc/src/mongoc/mongoc-structured-log-connection-private.h b/src/libmongoc/src/mongoc/mongoc-structured-log-connection-private.h new file mode 100644 index 00000000000..d1a9767c7b5 --- /dev/null +++ b/src/libmongoc/src/mongoc/mongoc-structured-log-connection-private.h @@ -0,0 +1,27 @@ +/* + * Copyright 2020 MongoDB, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "mongoc-prelude.h" +#include "mongoc-structured-log.h" +#include "mongoc-cmd-private.h" + +#ifndef MONGOC_STRUCTRURED_LOG_CONNECTION_PRIVATE_H +#define MONGOC_STRUCTRURED_LOG_CONNECTION_PRIVATE_H + +void +mongoc_structured_log_connection_client_created (); + +#endif /* MONGOC_STRUCTURED_LOG_COMMAND_PRIVATE_H */ diff --git a/src/libmongoc/src/mongoc/mongoc-structured-log-connection.c b/src/libmongoc/src/mongoc/mongoc-structured-log-connection.c new file mode 100644 index 00000000000..bf57d2b1d7f --- /dev/null +++ b/src/libmongoc/src/mongoc/mongoc-structured-log-connection.c @@ -0,0 +1,40 @@ +/* + * Copyright 2020 MongoDB, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +#if defined(__linux__) +#include +#elif defined(_WIN32) +#include +#elif defined(__FreeBSD__) +#include +#else +#include +#endif +#include + +#include "mongoc-structured-log-private.h" + +void +mongoc_structured_log_connection_client_created () +{ + mongoc_structured_log (MONGOC_STRUCTURED_LOG_LEVEL_DEBUG, + MONGOC_STRUCTURED_LOG_COMPONENT_CONNECTION, + "Client created", + NULL, + NULL); + +} From 4b580f01a1be3ab98bf8f2bee16b99fe64ff6195 Mon Sep 17 00:00:00 2001 From: Andreas Braun Date: Thu, 24 Sep 2020 13:53:00 +0200 Subject: [PATCH 020/139] Fix suggestions from code review --- .../mongoc-structured-log-command-private.h | 7 +- .../mongoc/mongoc-structured-log-command.c | 69 +++++++++---------- ...mongoc-structured-log-connection-private.h | 9 +-- .../mongoc/mongoc-structured-log-connection.c | 14 +--- .../mongoc/mongoc-structured-log-private.h | 7 +- .../src/mongoc/mongoc-structured-log.c | 21 ++---- 6 files changed, 53 insertions(+), 74 deletions(-) diff --git a/src/libmongoc/src/mongoc/mongoc-structured-log-command-private.h b/src/libmongoc/src/mongoc/mongoc-structured-log-command-private.h index a9560d5b8a9..e4ac4bfd400 100644 --- a/src/libmongoc/src/mongoc/mongoc-structured-log-command-private.h +++ b/src/libmongoc/src/mongoc/mongoc-structured-log-command-private.h @@ -15,12 +15,13 @@ */ #include "mongoc-prelude.h" + +#ifndef MONGOC_STRUCTURED_LOG_COMMAND_PRIVATE_H +#define MONGOC_STRUCTURED_LOG_COMMAND_PRIVATE_H + #include "mongoc-structured-log.h" #include "mongoc-cmd-private.h" -#ifndef MONGOC_STRUCTRURED_LOG_COMMAND_PRIVATE_H -#define MONGOC_STRUCTRURED_LOG_COMMAND_PRIVATE_H - typedef struct { const char *command_name; const char *db_name; diff --git a/src/libmongoc/src/mongoc/mongoc-structured-log-command.c b/src/libmongoc/src/mongoc/mongoc-structured-log-command.c index 4d797094579..bfb753d84ab 100644 --- a/src/libmongoc/src/mongoc/mongoc-structured-log-command.c +++ b/src/libmongoc/src/mongoc/mongoc-structured-log-command.c @@ -14,18 +14,6 @@ * limitations under the License. */ - -#if defined(__linux__) -#include -#elif defined(_WIN32) -#include -#elif defined(__FreeBSD__) -#include -#else -#include -#endif -#include - #include "mongoc-structured-log.h" #include "mongoc-structured-log-private.h" #include "mongoc-structured-log-command-private.h" @@ -130,14 +118,17 @@ mongoc_structured_log_command_started (const bson_t *command, bool explicit_session) { _mongoc_structured_log_command_t command_log = { - .command = command, - .command_name = command_name, - .db_name = db_name, - .operation_id = operation_id, - .request_id = request_id, - .driver_connection_id = driver_connection_id, - .server_connection_id = server_connection_id, - .explicit_session = explicit_session, + command_name, + db_name, + command, + NULL, + NULL, + 0, + operation_id, + request_id, + driver_connection_id, + server_connection_id, + explicit_session, }; mongoc_structured_log (MONGOC_STRUCTURED_LOG_LEVEL_INFO, @@ -158,14 +149,17 @@ mongoc_structured_log_command_success (const char *command_name, bool explicit_session) { _mongoc_structured_log_command_t command_log = { - .command_name = command_name, - .operation_id = operation_id, - .reply = reply, - .duration = duration, - .request_id = request_id, - .driver_connection_id = driver_connection_id, - .server_connection_id = server_connection_id, - .explicit_session = explicit_session, + command_name, + NULL, + NULL, + reply, + NULL, + duration, + operation_id, + request_id, + driver_connection_id, + server_connection_id, + explicit_session, }; mongoc_structured_log (MONGOC_STRUCTURED_LOG_LEVEL_INFO, @@ -186,14 +180,17 @@ mongoc_structured_log_command_failure (const char *command_name, bool explicit_session) { _mongoc_structured_log_command_t command_log = { - .command_name = command_name, - .operation_id = operation_id, - .reply = reply, - .error = error, - .request_id = request_id, - .driver_connection_id = driver_connection_id, - .server_connection_id = server_connection_id, - .explicit_session = explicit_session, + command_name, + NULL, + NULL, + reply, + error, + 0, + operation_id, + request_id, + driver_connection_id, + server_connection_id, + explicit_session, }; mongoc_structured_log (MONGOC_STRUCTURED_LOG_LEVEL_INFO, diff --git a/src/libmongoc/src/mongoc/mongoc-structured-log-connection-private.h b/src/libmongoc/src/mongoc/mongoc-structured-log-connection-private.h index d1a9767c7b5..f487457655b 100644 --- a/src/libmongoc/src/mongoc/mongoc-structured-log-connection-private.h +++ b/src/libmongoc/src/mongoc/mongoc-structured-log-connection-private.h @@ -15,13 +15,14 @@ */ #include "mongoc-prelude.h" + +#ifndef MONGOC_STRUCTURED_LOG_CONNECTION_PRIVATE_H +#define MONGOC_STRUCTURED_LOG_CONNECTION_PRIVATE_H + #include "mongoc-structured-log.h" #include "mongoc-cmd-private.h" -#ifndef MONGOC_STRUCTRURED_LOG_CONNECTION_PRIVATE_H -#define MONGOC_STRUCTRURED_LOG_CONNECTION_PRIVATE_H - void -mongoc_structured_log_connection_client_created (); +mongoc_structured_log_connection_client_created (void); #endif /* MONGOC_STRUCTURED_LOG_COMMAND_PRIVATE_H */ diff --git a/src/libmongoc/src/mongoc/mongoc-structured-log-connection.c b/src/libmongoc/src/mongoc/mongoc-structured-log-connection.c index bf57d2b1d7f..6967960cc34 100644 --- a/src/libmongoc/src/mongoc/mongoc-structured-log-connection.c +++ b/src/libmongoc/src/mongoc/mongoc-structured-log-connection.c @@ -14,22 +14,10 @@ * limitations under the License. */ - -#if defined(__linux__) -#include -#elif defined(_WIN32) -#include -#elif defined(__FreeBSD__) -#include -#else -#include -#endif -#include - #include "mongoc-structured-log-private.h" void -mongoc_structured_log_connection_client_created () +mongoc_structured_log_connection_client_created (void) { mongoc_structured_log (MONGOC_STRUCTURED_LOG_LEVEL_DEBUG, MONGOC_STRUCTURED_LOG_COMPONENT_CONNECTION, diff --git a/src/libmongoc/src/mongoc/mongoc-structured-log-private.h b/src/libmongoc/src/mongoc/mongoc-structured-log-private.h index 752e5b31423..f979733567f 100644 --- a/src/libmongoc/src/mongoc/mongoc-structured-log-private.h +++ b/src/libmongoc/src/mongoc/mongoc-structured-log-private.h @@ -15,13 +15,14 @@ */ #include "mongoc-prelude.h" + +#ifndef MONGOC_STRUCTURED_LOG_PRIVATE_H +#define MONGOC_STRUCTURED_LOG_PRIVATE_H + #include "mongoc-structured-log.h" #include "mongoc-cmd-private.h" #include "mongoc-structured-log-command-private.h" -#ifndef MONGOC_STRUCTRURED_LOG_PRIVATE_H -#define MONGOC_STRUCTRURED_LOG_PRIVATE_H - #define MONGOC_STRUCTURED_LOG_DEFAULT_LEVEL MONGOC_STRUCTURED_LOG_LEVEL_WARNING; typedef void (*mongoc_structured_log_build_message_t) ( diff --git a/src/libmongoc/src/mongoc/mongoc-structured-log.c b/src/libmongoc/src/mongoc/mongoc-structured-log.c index 5d337c4cad9..bf2d514d228 100644 --- a/src/libmongoc/src/mongoc/mongoc-structured-log.c +++ b/src/libmongoc/src/mongoc/mongoc-structured-log.c @@ -14,22 +14,10 @@ * limitations under the License. */ - -#if defined(__linux__) -#include -#elif defined(_WIN32) -#include -#elif defined(__FreeBSD__) -#include -#else -#include -#endif -#include -#include - #include "mongoc-structured-log.h" #include "mongoc-structured-log-private.h" #include "mongoc-thread-private.h" +#include "mongoc-util-private.h" static void mongoc_structured_log_default_handler (mongoc_structured_log_entry_t *entry, @@ -53,7 +41,7 @@ static void mongoc_structured_log_entry_destroy (mongoc_structured_log_entry_t *entry) { if (entry->structured_message) { - bson_free (entry->structured_message); + bson_destroy (entry->structured_message); } } @@ -114,11 +102,14 @@ mongoc_structured_log (mongoc_structured_log_level_t level, structured_message_data, }; + bson_once (&once, &_mongoc_ensure_mutex_once); + bson_mutex_lock (&gStructuredLogMutex); + if (!gStructuredLogger) { + bson_mutex_unlock (&gStructuredLogMutex); return; } - bson_mutex_lock (&gStructuredLogMutex); gStructuredLogger (&entry, gStructuredLoggerData); bson_mutex_unlock (&gStructuredLogMutex); From 5aa50e0c1467176841ea3ae140b55be7c2643536 Mon Sep 17 00:00:00 2001 From: Andreas Braun Date: Thu, 24 Sep 2020 14:16:12 +0200 Subject: [PATCH 021/139] Remove unused variable --- src/libmongoc/src/mongoc/mongoc-cursor-legacy.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/libmongoc/src/mongoc/mongoc-cursor-legacy.c b/src/libmongoc/src/mongoc/mongoc-cursor-legacy.c index c7fae54e10a..8a68b45f6f9 100644 --- a/src/libmongoc/src/mongoc/mongoc-cursor-legacy.c +++ b/src/libmongoc/src/mongoc/mongoc-cursor-legacy.c @@ -95,14 +95,11 @@ _mongoc_cursor_monitor_legacy_query (mongoc_cursor_t *cursor, mongoc_server_stream_t *server_stream) { bson_t doc; - mongoc_client_t *client; char *db; bool r; ENTRY; - client = cursor->client; - bson_init (&doc); db = bson_strndup (cursor->ns, cursor->dblen); From 810d3fda1d7798b92c315d927ec1e4e3f52a36bd Mon Sep 17 00:00:00 2001 From: Andreas Braun Date: Fri, 25 Sep 2020 13:16:30 +0200 Subject: [PATCH 022/139] Remove union in favour of void * --- .../mongoc/mongoc-structured-log-command.c | 51 ++++++++++++------- .../mongoc/mongoc-structured-log-connection.c | 1 - .../mongoc/mongoc-structured-log-private.h | 8 +-- .../src/mongoc/mongoc-structured-log.c | 4 +- .../tests/test-mongoc-structured-log.c | 45 ++++++++-------- 5 files changed, 60 insertions(+), 49 deletions(-) diff --git a/src/libmongoc/src/mongoc/mongoc-structured-log-command.c b/src/libmongoc/src/mongoc/mongoc-structured-log-command.c index bfb753d84ab..781063ad017 100644 --- a/src/libmongoc/src/mongoc/mongoc-structured-log-command.c +++ b/src/libmongoc/src/mongoc/mongoc-structured-log-command.c @@ -20,11 +20,12 @@ static void _mongoc_log_structured_append_command_data ( - mongoc_structured_log_entry_t *entry) + void *structured_log_data, bson_t *structured_message /* OUT */) { - _mongoc_structured_log_command_t *log_command = entry->command; + _mongoc_structured_log_command_t *log_command = + (_mongoc_structured_log_command_t *) structured_log_data; - BCON_APPEND (entry->structured_message, + BCON_APPEND (structured_message, "commandName", BCON_UTF8 (log_command->command_name), "requestId", @@ -41,18 +42,22 @@ _mongoc_log_structured_append_command_data ( static void mongoc_log_structured_build_command_started_message ( - mongoc_structured_log_entry_t *entry) + mongoc_structured_log_component_t component, + void *structured_log_data, + bson_t *structured_message /* OUT */) { char *cmd_json; - _mongoc_structured_log_command_t *log_command = entry->command; + _mongoc_structured_log_command_t *log_command = + (_mongoc_structured_log_command_t *) structured_log_data; - BSON_ASSERT (entry->component == MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND); + BSON_ASSERT (component == MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND); cmd_json = bson_as_canonical_extended_json (log_command->command, NULL); - _mongoc_log_structured_append_command_data (entry); + _mongoc_log_structured_append_command_data (structured_log_data, + structured_message); - BCON_APPEND (entry->structured_message, + BCON_APPEND (structured_message, "databaseName", BCON_UTF8 (log_command->db_name), "command", @@ -63,18 +68,22 @@ mongoc_log_structured_build_command_started_message ( static void mongoc_log_structured_build_command_succeeded_message ( - mongoc_structured_log_entry_t *entry) + mongoc_structured_log_component_t component, + void *structured_log_data, + bson_t *structured_message /* OUT */) { char *reply_json; - _mongoc_structured_log_command_t *log_command = entry->command; + _mongoc_structured_log_command_t *log_command = + (_mongoc_structured_log_command_t *) structured_log_data; - BSON_ASSERT (entry->component == MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND); + BSON_ASSERT (component == MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND); reply_json = bson_as_canonical_extended_json (log_command->reply, NULL); - _mongoc_log_structured_append_command_data (entry); + _mongoc_log_structured_append_command_data (structured_log_data, + structured_message); - BCON_APPEND (entry->structured_message, + BCON_APPEND (structured_message, "duration", BCON_INT64 (log_command->duration), "reply", @@ -85,21 +94,25 @@ mongoc_log_structured_build_command_succeeded_message ( static void mongoc_log_structured_build_command_failed_message ( - mongoc_structured_log_entry_t *entry) + mongoc_structured_log_component_t component, + void *structured_log_data, + bson_t *structured_message /* OUT */) { char *reply_json; - _mongoc_structured_log_command_t *log_command = entry->command; + _mongoc_structured_log_command_t *log_command = + (_mongoc_structured_log_command_t *) structured_log_data; - BSON_ASSERT (entry->component == MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND); + BSON_ASSERT (component == MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND); reply_json = bson_as_canonical_extended_json (log_command->reply, NULL); - _mongoc_log_structured_append_command_data (entry); + _mongoc_log_structured_append_command_data (structured_log_data, + structured_message); - BCON_APPEND (entry->structured_message, "reply", BCON_UTF8 (reply_json)); + BCON_APPEND (structured_message, "reply", BCON_UTF8 (reply_json)); if (log_command->error) { - BCON_APPEND (entry->structured_message, + BCON_APPEND (structured_message, "failure", BCON_UTF8 (log_command->error->message)); } diff --git a/src/libmongoc/src/mongoc/mongoc-structured-log-connection.c b/src/libmongoc/src/mongoc/mongoc-structured-log-connection.c index 6967960cc34..58205d9cb4e 100644 --- a/src/libmongoc/src/mongoc/mongoc-structured-log-connection.c +++ b/src/libmongoc/src/mongoc/mongoc-structured-log-connection.c @@ -24,5 +24,4 @@ mongoc_structured_log_connection_client_created (void) "Client created", NULL, NULL); - } diff --git a/src/libmongoc/src/mongoc/mongoc-structured-log-private.h b/src/libmongoc/src/mongoc/mongoc-structured-log-private.h index f979733567f..9cda8c8b753 100644 --- a/src/libmongoc/src/mongoc/mongoc-structured-log-private.h +++ b/src/libmongoc/src/mongoc/mongoc-structured-log-private.h @@ -26,7 +26,9 @@ #define MONGOC_STRUCTURED_LOG_DEFAULT_LEVEL MONGOC_STRUCTURED_LOG_LEVEL_WARNING; typedef void (*mongoc_structured_log_build_message_t) ( - mongoc_structured_log_entry_t *entry); + mongoc_structured_log_component_t component, + void *structured_log_data, + bson_t *structured_message /* OUT */); struct _mongoc_structured_log_entry_t { mongoc_structured_log_level_t level; @@ -34,9 +36,7 @@ struct _mongoc_structured_log_entry_t { const char *message; bson_t *structured_message; mongoc_structured_log_build_message_t build_message_func; - union { - _mongoc_structured_log_command_t *command; - }; + void *structured_log_data; }; void diff --git a/src/libmongoc/src/mongoc/mongoc-structured-log.c b/src/libmongoc/src/mongoc/mongoc-structured-log.c index bf2d514d228..0577be12b5b 100644 --- a/src/libmongoc/src/mongoc/mongoc-structured-log.c +++ b/src/libmongoc/src/mongoc/mongoc-structured-log.c @@ -53,7 +53,9 @@ mongoc_structured_log_entry_get_message (mongoc_structured_log_entry_t *entry) BCON_NEW ("message", BCON_UTF8 (entry->message)); if (entry->build_message_func) { - entry->build_message_func (entry); + entry->build_message_func (entry->component, + entry->structured_log_data, + entry->structured_message); } } diff --git a/src/libmongoc/tests/test-mongoc-structured-log.c b/src/libmongoc/tests/test-mongoc-structured-log.c index 200f79ed99b..b7d42aa625a 100644 --- a/src/libmongoc/tests/test-mongoc-structured-log.c +++ b/src/libmongoc/tests/test-mongoc-structured-log.c @@ -62,16 +62,14 @@ test_plain_log_entry () { struct structured_log_state old_state; struct log_assumption assumption = { - .expected_entry = - { - .level = MONGOC_STRUCTURED_LOG_LEVEL_WARNING, - .component = MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND, - .message = "Plain log entry", - .structured_message = - BCON_NEW ("message", BCON_UTF8 ("Plain log entry")), - }, - .expected_calls = 1, - .calls = 0, + { + MONGOC_STRUCTURED_LOG_LEVEL_WARNING, + MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND, + "Plain log entry", + BCON_NEW ("message", BCON_UTF8 ("Plain log entry")), + }, + 1, + 0, }; save_state (&old_state); @@ -90,9 +88,9 @@ test_plain_log_entry () } void -_test_append_extra_data (mongoc_structured_log_entry_t *entry) +_test_append_extra_data (mongoc_structured_log_component_t component, void *structured_log_data, bson_t *structured_message /* OUT */) { - BCON_APPEND (entry->structured_message, "extra", BCON_INT32 (1)); + BCON_APPEND (structured_message, "extra", BCON_INT32 (1)); } void @@ -100,18 +98,17 @@ test_log_entry_with_extra_data () { struct structured_log_state old_state; struct log_assumption assumption = { - .expected_entry = - { - .level = MONGOC_STRUCTURED_LOG_LEVEL_WARNING, - .component = MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND, - .message = "Plain log entry", - .structured_message = BCON_NEW ("message", - BCON_UTF8 ("Plain log entry"), - "extra", - BCON_INT32 (1)), - }, - .expected_calls = 1, - .calls = 0, + { + MONGOC_STRUCTURED_LOG_LEVEL_WARNING, + MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND, + "Plain log entry", + BCON_NEW ("message", + BCON_UTF8 ("Plain log entry"), + "extra", + BCON_INT32 (1)), + }, + 1, + 0, }; save_state (&old_state); From a8be342bc9a522ae0a183da6d1d305a5199de3a0 Mon Sep 17 00:00:00 2001 From: Andreas Braun Date: Fri, 25 Sep 2020 13:48:38 +0200 Subject: [PATCH 023/139] Use lowercase test names --- src/libmongoc/tests/test-mongoc-structured-log.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libmongoc/tests/test-mongoc-structured-log.c b/src/libmongoc/tests/test-mongoc-structured-log.c index b7d42aa625a..83f9047ee77 100644 --- a/src/libmongoc/tests/test-mongoc-structured-log.c +++ b/src/libmongoc/tests/test-mongoc-structured-log.c @@ -129,7 +129,7 @@ test_log_entry_with_extra_data () void test_structured_log_install (TestSuite *suite) { - TestSuite_Add (suite, "/Structured_Log/plain", test_plain_log_entry); + TestSuite_Add (suite, "/structured_log/plain", test_plain_log_entry); TestSuite_Add ( - suite, "/Structured_Log/with_extra_data", test_log_entry_with_extra_data); + suite, "/structured_log/with_extra_data", test_log_entry_with_extra_data); } From 2e73cb45c41232a5268ad04a29fa98fb55a7fc5f Mon Sep 17 00:00:00 2001 From: Andreas Braun Date: Fri, 25 Sep 2020 13:59:43 +0200 Subject: [PATCH 024/139] Free allocated memory in tests --- src/libmongoc/tests/test-mongoc-structured-log.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/libmongoc/tests/test-mongoc-structured-log.c b/src/libmongoc/tests/test-mongoc-structured-log.c index 83f9047ee77..f512bafcc9e 100644 --- a/src/libmongoc/tests/test-mongoc-structured-log.c +++ b/src/libmongoc/tests/test-mongoc-structured-log.c @@ -85,6 +85,8 @@ test_plain_log_entry () ASSERT_CMPINT (assumption.calls, =, 1); restore_state (&old_state); + + bson_destroy (assumption.expected_entry.structured_message); } void @@ -124,6 +126,8 @@ test_log_entry_with_extra_data () ASSERT_CMPINT (assumption.calls, =, 1); restore_state (&old_state); + + bson_destroy (assumption.expected_entry.structured_message); } void From c264c7e657e7b4b135050a97543f338cd001b540 Mon Sep 17 00:00:00 2001 From: Andreas Braun Date: Fri, 25 Sep 2020 15:41:53 +0200 Subject: [PATCH 025/139] Fix wrong location for imports And that's why you shouldn't trust your IDE to correctly add include statements --- src/libmongoc/src/mongoc/mongoc-client.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libmongoc/src/mongoc/mongoc-client.c b/src/libmongoc/src/mongoc/mongoc-client.c index 544f9f87880..3b55a18a880 100644 --- a/src/libmongoc/src/mongoc/mongoc-client.c +++ b/src/libmongoc/src/mongoc/mongoc-client.c @@ -57,14 +57,14 @@ #include "mongoc-change-stream-private.h" #include "mongoc-client-session-private.h" #include "mongoc-cursor-private.h" +#include "mongoc-structured-log-command-private.h" +#include "mongoc-structured-log-connection-private.h" #ifdef MONGOC_ENABLE_SSL #include "mongoc-stream-tls.h" #include "mongoc-ssl-private.h" #include "mongoc-cmd-private.h" #include "mongoc-opts-private.h" -#include "mongoc-structured-log-command-private.h" -#include "mongoc-structured-log-connection-private.h" #endif #if defined(MONGOC_ENABLE_SSL_OPENSSL) && OPENSSL_VERSION_NUMBER >= 0x10100000L From eae3b5156c1bac8ae33187d5e5d8d637c1bf7098 Mon Sep 17 00:00:00 2001 From: Andreas Braun Date: Fri, 25 Sep 2020 15:49:30 +0200 Subject: [PATCH 026/139] Attach OP_MSG document sequences when logging commands --- src/libmongoc/src/mongoc/mongoc-cluster.c | 13 +++----- .../mongoc-structured-log-command-private.h | 7 ++++ .../mongoc/mongoc-structured-log-command.c | 32 +++++++++++++++++++ 3 files changed, 44 insertions(+), 8 deletions(-) diff --git a/src/libmongoc/src/mongoc/mongoc-cluster.c b/src/libmongoc/src/mongoc/mongoc-cluster.c index 17e4e175225..4a28f6023b2 100644 --- a/src/libmongoc/src/mongoc/mongoc-cluster.c +++ b/src/libmongoc/src/mongoc/mongoc-cluster.c @@ -534,14 +534,11 @@ mongoc_cluster_run_command_monitored (mongoc_cluster_t *cluster, mongoc_cmd_t *c } // @todo Provide missing arguments - mongoc_structured_log_command_started (cmd->command, - cmd->command_name, - cmd->db_name, - cmd->operation_id, - request_id, - 0, - 0, - false); + mongoc_structured_log_command_started_with_cmd (cmd, + request_id, + 0, + 0, + false); if (callbacks->started) { mongoc_apm_command_started_init_with_cmd ( diff --git a/src/libmongoc/src/mongoc/mongoc-structured-log-command-private.h b/src/libmongoc/src/mongoc/mongoc-structured-log-command-private.h index e4ac4bfd400..c8913a0d322 100644 --- a/src/libmongoc/src/mongoc/mongoc-structured-log-command-private.h +++ b/src/libmongoc/src/mongoc/mongoc-structured-log-command-private.h @@ -46,6 +46,13 @@ mongoc_structured_log_command_started (const bson_t *command, uint32_t server_connection_id, bool explicit_session); +void +mongoc_structured_log_command_started_with_cmd (const mongoc_cmd_t *cmd, + uint32_t request_id, + uint32_t driver_connection_id, + uint32_t server_connection_id, + bool explicit_session); + void mongoc_structured_log_command_success (const char *command_name, int64_t operation_id, diff --git a/src/libmongoc/src/mongoc/mongoc-structured-log-command.c b/src/libmongoc/src/mongoc/mongoc-structured-log-command.c index 781063ad017..aa3ed884add 100644 --- a/src/libmongoc/src/mongoc/mongoc-structured-log-command.c +++ b/src/libmongoc/src/mongoc/mongoc-structured-log-command.c @@ -151,6 +151,38 @@ mongoc_structured_log_command_started (const bson_t *command, &command_log); } +void +mongoc_structured_log_command_started_with_cmd (const mongoc_cmd_t *cmd, + uint32_t request_id, + uint32_t driver_connection_id, + uint32_t server_connection_id, + bool explicit_session) +{ + // Discard const modifier, we promise we won't modify this + bson_t *command = (bson_t *) cmd->command; + bool command_owned = false; + + if (cmd->payload && !cmd->payload_size) { + command = bson_copy (cmd->command); + command_owned = true; + + _mongoc_cmd_append_payload_as_array (cmd, command); + } + + mongoc_structured_log_command_started (command, + cmd->command_name, + cmd->db_name, + cmd->operation_id, + request_id, + driver_connection_id, + server_connection_id, + explicit_session); + + if (command_owned) { + bson_destroy (command); + } +} + void mongoc_structured_log_command_success (const char *command_name, int64_t operation_id, From 42d50c01725a5d6b1a16ca0698b5872eeeeee838 Mon Sep 17 00:00:00 2001 From: Andreas Braun Date: Fri, 25 Sep 2020 15:51:03 +0200 Subject: [PATCH 027/139] Remove mixed declaration and code --- src/libmongoc/src/mongoc/mongoc-structured-log.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/libmongoc/src/mongoc/mongoc-structured-log.c b/src/libmongoc/src/mongoc/mongoc-structured-log.c index 0577be12b5b..6c2ca913c4c 100644 --- a/src/libmongoc/src/mongoc/mongoc-structured-log.c +++ b/src/libmongoc/src/mongoc/mongoc-structured-log.c @@ -201,6 +201,7 @@ static void mongoc_structured_log_default_handler (mongoc_structured_log_entry_t *entry, void *user_data) { + char *message; mongoc_structured_log_level_t log_level = _mongoc_structured_log_get_log_level ( mongoc_structured_log_entry_get_component (entry)); @@ -209,7 +210,7 @@ mongoc_structured_log_default_handler (mongoc_structured_log_entry_t *entry, return; } - char *message = + message = bson_as_json (mongoc_structured_log_entry_get_message (entry), NULL); fprintf (_mongoc_structured_log_get_stream (), From 794d51b57c94aa901cf75d3838d698cde3e10b27 Mon Sep 17 00:00:00 2001 From: Andreas Braun Date: Mon, 28 Sep 2020 16:00:45 +0200 Subject: [PATCH 028/139] Fix single-line comments --- src/libmongoc/src/mongoc/mongoc-client.c | 6 +++--- src/libmongoc/src/mongoc/mongoc-cluster.c | 6 +++--- src/libmongoc/src/mongoc/mongoc-cursor-legacy.c | 2 +- src/libmongoc/src/mongoc/mongoc-cursor.c | 6 +++--- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/libmongoc/src/mongoc/mongoc-client.c b/src/libmongoc/src/mongoc/mongoc-client.c index 3b55a18a880..b6d5c0cbc68 100644 --- a/src/libmongoc/src/mongoc/mongoc-client.c +++ b/src/libmongoc/src/mongoc/mongoc-client.c @@ -2166,7 +2166,7 @@ _mongoc_client_monitor_op_killcursors (mongoc_cluster_t *cluster, bson_init (&doc); _mongoc_client_prepare_killcursors_command (cursor_id, collection, &doc); - // @todo Provide missing arguments + /* @todo Provide missing arguments */ mongoc_structured_log_command_started ( &doc, "killCursors", db, operation_id, cluster->request_id, 0, 0, false); @@ -2220,7 +2220,7 @@ _mongoc_client_monitor_op_killcursors_succeeded (mongoc_cluster_t *cluster, bson_array_builder_append_int64 (cursors_unknown, cursor_id); bson_append_array_builder_end (&doc, cursors_unknown); - // @todo Provide missing arguments + /* @todo Provide missing arguments */ mongoc_structured_log_command_success ("killCursors", operation_id, &doc, @@ -2276,7 +2276,7 @@ _mongoc_client_monitor_op_killcursors_failed (mongoc_cluster_t *cluster, bson_init (&doc); bson_append_int32 (&doc, "ok", 2, 0); - // @todo Provide missing arguments + /* @todo Provide missing arguments */ mongoc_structured_log_command_failure ("killCursors", operation_id, &doc, diff --git a/src/libmongoc/src/mongoc/mongoc-cluster.c b/src/libmongoc/src/mongoc/mongoc-cluster.c index 4a28f6023b2..3a8a8187def 100644 --- a/src/libmongoc/src/mongoc/mongoc-cluster.c +++ b/src/libmongoc/src/mongoc/mongoc-cluster.c @@ -533,7 +533,7 @@ mongoc_cluster_run_command_monitored (mongoc_cluster_t *cluster, mongoc_cmd_t *c } } - // @todo Provide missing arguments + /* @todo Provide missing arguments */ mongoc_structured_log_command_started_with_cmd (cmd, request_id, 0, @@ -563,7 +563,7 @@ mongoc_cluster_run_command_monitored (mongoc_cluster_t *cluster, mongoc_cmd_t *c bson_append_int32 (&fake_reply, "ok", 2, 1); } - // @todo Provide missing arguments + /* @todo Provide missing arguments */ mongoc_structured_log_command_success (cmd->command_name, cmd->operation_id, cmd->is_acknowledged ? reply @@ -597,7 +597,7 @@ mongoc_cluster_run_command_monitored (mongoc_cluster_t *cluster, mongoc_cmd_t *c } else { int64_t duration = bson_get_monotonic_time () - started; - // @todo Provide missing arguments + /* @todo Provide missing arguments */ mongoc_structured_log_command_failure (cmd->command_name, cmd->operation_id, reply, diff --git a/src/libmongoc/src/mongoc/mongoc-cursor-legacy.c b/src/libmongoc/src/mongoc/mongoc-cursor-legacy.c index 8a68b45f6f9..93a438b2242 100644 --- a/src/libmongoc/src/mongoc/mongoc-cursor-legacy.c +++ b/src/libmongoc/src/mongoc/mongoc-cursor-legacy.c @@ -50,7 +50,7 @@ _mongoc_cursor_monitor_legacy_get_more (mongoc_cursor_t *cursor, mongoc_server_s _mongoc_cursor_prepare_getmore_command (cursor, &doc); db = bson_strndup (cursor->ns, cursor->dblen); - // @todo Provide missing arguments + /* @todo Provide missing arguments */ mongoc_structured_log_command_started (&doc, "getMore", db, diff --git a/src/libmongoc/src/mongoc/mongoc-cursor.c b/src/libmongoc/src/mongoc/mongoc-cursor.c index 47872536b0e..05ea665b72a 100644 --- a/src/libmongoc/src/mongoc/mongoc-cursor.c +++ b/src/libmongoc/src/mongoc/mongoc-cursor.c @@ -647,7 +647,7 @@ _mongoc_cursor_monitor_command (mongoc_cursor_t *cursor, client = cursor->client; db = bson_strndup (cursor->ns, cursor->dblen); - // @todo Provide missing arguments + /* @todo Provide missing arguments */ mongoc_structured_log_command_started (cmd, cmd_name, db, @@ -738,7 +738,7 @@ _mongoc_cursor_monitor_succeeded (mongoc_cursor_t *cursor, bson_destroy (&docs_array); - // @todo Provide missing arguments + /* @todo Provide missing arguments */ mongoc_structured_log_command_success (cmd_name, cursor->operation_id, &reply, @@ -796,7 +796,7 @@ _mongoc_cursor_monitor_failed (mongoc_cursor_t *cursor, bsonBuildDecl (reply, kv ("ok", int32 (0))); char *db = bson_strndup (cursor->ns, cursor->dblen); - // @todo Provide missing arguments + /* @todo Provide missing arguments */ mongoc_structured_log_command_failure (cmd_name, cursor->operation_id, &reply, From bfc69a1280a24de86a8739d620215fbba0ac5e97 Mon Sep 17 00:00:00 2001 From: Andreas Braun Date: Wed, 30 Sep 2020 14:09:13 +0200 Subject: [PATCH 029/139] Fix more single line comments --- src/libmongoc/src/mongoc/mongoc-structured-log-command.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libmongoc/src/mongoc/mongoc-structured-log-command.c b/src/libmongoc/src/mongoc/mongoc-structured-log-command.c index aa3ed884add..64697cea452 100644 --- a/src/libmongoc/src/mongoc/mongoc-structured-log-command.c +++ b/src/libmongoc/src/mongoc/mongoc-structured-log-command.c @@ -158,7 +158,7 @@ mongoc_structured_log_command_started_with_cmd (const mongoc_cmd_t *cmd, uint32_t server_connection_id, bool explicit_session) { - // Discard const modifier, we promise we won't modify this + /* Discard const modifier, we promise we won't modify this */ bson_t *command = (bson_t *) cmd->command; bool command_owned = false; From 4e846efb9fee12386285e2a4527ae0e75e42c545 Mon Sep 17 00:00:00 2001 From: Andreas Braun Date: Thu, 8 Oct 2020 14:48:55 +0200 Subject: [PATCH 030/139] Update logged fields to latest spec version --- src/libmongoc/src/mongoc/mongoc-client.c | 14 +++++-- src/libmongoc/src/mongoc/mongoc-cluster.c | 10 ++--- .../src/mongoc/mongoc-cursor-legacy.c | 2 +- src/libmongoc/src/mongoc/mongoc-cursor.c | 6 +-- .../mongoc-structured-log-command-private.h | 36 +++++++++------- .../mongoc/mongoc-structured-log-command.c | 41 ++++++++++++++----- 6 files changed, 69 insertions(+), 40 deletions(-) diff --git a/src/libmongoc/src/mongoc/mongoc-client.c b/src/libmongoc/src/mongoc/mongoc-client.c index b6d5c0cbc68..c34a455191b 100644 --- a/src/libmongoc/src/mongoc/mongoc-client.c +++ b/src/libmongoc/src/mongoc/mongoc-client.c @@ -2167,8 +2167,14 @@ _mongoc_client_monitor_op_killcursors (mongoc_cluster_t *cluster, _mongoc_client_prepare_killcursors_command (cursor_id, collection, &doc); /* @todo Provide missing arguments */ - mongoc_structured_log_command_started ( - &doc, "killCursors", db, operation_id, cluster->request_id, 0, 0, false); + mongoc_structured_log_command_started (&doc, + "killCursors", + db, + operation_id, + cluster->request_id, + &server_stream->sd->host, + 0, + false); if (!client->apm_callbacks.started) { bson_destroy (&doc); @@ -2226,7 +2232,7 @@ _mongoc_client_monitor_op_killcursors_succeeded (mongoc_cluster_t *cluster, &doc, duration, cluster->request_id, - 0, + &server_stream->sd->host, 0, false); @@ -2282,7 +2288,7 @@ _mongoc_client_monitor_op_killcursors_failed (mongoc_cluster_t *cluster, &doc, error, cluster->request_id, - 0, + &server_stream->sd->host, 0, false); diff --git a/src/libmongoc/src/mongoc/mongoc-cluster.c b/src/libmongoc/src/mongoc/mongoc-cluster.c index 3a8a8187def..305a9175af9 100644 --- a/src/libmongoc/src/mongoc/mongoc-cluster.c +++ b/src/libmongoc/src/mongoc/mongoc-cluster.c @@ -534,11 +534,7 @@ mongoc_cluster_run_command_monitored (mongoc_cluster_t *cluster, mongoc_cmd_t *c } /* @todo Provide missing arguments */ - mongoc_structured_log_command_started_with_cmd (cmd, - request_id, - 0, - 0, - false); + mongoc_structured_log_command_started_with_cmd (cmd, request_id, 0, false); if (callbacks->started) { mongoc_apm_command_started_init_with_cmd ( @@ -570,7 +566,7 @@ mongoc_cluster_run_command_monitored (mongoc_cluster_t *cluster, mongoc_cmd_t *c : &fake_reply, duration, request_id, - 0, + &server_stream->sd->host, 0, false); @@ -603,7 +599,7 @@ mongoc_cluster_run_command_monitored (mongoc_cluster_t *cluster, mongoc_cmd_t *c reply, error, cluster->request_id, - 0, + &server_stream->sd->host, 0, false); diff --git a/src/libmongoc/src/mongoc/mongoc-cursor-legacy.c b/src/libmongoc/src/mongoc/mongoc-cursor-legacy.c index 93a438b2242..8a52ff4758e 100644 --- a/src/libmongoc/src/mongoc/mongoc-cursor-legacy.c +++ b/src/libmongoc/src/mongoc/mongoc-cursor-legacy.c @@ -56,7 +56,7 @@ _mongoc_cursor_monitor_legacy_get_more (mongoc_cursor_t *cursor, mongoc_server_s db, cursor->operation_id, client->cluster.request_id, - 0, + &server_stream->sd->host, 0, false); diff --git a/src/libmongoc/src/mongoc/mongoc-cursor.c b/src/libmongoc/src/mongoc/mongoc-cursor.c index 05ea665b72a..eb2609ecb58 100644 --- a/src/libmongoc/src/mongoc/mongoc-cursor.c +++ b/src/libmongoc/src/mongoc/mongoc-cursor.c @@ -653,7 +653,7 @@ _mongoc_cursor_monitor_command (mongoc_cursor_t *cursor, db, cursor->operation_id, client->cluster.request_id, - 0, + &server_stream->sd->host, 0, false); @@ -744,7 +744,7 @@ _mongoc_cursor_monitor_succeeded (mongoc_cursor_t *cursor, &reply, duration, client->cluster.request_id, - 0, + &stream->sd->host, 0, false); @@ -802,7 +802,7 @@ _mongoc_cursor_monitor_failed (mongoc_cursor_t *cursor, &reply, &cursor->error, client->cluster.request_id, - 0, + &stream->sd->host, 0, false); diff --git a/src/libmongoc/src/mongoc/mongoc-structured-log-command-private.h b/src/libmongoc/src/mongoc/mongoc-structured-log-command-private.h index c8913a0d322..72dbc63c70a 100644 --- a/src/libmongoc/src/mongoc/mongoc-structured-log-command-private.h +++ b/src/libmongoc/src/mongoc/mongoc-structured-log-command-private.h @@ -23,17 +23,24 @@ #include "mongoc-cmd-private.h" typedef struct { - const char *command_name; - const char *db_name; - const bson_t *command; - const bson_t *reply; - const bson_error_t *error; - int64_t duration; - int64_t operation_id; - uint32_t request_id; - uint32_t driver_connection_id; - uint32_t server_connection_id; - bool explicit_session; + /* Needed for: - Started + * / - Succeeded + * | / - Failed + * | | / + * | | | */ + const char *command_name; /* x x x */ + const char *db_name; /* x - - */ + const bson_t *command; /* x - - */ + const bson_t *reply; /* - x x */ + const bson_error_t *error; /* - - x */ + int64_t duration; /* - x x */ + int64_t operation_id; /* x x x */ + uint32_t request_id; /* x x x */ + const mongoc_host_list_t *host; /* x x x */ + char *server_resolved_ip; /* x x x */ + uint16_t client_port; /* x x x */ + uint32_t server_connection_id; /* x x x */ + bool explicit_session; /* x x x */ } _mongoc_structured_log_command_t; void @@ -42,14 +49,13 @@ mongoc_structured_log_command_started (const bson_t *command, const char *db_name, int64_t operation_id, uint32_t request_id, - uint32_t driver_connection_id, + const mongoc_host_list_t *host, uint32_t server_connection_id, bool explicit_session); void mongoc_structured_log_command_started_with_cmd (const mongoc_cmd_t *cmd, uint32_t request_id, - uint32_t driver_connection_id, uint32_t server_connection_id, bool explicit_session); @@ -59,7 +65,7 @@ mongoc_structured_log_command_success (const char *command_name, const bson_t *reply, uint64_t duration, uint32_t request_id, - uint32_t driver_connection_id, + const mongoc_host_list_t *host, uint32_t server_connection_id, bool explicit_session); @@ -69,7 +75,7 @@ mongoc_structured_log_command_failure (const char *command_name, const bson_t *reply, const bson_error_t *error, uint32_t request_id, - uint32_t driver_connection_id, + const mongoc_host_list_t *host, uint32_t server_connection_id, bool explicit_session); diff --git a/src/libmongoc/src/mongoc/mongoc-structured-log-command.c b/src/libmongoc/src/mongoc/mongoc-structured-log-command.c index 64697cea452..24c58753637 100644 --- a/src/libmongoc/src/mongoc/mongoc-structured-log-command.c +++ b/src/libmongoc/src/mongoc/mongoc-structured-log-command.c @@ -32,8 +32,21 @@ _mongoc_log_structured_append_command_data ( BCON_INT32 (log_command->request_id), "operationId", BCON_INT64 (log_command->operation_id), - "driverConnectionId", - BCON_INT32 (log_command->driver_connection_id), + "serverHostname", + BCON_UTF8 (log_command->host->host), + "serverResolvedIPAddress", + BCON_UTF8 (log_command->server_resolved_ip), + "serverPort", + BCON_INT32 (log_command->host->port)); + + // Append client port only if it was provided + if (log_command->client_port) { + BCON_APPEND (structured_message, + "clientPort", + BCON_INT32 (log_command->client_port)); + } + + BCON_APPEND (structured_message, "serverConnectionId", BCON_INT32 (log_command->server_connection_id), "explicitSession", @@ -52,6 +65,7 @@ mongoc_log_structured_build_command_started_message ( BSON_ASSERT (component == MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND); + /* @todo Use MONGODB_MAX_DOCUMENT_LENGTH */ cmd_json = bson_as_canonical_extended_json (log_command->command, NULL); _mongoc_log_structured_append_command_data (structured_log_data, @@ -78,6 +92,7 @@ mongoc_log_structured_build_command_succeeded_message ( BSON_ASSERT (component == MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND); + /* @todo Use MONGODB_MAX_DOCUMENT_LENGTH */ reply_json = bson_as_canonical_extended_json (log_command->reply, NULL); _mongoc_log_structured_append_command_data (structured_log_data, @@ -104,6 +119,7 @@ mongoc_log_structured_build_command_failed_message ( BSON_ASSERT (component == MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND); + /* @todo Use MONGODB_MAX_DOCUMENT_LENGTH */ reply_json = bson_as_canonical_extended_json (log_command->reply, NULL); _mongoc_log_structured_append_command_data (structured_log_data, @@ -126,7 +142,7 @@ mongoc_structured_log_command_started (const bson_t *command, const char *db_name, int64_t operation_id, uint32_t request_id, - uint32_t driver_connection_id, + const mongoc_host_list_t *host, uint32_t server_connection_id, bool explicit_session) { @@ -139,7 +155,9 @@ mongoc_structured_log_command_started (const bson_t *command, 0, operation_id, request_id, - driver_connection_id, + host, + NULL, + 0, server_connection_id, explicit_session, }; @@ -154,7 +172,6 @@ mongoc_structured_log_command_started (const bson_t *command, void mongoc_structured_log_command_started_with_cmd (const mongoc_cmd_t *cmd, uint32_t request_id, - uint32_t driver_connection_id, uint32_t server_connection_id, bool explicit_session) { @@ -174,7 +191,7 @@ mongoc_structured_log_command_started_with_cmd (const mongoc_cmd_t *cmd, cmd->db_name, cmd->operation_id, request_id, - driver_connection_id, + &cmd->server_stream->sd->host, server_connection_id, explicit_session); @@ -189,7 +206,7 @@ mongoc_structured_log_command_success (const char *command_name, const bson_t *reply, uint64_t duration, uint32_t request_id, - uint32_t driver_connection_id, + const mongoc_host_list_t *host, uint32_t server_connection_id, bool explicit_session) { @@ -202,7 +219,9 @@ mongoc_structured_log_command_success (const char *command_name, duration, operation_id, request_id, - driver_connection_id, + host, + NULL, + 0, server_connection_id, explicit_session, }; @@ -220,7 +239,7 @@ mongoc_structured_log_command_failure (const char *command_name, const bson_t *reply, const bson_error_t *error, uint32_t request_id, - uint32_t driver_connection_id, + const mongoc_host_list_t *host, uint32_t server_connection_id, bool explicit_session) { @@ -233,7 +252,9 @@ mongoc_structured_log_command_failure (const char *command_name, 0, operation_id, request_id, - driver_connection_id, + host, + NULL, + 0, server_connection_id, explicit_session, }; From ca10a2a28432312ff4e4018fb03b112ed8fb4ce4 Mon Sep 17 00:00:00 2001 From: Andreas Braun Date: Tue, 13 Oct 2020 10:49:31 +0200 Subject: [PATCH 031/139] Use maximum document length setting when logging documents --- .../mongoc/mongoc-structured-log-command.c | 9 +++---- .../mongoc/mongoc-structured-log-private.h | 1 + .../src/mongoc/mongoc-structured-log.c | 24 +++++++++++++++++++ .../src/mongoc/mongoc-structured-log.h | 10 ++++++++ 4 files changed, 38 insertions(+), 6 deletions(-) diff --git a/src/libmongoc/src/mongoc/mongoc-structured-log-command.c b/src/libmongoc/src/mongoc/mongoc-structured-log-command.c index 24c58753637..7ee5b21638e 100644 --- a/src/libmongoc/src/mongoc/mongoc-structured-log-command.c +++ b/src/libmongoc/src/mongoc/mongoc-structured-log-command.c @@ -65,8 +65,7 @@ mongoc_log_structured_build_command_started_message ( BSON_ASSERT (component == MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND); - /* @todo Use MONGODB_MAX_DOCUMENT_LENGTH */ - cmd_json = bson_as_canonical_extended_json (log_command->command, NULL); + cmd_json = mongoc_structured_log_document_to_json (log_command->command); _mongoc_log_structured_append_command_data (structured_log_data, structured_message); @@ -92,8 +91,7 @@ mongoc_log_structured_build_command_succeeded_message ( BSON_ASSERT (component == MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND); - /* @todo Use MONGODB_MAX_DOCUMENT_LENGTH */ - reply_json = bson_as_canonical_extended_json (log_command->reply, NULL); + reply_json = mongoc_structured_log_document_to_json (log_command->reply); _mongoc_log_structured_append_command_data (structured_log_data, structured_message); @@ -119,8 +117,7 @@ mongoc_log_structured_build_command_failed_message ( BSON_ASSERT (component == MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND); - /* @todo Use MONGODB_MAX_DOCUMENT_LENGTH */ - reply_json = bson_as_canonical_extended_json (log_command->reply, NULL); + reply_json = mongoc_structured_log_document_to_json (log_command->reply); _mongoc_log_structured_append_command_data (structured_log_data, structured_message); diff --git a/src/libmongoc/src/mongoc/mongoc-structured-log-private.h b/src/libmongoc/src/mongoc/mongoc-structured-log-private.h index 9cda8c8b753..d3981b80e5c 100644 --- a/src/libmongoc/src/mongoc/mongoc-structured-log-private.h +++ b/src/libmongoc/src/mongoc/mongoc-structured-log-private.h @@ -24,6 +24,7 @@ #include "mongoc-structured-log-command-private.h" #define MONGOC_STRUCTURED_LOG_DEFAULT_LEVEL MONGOC_STRUCTURED_LOG_LEVEL_WARNING; +#define MONGOC_STRUCTURED_LOG_DEFAULT_MAX_DOCUMENT_LENGTH 1000; typedef void (*mongoc_structured_log_build_message_t) ( mongoc_structured_log_component_t component, diff --git a/src/libmongoc/src/mongoc/mongoc-structured-log.c b/src/libmongoc/src/mongoc/mongoc-structured-log.c index 6c2ca913c4c..01cc2171e74 100644 --- a/src/libmongoc/src/mongoc/mongoc-structured-log.c +++ b/src/libmongoc/src/mongoc/mongoc-structured-log.c @@ -222,6 +222,30 @@ mongoc_structured_log_default_handler (mongoc_structured_log_entry_t *entry, bson_free (message); } +static int32_t +mongoc_structured_log_get_max_length (void) +{ + const char *max_length_str = getenv ("MONGODB_LOGGING_MAX_DOCUMENT_LENGTH"); + + if (!max_length_str) { + return MONGOC_STRUCTURED_LOG_DEFAULT_MAX_DOCUMENT_LENGTH; + } + + if (!strcmp (max_length_str, "unlimited")) { + return BSON_MAX_LEN_UNLIMITED; + } + + return strtoul (max_length_str, NULL, 10); +} + +char * +mongoc_structured_log_document_to_json (const bson_t *document) +{ + bson_json_opts_t opts = {BSON_JSON_MODE_CANONICAL, mongoc_structured_log_get_max_length ()}; + + return bson_as_json_with_opts (document, NULL, &opts); +} + /* just for testing */ void _mongoc_structured_log_get_handler (mongoc_structured_log_func_t *log_func, diff --git a/src/libmongoc/src/mongoc/mongoc-structured-log.h b/src/libmongoc/src/mongoc/mongoc-structured-log.h index 3b9dae259bf..26a00bb40b6 100644 --- a/src/libmongoc/src/mongoc/mongoc-structured-log.h +++ b/src/libmongoc/src/mongoc/mongoc-structured-log.h @@ -94,6 +94,16 @@ MONGOC_EXPORT (mongoc_structured_log_component_t) mongoc_structured_log_entry_get_component ( const mongoc_structured_log_entry_t *entry); +/** + * mongoc_structured_log_document_to_json: + * @document: A BSON document to be serialized + * + * Returns the extended JSON representation of the given BSON document, + * respecting maximum logging length settings. + */ +MONGOC_EXPORT (char *) +mongoc_structured_log_document_to_json (const bson_t *document); + BSON_END_DECLS #endif /* MONGOC_STRUCTURED_LOG_H */ From 33b8ab0ffb8827568b4ce9a704fc6f8084139ad3 Mon Sep 17 00:00:00 2001 From: Andreas Braun Date: Tue, 13 Oct 2020 16:44:41 +0200 Subject: [PATCH 032/139] Fix wrong comment style --- src/libmongoc/src/mongoc/mongoc-structured-log-command.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libmongoc/src/mongoc/mongoc-structured-log-command.c b/src/libmongoc/src/mongoc/mongoc-structured-log-command.c index 7ee5b21638e..b68f2362476 100644 --- a/src/libmongoc/src/mongoc/mongoc-structured-log-command.c +++ b/src/libmongoc/src/mongoc/mongoc-structured-log-command.c @@ -39,7 +39,7 @@ _mongoc_log_structured_append_command_data ( "serverPort", BCON_INT32 (log_command->host->port)); - // Append client port only if it was provided + /* Append client port only if it was provided */ if (log_command->client_port) { BCON_APPEND (structured_message, "clientPort", From fbf270f7906588a6096f5e5c83179c9ba3cbdbde Mon Sep 17 00:00:00 2001 From: Micah Scott Date: Tue, 5 Nov 2024 18:04:51 -0800 Subject: [PATCH 033/139] clang-format-all --- src/libmongoc/src/mongoc/mongoc-client.c | 30 ++------- src/libmongoc/src/mongoc/mongoc-cluster.c | 21 ++---- .../src/mongoc/mongoc-cursor-legacy.c | 10 +-- src/libmongoc/src/mongoc/mongoc-cursor.c | 30 ++------- .../mongoc/mongoc-structured-log-command.c | 65 ++++++------------- .../mongoc/mongoc-structured-log-connection.c | 7 +- .../mongoc/mongoc-structured-log-private.h | 10 ++- .../src/mongoc/mongoc-structured-log.c | 55 +++++----------- .../src/mongoc/mongoc-structured-log.h | 12 ++-- .../tests/test-mongoc-structured-log.c | 22 +++---- 10 files changed, 78 insertions(+), 184 deletions(-) diff --git a/src/libmongoc/src/mongoc/mongoc-client.c b/src/libmongoc/src/mongoc/mongoc-client.c index c34a455191b..b54d50417b5 100644 --- a/src/libmongoc/src/mongoc/mongoc-client.c +++ b/src/libmongoc/src/mongoc/mongoc-client.c @@ -2167,14 +2167,8 @@ _mongoc_client_monitor_op_killcursors (mongoc_cluster_t *cluster, _mongoc_client_prepare_killcursors_command (cursor_id, collection, &doc); /* @todo Provide missing arguments */ - mongoc_structured_log_command_started (&doc, - "killCursors", - db, - operation_id, - cluster->request_id, - &server_stream->sd->host, - 0, - false); + mongoc_structured_log_command_started ( + &doc, "killCursors", db, operation_id, cluster->request_id, &server_stream->sd->host, 0, false); if (!client->apm_callbacks.started) { bson_destroy (&doc); @@ -2227,14 +2221,8 @@ _mongoc_client_monitor_op_killcursors_succeeded (mongoc_cluster_t *cluster, bson_append_array_builder_end (&doc, cursors_unknown); /* @todo Provide missing arguments */ - mongoc_structured_log_command_success ("killCursors", - operation_id, - &doc, - duration, - cluster->request_id, - &server_stream->sd->host, - 0, - false); + mongoc_structured_log_command_success ( + "killCursors", operation_id, &doc, duration, cluster->request_id, &server_stream->sd->host, 0, false); if (!client->apm_callbacks.succeeded) { bson_destroy (&doc); @@ -2283,14 +2271,8 @@ _mongoc_client_monitor_op_killcursors_failed (mongoc_cluster_t *cluster, bson_append_int32 (&doc, "ok", 2, 0); /* @todo Provide missing arguments */ - mongoc_structured_log_command_failure ("killCursors", - operation_id, - &doc, - error, - cluster->request_id, - &server_stream->sd->host, - 0, - false); + mongoc_structured_log_command_failure ( + "killCursors", operation_id, &doc, error, cluster->request_id, &server_stream->sd->host, 0, false); if (!client->apm_callbacks.failed) { bson_destroy (&doc); diff --git a/src/libmongoc/src/mongoc/mongoc-cluster.c b/src/libmongoc/src/mongoc/mongoc-cluster.c index 305a9175af9..c23fbd79511 100644 --- a/src/libmongoc/src/mongoc/mongoc-cluster.c +++ b/src/libmongoc/src/mongoc/mongoc-cluster.c @@ -562,8 +562,7 @@ mongoc_cluster_run_command_monitored (mongoc_cluster_t *cluster, mongoc_cmd_t *c /* @todo Provide missing arguments */ mongoc_structured_log_command_success (cmd->command_name, cmd->operation_id, - cmd->is_acknowledged ? reply - : &fake_reply, + cmd->is_acknowledged ? reply : &fake_reply, duration, request_id, &server_stream->sd->host, @@ -594,30 +593,24 @@ mongoc_cluster_run_command_monitored (mongoc_cluster_t *cluster, mongoc_cmd_t *c int64_t duration = bson_get_monotonic_time () - started; /* @todo Provide missing arguments */ - mongoc_structured_log_command_failure (cmd->command_name, - cmd->operation_id, - reply, - error, - cluster->request_id, - &server_stream->sd->host, - 0, - false); + mongoc_structured_log_command_failure ( + cmd->command_name, cmd->operation_id, reply, error, cluster->request_id, &server_stream->sd->host, 0, false); if (callbacks->failed) { mongoc_apm_command_failed_init (&failed_event, duration, cmd->command_name, cmd->db_name, - error, + error, reply, request_id, cmd->operation_id, &server_stream->sd->host, server_id, &server_stream->sd->service_id, - server_stream->sd->server_connection_id, - is_redacted, - cluster->client->apm_context); + server_stream->sd->server_connection_id, + is_redacted, + cluster->client->apm_context); callbacks->failed (&failed_event); mongoc_apm_command_failed_cleanup (&failed_event); diff --git a/src/libmongoc/src/mongoc/mongoc-cursor-legacy.c b/src/libmongoc/src/mongoc/mongoc-cursor-legacy.c index 8a52ff4758e..c43339548c8 100644 --- a/src/libmongoc/src/mongoc/mongoc-cursor-legacy.c +++ b/src/libmongoc/src/mongoc/mongoc-cursor-legacy.c @@ -51,14 +51,8 @@ _mongoc_cursor_monitor_legacy_get_more (mongoc_cursor_t *cursor, mongoc_server_s db = bson_strndup (cursor->ns, cursor->dblen); /* @todo Provide missing arguments */ - mongoc_structured_log_command_started (&doc, - "getMore", - db, - cursor->operation_id, - client->cluster.request_id, - &server_stream->sd->host, - 0, - false); + mongoc_structured_log_command_started ( + &doc, "getMore", db, cursor->operation_id, client->cluster.request_id, &server_stream->sd->host, 0, false); if (!client->apm_callbacks.started) { /* successful */ diff --git a/src/libmongoc/src/mongoc/mongoc-cursor.c b/src/libmongoc/src/mongoc/mongoc-cursor.c index eb2609ecb58..41c7ede3101 100644 --- a/src/libmongoc/src/mongoc/mongoc-cursor.c +++ b/src/libmongoc/src/mongoc/mongoc-cursor.c @@ -648,14 +648,8 @@ _mongoc_cursor_monitor_command (mongoc_cursor_t *cursor, db = bson_strndup (cursor->ns, cursor->dblen); /* @todo Provide missing arguments */ - mongoc_structured_log_command_started (cmd, - cmd_name, - db, - cursor->operation_id, - client->cluster.request_id, - &server_stream->sd->host, - 0, - false); + mongoc_structured_log_command_started ( + cmd, cmd_name, db, cursor->operation_id, client->cluster.request_id, &server_stream->sd->host, 0, false); if (!client->apm_callbacks.started) { /* successful */ @@ -739,14 +733,8 @@ _mongoc_cursor_monitor_succeeded (mongoc_cursor_t *cursor, bson_destroy (&docs_array); /* @todo Provide missing arguments */ - mongoc_structured_log_command_success (cmd_name, - cursor->operation_id, - &reply, - duration, - client->cluster.request_id, - &stream->sd->host, - 0, - false); + mongoc_structured_log_command_success ( + cmd_name, cursor->operation_id, &reply, duration, client->cluster.request_id, &stream->sd->host, 0, false); if (!client->apm_callbacks.succeeded) { bson_destroy (&reply); @@ -797,14 +785,8 @@ _mongoc_cursor_monitor_failed (mongoc_cursor_t *cursor, char *db = bson_strndup (cursor->ns, cursor->dblen); /* @todo Provide missing arguments */ - mongoc_structured_log_command_failure (cmd_name, - cursor->operation_id, - &reply, - &cursor->error, - client->cluster.request_id, - &stream->sd->host, - 0, - false); + mongoc_structured_log_command_failure ( + cmd_name, cursor->operation_id, &reply, &cursor->error, client->cluster.request_id, &stream->sd->host, 0, false); if (!client->apm_callbacks.failed) { bson_destroy (&reply); diff --git a/src/libmongoc/src/mongoc/mongoc-structured-log-command.c b/src/libmongoc/src/mongoc/mongoc-structured-log-command.c index b68f2362476..20adc6b1586 100644 --- a/src/libmongoc/src/mongoc/mongoc-structured-log-command.c +++ b/src/libmongoc/src/mongoc/mongoc-structured-log-command.c @@ -19,11 +19,9 @@ #include "mongoc-structured-log-command-private.h" static void -_mongoc_log_structured_append_command_data ( - void *structured_log_data, bson_t *structured_message /* OUT */) +_mongoc_log_structured_append_command_data (void *structured_log_data, bson_t *structured_message /* OUT */) { - _mongoc_structured_log_command_t *log_command = - (_mongoc_structured_log_command_t *) structured_log_data; + _mongoc_structured_log_command_t *log_command = (_mongoc_structured_log_command_t *) structured_log_data; BCON_APPEND (structured_message, "commandName", @@ -41,9 +39,7 @@ _mongoc_log_structured_append_command_data ( /* Append client port only if it was provided */ if (log_command->client_port) { - BCON_APPEND (structured_message, - "clientPort", - BCON_INT32 (log_command->client_port)); + BCON_APPEND (structured_message, "clientPort", BCON_INT32 (log_command->client_port)); } BCON_APPEND (structured_message, @@ -54,80 +50,61 @@ _mongoc_log_structured_append_command_data ( } static void -mongoc_log_structured_build_command_started_message ( - mongoc_structured_log_component_t component, - void *structured_log_data, - bson_t *structured_message /* OUT */) +mongoc_log_structured_build_command_started_message (mongoc_structured_log_component_t component, + void *structured_log_data, + bson_t *structured_message /* OUT */) { char *cmd_json; - _mongoc_structured_log_command_t *log_command = - (_mongoc_structured_log_command_t *) structured_log_data; + _mongoc_structured_log_command_t *log_command = (_mongoc_structured_log_command_t *) structured_log_data; BSON_ASSERT (component == MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND); cmd_json = mongoc_structured_log_document_to_json (log_command->command); - _mongoc_log_structured_append_command_data (structured_log_data, - structured_message); + _mongoc_log_structured_append_command_data (structured_log_data, structured_message); - BCON_APPEND (structured_message, - "databaseName", - BCON_UTF8 (log_command->db_name), - "command", - BCON_UTF8 (cmd_json)); + BCON_APPEND (structured_message, "databaseName", BCON_UTF8 (log_command->db_name), "command", BCON_UTF8 (cmd_json)); bson_free (cmd_json); } static void -mongoc_log_structured_build_command_succeeded_message ( - mongoc_structured_log_component_t component, - void *structured_log_data, - bson_t *structured_message /* OUT */) +mongoc_log_structured_build_command_succeeded_message (mongoc_structured_log_component_t component, + void *structured_log_data, + bson_t *structured_message /* OUT */) { char *reply_json; - _mongoc_structured_log_command_t *log_command = - (_mongoc_structured_log_command_t *) structured_log_data; + _mongoc_structured_log_command_t *log_command = (_mongoc_structured_log_command_t *) structured_log_data; BSON_ASSERT (component == MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND); reply_json = mongoc_structured_log_document_to_json (log_command->reply); - _mongoc_log_structured_append_command_data (structured_log_data, - structured_message); + _mongoc_log_structured_append_command_data (structured_log_data, structured_message); - BCON_APPEND (structured_message, - "duration", - BCON_INT64 (log_command->duration), - "reply", - BCON_UTF8 (reply_json)); + BCON_APPEND (structured_message, "duration", BCON_INT64 (log_command->duration), "reply", BCON_UTF8 (reply_json)); bson_free (reply_json); } static void -mongoc_log_structured_build_command_failed_message ( - mongoc_structured_log_component_t component, - void *structured_log_data, - bson_t *structured_message /* OUT */) +mongoc_log_structured_build_command_failed_message (mongoc_structured_log_component_t component, + void *structured_log_data, + bson_t *structured_message /* OUT */) { char *reply_json; - _mongoc_structured_log_command_t *log_command = - (_mongoc_structured_log_command_t *) structured_log_data; + _mongoc_structured_log_command_t *log_command = (_mongoc_structured_log_command_t *) structured_log_data; BSON_ASSERT (component == MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND); reply_json = mongoc_structured_log_document_to_json (log_command->reply); - _mongoc_log_structured_append_command_data (structured_log_data, - structured_message); + _mongoc_log_structured_append_command_data (structured_log_data, structured_message); BCON_APPEND (structured_message, "reply", BCON_UTF8 (reply_json)); if (log_command->error) { - BCON_APPEND (structured_message, - "failure", - BCON_UTF8 (log_command->error->message)); + BCON_APPEND (structured_message, "failure", BCON_UTF8 (log_command->error->message)); } bson_free (reply_json); diff --git a/src/libmongoc/src/mongoc/mongoc-structured-log-connection.c b/src/libmongoc/src/mongoc/mongoc-structured-log-connection.c index 58205d9cb4e..0158f22c61f 100644 --- a/src/libmongoc/src/mongoc/mongoc-structured-log-connection.c +++ b/src/libmongoc/src/mongoc/mongoc-structured-log-connection.c @@ -19,9 +19,6 @@ void mongoc_structured_log_connection_client_created (void) { - mongoc_structured_log (MONGOC_STRUCTURED_LOG_LEVEL_DEBUG, - MONGOC_STRUCTURED_LOG_COMPONENT_CONNECTION, - "Client created", - NULL, - NULL); + mongoc_structured_log ( + MONGOC_STRUCTURED_LOG_LEVEL_DEBUG, MONGOC_STRUCTURED_LOG_COMPONENT_CONNECTION, "Client created", NULL, NULL); } diff --git a/src/libmongoc/src/mongoc/mongoc-structured-log-private.h b/src/libmongoc/src/mongoc/mongoc-structured-log-private.h index d3981b80e5c..c9e95dfa9b6 100644 --- a/src/libmongoc/src/mongoc/mongoc-structured-log-private.h +++ b/src/libmongoc/src/mongoc/mongoc-structured-log-private.h @@ -26,10 +26,9 @@ #define MONGOC_STRUCTURED_LOG_DEFAULT_LEVEL MONGOC_STRUCTURED_LOG_LEVEL_WARNING; #define MONGOC_STRUCTURED_LOG_DEFAULT_MAX_DOCUMENT_LENGTH 1000; -typedef void (*mongoc_structured_log_build_message_t) ( - mongoc_structured_log_component_t component, - void *structured_log_data, - bson_t *structured_message /* OUT */); +typedef void (*mongoc_structured_log_build_message_t) (mongoc_structured_log_component_t component, + void *structured_log_data, + bson_t *structured_message /* OUT */); struct _mongoc_structured_log_entry_t { mongoc_structured_log_level_t level; @@ -48,7 +47,6 @@ mongoc_structured_log (mongoc_structured_log_level_t level, void *structured_message_data); void -_mongoc_structured_log_get_handler (mongoc_structured_log_func_t *log_func, - void **user_data); +_mongoc_structured_log_get_handler (mongoc_structured_log_func_t *log_func, void **user_data); #endif /* MONGOC_STRUCTURED_LOG_PRIVATE_H */ diff --git a/src/libmongoc/src/mongoc/mongoc-structured-log.c b/src/libmongoc/src/mongoc/mongoc-structured-log.c index 01cc2171e74..5526120eaf2 100644 --- a/src/libmongoc/src/mongoc/mongoc-structured-log.c +++ b/src/libmongoc/src/mongoc/mongoc-structured-log.c @@ -20,13 +20,11 @@ #include "mongoc-util-private.h" static void -mongoc_structured_log_default_handler (mongoc_structured_log_entry_t *entry, - void *user_data); +mongoc_structured_log_default_handler (mongoc_structured_log_entry_t *entry, void *user_data); static bson_once_t once = BSON_ONCE_INIT; static bson_mutex_t gStructuredLogMutex; -static mongoc_structured_log_func_t gStructuredLogger = - mongoc_structured_log_default_handler; +static mongoc_structured_log_func_t gStructuredLogger = mongoc_structured_log_default_handler; static void *gStructuredLoggerData; static FILE *log_stream; @@ -49,13 +47,10 @@ const bson_t * mongoc_structured_log_entry_get_message (mongoc_structured_log_entry_t *entry) { if (!entry->structured_message) { - entry->structured_message = - BCON_NEW ("message", BCON_UTF8 (entry->message)); + entry->structured_message = BCON_NEW ("message", BCON_UTF8 (entry->message)); if (entry->build_message_func) { - entry->build_message_func (entry->component, - entry->structured_log_data, - entry->structured_message); + entry->build_message_func (entry->component, entry->structured_log_data, entry->structured_message); } } @@ -63,22 +58,19 @@ mongoc_structured_log_entry_get_message (mongoc_structured_log_entry_t *entry) } mongoc_structured_log_level_t -mongoc_structured_log_entry_get_level ( - const mongoc_structured_log_entry_t *entry) +mongoc_structured_log_entry_get_level (const mongoc_structured_log_entry_t *entry) { return entry->level; } mongoc_structured_log_component_t -mongoc_structured_log_entry_get_component ( - const mongoc_structured_log_entry_t *entry) +mongoc_structured_log_entry_get_component (const mongoc_structured_log_entry_t *entry) { return entry->component; } void -mongoc_structured_log_set_handler (mongoc_structured_log_func_t log_func, - void *user_data) +mongoc_structured_log_set_handler (mongoc_structured_log_func_t log_func, void *user_data) { bson_once (&once, &_mongoc_ensure_mutex_once); @@ -144,32 +136,25 @@ _mongoc_structured_log_get_log_level_from_env (const char *variable) } else if (!strcasecmp (level, "emergency")) { return MONGOC_STRUCTURED_LOG_LEVEL_EMERGENCY; } else { - MONGOC_ERROR ( - "Invalid log level %s read for variable %s", level, variable); + MONGOC_ERROR ("Invalid log level %s read for variable %s", level, variable); exit (EXIT_FAILURE); } } static mongoc_structured_log_level_t -_mongoc_structured_log_get_log_level ( - mongoc_structured_log_component_t component) +_mongoc_structured_log_get_log_level (mongoc_structured_log_component_t component) { switch (component) { case MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND: - return _mongoc_structured_log_get_log_level_from_env ( - "MONGODB_LOGGING_COMMAND"); + return _mongoc_structured_log_get_log_level_from_env ("MONGODB_LOGGING_COMMAND"); case MONGOC_STRUCTURED_LOG_COMPONENT_CONNECTION: - return _mongoc_structured_log_get_log_level_from_env ( - "MONGODB_LOGGING_CONNECTION"); + return _mongoc_structured_log_get_log_level_from_env ("MONGODB_LOGGING_CONNECTION"); case MONGOC_STRUCTURED_LOG_COMPONENT_SDAM: - return _mongoc_structured_log_get_log_level_from_env ( - "MONGODB_LOGGING_SDAM"); + return _mongoc_structured_log_get_log_level_from_env ("MONGODB_LOGGING_SDAM"); case MONGOC_STRUCTURED_LOG_COMPONENT_SERVER_SELECTION: - return _mongoc_structured_log_get_log_level_from_env ( - "MONGODB_LOGGING_SERVER_SELECTION"); + return _mongoc_structured_log_get_log_level_from_env ("MONGODB_LOGGING_SERVER_SELECTION"); default: - MONGOC_ERROR ("Requesting log level for unsupported component %d", - component); + MONGOC_ERROR ("Requesting log level for unsupported component %d", component); exit (EXIT_FAILURE); } } @@ -198,20 +183,17 @@ _mongoc_structured_log_get_stream () } static void -mongoc_structured_log_default_handler (mongoc_structured_log_entry_t *entry, - void *user_data) +mongoc_structured_log_default_handler (mongoc_structured_log_entry_t *entry, void *user_data) { char *message; mongoc_structured_log_level_t log_level = - _mongoc_structured_log_get_log_level ( - mongoc_structured_log_entry_get_component (entry)); + _mongoc_structured_log_get_log_level (mongoc_structured_log_entry_get_component (entry)); if (log_level < mongoc_structured_log_entry_get_level (entry)) { return; } - message = - bson_as_json (mongoc_structured_log_entry_get_message (entry), NULL); + message = bson_as_json (mongoc_structured_log_entry_get_message (entry), NULL); fprintf (_mongoc_structured_log_get_stream (), "Structured log: %d, %d, %s\n", @@ -248,8 +230,7 @@ mongoc_structured_log_document_to_json (const bson_t *document) /* just for testing */ void -_mongoc_structured_log_get_handler (mongoc_structured_log_func_t *log_func, - void **user_data) +_mongoc_structured_log_get_handler (mongoc_structured_log_func_t *log_func, void **user_data) { *log_func = gStructuredLogger; *user_data = gStructuredLoggerData; diff --git a/src/libmongoc/src/mongoc/mongoc-structured-log.h b/src/libmongoc/src/mongoc/mongoc-structured-log.h index 26a00bb40b6..c157eb52822 100644 --- a/src/libmongoc/src/mongoc/mongoc-structured-log.h +++ b/src/libmongoc/src/mongoc/mongoc-structured-log.h @@ -46,8 +46,7 @@ typedef enum { typedef struct _mongoc_structured_log_entry_t mongoc_structured_log_entry_t; -typedef void (*mongoc_structured_log_func_t) ( - mongoc_structured_log_entry_t *entry, void *user_data); +typedef void (*mongoc_structured_log_func_t) (mongoc_structured_log_entry_t *entry, void *user_data); /** * mongoc_structured_log_set_handler: @@ -57,8 +56,7 @@ typedef void (*mongoc_structured_log_func_t) ( * Sets the function to be called to handle structured log messages. */ MONGOC_EXPORT (void) -mongoc_structured_log_set_handler (mongoc_structured_log_func_t log_func, - void *user_data); +mongoc_structured_log_set_handler (mongoc_structured_log_func_t log_func, void *user_data); /** * mongoc_structured_log_entry_get_message: @@ -81,8 +79,7 @@ mongoc_structured_log_entry_get_message (mongoc_structured_log_entry_t *entry); * Returns the severity level of the structured log entry */ MONGOC_EXPORT (mongoc_structured_log_level_t) -mongoc_structured_log_entry_get_level ( - const mongoc_structured_log_entry_t *entry); +mongoc_structured_log_entry_get_level (const mongoc_structured_log_entry_t *entry); /** * mongoc_structured_log_entry_get_component: @@ -91,8 +88,7 @@ mongoc_structured_log_entry_get_level ( * Returns the component of the structured log entry */ MONGOC_EXPORT (mongoc_structured_log_component_t) -mongoc_structured_log_entry_get_component ( - const mongoc_structured_log_entry_t *entry); +mongoc_structured_log_entry_get_component (const mongoc_structured_log_entry_t *entry); /** * mongoc_structured_log_document_to_json: diff --git a/src/libmongoc/tests/test-mongoc-structured-log.c b/src/libmongoc/tests/test-mongoc-structured-log.c index f512bafcc9e..5b06f4feeaf 100644 --- a/src/libmongoc/tests/test-mongoc-structured-log.c +++ b/src/libmongoc/tests/test-mongoc-structured-log.c @@ -53,8 +53,7 @@ structured_log_func (mongoc_structured_log_entry_t *entry, void *user_data) ASSERT_CMPINT (entry->level, ==, assumption->expected_entry.level); ASSERT_CMPINT (entry->component, ==, assumption->expected_entry.component); - ASSERT (bson_equal (mongoc_structured_log_entry_get_message (entry), - assumption->expected_entry.structured_message)); + ASSERT (bson_equal (mongoc_structured_log_entry_get_message (entry), assumption->expected_entry.structured_message)); } void @@ -76,11 +75,8 @@ test_plain_log_entry () mongoc_structured_log_set_handler (structured_log_func, &assumption); - mongoc_structured_log (MONGOC_STRUCTURED_LOG_LEVEL_WARNING, - MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND, - "Plain log entry", - NULL, - NULL); + mongoc_structured_log ( + MONGOC_STRUCTURED_LOG_LEVEL_WARNING, MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND, "Plain log entry", NULL, NULL); ASSERT_CMPINT (assumption.calls, =, 1); @@ -90,7 +86,9 @@ test_plain_log_entry () } void -_test_append_extra_data (mongoc_structured_log_component_t component, void *structured_log_data, bson_t *structured_message /* OUT */) +_test_append_extra_data (mongoc_structured_log_component_t component, + void *structured_log_data, + bson_t *structured_message /* OUT */) { BCON_APPEND (structured_message, "extra", BCON_INT32 (1)); } @@ -104,10 +102,7 @@ test_log_entry_with_extra_data () MONGOC_STRUCTURED_LOG_LEVEL_WARNING, MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND, "Plain log entry", - BCON_NEW ("message", - BCON_UTF8 ("Plain log entry"), - "extra", - BCON_INT32 (1)), + BCON_NEW ("message", BCON_UTF8 ("Plain log entry"), "extra", BCON_INT32 (1)), }, 1, 0, @@ -134,6 +129,5 @@ void test_structured_log_install (TestSuite *suite) { TestSuite_Add (suite, "/structured_log/plain", test_plain_log_entry); - TestSuite_Add ( - suite, "/structured_log/with_extra_data", test_log_entry_with_extra_data); + TestSuite_Add (suite, "/structured_log/with_extra_data", test_log_entry_with_extra_data); } From 30589f4e30c9603bd4cb59f07dbfc89310439192 Mon Sep 17 00:00:00 2001 From: Micah Scott Date: Tue, 5 Nov 2024 19:49:13 -0800 Subject: [PATCH 034/139] CDRIVER-4044 consistent copyright dates for new files Best practice in the repo is to use "2009-present" on all files, see: https://github.com/mongodb/mongo-c-driver/pull/1655 --- .../src/mongoc/mongoc-structured-log-command-private.h | 2 +- src/libmongoc/src/mongoc/mongoc-structured-log-command.c | 2 +- .../src/mongoc/mongoc-structured-log-connection-private.h | 2 +- src/libmongoc/src/mongoc/mongoc-structured-log-connection.c | 2 +- src/libmongoc/src/mongoc/mongoc-structured-log-private.h | 2 +- src/libmongoc/src/mongoc/mongoc-structured-log.c | 2 +- src/libmongoc/src/mongoc/mongoc-structured-log.h | 2 +- src/libmongoc/tests/test-mongoc-structured-log.c | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/libmongoc/src/mongoc/mongoc-structured-log-command-private.h b/src/libmongoc/src/mongoc/mongoc-structured-log-command-private.h index 72dbc63c70a..676c77d4703 100644 --- a/src/libmongoc/src/mongoc/mongoc-structured-log-command-private.h +++ b/src/libmongoc/src/mongoc/mongoc-structured-log-command-private.h @@ -1,5 +1,5 @@ /* - * Copyright 2020 MongoDB, Inc. + * Copyright 2009-present MongoDB, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/src/libmongoc/src/mongoc/mongoc-structured-log-command.c b/src/libmongoc/src/mongoc/mongoc-structured-log-command.c index 20adc6b1586..a86b6db59c2 100644 --- a/src/libmongoc/src/mongoc/mongoc-structured-log-command.c +++ b/src/libmongoc/src/mongoc/mongoc-structured-log-command.c @@ -1,5 +1,5 @@ /* - * Copyright 2020 MongoDB, Inc. + * Copyright 2009-present MongoDB, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/src/libmongoc/src/mongoc/mongoc-structured-log-connection-private.h b/src/libmongoc/src/mongoc/mongoc-structured-log-connection-private.h index f487457655b..00197ef20dd 100644 --- a/src/libmongoc/src/mongoc/mongoc-structured-log-connection-private.h +++ b/src/libmongoc/src/mongoc/mongoc-structured-log-connection-private.h @@ -1,5 +1,5 @@ /* - * Copyright 2020 MongoDB, Inc. + * Copyright 2009-present MongoDB, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/src/libmongoc/src/mongoc/mongoc-structured-log-connection.c b/src/libmongoc/src/mongoc/mongoc-structured-log-connection.c index 0158f22c61f..7b7de41e0d7 100644 --- a/src/libmongoc/src/mongoc/mongoc-structured-log-connection.c +++ b/src/libmongoc/src/mongoc/mongoc-structured-log-connection.c @@ -1,5 +1,5 @@ /* - * Copyright 2020 MongoDB, Inc. + * Copyright 2009-present MongoDB, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/src/libmongoc/src/mongoc/mongoc-structured-log-private.h b/src/libmongoc/src/mongoc/mongoc-structured-log-private.h index c9e95dfa9b6..dbafb744550 100644 --- a/src/libmongoc/src/mongoc/mongoc-structured-log-private.h +++ b/src/libmongoc/src/mongoc/mongoc-structured-log-private.h @@ -1,5 +1,5 @@ /* - * Copyright 2020 MongoDB, Inc. + * Copyright 2009-present MongoDB, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/src/libmongoc/src/mongoc/mongoc-structured-log.c b/src/libmongoc/src/mongoc/mongoc-structured-log.c index 5526120eaf2..b8119749a1d 100644 --- a/src/libmongoc/src/mongoc/mongoc-structured-log.c +++ b/src/libmongoc/src/mongoc/mongoc-structured-log.c @@ -1,5 +1,5 @@ /* - * Copyright 2020 MongoDB, Inc. + * Copyright 2009-present MongoDB, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/src/libmongoc/src/mongoc/mongoc-structured-log.h b/src/libmongoc/src/mongoc/mongoc-structured-log.h index c157eb52822..9dfca942d54 100644 --- a/src/libmongoc/src/mongoc/mongoc-structured-log.h +++ b/src/libmongoc/src/mongoc/mongoc-structured-log.h @@ -1,5 +1,5 @@ /* - * Copyright 2020 MongoDB, Inc. + * Copyright 2009-present MongoDB, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/src/libmongoc/tests/test-mongoc-structured-log.c b/src/libmongoc/tests/test-mongoc-structured-log.c index 5b06f4feeaf..a9aac469ba4 100644 --- a/src/libmongoc/tests/test-mongoc-structured-log.c +++ b/src/libmongoc/tests/test-mongoc-structured-log.c @@ -1,5 +1,5 @@ /* - * Copyright 2020 MongoDB, Inc. + * Copyright 2009-present MongoDB, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. From df6247f8cf23845e152e1808dbc06e13281971c6 Mon Sep 17 00:00:00 2001 From: Micah Scott Date: Wed, 6 Nov 2024 10:12:05 -0800 Subject: [PATCH 035/139] fix build without altering functionality --- .../mongoc/mongoc-structured-log-command.c | 2 +- .../src/mongoc/mongoc-structured-log.c | 19 ++++++++++--------- .../tests/test-mongoc-structured-log.c | 4 ++-- 3 files changed, 13 insertions(+), 12 deletions(-) diff --git a/src/libmongoc/src/mongoc/mongoc-structured-log-command.c b/src/libmongoc/src/mongoc/mongoc-structured-log-command.c index a86b6db59c2..7f2abeabc22 100644 --- a/src/libmongoc/src/mongoc/mongoc-structured-log-command.c +++ b/src/libmongoc/src/mongoc/mongoc-structured-log-command.c @@ -153,7 +153,7 @@ mongoc_structured_log_command_started_with_cmd (const mongoc_cmd_t *cmd, bson_t *command = (bson_t *) cmd->command; bool command_owned = false; - if (cmd->payload && !cmd->payload_size) { + if (cmd->payloads_count != 0) { command = bson_copy (cmd->command); command_owned = true; diff --git a/src/libmongoc/src/mongoc/mongoc-structured-log.c b/src/libmongoc/src/mongoc/mongoc-structured-log.c index b8119749a1d..9880a9a665b 100644 --- a/src/libmongoc/src/mongoc/mongoc-structured-log.c +++ b/src/libmongoc/src/mongoc/mongoc-structured-log.c @@ -154,13 +154,13 @@ _mongoc_structured_log_get_log_level (mongoc_structured_log_component_t componen case MONGOC_STRUCTURED_LOG_COMPONENT_SERVER_SELECTION: return _mongoc_structured_log_get_log_level_from_env ("MONGODB_LOGGING_SERVER_SELECTION"); default: - MONGOC_ERROR ("Requesting log level for unsupported component %d", component); + MONGOC_ERROR ("Requesting log level for unsupported component %d", (int) component); exit (EXIT_FAILURE); } } static void -_mongoc_structured_log_initialize_stream () +_mongoc_structured_log_initialize_stream (void) { const char *log_target = getenv ("MONGODB_LOGGING_PATH"); bool log_to_stderr = !log_target || !strcmp (log_target, "stderr"); @@ -173,7 +173,7 @@ _mongoc_structured_log_initialize_stream () } static FILE * -_mongoc_structured_log_get_stream () +_mongoc_structured_log_get_stream (void) { if (!log_stream) { _mongoc_structured_log_initialize_stream (); @@ -193,12 +193,12 @@ mongoc_structured_log_default_handler (mongoc_structured_log_entry_t *entry, voi return; } - message = bson_as_json (mongoc_structured_log_entry_get_message (entry), NULL); + message = bson_as_legacy_extended_json (mongoc_structured_log_entry_get_message (entry), NULL); fprintf (_mongoc_structured_log_get_stream (), "Structured log: %d, %d, %s\n", - mongoc_structured_log_entry_get_level (entry), - mongoc_structured_log_entry_get_component (entry), + (int) mongoc_structured_log_entry_get_level (entry), + (int) mongoc_structured_log_entry_get_component (entry), message); bson_free (message); @@ -223,9 +223,10 @@ mongoc_structured_log_get_max_length (void) char * mongoc_structured_log_document_to_json (const bson_t *document) { - bson_json_opts_t opts = {BSON_JSON_MODE_CANONICAL, mongoc_structured_log_get_max_length ()}; - - return bson_as_json_with_opts (document, NULL, &opts); + bson_json_opts_t *opts = bson_json_opts_new (BSON_JSON_MODE_CANONICAL, mongoc_structured_log_get_max_length ()); + char *json = bson_as_json_with_opts (document, NULL, opts); + bson_json_opts_destroy (opts); + return json; } /* just for testing */ diff --git a/src/libmongoc/tests/test-mongoc-structured-log.c b/src/libmongoc/tests/test-mongoc-structured-log.c index a9aac469ba4..feba3b1ae12 100644 --- a/src/libmongoc/tests/test-mongoc-structured-log.c +++ b/src/libmongoc/tests/test-mongoc-structured-log.c @@ -57,7 +57,7 @@ structured_log_func (mongoc_structured_log_entry_t *entry, void *user_data) } void -test_plain_log_entry () +test_plain_log_entry (void) { struct structured_log_state old_state; struct log_assumption assumption = { @@ -94,7 +94,7 @@ _test_append_extra_data (mongoc_structured_log_component_t component, } void -test_log_entry_with_extra_data () +test_log_entry_with_extra_data (void) { struct structured_log_state old_state; struct log_assumption assumption = { From 7a4d81f1fd7d7f321608b9ea5a07377d012b99e9 Mon Sep 17 00:00:00 2001 From: Micah Scott Date: Wed, 6 Nov 2024 12:41:24 -0800 Subject: [PATCH 036/139] mongoc_cursor_monitor memory leak fix --- src/libmongoc/src/mongoc/mongoc-cursor.c | 78 +++++++++++------------- 1 file changed, 37 insertions(+), 41 deletions(-) diff --git a/src/libmongoc/src/mongoc/mongoc-cursor.c b/src/libmongoc/src/mongoc/mongoc-cursor.c index 41c7ede3101..7ab4b8d26f9 100644 --- a/src/libmongoc/src/mongoc/mongoc-cursor.c +++ b/src/libmongoc/src/mongoc/mongoc-cursor.c @@ -736,28 +736,26 @@ _mongoc_cursor_monitor_succeeded (mongoc_cursor_t *cursor, mongoc_structured_log_command_success ( cmd_name, cursor->operation_id, &reply, duration, client->cluster.request_id, &stream->sd->host, 0, false); - if (!client->apm_callbacks.succeeded) { - bson_destroy (&reply); - EXIT; + if (client->apm_callbacks.succeeded) { + mongoc_apm_command_succeeded_init (&event, + duration, + &reply, + cmd_name, + db, + client->cluster.request_id, + cursor->operation_id, + &stream->sd->host, + stream->sd->id, + &stream->sd->service_id, + stream->sd->server_connection_id, + false, + client->apm_context); + + client->apm_callbacks.succeeded (&event); + + mongoc_apm_command_succeeded_cleanup (&event); } - mongoc_apm_command_succeeded_init (&event, - duration, - &reply, - cmd_name, - db, - client->cluster.request_id, - cursor->operation_id, - &stream->sd->host, - stream->sd->id, - &stream->sd->service_id, - stream->sd->server_connection_id, - false, - client->apm_context); - - client->apm_callbacks.succeeded (&event); - - mongoc_apm_command_succeeded_cleanup (&event); bson_destroy (&reply); bson_free (db); @@ -788,29 +786,27 @@ _mongoc_cursor_monitor_failed (mongoc_cursor_t *cursor, mongoc_structured_log_command_failure ( cmd_name, cursor->operation_id, &reply, &cursor->error, client->cluster.request_id, &stream->sd->host, 0, false); - if (!client->apm_callbacks.failed) { - bson_destroy (&reply); - EXIT; + if (client->apm_callbacks.failed) { + mongoc_apm_command_failed_init (&event, + duration, + cmd_name, + db, + &cursor->error, + &reply, + client->cluster.request_id, + cursor->operation_id, + &stream->sd->host, + stream->sd->id, + &stream->sd->service_id, + stream->sd->server_connection_id, + false, + client->apm_context); + + client->apm_callbacks.failed (&event); + + mongoc_apm_command_failed_cleanup (&event); } - mongoc_apm_command_failed_init (&event, - duration, - cmd_name, - db, - &cursor->error, - &reply, - client->cluster.request_id, - cursor->operation_id, - &stream->sd->host, - stream->sd->id, - &stream->sd->service_id, - stream->sd->server_connection_id, - false, - client->apm_context); - - client->apm_callbacks.failed (&event); - - mongoc_apm_command_failed_cleanup (&event); bson_destroy (&reply); bson_free (db); From 69ddf0d6117676fbed5bbe9c53e61607aaddbab5 Mon Sep 17 00:00:00 2001 From: Micah Scott Date: Fri, 8 Nov 2024 16:13:13 -0800 Subject: [PATCH 037/139] mongoc_structured_log using a macro-generated table of function calls --- src/libmongoc/CMakeLists.txt | 2 - src/libmongoc/src/mongoc/mongoc-client.c | 59 ++++- src/libmongoc/src/mongoc/mongoc-cluster.c | 59 +++-- .../src/mongoc/mongoc-cursor-legacy.c | 19 +- src/libmongoc/src/mongoc/mongoc-cursor.c | 55 +++- .../mongoc-structured-log-command-private.h | 82 ------ .../mongoc/mongoc-structured-log-command.c | 241 ------------------ ...mongoc-structured-log-connection-private.h | 28 -- .../mongoc/mongoc-structured-log-connection.c | 24 -- .../mongoc/mongoc-structured-log-private.h | 154 +++++++++-- .../src/mongoc/mongoc-structured-log.c | 204 +++++++++++---- .../src/mongoc/mongoc-structured-log.h | 51 ++-- .../tests/test-mongoc-structured-log.c | 218 ++++++++++++---- 13 files changed, 632 insertions(+), 564 deletions(-) delete mode 100644 src/libmongoc/src/mongoc/mongoc-structured-log-command-private.h delete mode 100644 src/libmongoc/src/mongoc/mongoc-structured-log-command.c delete mode 100644 src/libmongoc/src/mongoc/mongoc-structured-log-connection-private.h delete mode 100644 src/libmongoc/src/mongoc/mongoc-structured-log-connection.c diff --git a/src/libmongoc/CMakeLists.txt b/src/libmongoc/CMakeLists.txt index b415bcbd486..c03a368e6dd 100644 --- a/src/libmongoc/CMakeLists.txt +++ b/src/libmongoc/CMakeLists.txt @@ -648,8 +648,6 @@ set (MONGOC_SOURCES ${PROJECT_SOURCE_DIR}/src/mongoc/mongoc-stream-gridfs-upload.c ${PROJECT_SOURCE_DIR}/src/mongoc/mongoc-stream-socket.c ${PROJECT_SOURCE_DIR}/src/mongoc/mongoc-structured-log.c - ${PROJECT_SOURCE_DIR}/src/mongoc/mongoc-structured-log-command.c - ${PROJECT_SOURCE_DIR}/src/mongoc/mongoc-structured-log-connection.c ${PROJECT_SOURCE_DIR}/src/mongoc/mongoc-timeout.c ${PROJECT_SOURCE_DIR}/src/mongoc/mongoc-topology.c ${PROJECT_SOURCE_DIR}/src/mongoc/mongoc-topology-background-monitoring.c diff --git a/src/libmongoc/src/mongoc/mongoc-client.c b/src/libmongoc/src/mongoc/mongoc-client.c index b54d50417b5..fb95495cfd7 100644 --- a/src/libmongoc/src/mongoc/mongoc-client.c +++ b/src/libmongoc/src/mongoc/mongoc-client.c @@ -57,8 +57,7 @@ #include "mongoc-change-stream-private.h" #include "mongoc-client-session-private.h" #include "mongoc-cursor-private.h" -#include "mongoc-structured-log-command-private.h" -#include "mongoc-structured-log-connection-private.h" +#include "mongoc-structured-log-private.h" #ifdef MONGOC_ENABLE_SSL #include "mongoc-stream-tls.h" @@ -1124,7 +1123,8 @@ _mongoc_client_new_from_topology (mongoc_topology_t *topology) } #endif - mongoc_structured_log_connection_client_created (); + mongoc_structured_log ( + MONGOC_STRUCTURED_LOG_LEVEL_DEBUG, MONGOC_STRUCTURED_LOG_COMPONENT_CONNECTION, "Client created"); mongoc_counter_clients_active_inc (); @@ -2166,9 +2166,20 @@ _mongoc_client_monitor_op_killcursors (mongoc_cluster_t *cluster, bson_init (&doc); _mongoc_client_prepare_killcursors_command (cursor_id, collection, &doc); - /* @todo Provide missing arguments */ - mongoc_structured_log_command_started ( - &doc, "killCursors", db, operation_id, cluster->request_id, &server_stream->sd->host, 0, false); + mongoc_structured_log ( + MONGOC_STRUCTURED_LOG_LEVEL_INFO, + MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND, + "Command started", + MONGOC_STRUCTURED_LOG_INT32 ("requestId", cluster->request_id), + MONGOC_STRUCTURED_LOG_SERVER_DESCRIPTION (server_stream->sd, + (MONGOC_STRUCTURED_LOG_SERVER_DESCRIPTION_SERVER_HOST | + MONGOC_STRUCTURED_LOG_SERVER_DESCRIPTION_SERVER_PORT | + MONGOC_STRUCTURED_LOG_SERVER_DESCRIPTION_SERVER_CONNECTION_ID | + MONGOC_STRUCTURED_LOG_SERVER_DESCRIPTION_SERVICE_ID)), + MONGOC_STRUCTURED_LOG_UTF8 ("databaseName", db), + MONGOC_STRUCTURED_LOG_UTF8 ("commandName", "killCursors"), + MONGOC_STRUCTURED_LOG_INT64 ("operationId", operation_id), + MONGOC_STRUCTURED_LOG_BSON_AS_JSON ("command", &doc)); if (!client->apm_callbacks.started) { bson_destroy (&doc); @@ -2220,9 +2231,21 @@ _mongoc_client_monitor_op_killcursors_succeeded (mongoc_cluster_t *cluster, bson_array_builder_append_int64 (cursors_unknown, cursor_id); bson_append_array_builder_end (&doc, cursors_unknown); - /* @todo Provide missing arguments */ - mongoc_structured_log_command_success ( - "killCursors", operation_id, &doc, duration, cluster->request_id, &server_stream->sd->host, 0, false); + mongoc_structured_log ( + MONGOC_STRUCTURED_LOG_LEVEL_INFO, + MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND, + "Command succeeded", + MONGOC_STRUCTURED_LOG_INT32 ("requestId", cluster->request_id), + MONGOC_STRUCTURED_LOG_SERVER_DESCRIPTION (server_stream->sd, + (MONGOC_STRUCTURED_LOG_SERVER_DESCRIPTION_SERVER_HOST | + MONGOC_STRUCTURED_LOG_SERVER_DESCRIPTION_SERVER_PORT | + MONGOC_STRUCTURED_LOG_SERVER_DESCRIPTION_SERVER_CONNECTION_ID | + MONGOC_STRUCTURED_LOG_SERVER_DESCRIPTION_SERVICE_ID)), + MONGOC_STRUCTURED_LOG_UTF8 ("databaseName", db), + MONGOC_STRUCTURED_LOG_UTF8 ("commandName", "killCursors"), + MONGOC_STRUCTURED_LOG_INT64 ("operationId", operation_id), + MONGOC_STRUCTURED_LOG_INT64 ("durationMS", duration), + MONGOC_STRUCTURED_LOG_BSON_AS_JSON ("reply", &doc)); if (!client->apm_callbacks.succeeded) { bson_destroy (&doc); @@ -2270,9 +2293,21 @@ _mongoc_client_monitor_op_killcursors_failed (mongoc_cluster_t *cluster, bson_init (&doc); bson_append_int32 (&doc, "ok", 2, 0); - /* @todo Provide missing arguments */ - mongoc_structured_log_command_failure ( - "killCursors", operation_id, &doc, error, cluster->request_id, &server_stream->sd->host, 0, false); + mongoc_structured_log ( + MONGOC_STRUCTURED_LOG_LEVEL_INFO, + MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND, + "Command failed", + MONGOC_STRUCTURED_LOG_INT32 ("requestId", cluster->request_id), + MONGOC_STRUCTURED_LOG_SERVER_DESCRIPTION (server_stream->sd, + (MONGOC_STRUCTURED_LOG_SERVER_DESCRIPTION_SERVER_HOST | + MONGOC_STRUCTURED_LOG_SERVER_DESCRIPTION_SERVER_PORT | + MONGOC_STRUCTURED_LOG_SERVER_DESCRIPTION_SERVER_CONNECTION_ID | + MONGOC_STRUCTURED_LOG_SERVER_DESCRIPTION_SERVICE_ID)), + MONGOC_STRUCTURED_LOG_UTF8 ("databaseName", db), + MONGOC_STRUCTURED_LOG_UTF8 ("commandName", "killCursors"), + MONGOC_STRUCTURED_LOG_INT64 ("operationId", operation_id), + MONGOC_STRUCTURED_LOG_INT64 ("durationMS", duration), + MONGOC_STRUCTURED_LOG_BSON_AS_JSON ("failure", &doc)); if (!client->apm_callbacks.failed) { bson_destroy (&doc); diff --git a/src/libmongoc/src/mongoc/mongoc-cluster.c b/src/libmongoc/src/mongoc/mongoc-cluster.c index c23fbd79511..79109a0da67 100644 --- a/src/libmongoc/src/mongoc/mongoc-cluster.c +++ b/src/libmongoc/src/mongoc/mongoc-cluster.c @@ -56,7 +56,7 @@ #include "mongoc-handshake-private.h" #include "mongoc-cluster-aws-private.h" #include "mongoc-error-private.h" -#include "mongoc-structured-log-command-private.h" +#include "mongoc-structured-log-private.h" #include #include @@ -533,8 +533,19 @@ mongoc_cluster_run_command_monitored (mongoc_cluster_t *cluster, mongoc_cmd_t *c } } - /* @todo Provide missing arguments */ - mongoc_structured_log_command_started_with_cmd (cmd, request_id, 0, false); + mongoc_structured_log ( + MONGOC_STRUCTURED_LOG_LEVEL_INFO, + MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND, + "Command started", + MONGOC_STRUCTURED_LOG_INT32 ("requestId", request_id), + MONGOC_STRUCTURED_LOG_SERVER_DESCRIPTION (server_stream->sd, + (MONGOC_STRUCTURED_LOG_SERVER_DESCRIPTION_SERVER_HOST | + MONGOC_STRUCTURED_LOG_SERVER_DESCRIPTION_SERVER_PORT | + MONGOC_STRUCTURED_LOG_SERVER_DESCRIPTION_SERVER_CONNECTION_ID | + MONGOC_STRUCTURED_LOG_SERVER_DESCRIPTION_SERVICE_ID)), + MONGOC_STRUCTURED_LOG_CMD (cmd, + (MONGOC_STRUCTURED_LOG_CMD_DATABASE_NAME | MONGOC_STRUCTURED_LOG_CMD_COMMAND_NAME | + MONGOC_STRUCTURED_LOG_CMD_OPERATION_ID | MONGOC_STRUCTURED_LOG_CMD_COMMAND))); if (callbacks->started) { mongoc_apm_command_started_init_with_cmd ( @@ -559,15 +570,21 @@ mongoc_cluster_run_command_monitored (mongoc_cluster_t *cluster, mongoc_cmd_t *c bson_append_int32 (&fake_reply, "ok", 2, 1); } - /* @todo Provide missing arguments */ - mongoc_structured_log_command_success (cmd->command_name, - cmd->operation_id, - cmd->is_acknowledged ? reply : &fake_reply, - duration, - request_id, - &server_stream->sd->host, - 0, - false); + mongoc_structured_log ( + MONGOC_STRUCTURED_LOG_LEVEL_INFO, + MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND, + "Command succeeded", + MONGOC_STRUCTURED_LOG_INT32 ("requestId", request_id), + MONGOC_STRUCTURED_LOG_SERVER_DESCRIPTION (server_stream->sd, + (MONGOC_STRUCTURED_LOG_SERVER_DESCRIPTION_SERVER_HOST | + MONGOC_STRUCTURED_LOG_SERVER_DESCRIPTION_SERVER_PORT | + MONGOC_STRUCTURED_LOG_SERVER_DESCRIPTION_SERVER_CONNECTION_ID | + MONGOC_STRUCTURED_LOG_SERVER_DESCRIPTION_SERVICE_ID)), + MONGOC_STRUCTURED_LOG_CMD (cmd, + (MONGOC_STRUCTURED_LOG_CMD_DATABASE_NAME | MONGOC_STRUCTURED_LOG_CMD_COMMAND_NAME | + MONGOC_STRUCTURED_LOG_CMD_OPERATION_ID)), + MONGOC_STRUCTURED_LOG_INT64 ("durationMS", duration), + MONGOC_STRUCTURED_LOG_BSON_AS_JSON ("reply", cmd->is_acknowledged ? reply : &fake_reply)); if (callbacks->succeeded) { mongoc_apm_command_succeeded_init (&succeeded_event, @@ -592,9 +609,21 @@ mongoc_cluster_run_command_monitored (mongoc_cluster_t *cluster, mongoc_cmd_t *c } else { int64_t duration = bson_get_monotonic_time () - started; - /* @todo Provide missing arguments */ - mongoc_structured_log_command_failure ( - cmd->command_name, cmd->operation_id, reply, error, cluster->request_id, &server_stream->sd->host, 0, false); + mongoc_structured_log ( + MONGOC_STRUCTURED_LOG_LEVEL_INFO, + MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND, + "Command failed", + MONGOC_STRUCTURED_LOG_INT32 ("requestId", request_id), + MONGOC_STRUCTURED_LOG_SERVER_DESCRIPTION (server_stream->sd, + (MONGOC_STRUCTURED_LOG_SERVER_DESCRIPTION_SERVER_HOST | + MONGOC_STRUCTURED_LOG_SERVER_DESCRIPTION_SERVER_PORT | + MONGOC_STRUCTURED_LOG_SERVER_DESCRIPTION_SERVER_CONNECTION_ID | + MONGOC_STRUCTURED_LOG_SERVER_DESCRIPTION_SERVICE_ID)), + MONGOC_STRUCTURED_LOG_CMD (cmd, + (MONGOC_STRUCTURED_LOG_CMD_DATABASE_NAME | MONGOC_STRUCTURED_LOG_CMD_COMMAND_NAME | + MONGOC_STRUCTURED_LOG_CMD_OPERATION_ID)), + MONGOC_STRUCTURED_LOG_INT64 ("durationMS", duration), + MONGOC_STRUCTURED_LOG_BSON_AS_JSON ("failure", reply)); if (callbacks->failed) { mongoc_apm_command_failed_init (&failed_event, diff --git a/src/libmongoc/src/mongoc/mongoc-cursor-legacy.c b/src/libmongoc/src/mongoc/mongoc-cursor-legacy.c index c43339548c8..fcf83ef56c0 100644 --- a/src/libmongoc/src/mongoc/mongoc-cursor-legacy.c +++ b/src/libmongoc/src/mongoc/mongoc-cursor-legacy.c @@ -32,7 +32,7 @@ #include "mongoc-write-concern-private.h" #include "mongoc-read-prefs-private.h" #include "mongoc-rpc-private.h" -#include "mongoc-structured-log-command-private.h" +#include "mongoc-structured-log-private.h" #include @@ -50,9 +50,20 @@ _mongoc_cursor_monitor_legacy_get_more (mongoc_cursor_t *cursor, mongoc_server_s _mongoc_cursor_prepare_getmore_command (cursor, &doc); db = bson_strndup (cursor->ns, cursor->dblen); - /* @todo Provide missing arguments */ - mongoc_structured_log_command_started ( - &doc, "getMore", db, cursor->operation_id, client->cluster.request_id, &server_stream->sd->host, 0, false); + mongoc_structured_log ( + MONGOC_STRUCTURED_LOG_LEVEL_INFO, + MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND, + "Command started", + MONGOC_STRUCTURED_LOG_INT32 ("requestId", client->cluster.request_id), + MONGOC_STRUCTURED_LOG_SERVER_DESCRIPTION (server_stream->sd, + (MONGOC_STRUCTURED_LOG_SERVER_DESCRIPTION_SERVER_HOST | + MONGOC_STRUCTURED_LOG_SERVER_DESCRIPTION_SERVER_PORT | + MONGOC_STRUCTURED_LOG_SERVER_DESCRIPTION_SERVER_CONNECTION_ID | + MONGOC_STRUCTURED_LOG_SERVER_DESCRIPTION_SERVICE_ID)), + MONGOC_STRUCTURED_LOG_UTF8 ("databaseName", db), + MONGOC_STRUCTURED_LOG_UTF8 ("commandName", "getMore"), + MONGOC_STRUCTURED_LOG_INT64 ("operationId", cursor->operation_id), + MONGOC_STRUCTURED_LOG_BSON_AS_JSON ("command", &doc)); if (!client->apm_callbacks.started) { /* successful */ diff --git a/src/libmongoc/src/mongoc/mongoc-cursor.c b/src/libmongoc/src/mongoc/mongoc-cursor.c index 7ab4b8d26f9..1fe9cb0a8d9 100644 --- a/src/libmongoc/src/mongoc/mongoc-cursor.c +++ b/src/libmongoc/src/mongoc/mongoc-cursor.c @@ -29,7 +29,7 @@ #include "mongoc-write-concern-private.h" #include "mongoc-read-prefs-private.h" #include "mongoc-aggregate-private.h" -#include "mongoc-structured-log-command-private.h" +#include "mongoc-structured-log-private.h" #include #include @@ -647,9 +647,20 @@ _mongoc_cursor_monitor_command (mongoc_cursor_t *cursor, client = cursor->client; db = bson_strndup (cursor->ns, cursor->dblen); - /* @todo Provide missing arguments */ - mongoc_structured_log_command_started ( - cmd, cmd_name, db, cursor->operation_id, client->cluster.request_id, &server_stream->sd->host, 0, false); + mongoc_structured_log ( + MONGOC_STRUCTURED_LOG_LEVEL_INFO, + MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND, + "Command started", + MONGOC_STRUCTURED_LOG_INT32 ("requestId", client->cluster.request_id), + MONGOC_STRUCTURED_LOG_SERVER_DESCRIPTION (server_stream->sd, + (MONGOC_STRUCTURED_LOG_SERVER_DESCRIPTION_SERVER_HOST | + MONGOC_STRUCTURED_LOG_SERVER_DESCRIPTION_SERVER_PORT | + MONGOC_STRUCTURED_LOG_SERVER_DESCRIPTION_SERVER_CONNECTION_ID | + MONGOC_STRUCTURED_LOG_SERVER_DESCRIPTION_SERVICE_ID)), + MONGOC_STRUCTURED_LOG_UTF8 ("databaseName", db), + MONGOC_STRUCTURED_LOG_UTF8 ("commandName", cmd_name), + MONGOC_STRUCTURED_LOG_INT64 ("operationId", cursor->operation_id), + MONGOC_STRUCTURED_LOG_BSON_AS_JSON ("command", cmd)); if (!client->apm_callbacks.started) { /* successful */ @@ -732,9 +743,21 @@ _mongoc_cursor_monitor_succeeded (mongoc_cursor_t *cursor, bson_destroy (&docs_array); - /* @todo Provide missing arguments */ - mongoc_structured_log_command_success ( - cmd_name, cursor->operation_id, &reply, duration, client->cluster.request_id, &stream->sd->host, 0, false); + mongoc_structured_log ( + MONGOC_STRUCTURED_LOG_LEVEL_INFO, + MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND, + "Command succeeded", + MONGOC_STRUCTURED_LOG_INT32 ("requestId", client->cluster.request_id), + MONGOC_STRUCTURED_LOG_SERVER_DESCRIPTION (stream->sd, + (MONGOC_STRUCTURED_LOG_SERVER_DESCRIPTION_SERVER_HOST | + MONGOC_STRUCTURED_LOG_SERVER_DESCRIPTION_SERVER_PORT | + MONGOC_STRUCTURED_LOG_SERVER_DESCRIPTION_SERVER_CONNECTION_ID | + MONGOC_STRUCTURED_LOG_SERVER_DESCRIPTION_SERVICE_ID)), + MONGOC_STRUCTURED_LOG_UTF8 ("databaseName", db), + MONGOC_STRUCTURED_LOG_UTF8 ("commandName", cmd_name), + MONGOC_STRUCTURED_LOG_INT64 ("operationId", cursor->operation_id), + MONGOC_STRUCTURED_LOG_INT64 ("durationMS", duration), + MONGOC_STRUCTURED_LOG_BSON_AS_JSON ("reply", &reply)); if (client->apm_callbacks.succeeded) { mongoc_apm_command_succeeded_init (&event, @@ -782,9 +805,21 @@ _mongoc_cursor_monitor_failed (mongoc_cursor_t *cursor, bsonBuildDecl (reply, kv ("ok", int32 (0))); char *db = bson_strndup (cursor->ns, cursor->dblen); - /* @todo Provide missing arguments */ - mongoc_structured_log_command_failure ( - cmd_name, cursor->operation_id, &reply, &cursor->error, client->cluster.request_id, &stream->sd->host, 0, false); + mongoc_structured_log ( + MONGOC_STRUCTURED_LOG_LEVEL_INFO, + MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND, + "Command failed", + MONGOC_STRUCTURED_LOG_INT32 ("requestId", client->cluster.request_id), + MONGOC_STRUCTURED_LOG_SERVER_DESCRIPTION (stream->sd, + (MONGOC_STRUCTURED_LOG_SERVER_DESCRIPTION_SERVER_HOST | + MONGOC_STRUCTURED_LOG_SERVER_DESCRIPTION_SERVER_PORT | + MONGOC_STRUCTURED_LOG_SERVER_DESCRIPTION_SERVER_CONNECTION_ID | + MONGOC_STRUCTURED_LOG_SERVER_DESCRIPTION_SERVICE_ID)), + MONGOC_STRUCTURED_LOG_UTF8 ("databaseName", db), + MONGOC_STRUCTURED_LOG_UTF8 ("commandName", cmd_name), + MONGOC_STRUCTURED_LOG_INT64 ("operationId", cursor->operation_id), + MONGOC_STRUCTURED_LOG_INT64 ("durationMS", duration), + MONGOC_STRUCTURED_LOG_BSON_AS_JSON ("failure", &reply)); if (client->apm_callbacks.failed) { mongoc_apm_command_failed_init (&event, diff --git a/src/libmongoc/src/mongoc/mongoc-structured-log-command-private.h b/src/libmongoc/src/mongoc/mongoc-structured-log-command-private.h deleted file mode 100644 index 676c77d4703..00000000000 --- a/src/libmongoc/src/mongoc/mongoc-structured-log-command-private.h +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Copyright 2009-present MongoDB, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "mongoc-prelude.h" - -#ifndef MONGOC_STRUCTURED_LOG_COMMAND_PRIVATE_H -#define MONGOC_STRUCTURED_LOG_COMMAND_PRIVATE_H - -#include "mongoc-structured-log.h" -#include "mongoc-cmd-private.h" - -typedef struct { - /* Needed for: - Started - * / - Succeeded - * | / - Failed - * | | / - * | | | */ - const char *command_name; /* x x x */ - const char *db_name; /* x - - */ - const bson_t *command; /* x - - */ - const bson_t *reply; /* - x x */ - const bson_error_t *error; /* - - x */ - int64_t duration; /* - x x */ - int64_t operation_id; /* x x x */ - uint32_t request_id; /* x x x */ - const mongoc_host_list_t *host; /* x x x */ - char *server_resolved_ip; /* x x x */ - uint16_t client_port; /* x x x */ - uint32_t server_connection_id; /* x x x */ - bool explicit_session; /* x x x */ -} _mongoc_structured_log_command_t; - -void -mongoc_structured_log_command_started (const bson_t *command, - const char *command_name, - const char *db_name, - int64_t operation_id, - uint32_t request_id, - const mongoc_host_list_t *host, - uint32_t server_connection_id, - bool explicit_session); - -void -mongoc_structured_log_command_started_with_cmd (const mongoc_cmd_t *cmd, - uint32_t request_id, - uint32_t server_connection_id, - bool explicit_session); - -void -mongoc_structured_log_command_success (const char *command_name, - int64_t operation_id, - const bson_t *reply, - uint64_t duration, - uint32_t request_id, - const mongoc_host_list_t *host, - uint32_t server_connection_id, - bool explicit_session); - -void -mongoc_structured_log_command_failure (const char *command_name, - int64_t operation_id, - const bson_t *reply, - const bson_error_t *error, - uint32_t request_id, - const mongoc_host_list_t *host, - uint32_t server_connection_id, - bool explicit_session); - -#endif /* MONGOC_STRUCTURED_LOG_COMMAND_PRIVATE_H */ diff --git a/src/libmongoc/src/mongoc/mongoc-structured-log-command.c b/src/libmongoc/src/mongoc/mongoc-structured-log-command.c deleted file mode 100644 index 7f2abeabc22..00000000000 --- a/src/libmongoc/src/mongoc/mongoc-structured-log-command.c +++ /dev/null @@ -1,241 +0,0 @@ -/* - * Copyright 2009-present MongoDB, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "mongoc-structured-log.h" -#include "mongoc-structured-log-private.h" -#include "mongoc-structured-log-command-private.h" - -static void -_mongoc_log_structured_append_command_data (void *structured_log_data, bson_t *structured_message /* OUT */) -{ - _mongoc_structured_log_command_t *log_command = (_mongoc_structured_log_command_t *) structured_log_data; - - BCON_APPEND (structured_message, - "commandName", - BCON_UTF8 (log_command->command_name), - "requestId", - BCON_INT32 (log_command->request_id), - "operationId", - BCON_INT64 (log_command->operation_id), - "serverHostname", - BCON_UTF8 (log_command->host->host), - "serverResolvedIPAddress", - BCON_UTF8 (log_command->server_resolved_ip), - "serverPort", - BCON_INT32 (log_command->host->port)); - - /* Append client port only if it was provided */ - if (log_command->client_port) { - BCON_APPEND (structured_message, "clientPort", BCON_INT32 (log_command->client_port)); - } - - BCON_APPEND (structured_message, - "serverConnectionId", - BCON_INT32 (log_command->server_connection_id), - "explicitSession", - BCON_BOOL (log_command->explicit_session)); -} - -static void -mongoc_log_structured_build_command_started_message (mongoc_structured_log_component_t component, - void *structured_log_data, - bson_t *structured_message /* OUT */) -{ - char *cmd_json; - _mongoc_structured_log_command_t *log_command = (_mongoc_structured_log_command_t *) structured_log_data; - - BSON_ASSERT (component == MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND); - - cmd_json = mongoc_structured_log_document_to_json (log_command->command); - - _mongoc_log_structured_append_command_data (structured_log_data, structured_message); - - BCON_APPEND (structured_message, "databaseName", BCON_UTF8 (log_command->db_name), "command", BCON_UTF8 (cmd_json)); - - bson_free (cmd_json); -} - -static void -mongoc_log_structured_build_command_succeeded_message (mongoc_structured_log_component_t component, - void *structured_log_data, - bson_t *structured_message /* OUT */) -{ - char *reply_json; - _mongoc_structured_log_command_t *log_command = (_mongoc_structured_log_command_t *) structured_log_data; - - BSON_ASSERT (component == MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND); - - reply_json = mongoc_structured_log_document_to_json (log_command->reply); - - _mongoc_log_structured_append_command_data (structured_log_data, structured_message); - - BCON_APPEND (structured_message, "duration", BCON_INT64 (log_command->duration), "reply", BCON_UTF8 (reply_json)); - - bson_free (reply_json); -} - -static void -mongoc_log_structured_build_command_failed_message (mongoc_structured_log_component_t component, - void *structured_log_data, - bson_t *structured_message /* OUT */) -{ - char *reply_json; - _mongoc_structured_log_command_t *log_command = (_mongoc_structured_log_command_t *) structured_log_data; - - BSON_ASSERT (component == MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND); - - reply_json = mongoc_structured_log_document_to_json (log_command->reply); - - _mongoc_log_structured_append_command_data (structured_log_data, structured_message); - - BCON_APPEND (structured_message, "reply", BCON_UTF8 (reply_json)); - - if (log_command->error) { - BCON_APPEND (structured_message, "failure", BCON_UTF8 (log_command->error->message)); - } - - bson_free (reply_json); -} - -void -mongoc_structured_log_command_started (const bson_t *command, - const char *command_name, - const char *db_name, - int64_t operation_id, - uint32_t request_id, - const mongoc_host_list_t *host, - uint32_t server_connection_id, - bool explicit_session) -{ - _mongoc_structured_log_command_t command_log = { - command_name, - db_name, - command, - NULL, - NULL, - 0, - operation_id, - request_id, - host, - NULL, - 0, - server_connection_id, - explicit_session, - }; - - mongoc_structured_log (MONGOC_STRUCTURED_LOG_LEVEL_INFO, - MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND, - "Command started", - mongoc_log_structured_build_command_started_message, - &command_log); -} - -void -mongoc_structured_log_command_started_with_cmd (const mongoc_cmd_t *cmd, - uint32_t request_id, - uint32_t server_connection_id, - bool explicit_session) -{ - /* Discard const modifier, we promise we won't modify this */ - bson_t *command = (bson_t *) cmd->command; - bool command_owned = false; - - if (cmd->payloads_count != 0) { - command = bson_copy (cmd->command); - command_owned = true; - - _mongoc_cmd_append_payload_as_array (cmd, command); - } - - mongoc_structured_log_command_started (command, - cmd->command_name, - cmd->db_name, - cmd->operation_id, - request_id, - &cmd->server_stream->sd->host, - server_connection_id, - explicit_session); - - if (command_owned) { - bson_destroy (command); - } -} - -void -mongoc_structured_log_command_success (const char *command_name, - int64_t operation_id, - const bson_t *reply, - uint64_t duration, - uint32_t request_id, - const mongoc_host_list_t *host, - uint32_t server_connection_id, - bool explicit_session) -{ - _mongoc_structured_log_command_t command_log = { - command_name, - NULL, - NULL, - reply, - NULL, - duration, - operation_id, - request_id, - host, - NULL, - 0, - server_connection_id, - explicit_session, - }; - - mongoc_structured_log (MONGOC_STRUCTURED_LOG_LEVEL_INFO, - MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND, - "Command succeeded", - mongoc_log_structured_build_command_succeeded_message, - &command_log); -} - -void -mongoc_structured_log_command_failure (const char *command_name, - int64_t operation_id, - const bson_t *reply, - const bson_error_t *error, - uint32_t request_id, - const mongoc_host_list_t *host, - uint32_t server_connection_id, - bool explicit_session) -{ - _mongoc_structured_log_command_t command_log = { - command_name, - NULL, - NULL, - reply, - error, - 0, - operation_id, - request_id, - host, - NULL, - 0, - server_connection_id, - explicit_session, - }; - - mongoc_structured_log (MONGOC_STRUCTURED_LOG_LEVEL_INFO, - MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND, - "Command failed", - mongoc_log_structured_build_command_failed_message, - &command_log); -} diff --git a/src/libmongoc/src/mongoc/mongoc-structured-log-connection-private.h b/src/libmongoc/src/mongoc/mongoc-structured-log-connection-private.h deleted file mode 100644 index 00197ef20dd..00000000000 --- a/src/libmongoc/src/mongoc/mongoc-structured-log-connection-private.h +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright 2009-present MongoDB, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "mongoc-prelude.h" - -#ifndef MONGOC_STRUCTURED_LOG_CONNECTION_PRIVATE_H -#define MONGOC_STRUCTURED_LOG_CONNECTION_PRIVATE_H - -#include "mongoc-structured-log.h" -#include "mongoc-cmd-private.h" - -void -mongoc_structured_log_connection_client_created (void); - -#endif /* MONGOC_STRUCTURED_LOG_COMMAND_PRIVATE_H */ diff --git a/src/libmongoc/src/mongoc/mongoc-structured-log-connection.c b/src/libmongoc/src/mongoc/mongoc-structured-log-connection.c deleted file mode 100644 index 7b7de41e0d7..00000000000 --- a/src/libmongoc/src/mongoc/mongoc-structured-log-connection.c +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Copyright 2009-present MongoDB, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "mongoc-structured-log-private.h" - -void -mongoc_structured_log_connection_client_created (void) -{ - mongoc_structured_log ( - MONGOC_STRUCTURED_LOG_LEVEL_DEBUG, MONGOC_STRUCTURED_LOG_COMPONENT_CONNECTION, "Client created", NULL, NULL); -} diff --git a/src/libmongoc/src/mongoc/mongoc-structured-log-private.h b/src/libmongoc/src/mongoc/mongoc-structured-log-private.h index dbafb744550..2094a492a8d 100644 --- a/src/libmongoc/src/mongoc/mongoc-structured-log-private.h +++ b/src/libmongoc/src/mongoc/mongoc-structured-log-private.h @@ -19,34 +19,156 @@ #ifndef MONGOC_STRUCTURED_LOG_PRIVATE_H #define MONGOC_STRUCTURED_LOG_PRIVATE_H +#include #include "mongoc-structured-log.h" #include "mongoc-cmd-private.h" -#include "mongoc-structured-log-command-private.h" +#include "mongoc-server-description-private.h" -#define MONGOC_STRUCTURED_LOG_DEFAULT_LEVEL MONGOC_STRUCTURED_LOG_LEVEL_WARNING; -#define MONGOC_STRUCTURED_LOG_DEFAULT_MAX_DOCUMENT_LENGTH 1000; +BSON_BEGIN_DECLS -typedef void (*mongoc_structured_log_build_message_t) (mongoc_structured_log_component_t component, - void *structured_log_data, - bson_t *structured_message /* OUT */); +#define MONGOC_STRUCTURED_LOG_DEFAULT_LEVEL MONGOC_STRUCTURED_LOG_LEVEL_WARNING +#define MONGOC_STRUCTURED_LOG_DEFAULT_MAX_DOCUMENT_LENGTH 1000 -struct _mongoc_structured_log_entry_t { +#define mongoc_structured_log(_level, _component, ...) \ + _MONGOC_STRUCTURED_LOG (_level, _component, __VA_ARGS__, {.func = NULL}) + +#define _MONGOC_STRUCTURED_LOG(_level, _component, _message, ...) \ + do { \ + mongoc_structured_log_entry_t _entry = { \ + .envelope.level = (_level), .envelope.component = (_component), .envelope.message = (_message)}; \ + if (_mongoc_structured_log_should_log (&_entry.envelope)) { \ + const mongoc_structured_log_builder_stage_t _builder[] = {__VA_ARGS__}; \ + _entry.builder = _builder; \ + _mongoc_structured_log_with_entry (&_entry); \ + } \ + } while (0) + +#define MONGOC_STRUCTURED_LOG_UTF8(_key_or_null, _value_utf8) \ + { \ + .func = _mongoc_structured_log_append_utf8, .arg1.utf8 = (_key_or_null), .arg2.utf8 = (_value_utf8), \ + } + +#define MONGOC_STRUCTURED_LOG_INT32(_key_or_null, _value_int32) \ + { \ + .func = _mongoc_structured_log_append_int32, .arg1.utf8 = (_key_or_null), .arg2.int32 = (_value_int32), \ + } + +#define MONGOC_STRUCTURED_LOG_INT64(_key_or_null, _value_int64) \ + { \ + .func = _mongoc_structured_log_append_int64, .arg1.utf8 = (_key_or_null), .arg2.int64 = (_value_int64), \ + } + +#define MONGOC_STRUCTURED_LOG_BOOL(_key_or_null, _value_bool) \ + { \ + .func = _mongoc_structured_log_append_bool, .arg1.utf8 = (_key_or_null), .arg2.boolean = (_value_bool), \ + } + +#define MONGOC_STRUCTURED_LOG_OID_AS_HEX(_key_or_null, _value_oid) \ + { \ + .func = _mongoc_structured_log_append_oid_as_hex, .arg1.utf8 = (_key_or_null), .arg2.oid = (_value_oid), \ + } + +#define MONGOC_STRUCTURED_LOG_BSON_AS_JSON(_key_or_null, _value_bson) \ + { \ + .func = _mongoc_structured_log_append_bson_as_json, .arg1.utf8 = (_key_or_null), .arg2.bson = (_value_bson), \ + } + +#define MONGOC_STRUCTURED_LOG_CMD(_cmd, _flags) \ + { \ + .func = _mongoc_structured_log_append_cmd, .arg1.cmd = (_cmd), .arg2.cmd_flags = (_flags) \ + } + +#define MONGOC_STRUCTURED_LOG_SERVER_DESCRIPTION(_server_description, _flags) \ + { \ + .func = _mongoc_structured_log_append_server_description, .arg1.server_description = (_server_description), \ + .arg2.server_description_flags = (_flags) \ + } + +typedef struct mongoc_structured_log_builder_stage_t mongoc_structured_log_builder_stage_t; + +typedef void (*mongoc_structured_log_builder_func_t) (bson_t *bson, const mongoc_structured_log_builder_stage_t *stage); + +typedef enum { + MONGOC_STRUCTURED_LOG_CMD_COMMAND = (1 << 0), + MONGOC_STRUCTURED_LOG_CMD_DATABASE_NAME = (1 << 1), + MONGOC_STRUCTURED_LOG_CMD_COMMAND_NAME = (1 << 2), + MONGOC_STRUCTURED_LOG_CMD_OPERATION_ID = (1 << 3), +} mongoc_structured_log_cmd_flags_t; + +typedef enum { + MONGOC_STRUCTURED_LOG_SERVER_DESCRIPTION_SERVER_HOST = (1 << 0), + MONGOC_STRUCTURED_LOG_SERVER_DESCRIPTION_SERVER_PORT = (1 << 1), + MONGOC_STRUCTURED_LOG_SERVER_DESCRIPTION_SERVER_CONNECTION_ID = (1 << 2), + MONGOC_STRUCTURED_LOG_SERVER_DESCRIPTION_SERVICE_ID = (1 << 3), +} mongoc_structured_log_server_description_flags_t; + +struct mongoc_structured_log_builder_stage_t { + mongoc_structured_log_builder_func_t func; // NULL sentinel here + union { + const mongoc_cmd_t *cmd; + const mongoc_server_description_t *server_description; + const void *utf8; + } arg1; + union { + bool boolean; + bson_oid_t *oid; + const bson_t *bson; + const void *utf8; + int32_t int32; + int64_t int64; + mongoc_structured_log_cmd_flags_t cmd_flags; + mongoc_structured_log_server_description_flags_t server_description_flags; + } arg2; + // Avoid adding an arg3, prefer to keep sizeof stage small +}; + +typedef struct mongoc_structured_log_envelope_t { mongoc_structured_log_level_t level; mongoc_structured_log_component_t component; const char *message; - bson_t *structured_message; - mongoc_structured_log_build_message_t build_message_func; - void *structured_log_data; +} mongoc_structured_log_envelope_t; + +struct mongoc_structured_log_entry_t { + mongoc_structured_log_envelope_t envelope; + const mongoc_structured_log_builder_stage_t *builder; // Required }; +char * +mongoc_structured_log_document_to_json (const bson_t *document, size_t *length); + void -mongoc_structured_log (mongoc_structured_log_level_t level, - mongoc_structured_log_component_t component, - const char *message, - mongoc_structured_log_build_message_t build_message_func, - void *structured_message_data); +mongoc_structured_log_get_handler (mongoc_structured_log_func_t *log_func, void **user_data); + +bool +_mongoc_structured_log_should_log (const mongoc_structured_log_envelope_t *envelope); + +void +_mongoc_structured_log_with_entry (const mongoc_structured_log_entry_t *entry); + +void +_mongoc_structured_log_append_utf8 (bson_t *bson, const mongoc_structured_log_builder_stage_t *stage); + +void +_mongoc_structured_log_append_int32 (bson_t *bson, const mongoc_structured_log_builder_stage_t *stage); + +void +_mongoc_structured_log_append_int64 (bson_t *bson, const mongoc_structured_log_builder_stage_t *stage); void -_mongoc_structured_log_get_handler (mongoc_structured_log_func_t *log_func, void **user_data); +_mongoc_structured_log_append_bool (bson_t *bson, const mongoc_structured_log_builder_stage_t *stage); + +void +_mongoc_structured_log_append_oid_as_hex (bson_t *bson, const mongoc_structured_log_builder_stage_t *stage); + +void +_mongoc_structured_log_append_bson_as_json (bson_t *bson, const mongoc_structured_log_builder_stage_t *stage); + +void +_mongoc_structured_log_append_cmd (bson_t *bson, const mongoc_structured_log_builder_stage_t *stage); + +void +_mongoc_structured_log_append_server_description (bson_t *bson, const mongoc_structured_log_builder_stage_t *stage); + +BSON_END_DECLS #endif /* MONGOC_STRUCTURED_LOG_PRIVATE_H */ diff --git a/src/libmongoc/src/mongoc/mongoc-structured-log.c b/src/libmongoc/src/mongoc/mongoc-structured-log.c index 9880a9a665b..0c2fd33336d 100644 --- a/src/libmongoc/src/mongoc/mongoc-structured-log.c +++ b/src/libmongoc/src/mongoc/mongoc-structured-log.c @@ -20,7 +20,7 @@ #include "mongoc-util-private.h" static void -mongoc_structured_log_default_handler (mongoc_structured_log_entry_t *entry, void *user_data); +mongoc_structured_log_default_handler (const mongoc_structured_log_entry_t *entry, void *user_data); static bson_once_t once = BSON_ONCE_INIT; static bson_mutex_t gStructuredLogMutex; @@ -35,38 +35,27 @@ static BSON_ONCE_FUN (_mongoc_ensure_mutex_once) BSON_ONCE_RETURN; } -static void -mongoc_structured_log_entry_destroy (mongoc_structured_log_entry_t *entry) +bson_t * +mongoc_structured_log_entry_message_as_bson (const mongoc_structured_log_entry_t *entry) { - if (entry->structured_message) { - bson_destroy (entry->structured_message); + bson_t *bson = bson_new (); + BSON_APPEND_UTF8 (bson, "message", entry->envelope.message); + for (const mongoc_structured_log_builder_stage_t *stage = entry->builder; stage->func; stage++) { + stage->func (bson, stage); } -} - -const bson_t * -mongoc_structured_log_entry_get_message (mongoc_structured_log_entry_t *entry) -{ - if (!entry->structured_message) { - entry->structured_message = BCON_NEW ("message", BCON_UTF8 (entry->message)); - - if (entry->build_message_func) { - entry->build_message_func (entry->component, entry->structured_log_data, entry->structured_message); - } - } - - return entry->structured_message; + return bson; } mongoc_structured_log_level_t mongoc_structured_log_entry_get_level (const mongoc_structured_log_entry_t *entry) { - return entry->level; + return entry->envelope.level; } mongoc_structured_log_component_t mongoc_structured_log_entry_get_component (const mongoc_structured_log_entry_t *entry) { - return entry->component; + return entry->envelope.component; } void @@ -80,22 +69,19 @@ mongoc_structured_log_set_handler (mongoc_structured_log_func_t log_func, void * bson_mutex_unlock (&gStructuredLogMutex); } -void -mongoc_structured_log (mongoc_structured_log_level_t level, - mongoc_structured_log_component_t component, - const char *message, - mongoc_structured_log_build_message_t build_message_func, - void *structured_message_data) -{ - mongoc_structured_log_entry_t entry = { - level, - component, - message, - NULL, - build_message_func, - structured_message_data, - }; +bool +_mongoc_structured_log_should_log (const mongoc_structured_log_envelope_t *envelope) +{ + // @todo Implement early-out settings for limiting max log level + (void) envelope; + // Don't take mutex, no need for atomicity. + // This should be a low cost early-out when logging is disabled. + return gStructuredLogger != NULL; +} +void +_mongoc_structured_log_with_entry (const mongoc_structured_log_entry_t *entry) +{ bson_once (&once, &_mongoc_ensure_mutex_once); bson_mutex_lock (&gStructuredLogMutex); @@ -104,10 +90,8 @@ mongoc_structured_log (mongoc_structured_log_level_t level, return; } - gStructuredLogger (&entry, gStructuredLoggerData); + gStructuredLogger (entry, gStructuredLoggerData); bson_mutex_unlock (&gStructuredLogMutex); - - mongoc_structured_log_entry_destroy (&entry); } static mongoc_structured_log_level_t @@ -183,9 +167,9 @@ _mongoc_structured_log_get_stream (void) } static void -mongoc_structured_log_default_handler (mongoc_structured_log_entry_t *entry, void *user_data) +mongoc_structured_log_default_handler (const mongoc_structured_log_entry_t *entry, void *user_data) { - char *message; + // @todo This really needs a cache, we shouldn't be parsing env vars for each should_log check mongoc_structured_log_level_t log_level = _mongoc_structured_log_get_log_level (mongoc_structured_log_entry_get_component (entry)); @@ -193,15 +177,17 @@ mongoc_structured_log_default_handler (mongoc_structured_log_entry_t *entry, voi return; } - message = bson_as_legacy_extended_json (mongoc_structured_log_entry_get_message (entry), NULL); + bson_t *bson_message = mongoc_structured_log_entry_message_as_bson (entry); + char *json_message = bson_as_relaxed_extended_json (bson_message, NULL); fprintf (_mongoc_structured_log_get_stream (), "Structured log: %d, %d, %s\n", (int) mongoc_structured_log_entry_get_level (entry), (int) mongoc_structured_log_entry_get_component (entry), - message); + json_message); - bson_free (message); + bson_free (json_message); + bson_destroy (bson_message); } static int32_t @@ -221,18 +207,140 @@ mongoc_structured_log_get_max_length (void) } char * -mongoc_structured_log_document_to_json (const bson_t *document) +mongoc_structured_log_document_to_json (const bson_t *document, size_t *length) { bson_json_opts_t *opts = bson_json_opts_new (BSON_JSON_MODE_CANONICAL, mongoc_structured_log_get_max_length ()); - char *json = bson_as_json_with_opts (document, NULL, opts); + char *json = bson_as_json_with_opts (document, length, opts); bson_json_opts_destroy (opts); return json; } -/* just for testing */ void -_mongoc_structured_log_get_handler (mongoc_structured_log_func_t *log_func, void **user_data) +mongoc_structured_log_get_handler (mongoc_structured_log_func_t *log_func, void **user_data) { *log_func = gStructuredLogger; *user_data = gStructuredLoggerData; } + +void +_mongoc_structured_log_append_utf8 (bson_t *bson, const mongoc_structured_log_builder_stage_t *stage) +{ + const char *key_or_null = stage->arg1.utf8; + if (key_or_null) { + bson_append_utf8 (bson, key_or_null, -1, stage->arg2.utf8, -1); + } +} + +void +_mongoc_structured_log_append_int32 (bson_t *bson, const mongoc_structured_log_builder_stage_t *stage) +{ + const char *key_or_null = stage->arg1.utf8; + if (key_or_null) { + bson_append_int32 (bson, key_or_null, -1, stage->arg2.int32); + } +} + +void +_mongoc_structured_log_append_int64 (bson_t *bson, const mongoc_structured_log_builder_stage_t *stage) +{ + const char *key_or_null = stage->arg1.utf8; + if (key_or_null) { + bson_append_int64 (bson, key_or_null, -1, stage->arg2.int64); + } +} + +void +_mongoc_structured_log_append_bool (bson_t *bson, const mongoc_structured_log_builder_stage_t *stage) +{ + const char *key_or_null = stage->arg1.utf8; + if (key_or_null) { + bson_append_bool (bson, key_or_null, -1, stage->arg2.boolean); + } +} + +void +_mongoc_structured_log_append_oid_as_hex (bson_t *bson, const mongoc_structured_log_builder_stage_t *stage) +{ + const char *key_or_null = stage->arg1.utf8; + if (key_or_null) { + char str[25]; + bson_oid_to_string (stage->arg2.oid, str); + bson_append_utf8 (bson, key_or_null, -1, str, 24); + } +} + +void +_mongoc_structured_log_append_bson_as_json (bson_t *bson, const mongoc_structured_log_builder_stage_t *stage) +{ + const char *key_or_null = stage->arg1.utf8; + if (key_or_null) { + size_t json_length; + char *json = mongoc_structured_log_document_to_json (stage->arg2.bson, &json_length); + if (json) { + bson_append_utf8 (bson, key_or_null, -1, json, json_length); + bson_free (json); + } + } +} + +void +_mongoc_structured_log_append_cmd (bson_t *bson, const mongoc_structured_log_builder_stage_t *stage) +{ + const mongoc_cmd_t *cmd = stage->arg1.cmd; + const mongoc_structured_log_cmd_flags_t flags = stage->arg2.cmd_flags; + + if (flags & MONGOC_STRUCTURED_LOG_CMD_DATABASE_NAME) { + BSON_APPEND_UTF8 (bson, "databaseName", cmd->db_name); + } + if (flags & MONGOC_STRUCTURED_LOG_CMD_COMMAND_NAME) { + BSON_APPEND_UTF8 (bson, "commandName", cmd->command_name); + } + if (flags & MONGOC_STRUCTURED_LOG_CMD_OPERATION_ID) { + BSON_APPEND_INT64 (bson, "operationId", cmd->operation_id); + } + if (flags & MONGOC_STRUCTURED_LOG_CMD_COMMAND) { + bson_t *command_copy = NULL; + + // @todo This is a performance bottleneck, we shouldn't be copying + // a potentially large command to serialize a potentially very + // small part of it. We should be outputting JSON, constrained + // by length limit, while visiting borrowed references to each + // command attribute and each payload. CDRIVER-4814 + if (cmd->payloads_count > 0) { + command_copy = bson_copy (cmd->command); + _mongoc_cmd_append_payload_as_array (cmd, command_copy); + } + + size_t json_length; + char *json = mongoc_structured_log_document_to_json (command_copy ? command_copy : cmd->command, &json_length); + if (json) { + const char *key = "command"; + bson_append_utf8 (bson, key, strlen (key), json, json_length); + bson_free (json); + } + + bson_destroy (command_copy); + } +} + +void +_mongoc_structured_log_append_server_description (bson_t *bson, const mongoc_structured_log_builder_stage_t *stage) +{ + const mongoc_server_description_t *sd = stage->arg1.server_description; + const mongoc_structured_log_server_description_flags_t flags = stage->arg2.server_description_flags; + + if (flags & MONGOC_STRUCTURED_LOG_SERVER_DESCRIPTION_SERVER_HOST) { + BSON_APPEND_UTF8 (bson, "serverHost", sd->host.host); + } + if (flags & MONGOC_STRUCTURED_LOG_SERVER_DESCRIPTION_SERVER_PORT) { + BSON_APPEND_INT32 (bson, "serverPort", sd->host.port); + } + if (flags & MONGOC_STRUCTURED_LOG_SERVER_DESCRIPTION_SERVER_CONNECTION_ID) { + BSON_APPEND_INT64 (bson, "serverConnectionId", sd->server_connection_id); + } + if (flags & MONGOC_STRUCTURED_LOG_SERVER_DESCRIPTION_SERVICE_ID) { + char str[25]; + bson_oid_to_string (&sd->service_id, str); + BSON_APPEND_UTF8 (bson, "serviceId", str); + } +} diff --git a/src/libmongoc/src/mongoc/mongoc-structured-log.h b/src/libmongoc/src/mongoc/mongoc-structured-log.h index 9dfca942d54..5ab156ad499 100644 --- a/src/libmongoc/src/mongoc/mongoc-structured-log.h +++ b/src/libmongoc/src/mongoc/mongoc-structured-log.h @@ -44,9 +44,9 @@ typedef enum { MONGOC_STRUCTURED_LOG_COMPONENT_CONNECTION, } mongoc_structured_log_component_t; -typedef struct _mongoc_structured_log_entry_t mongoc_structured_log_entry_t; +typedef struct mongoc_structured_log_entry_t mongoc_structured_log_entry_t; -typedef void (*mongoc_structured_log_func_t) (mongoc_structured_log_entry_t *entry, void *user_data); +typedef void (*mongoc_structured_log_func_t) (const mongoc_structured_log_entry_t *entry, void *user_data); /** * mongoc_structured_log_set_handler: @@ -54,52 +54,51 @@ typedef void (*mongoc_structured_log_func_t) (mongoc_structured_log_entry_t *ent * @user_data: User data for @log_func. * * Sets the function to be called to handle structured log messages. + * + * The callback is given a mongoc_structured_log_entry_t* as a handle for + * obtaining additional information about the log message. + * + * The entry pointer is only valid during a callback, because it's a low + * cost reference to temporary data. When the log message is extracted + * as a bson_t a new owned copy of this data must be made, and deferred + * transformations like bson_as_json take place. + * + * Callbacks may use mongoc_structured_log_entry_get_level and + * mongoc_structured_log_entry_get_component to filter log messages, and + * selectively call mongoc_structured_log_entry_message_as_bson to create + * a bson_t for log messages only when needed. */ MONGOC_EXPORT (void) mongoc_structured_log_set_handler (mongoc_structured_log_func_t log_func, void *user_data); /** - * mongoc_structured_log_entry_get_message: - * @entry: A log entry to extract the message from - * - * Returns the structured message as a bson_t pointer. + * mongoc_structured_log_entry_message_as_bson: + * @entry: A log entry to extract the message from. * - * When this function is called, the message is lazily generated if it hasn't - * already been generated. Note that it is not safe to call this method outside - * of a log handler, as the data needed to assemble the message may have been - * freed already. + * Returns the structured message as a new allocated bson_t + * that must be freed by bson_destroy(). */ -MONGOC_EXPORT (const bson_t *) -mongoc_structured_log_entry_get_message (mongoc_structured_log_entry_t *entry); +MONGOC_EXPORT (bson_t *) +mongoc_structured_log_entry_message_as_bson (const mongoc_structured_log_entry_t *entry); /** * mongoc_structured_log_entry_get_level: - * @entry: A log entry to read the level from + * @entry: A log entry to read the level from. * - * Returns the severity level of the structured log entry + * Returns the severity level of the structured log entry. */ MONGOC_EXPORT (mongoc_structured_log_level_t) mongoc_structured_log_entry_get_level (const mongoc_structured_log_entry_t *entry); /** * mongoc_structured_log_entry_get_component: - * @entry: A log entry to read the component from + * @entry: A log entry to read the component from. * - * Returns the component of the structured log entry + * Returns the component of the structured log entry. */ MONGOC_EXPORT (mongoc_structured_log_component_t) mongoc_structured_log_entry_get_component (const mongoc_structured_log_entry_t *entry); -/** - * mongoc_structured_log_document_to_json: - * @document: A BSON document to be serialized - * - * Returns the extended JSON representation of the given BSON document, - * respecting maximum logging length settings. - */ -MONGOC_EXPORT (char *) -mongoc_structured_log_document_to_json (const bson_t *document); - BSON_END_DECLS #endif /* MONGOC_STRUCTURED_LOG_H */ diff --git a/src/libmongoc/tests/test-mongoc-structured-log.c b/src/libmongoc/tests/test-mongoc-structured-log.c index feba3b1ae12..5231e5a12bc 100644 --- a/src/libmongoc/tests/test-mongoc-structured-log.c +++ b/src/libmongoc/tests/test-mongoc-structured-log.c @@ -19,31 +19,34 @@ #include "mongoc/mongoc-structured-log-private.h" #include "TestSuite.h" -struct log_assumption { - mongoc_structured_log_entry_t expected_entry; +typedef struct log_assumption { + mongoc_structured_log_envelope_t expected_envelope; + bson_t *expected_bson; int expected_calls; int calls; -}; +} log_assumption; -struct structured_log_state { +typedef struct structured_log_state { mongoc_structured_log_func_t handler; void *data; -}; +} structured_log_state; -static void -save_state (struct structured_log_state *state) +static BSON_INLINE structured_log_state +save_state (void) { - _mongoc_structured_log_get_handler (&state->handler, &state->data); + structured_log_state state; + mongoc_structured_log_get_handler (&state.handler, &state.data); + return state; } -static void -restore_state (const struct structured_log_state *state) +static BSON_INLINE void +restore_state (structured_log_state state) { - mongoc_structured_log_set_handler (state->handler, state->data); + mongoc_structured_log_set_handler (state.handler, state.data); } static void -structured_log_func (mongoc_structured_log_entry_t *entry, void *user_data) +structured_log_func (const mongoc_structured_log_entry_t *entry, void *user_data) { struct log_assumption *assumption = (struct log_assumption *) user_data; @@ -51,78 +54,180 @@ structured_log_func (mongoc_structured_log_entry_t *entry, void *user_data) ASSERT_CMPINT (assumption->calls, <=, assumption->expected_calls); - ASSERT_CMPINT (entry->level, ==, assumption->expected_entry.level); - ASSERT_CMPINT (entry->component, ==, assumption->expected_entry.component); - ASSERT (bson_equal (mongoc_structured_log_entry_get_message (entry), assumption->expected_entry.structured_message)); + ASSERT_CMPINT (entry->envelope.level, ==, assumption->expected_envelope.level); + ASSERT_CMPINT (entry->envelope.component, ==, assumption->expected_envelope.component); + ASSERT_CMPSTR (entry->envelope.message, assumption->expected_envelope.message); + + ASSERT_CMPINT (entry->envelope.level, ==, mongoc_structured_log_entry_get_level (entry)); + ASSERT_CMPINT (entry->envelope.component, ==, mongoc_structured_log_entry_get_component (entry)); + + // Each call to message_as_bson allocates an identical copy + bson_t *bson_1 = mongoc_structured_log_entry_message_as_bson (entry); + bson_t *bson_2 = mongoc_structured_log_entry_message_as_bson (entry); + + // Compare for exact bson equality *after* comparing json strings, to give a more user friendly error on most + // failures + char *json_actual = bson_as_relaxed_extended_json (bson_1, NULL); + char *json_expected = bson_as_relaxed_extended_json (assumption->expected_bson, NULL); + ASSERT_CMPSTR (json_actual, json_expected); + + ASSERT (bson_equal (bson_1, assumption->expected_bson)); + ASSERT (bson_equal (bson_2, assumption->expected_bson)); + bson_destroy (bson_2); + bson_destroy (bson_1); + bson_free (json_actual); + bson_free (json_expected); } void test_plain_log_entry (void) { - struct structured_log_state old_state; struct log_assumption assumption = { - { - MONGOC_STRUCTURED_LOG_LEVEL_WARNING, - MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND, - "Plain log entry", - BCON_NEW ("message", BCON_UTF8 ("Plain log entry")), - }, - 1, - 0, + .expected_envelope.level = MONGOC_STRUCTURED_LOG_LEVEL_WARNING, + .expected_envelope.component = MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND, + .expected_envelope.message = "Plain log entry", + .expected_bson = BCON_NEW ("message", BCON_UTF8 ("Plain log entry")), + .expected_calls = 1, }; - save_state (&old_state); - + structured_log_state old_state = save_state (); mongoc_structured_log_set_handler (structured_log_func, &assumption); mongoc_structured_log ( - MONGOC_STRUCTURED_LOG_LEVEL_WARNING, MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND, "Plain log entry", NULL, NULL); - - ASSERT_CMPINT (assumption.calls, =, 1); - - restore_state (&old_state); - - bson_destroy (assumption.expected_entry.structured_message); -} + MONGOC_STRUCTURED_LOG_LEVEL_WARNING, MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND, "Plain log entry"); -void -_test_append_extra_data (mongoc_structured_log_component_t component, - void *structured_log_data, - bson_t *structured_message /* OUT */) -{ - BCON_APPEND (structured_message, "extra", BCON_INT32 (1)); + ASSERT_CMPINT (assumption.calls, ==, 1); + restore_state (old_state); + bson_destroy (assumption.expected_bson); } void test_log_entry_with_extra_data (void) { - struct structured_log_state old_state; struct log_assumption assumption = { - { - MONGOC_STRUCTURED_LOG_LEVEL_WARNING, - MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND, - "Plain log entry", - BCON_NEW ("message", BCON_UTF8 ("Plain log entry"), "extra", BCON_INT32 (1)), - }, - 1, - 0, + .expected_envelope.level = MONGOC_STRUCTURED_LOG_LEVEL_WARNING, + .expected_envelope.component = MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND, + .expected_envelope.message = "Plain log entry", + .expected_bson = BCON_NEW ("message", BCON_UTF8 ("Plain log entry"), "extra", BCON_INT32 (1)), + .expected_calls = 1, }; - save_state (&old_state); - + structured_log_state old_state = save_state (); mongoc_structured_log_set_handler (structured_log_func, &assumption); mongoc_structured_log (MONGOC_STRUCTURED_LOG_LEVEL_WARNING, MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND, "Plain log entry", - _test_append_extra_data, - NULL); + MONGOC_STRUCTURED_LOG_INT32 ("extra", 1)); + + ASSERT_CMPINT (assumption.calls, ==, 1); + restore_state (old_state); + bson_destroy (assumption.expected_bson); +} - ASSERT_CMPINT (assumption.calls, =, 1); +void +test_log_entry_with_all_data_types (void) +{ + struct log_assumption assumption = { + .expected_envelope.level = MONGOC_STRUCTURED_LOG_LEVEL_WARNING, + .expected_envelope.component = MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND, + .expected_envelope.message = "Log entry with all data types", + .expected_bson = BCON_NEW ("message", + BCON_UTF8 ("Log entry with all data types"), + "k1", + BCON_UTF8 ("string value"), + "k2", + BCON_NULL, + "k3", + BCON_INT32 (-12345), + "k4", + BCON_INT64 (0x76543210aabbccdd), + "k5", + BCON_BOOL (true), + "k6", + BCON_BOOL (false), + "k7", + BCON_UTF8 ("{ \"k\" : \"v\" }"), + "k8", + BCON_UTF8 ("112233445566778899aabbcc"), + "databaseName", + BCON_UTF8 ("Some database"), + "commandName", + BCON_UTF8 ("Not a command"), + "operationId", + BCON_INT64 (0x12345678eeff0011), + "command", + BCON_UTF8 ("{ \"c\" : \"d\" }"), + "serverHost", + BCON_UTF8 ("db.example.com"), + "serverPort", + BCON_INT32 (2345), + "serverConnectionId", + BCON_INT64 (0x3deeff0011223345), + "serviceId", + BCON_UTF8 ("2233445566778899aabbccdd")), + .expected_calls = 1, + }; + + bson_t *json_doc = BCON_NEW ("k", BCON_UTF8 ("v")); + bson_t *cmd_doc = BCON_NEW ("c", BCON_UTF8 ("d")); + + bson_oid_t oid; + bson_oid_init_from_string (&oid, "112233445566778899aabbcc"); - restore_state (&old_state); + mongoc_cmd_t cmd = { + .db_name = "Some database", + .command_name = "Not a command", + .operation_id = 0x12345678eeff0011, + .command = cmd_doc, + }; + + mongoc_server_description_t server_description = { + .host.host = "db.example.com", + .host.port = 2345, + .server_connection_id = 0x3deeff0011223345, + }; + bson_oid_init_from_string (&server_description.service_id, "2233445566778899aabbccdd"); - bson_destroy (assumption.expected_entry.structured_message); + structured_log_state old_state = save_state (); + mongoc_structured_log_set_handler (structured_log_func, &assumption); + + mongoc_structured_log ( + MONGOC_STRUCTURED_LOG_LEVEL_WARNING, + MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND, + "Log entry with all data types", + // Basic BSON types. + // Supports optional values (skip when key is NULL) + MONGOC_STRUCTURED_LOG_UTF8 ("k1", "string value"), + MONGOC_STRUCTURED_LOG_UTF8 ("k2", NULL), + MONGOC_STRUCTURED_LOG_UTF8 (NULL, NULL), + MONGOC_STRUCTURED_LOG_INT32 ("k3", -12345), + MONGOC_STRUCTURED_LOG_INT32 (NULL, 9999), + MONGOC_STRUCTURED_LOG_INT64 ("k4", 0x76543210aabbccdd), + MONGOC_STRUCTURED_LOG_INT64 (NULL, -1), + MONGOC_STRUCTURED_LOG_BOOL ("k5", true), + MONGOC_STRUCTURED_LOG_BOOL ("k6", false), + MONGOC_STRUCTURED_LOG_BOOL (NULL, true), + // Deferred conversions + MONGOC_STRUCTURED_LOG_BSON_AS_JSON ("k7", json_doc), + MONGOC_STRUCTURED_LOG_BSON_AS_JSON (NULL, NULL), + MONGOC_STRUCTURED_LOG_OID_AS_HEX ("k8", &oid), + MONGOC_STRUCTURED_LOG_OID_AS_HEX (NULL, NULL), + // Common structures, with explicit set of keys to include + MONGOC_STRUCTURED_LOG_CMD (&cmd, + (MONGOC_STRUCTURED_LOG_CMD_COMMAND | MONGOC_STRUCTURED_LOG_CMD_DATABASE_NAME | + MONGOC_STRUCTURED_LOG_CMD_COMMAND_NAME | MONGOC_STRUCTURED_LOG_CMD_OPERATION_ID)), + MONGOC_STRUCTURED_LOG_SERVER_DESCRIPTION (&server_description, + (MONGOC_STRUCTURED_LOG_SERVER_DESCRIPTION_SERVER_HOST | + MONGOC_STRUCTURED_LOG_SERVER_DESCRIPTION_SERVER_PORT | + MONGOC_STRUCTURED_LOG_SERVER_DESCRIPTION_SERVER_CONNECTION_ID | + MONGOC_STRUCTURED_LOG_SERVER_DESCRIPTION_SERVICE_ID))); + + ASSERT_CMPINT (assumption.calls, ==, 1); + restore_state (old_state); + bson_destroy (assumption.expected_bson); + bson_destroy (json_doc); + bson_destroy (cmd_doc); } void @@ -130,4 +235,5 @@ test_structured_log_install (TestSuite *suite) { TestSuite_Add (suite, "/structured_log/plain", test_plain_log_entry); TestSuite_Add (suite, "/structured_log/with_extra_data", test_log_entry_with_extra_data); + TestSuite_Add (suite, "/structured_log/with_all_data_types", test_log_entry_with_all_data_types); } From 92c7596743a87a2a5f99ae08f6857c3b585fda7c Mon Sep 17 00:00:00 2001 From: Micah Scott Date: Mon, 11 Nov 2024 11:53:12 -0800 Subject: [PATCH 038/139] structured logging support for sized non-terminated strings --- .../src/mongoc/mongoc-cursor-legacy.c | 5 +- src/libmongoc/src/mongoc/mongoc-cursor.c | 6 +- .../mongoc/mongoc-structured-log-private.h | 72 +++++++++++-------- .../src/mongoc/mongoc-structured-log.c | 55 +++++++++++--- .../tests/test-mongoc-structured-log.c | 53 +++++++++----- 5 files changed, 129 insertions(+), 62 deletions(-) diff --git a/src/libmongoc/src/mongoc/mongoc-cursor-legacy.c b/src/libmongoc/src/mongoc/mongoc-cursor-legacy.c index fcf83ef56c0..57142fad712 100644 --- a/src/libmongoc/src/mongoc/mongoc-cursor-legacy.c +++ b/src/libmongoc/src/mongoc/mongoc-cursor-legacy.c @@ -48,7 +48,6 @@ _mongoc_cursor_monitor_legacy_get_more (mongoc_cursor_t *cursor, mongoc_server_s client = cursor->client; _mongoc_cursor_prepare_getmore_command (cursor, &doc); - db = bson_strndup (cursor->ns, cursor->dblen); mongoc_structured_log ( MONGOC_STRUCTURED_LOG_LEVEL_INFO, @@ -60,7 +59,7 @@ _mongoc_cursor_monitor_legacy_get_more (mongoc_cursor_t *cursor, mongoc_server_s MONGOC_STRUCTURED_LOG_SERVER_DESCRIPTION_SERVER_PORT | MONGOC_STRUCTURED_LOG_SERVER_DESCRIPTION_SERVER_CONNECTION_ID | MONGOC_STRUCTURED_LOG_SERVER_DESCRIPTION_SERVICE_ID)), - MONGOC_STRUCTURED_LOG_UTF8 ("databaseName", db), + MONGOC_STRUCTURED_LOG_UTF8_N ("databaseName", cursor->ns, cursor->dblen), MONGOC_STRUCTURED_LOG_UTF8 ("commandName", "getMore"), MONGOC_STRUCTURED_LOG_INT64 ("operationId", cursor->operation_id), MONGOC_STRUCTURED_LOG_BSON_AS_JSON ("command", &doc)); @@ -68,10 +67,10 @@ _mongoc_cursor_monitor_legacy_get_more (mongoc_cursor_t *cursor, mongoc_server_s if (!client->apm_callbacks.started) { /* successful */ bson_destroy (&doc); - bson_free (db); RETURN (true); } + db = bson_strndup (cursor->ns, cursor->dblen); mongoc_apm_command_started_init (&event, &doc, db, diff --git a/src/libmongoc/src/mongoc/mongoc-cursor.c b/src/libmongoc/src/mongoc/mongoc-cursor.c index 1fe9cb0a8d9..097055a92f3 100644 --- a/src/libmongoc/src/mongoc/mongoc-cursor.c +++ b/src/libmongoc/src/mongoc/mongoc-cursor.c @@ -645,7 +645,6 @@ _mongoc_cursor_monitor_command (mongoc_cursor_t *cursor, ENTRY; client = cursor->client; - db = bson_strndup (cursor->ns, cursor->dblen); mongoc_structured_log ( MONGOC_STRUCTURED_LOG_LEVEL_INFO, @@ -657,17 +656,18 @@ _mongoc_cursor_monitor_command (mongoc_cursor_t *cursor, MONGOC_STRUCTURED_LOG_SERVER_DESCRIPTION_SERVER_PORT | MONGOC_STRUCTURED_LOG_SERVER_DESCRIPTION_SERVER_CONNECTION_ID | MONGOC_STRUCTURED_LOG_SERVER_DESCRIPTION_SERVICE_ID)), - MONGOC_STRUCTURED_LOG_UTF8 ("databaseName", db), + MONGOC_STRUCTURED_LOG_UTF8_N ("databaseName", cursor->ns, cursor->dblen), MONGOC_STRUCTURED_LOG_UTF8 ("commandName", cmd_name), MONGOC_STRUCTURED_LOG_INT64 ("operationId", cursor->operation_id), MONGOC_STRUCTURED_LOG_BSON_AS_JSON ("command", cmd)); if (!client->apm_callbacks.started) { /* successful */ - bson_free (db); RETURN (true); } + db = bson_strndup (cursor->ns, cursor->dblen); + mongoc_apm_command_started_init (&event, cmd, db, diff --git a/src/libmongoc/src/mongoc/mongoc-structured-log-private.h b/src/libmongoc/src/mongoc/mongoc-structured-log-private.h index 2094a492a8d..ed9489d9add 100644 --- a/src/libmongoc/src/mongoc/mongoc-structured-log-private.h +++ b/src/libmongoc/src/mongoc/mongoc-structured-log-private.h @@ -43,34 +43,43 @@ BSON_BEGIN_DECLS } \ } while (0) -#define MONGOC_STRUCTURED_LOG_UTF8(_key_or_null, _value_utf8) \ - { \ - .func = _mongoc_structured_log_append_utf8, .arg1.utf8 = (_key_or_null), .arg2.utf8 = (_value_utf8), \ +#define MONGOC_STRUCTURED_LOG_UTF8(_key_or_null, _value_utf8) \ + { \ + .func = _mongoc_structured_log_append_utf8, .arg1.utf8 = (_key_or_null), .arg2.utf8 = (_value_utf8) \ } -#define MONGOC_STRUCTURED_LOG_INT32(_key_or_null, _value_int32) \ - { \ - .func = _mongoc_structured_log_append_int32, .arg1.utf8 = (_key_or_null), .arg2.int32 = (_value_int32), \ +#define MONGOC_STRUCTURED_LOG_UTF8_N(_key_literal, _value_utf8, _value_len) \ + MONGOC_STRUCTURED_LOG_UTF8_NN (_key_literal, strlen (_key_literal), _value_utf8, _value_len) + +#define MONGOC_STRUCTURED_LOG_UTF8_NN(_key_or_null, _key_len, _value_utf8, _value_len) \ + {.func = _mongoc_structured_log_append_utf8_n_stage0, .arg1.utf8 = (_key_or_null), .arg2.int32 = (_key_len)}, \ + { \ + .func = _mongoc_structured_log_append_utf8_n_stage1, .arg1.utf8 = (_value_utf8), .arg2.int32 = (_value_len) \ } -#define MONGOC_STRUCTURED_LOG_INT64(_key_or_null, _value_int64) \ - { \ - .func = _mongoc_structured_log_append_int64, .arg1.utf8 = (_key_or_null), .arg2.int64 = (_value_int64), \ +#define MONGOC_STRUCTURED_LOG_INT32(_key_or_null, _value_int32) \ + { \ + .func = _mongoc_structured_log_append_int32, .arg1.utf8 = (_key_or_null), .arg2.int32 = (_value_int32) \ } -#define MONGOC_STRUCTURED_LOG_BOOL(_key_or_null, _value_bool) \ - { \ - .func = _mongoc_structured_log_append_bool, .arg1.utf8 = (_key_or_null), .arg2.boolean = (_value_bool), \ +#define MONGOC_STRUCTURED_LOG_INT64(_key_or_null, _value_int64) \ + { \ + .func = _mongoc_structured_log_append_int64, .arg1.utf8 = (_key_or_null), .arg2.int64 = (_value_int64) \ } -#define MONGOC_STRUCTURED_LOG_OID_AS_HEX(_key_or_null, _value_oid) \ - { \ - .func = _mongoc_structured_log_append_oid_as_hex, .arg1.utf8 = (_key_or_null), .arg2.oid = (_value_oid), \ +#define MONGOC_STRUCTURED_LOG_BOOL(_key_or_null, _value_bool) \ + { \ + .func = _mongoc_structured_log_append_bool, .arg1.utf8 = (_key_or_null), .arg2.boolean = (_value_bool) \ } -#define MONGOC_STRUCTURED_LOG_BSON_AS_JSON(_key_or_null, _value_bson) \ - { \ - .func = _mongoc_structured_log_append_bson_as_json, .arg1.utf8 = (_key_or_null), .arg2.bson = (_value_bson), \ +#define MONGOC_STRUCTURED_LOG_OID_AS_HEX(_key_or_null, _value_oid) \ + { \ + .func = _mongoc_structured_log_append_oid_as_hex, .arg1.utf8 = (_key_or_null), .arg2.oid = (_value_oid) \ + } + +#define MONGOC_STRUCTURED_LOG_BSON_AS_JSON(_key_or_null, _value_bson) \ + { \ + .func = _mongoc_structured_log_append_bson_as_json, .arg1.utf8 = (_key_or_null), .arg2.bson = (_value_bson) \ } #define MONGOC_STRUCTURED_LOG_CMD(_cmd, _flags) \ @@ -86,7 +95,8 @@ BSON_BEGIN_DECLS typedef struct mongoc_structured_log_builder_stage_t mongoc_structured_log_builder_stage_t; -typedef void (*mongoc_structured_log_builder_func_t) (bson_t *bson, const mongoc_structured_log_builder_stage_t *stage); +typedef const mongoc_structured_log_builder_stage_t *(*mongoc_structured_log_builder_func_t) ( + bson_t *bson, const mongoc_structured_log_builder_stage_t *stage); typedef enum { MONGOC_STRUCTURED_LOG_CMD_COMMAND = (1 << 0), @@ -119,7 +129,7 @@ struct mongoc_structured_log_builder_stage_t { mongoc_structured_log_cmd_flags_t cmd_flags; mongoc_structured_log_server_description_flags_t server_description_flags; } arg2; - // Avoid adding an arg3, prefer to keep sizeof stage small + // Avoid adding an arg3, prefer to use additional stages }; typedef struct mongoc_structured_log_envelope_t { @@ -145,28 +155,34 @@ _mongoc_structured_log_should_log (const mongoc_structured_log_envelope_t *envel void _mongoc_structured_log_with_entry (const mongoc_structured_log_entry_t *entry); -void +const mongoc_structured_log_builder_stage_t * _mongoc_structured_log_append_utf8 (bson_t *bson, const mongoc_structured_log_builder_stage_t *stage); -void +const mongoc_structured_log_builder_stage_t * +_mongoc_structured_log_append_utf8_n_stage0 (bson_t *bson, const mongoc_structured_log_builder_stage_t *stage); + +const mongoc_structured_log_builder_stage_t * +_mongoc_structured_log_append_utf8_n_stage1 (bson_t *bson, const mongoc_structured_log_builder_stage_t *stage); + +const mongoc_structured_log_builder_stage_t * _mongoc_structured_log_append_int32 (bson_t *bson, const mongoc_structured_log_builder_stage_t *stage); -void +const mongoc_structured_log_builder_stage_t * _mongoc_structured_log_append_int64 (bson_t *bson, const mongoc_structured_log_builder_stage_t *stage); -void +const mongoc_structured_log_builder_stage_t * _mongoc_structured_log_append_bool (bson_t *bson, const mongoc_structured_log_builder_stage_t *stage); -void +const mongoc_structured_log_builder_stage_t * _mongoc_structured_log_append_oid_as_hex (bson_t *bson, const mongoc_structured_log_builder_stage_t *stage); -void +const mongoc_structured_log_builder_stage_t * _mongoc_structured_log_append_bson_as_json (bson_t *bson, const mongoc_structured_log_builder_stage_t *stage); -void +const mongoc_structured_log_builder_stage_t * _mongoc_structured_log_append_cmd (bson_t *bson, const mongoc_structured_log_builder_stage_t *stage); -void +const mongoc_structured_log_builder_stage_t * _mongoc_structured_log_append_server_description (bson_t *bson, const mongoc_structured_log_builder_stage_t *stage); BSON_END_DECLS diff --git a/src/libmongoc/src/mongoc/mongoc-structured-log.c b/src/libmongoc/src/mongoc/mongoc-structured-log.c index 0c2fd33336d..f4398db762e 100644 --- a/src/libmongoc/src/mongoc/mongoc-structured-log.c +++ b/src/libmongoc/src/mongoc/mongoc-structured-log.c @@ -40,8 +40,9 @@ mongoc_structured_log_entry_message_as_bson (const mongoc_structured_log_entry_t { bson_t *bson = bson_new (); BSON_APPEND_UTF8 (bson, "message", entry->envelope.message); - for (const mongoc_structured_log_builder_stage_t *stage = entry->builder; stage->func; stage++) { - stage->func (bson, stage); + const mongoc_structured_log_builder_stage_t *stage = entry->builder; + while (stage->func) { + stage = stage->func (bson, stage); } return bson; } @@ -73,7 +74,7 @@ bool _mongoc_structured_log_should_log (const mongoc_structured_log_envelope_t *envelope) { // @todo Implement early-out settings for limiting max log level - (void) envelope; + BSON_UNUSED (envelope); // Don't take mutex, no need for atomicity. // This should be a low cost early-out when logging is disabled. return gStructuredLogger != NULL; @@ -222,43 +223,71 @@ mongoc_structured_log_get_handler (mongoc_structured_log_func_t *log_func, void *user_data = gStructuredLoggerData; } -void +const mongoc_structured_log_builder_stage_t * _mongoc_structured_log_append_utf8 (bson_t *bson, const mongoc_structured_log_builder_stage_t *stage) { const char *key_or_null = stage->arg1.utf8; if (key_or_null) { bson_append_utf8 (bson, key_or_null, -1, stage->arg2.utf8, -1); } + return stage + 1; } -void +const mongoc_structured_log_builder_stage_t * +_mongoc_structured_log_append_utf8_n_stage0 (bson_t *bson, const mongoc_structured_log_builder_stage_t *stage) +{ + BSON_ASSERT (stage[1].func == _mongoc_structured_log_append_utf8_n_stage1); + const char *key_or_null = stage[0].arg1.utf8; + int32_t key_len = stage[0].arg2.int32; + const char *value = stage[1].arg1.utf8; + int32_t value_len = stage[1].arg2.int32; + if (key_or_null) { + bson_append_utf8 (bson, key_or_null, key_len, value, value_len); + } + return stage + 2; +} + +const mongoc_structured_log_builder_stage_t * +_mongoc_structured_log_append_utf8_n_stage1 (bson_t *bson, const mongoc_structured_log_builder_stage_t *stage) +{ + // Never called, marks the second stage in a two-stage utf8_n + BSON_UNUSED (bson); + BSON_UNUSED (stage); + BSON_ASSERT (false); + return NULL; +} + +const mongoc_structured_log_builder_stage_t * _mongoc_structured_log_append_int32 (bson_t *bson, const mongoc_structured_log_builder_stage_t *stage) { const char *key_or_null = stage->arg1.utf8; if (key_or_null) { bson_append_int32 (bson, key_or_null, -1, stage->arg2.int32); } + return stage + 1; } -void +const mongoc_structured_log_builder_stage_t * _mongoc_structured_log_append_int64 (bson_t *bson, const mongoc_structured_log_builder_stage_t *stage) { const char *key_or_null = stage->arg1.utf8; if (key_or_null) { bson_append_int64 (bson, key_or_null, -1, stage->arg2.int64); } + return stage + 1; } -void +const mongoc_structured_log_builder_stage_t * _mongoc_structured_log_append_bool (bson_t *bson, const mongoc_structured_log_builder_stage_t *stage) { const char *key_or_null = stage->arg1.utf8; if (key_or_null) { bson_append_bool (bson, key_or_null, -1, stage->arg2.boolean); } + return stage + 1; } -void +const mongoc_structured_log_builder_stage_t * _mongoc_structured_log_append_oid_as_hex (bson_t *bson, const mongoc_structured_log_builder_stage_t *stage) { const char *key_or_null = stage->arg1.utf8; @@ -267,9 +296,10 @@ _mongoc_structured_log_append_oid_as_hex (bson_t *bson, const mongoc_structured_ bson_oid_to_string (stage->arg2.oid, str); bson_append_utf8 (bson, key_or_null, -1, str, 24); } + return stage + 1; } -void +const mongoc_structured_log_builder_stage_t * _mongoc_structured_log_append_bson_as_json (bson_t *bson, const mongoc_structured_log_builder_stage_t *stage) { const char *key_or_null = stage->arg1.utf8; @@ -281,9 +311,10 @@ _mongoc_structured_log_append_bson_as_json (bson_t *bson, const mongoc_structure bson_free (json); } } + return stage + 1; } -void +const mongoc_structured_log_builder_stage_t * _mongoc_structured_log_append_cmd (bson_t *bson, const mongoc_structured_log_builder_stage_t *stage) { const mongoc_cmd_t *cmd = stage->arg1.cmd; @@ -321,9 +352,10 @@ _mongoc_structured_log_append_cmd (bson_t *bson, const mongoc_structured_log_bui bson_destroy (command_copy); } + return stage + 1; } -void +const mongoc_structured_log_builder_stage_t * _mongoc_structured_log_append_server_description (bson_t *bson, const mongoc_structured_log_builder_stage_t *stage) { const mongoc_server_description_t *sd = stage->arg1.server_description; @@ -343,4 +375,5 @@ _mongoc_structured_log_append_server_description (bson_t *bson, const mongoc_str bson_oid_to_string (&sd->service_id, str); BSON_APPEND_UTF8 (bson, "serviceId", str); } + return stage + 1; } diff --git a/src/libmongoc/tests/test-mongoc-structured-log.c b/src/libmongoc/tests/test-mongoc-structured-log.c index 5231e5a12bc..bf79be8ad71 100644 --- a/src/libmongoc/tests/test-mongoc-structured-log.c +++ b/src/libmongoc/tests/test-mongoc-structured-log.c @@ -128,27 +128,39 @@ test_log_entry_with_extra_data (void) void test_log_entry_with_all_data_types (void) { + const char non_terminated_test_string[] = {0, 1, 2, 3, 'a', '\\'}; + bson_t *bson_str_n = bson_new (); + bson_append_utf8 (bson_str_n, "kStrN1", -1, non_terminated_test_string, sizeof non_terminated_test_string); + bson_append_utf8 (bson_str_n, "kStrN2", -1, non_terminated_test_string, sizeof non_terminated_test_string); + struct log_assumption assumption = { .expected_envelope.level = MONGOC_STRUCTURED_LOG_LEVEL_WARNING, .expected_envelope.component = MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND, .expected_envelope.message = "Log entry with all data types", .expected_bson = BCON_NEW ("message", BCON_UTF8 ("Log entry with all data types"), - "k1", + "kStr", BCON_UTF8 ("string value"), - "k2", + "kNullStr", + BCON_NULL, + BCON (bson_str_n), + "kNullStrN1", + BCON_NULL, + "kNullStrN2", + BCON_NULL, + "kNullStrN3", BCON_NULL, - "k3", + "kInt32", BCON_INT32 (-12345), - "k4", + "kInt64", BCON_INT64 (0x76543210aabbccdd), - "k5", + "kTrue", BCON_BOOL (true), - "k6", + "kFalse", BCON_BOOL (false), - "k7", + "kJSON", BCON_UTF8 ("{ \"k\" : \"v\" }"), - "k8", + "kOID", BCON_UTF8 ("112233445566778899aabbcc"), "databaseName", BCON_UTF8 ("Some database"), @@ -197,21 +209,27 @@ test_log_entry_with_all_data_types (void) MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND, "Log entry with all data types", // Basic BSON types. - // Supports optional values (skip when key is NULL) - MONGOC_STRUCTURED_LOG_UTF8 ("k1", "string value"), - MONGOC_STRUCTURED_LOG_UTF8 ("k2", NULL), + // Most support optional values (skip when key is NULL) + MONGOC_STRUCTURED_LOG_UTF8 ("kStr", "string value"), + MONGOC_STRUCTURED_LOG_UTF8 ("kNullStr", NULL), MONGOC_STRUCTURED_LOG_UTF8 (NULL, NULL), - MONGOC_STRUCTURED_LOG_INT32 ("k3", -12345), + MONGOC_STRUCTURED_LOG_UTF8_NN ("kStrN1ZZZ", 6, non_terminated_test_string, sizeof non_terminated_test_string), + MONGOC_STRUCTURED_LOG_UTF8_N ("kStrN2", non_terminated_test_string, sizeof non_terminated_test_string), + MONGOC_STRUCTURED_LOG_UTF8_NN ("kNullStrN1ZZZ", 10, NULL, 12345), + MONGOC_STRUCTURED_LOG_UTF8_NN ("kNullStrN2", -1, NULL, 12345), + MONGOC_STRUCTURED_LOG_UTF8_NN (NULL, 999, NULL, 999), + MONGOC_STRUCTURED_LOG_UTF8_N ("kNullStrN3", NULL, 12345), + MONGOC_STRUCTURED_LOG_INT32 ("kInt32", -12345), MONGOC_STRUCTURED_LOG_INT32 (NULL, 9999), - MONGOC_STRUCTURED_LOG_INT64 ("k4", 0x76543210aabbccdd), + MONGOC_STRUCTURED_LOG_INT64 ("kInt64", 0x76543210aabbccdd), MONGOC_STRUCTURED_LOG_INT64 (NULL, -1), - MONGOC_STRUCTURED_LOG_BOOL ("k5", true), - MONGOC_STRUCTURED_LOG_BOOL ("k6", false), + MONGOC_STRUCTURED_LOG_BOOL ("kTrue", true), + MONGOC_STRUCTURED_LOG_BOOL ("kFalse", false), MONGOC_STRUCTURED_LOG_BOOL (NULL, true), // Deferred conversions - MONGOC_STRUCTURED_LOG_BSON_AS_JSON ("k7", json_doc), + MONGOC_STRUCTURED_LOG_BSON_AS_JSON ("kJSON", json_doc), MONGOC_STRUCTURED_LOG_BSON_AS_JSON (NULL, NULL), - MONGOC_STRUCTURED_LOG_OID_AS_HEX ("k8", &oid), + MONGOC_STRUCTURED_LOG_OID_AS_HEX ("kOID", &oid), MONGOC_STRUCTURED_LOG_OID_AS_HEX (NULL, NULL), // Common structures, with explicit set of keys to include MONGOC_STRUCTURED_LOG_CMD (&cmd, @@ -228,6 +246,7 @@ test_log_entry_with_all_data_types (void) bson_destroy (assumption.expected_bson); bson_destroy (json_doc); bson_destroy (cmd_doc); + bson_destroy (bson_str_n); } void From c1396e1cae6532c9821525f48e2586210314ea5d Mon Sep 17 00:00:00 2001 From: Micah Scott Date: Mon, 11 Nov 2024 14:24:33 -0800 Subject: [PATCH 039/139] sync command-logging-and-monitoring tests From specifications repo changeset 0a7a8b5202ebb708956147f4f19776a54cd1fb2d --- .../logging/command.json | 215 +++ .../logging/driver-connection-id.json | 146 ++ .../logging/no-handshake-messages.json | 94 ++ .../logging/no-heartbeat-messages.json | 91 ++ .../logging/operation-id.json | 198 +++ .../logging/pre-42-server-connection-id.json | 119 ++ .../logging/redacted-commands.json | 1438 +++++++++++++++++ .../logging/server-connection-id.json | 131 ++ .../logging/service-id.json | 207 +++ .../logging/unacknowledged-write.json | 151 ++ .../monitoring/bulkWrite.json | 154 ++ .../monitoring/command.json | 83 + .../monitoring/deleteMany.json | 162 ++ .../monitoring/deleteOne.json | 162 ++ .../monitoring/find.json | 558 +++++++ .../monitoring/insertMany.json | 148 ++ .../monitoring/insertOne.json | 144 ++ .../pre-42-server-connection-id.json | 101 ++ .../monitoring/redacted-commands.json | 679 ++++++++ .../monitoring/server-connection-id.json | 101 ++ .../monitoring/unacknowledgedBulkWrite.json | 117 ++ .../monitoring/updateMany.json | 188 +++ .../monitoring/updateOne.json | 260 +++ .../monitoring/writeConcernError.json | 155 ++ 24 files changed, 5802 insertions(+) create mode 100644 src/libmongoc/tests/json/command-logging-and-monitoring/logging/command.json create mode 100644 src/libmongoc/tests/json/command-logging-and-monitoring/logging/driver-connection-id.json create mode 100644 src/libmongoc/tests/json/command-logging-and-monitoring/logging/no-handshake-messages.json create mode 100644 src/libmongoc/tests/json/command-logging-and-monitoring/logging/no-heartbeat-messages.json create mode 100644 src/libmongoc/tests/json/command-logging-and-monitoring/logging/operation-id.json create mode 100644 src/libmongoc/tests/json/command-logging-and-monitoring/logging/pre-42-server-connection-id.json create mode 100644 src/libmongoc/tests/json/command-logging-and-monitoring/logging/redacted-commands.json create mode 100644 src/libmongoc/tests/json/command-logging-and-monitoring/logging/server-connection-id.json create mode 100644 src/libmongoc/tests/json/command-logging-and-monitoring/logging/service-id.json create mode 100644 src/libmongoc/tests/json/command-logging-and-monitoring/logging/unacknowledged-write.json create mode 100644 src/libmongoc/tests/json/command-logging-and-monitoring/monitoring/bulkWrite.json create mode 100644 src/libmongoc/tests/json/command-logging-and-monitoring/monitoring/command.json create mode 100644 src/libmongoc/tests/json/command-logging-and-monitoring/monitoring/deleteMany.json create mode 100644 src/libmongoc/tests/json/command-logging-and-monitoring/monitoring/deleteOne.json create mode 100644 src/libmongoc/tests/json/command-logging-and-monitoring/monitoring/find.json create mode 100644 src/libmongoc/tests/json/command-logging-and-monitoring/monitoring/insertMany.json create mode 100644 src/libmongoc/tests/json/command-logging-and-monitoring/monitoring/insertOne.json create mode 100644 src/libmongoc/tests/json/command-logging-and-monitoring/monitoring/pre-42-server-connection-id.json create mode 100644 src/libmongoc/tests/json/command-logging-and-monitoring/monitoring/redacted-commands.json create mode 100644 src/libmongoc/tests/json/command-logging-and-monitoring/monitoring/server-connection-id.json create mode 100644 src/libmongoc/tests/json/command-logging-and-monitoring/monitoring/unacknowledgedBulkWrite.json create mode 100644 src/libmongoc/tests/json/command-logging-and-monitoring/monitoring/updateMany.json create mode 100644 src/libmongoc/tests/json/command-logging-and-monitoring/monitoring/updateOne.json create mode 100644 src/libmongoc/tests/json/command-logging-and-monitoring/monitoring/writeConcernError.json diff --git a/src/libmongoc/tests/json/command-logging-and-monitoring/logging/command.json b/src/libmongoc/tests/json/command-logging-and-monitoring/logging/command.json new file mode 100644 index 00000000000..d2970df692f --- /dev/null +++ b/src/libmongoc/tests/json/command-logging-and-monitoring/logging/command.json @@ -0,0 +1,215 @@ +{ + "description": "command-logging", + "schemaVersion": "1.13", + "createEntities": [ + { + "client": { + "id": "client", + "observeLogMessages": { + "command": "debug" + } + } + }, + { + "database": { + "id": "database", + "client": "client", + "databaseName": "logging-tests" + } + }, + { + "collection": { + "id": "collection", + "database": "database", + "collectionName": "logging-tests-collection" + } + } + ], + "initialData": [ + { + "collectionName": "logging-tests-collection", + "databaseName": "logging-tests", + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + ], + "tests": [ + { + "description": "A successful command", + "operations": [ + { + "name": "runCommand", + "object": "database", + "arguments": { + "command": { + "ping": 1 + }, + "commandName": "ping" + } + } + ], + "expectLogMessages": [ + { + "client": "client", + "messages": [ + { + "level": "debug", + "component": "command", + "data": { + "message": "Command started", + "databaseName": "logging-tests", + "commandName": "ping", + "command": { + "$$matchAsDocument": { + "$$matchAsRoot": { + "ping": 1, + "$db": "logging-tests" + } + } + }, + "requestId": { + "$$type": [ + "int", + "long" + ] + }, + "serverHost": { + "$$type": "string" + }, + "serverPort": { + "$$type": [ + "int", + "long" + ] + } + } + }, + { + "level": "debug", + "component": "command", + "data": { + "message": "Command succeeded", + "databaseName": "logging-tests", + "commandName": "ping", + "reply": { + "$$type": "string" + }, + "requestId": { + "$$type": [ + "int", + "long" + ] + }, + "serverHost": { + "$$type": "string" + }, + "serverPort": { + "$$type": [ + "int", + "long" + ] + }, + "durationMS": { + "$$type": [ + "double", + "int", + "long" + ] + } + } + } + ] + } + ] + }, + { + "description": "A failed command", + "operations": [ + { + "name": "find", + "object": "collection", + "arguments": { + "filter": { + "$or": true + } + }, + "expectError": { + "isClientError": false + } + } + ], + "expectLogMessages": [ + { + "client": "client", + "messages": [ + { + "level": "debug", + "component": "command", + "data": { + "message": "Command started", + "databaseName": "logging-tests", + "commandName": "find", + "command": { + "$$type": "string" + }, + "requestId": { + "$$type": [ + "int", + "long" + ] + }, + "serverHost": { + "$$type": "string" + }, + "serverPort": { + "$$type": [ + "int", + "long" + ] + } + } + }, + { + "level": "debug", + "component": "command", + "data": { + "message": "Command failed", + "databaseName": "logging-tests", + "commandName": "find", + "failure": { + "$$exists": true + }, + "requestId": { + "$$type": [ + "int", + "long" + ] + }, + "serverHost": { + "$$type": "string" + }, + "serverPort": { + "$$type": [ + "int", + "long" + ] + }, + "durationMS": { + "$$type": [ + "double", + "int", + "long" + ] + } + } + } + ] + } + ] + } + ] +} diff --git a/src/libmongoc/tests/json/command-logging-and-monitoring/logging/driver-connection-id.json b/src/libmongoc/tests/json/command-logging-and-monitoring/logging/driver-connection-id.json new file mode 100644 index 00000000000..40db98d6fae --- /dev/null +++ b/src/libmongoc/tests/json/command-logging-and-monitoring/logging/driver-connection-id.json @@ -0,0 +1,146 @@ +{ + "description": "driver-connection-id", + "schemaVersion": "1.13", + "createEntities": [ + { + "client": { + "id": "client", + "observeLogMessages": { + "command": "debug" + } + } + }, + { + "database": { + "id": "database", + "client": "client", + "databaseName": "logging-tests" + } + }, + { + "collection": { + "id": "collection", + "database": "database", + "collectionName": "logging-tests-collection" + } + } + ], + "initialData": [ + { + "collectionName": "logging-tests-collection", + "databaseName": "logging-tests", + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + ], + "tests": [ + { + "description": "A successful command", + "operations": [ + { + "name": "runCommand", + "object": "database", + "arguments": { + "command": { + "ping": 1 + }, + "commandName": "ping" + } + } + ], + "expectLogMessages": [ + { + "client": "client", + "messages": [ + { + "level": "debug", + "component": "command", + "data": { + "message": "Command started", + "databaseName": "logging-tests", + "commandName": "ping", + "driverConnectionId": { + "$$type": [ + "int", + "long" + ] + } + } + }, + { + "level": "debug", + "component": "command", + "data": { + "message": "Command succeeded", + "commandName": "ping", + "driverConnectionId": { + "$$type": [ + "int", + "long" + ] + } + } + } + ] + } + ] + }, + { + "description": "A failed command", + "operations": [ + { + "name": "find", + "object": "collection", + "arguments": { + "filter": { + "$or": true + } + }, + "expectError": { + "isClientError": false + } + } + ], + "expectLogMessages": [ + { + "client": "client", + "messages": [ + { + "level": "debug", + "component": "command", + "data": { + "message": "Command started", + "databaseName": "logging-tests", + "commandName": "find", + "driverConnectionId": { + "$$type": [ + "int", + "long" + ] + } + } + }, + { + "level": "debug", + "component": "command", + "data": { + "message": "Command failed", + "commandName": "find", + "driverConnectionId": { + "$$type": [ + "int", + "long" + ] + } + } + } + ] + } + ] + } + ] +} diff --git a/src/libmongoc/tests/json/command-logging-and-monitoring/logging/no-handshake-messages.json b/src/libmongoc/tests/json/command-logging-and-monitoring/logging/no-handshake-messages.json new file mode 100644 index 00000000000..a61e208798c --- /dev/null +++ b/src/libmongoc/tests/json/command-logging-and-monitoring/logging/no-handshake-messages.json @@ -0,0 +1,94 @@ +{ + "description": "no-handshake-command-logs", + "schemaVersion": "1.13", + "tests": [ + { + "description": "Handshake commands should not generate log messages", + "operations": [ + { + "name": "createEntities", + "object": "testRunner", + "arguments": { + "entities": [ + { + "client": { + "id": "client", + "observeLogMessages": { + "command": "debug" + }, + "observeEvents": [ + "connectionCreatedEvent", + "connectionReadyEvent" + ] + } + }, + { + "database": { + "id": "database", + "client": "client", + "databaseName": "logging-tests" + } + } + ] + } + }, + { + "name": "runCommand", + "object": "database", + "arguments": { + "command": { + "ping": 1 + }, + "commandName": "ping" + } + }, + { + "name": "waitForEvent", + "object": "testRunner", + "arguments": { + "client": "client", + "event": { + "connectionCreatedEvent": {} + }, + "count": 1 + } + }, + { + "name": "waitForEvent", + "object": "testRunner", + "arguments": { + "client": "client", + "event": { + "connectionReadyEvent": {} + }, + "count": 1 + } + } + ], + "expectLogMessages": [ + { + "client": "client", + "messages": [ + { + "level": "debug", + "component": "command", + "data": { + "message": "Command started", + "databaseName": "logging-tests", + "commandName": "ping" + } + }, + { + "level": "debug", + "component": "command", + "data": { + "message": "Command succeeded", + "commandName": "ping" + } + } + ] + } + ] + } + ] +} diff --git a/src/libmongoc/tests/json/command-logging-and-monitoring/logging/no-heartbeat-messages.json b/src/libmongoc/tests/json/command-logging-and-monitoring/logging/no-heartbeat-messages.json new file mode 100644 index 00000000000..525be9171df --- /dev/null +++ b/src/libmongoc/tests/json/command-logging-and-monitoring/logging/no-heartbeat-messages.json @@ -0,0 +1,91 @@ +{ + "description": "no-heartbeat-command-logs", + "schemaVersion": "1.13", + "runOnRequirements": [ + { + "topologies": [ + "single", + "replicaset", + "sharded" + ] + } + ], + "tests": [ + { + "description": "Heartbeat commands should not generate log messages", + "operations": [ + { + "name": "createEntities", + "object": "testRunner", + "arguments": { + "entities": [ + { + "client": { + "id": "client", + "observeLogMessages": { + "command": "debug" + }, + "observeEvents": [ + "serverDescriptionChangedEvent" + ] + } + }, + { + "database": { + "id": "database", + "client": "client", + "databaseName": "logging-tests" + } + } + ] + } + }, + { + "name": "waitForEvent", + "object": "testRunner", + "arguments": { + "client": "client", + "event": { + "serverDescriptionChangedEvent": {} + }, + "count": 1 + } + }, + { + "name": "runCommand", + "object": "database", + "arguments": { + "command": { + "ping": 1 + }, + "commandName": "ping" + } + } + ], + "expectLogMessages": [ + { + "client": "client", + "messages": [ + { + "level": "debug", + "component": "command", + "data": { + "message": "Command started", + "databaseName": "logging-tests", + "commandName": "ping" + } + }, + { + "level": "debug", + "component": "command", + "data": { + "message": "Command succeeded", + "commandName": "ping" + } + } + ] + } + ] + } + ] +} diff --git a/src/libmongoc/tests/json/command-logging-and-monitoring/logging/operation-id.json b/src/libmongoc/tests/json/command-logging-and-monitoring/logging/operation-id.json new file mode 100644 index 00000000000..b1a3cec3d91 --- /dev/null +++ b/src/libmongoc/tests/json/command-logging-and-monitoring/logging/operation-id.json @@ -0,0 +1,198 @@ +{ + "description": "operation-id", + "schemaVersion": "1.13", + "createEntities": [ + { + "client": { + "id": "client", + "observeLogMessages": { + "command": "debug" + } + } + }, + { + "database": { + "id": "database", + "client": "client", + "databaseName": "logging-tests" + } + }, + { + "collection": { + "id": "collection", + "database": "database", + "collectionName": "logging-tests-collection" + } + } + ], + "initialData": [ + { + "collectionName": "logging-tests-collection", + "databaseName": "logging-tests", + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + ], + "tests": [ + { + "description": "Successful bulk write command log messages include operationIds", + "operations": [ + { + "name": "bulkWrite", + "object": "collection", + "arguments": { + "requests": [ + { + "insertOne": { + "document": { + "x": 1 + } + } + }, + { + "deleteOne": { + "filter": { + "x": 1 + } + } + } + ] + } + } + ], + "expectLogMessages": [ + { + "client": "client", + "messages": [ + { + "level": "debug", + "component": "command", + "data": { + "message": "Command started", + "databaseName": "logging-tests", + "commandName": "insert", + "operationId": { + "$$type": [ + "int", + "long" + ] + } + } + }, + { + "level": "debug", + "component": "command", + "data": { + "message": "Command succeeded", + "commandName": "insert", + "operationId": { + "$$type": [ + "int", + "long" + ] + } + } + }, + { + "level": "debug", + "component": "command", + "data": { + "message": "Command started", + "databaseName": "logging-tests", + "commandName": "delete", + "operationId": { + "$$type": [ + "int", + "long" + ] + } + } + }, + { + "level": "debug", + "component": "command", + "data": { + "message": "Command succeeded", + "commandName": "delete", + "operationId": { + "$$type": [ + "int", + "long" + ] + } + } + } + ] + } + ] + }, + { + "description": "Failed bulk write command log message includes operationId", + "operations": [ + { + "name": "bulkWrite", + "object": "collection", + "arguments": { + "requests": [ + { + "updateOne": { + "filter": { + "x": 1 + }, + "update": [ + { + "$invalidOperator": true + } + ] + } + } + ] + }, + "expectError": { + "isClientError": false + } + } + ], + "expectLogMessages": [ + { + "client": "client", + "messages": [ + { + "level": "debug", + "component": "command", + "data": { + "message": "Command started", + "databaseName": "logging-tests", + "commandName": "update", + "operationId": { + "$$type": [ + "int", + "long" + ] + } + } + }, + { + "level": "debug", + "component": "command", + "data": { + "message": "Command failed", + "commandName": "update", + "operationId": { + "$$type": [ + "int", + "long" + ] + } + } + } + ] + } + ] + } + ] +} diff --git a/src/libmongoc/tests/json/command-logging-and-monitoring/logging/pre-42-server-connection-id.json b/src/libmongoc/tests/json/command-logging-and-monitoring/logging/pre-42-server-connection-id.json new file mode 100644 index 00000000000..d5ebd865906 --- /dev/null +++ b/src/libmongoc/tests/json/command-logging-and-monitoring/logging/pre-42-server-connection-id.json @@ -0,0 +1,119 @@ +{ + "description": "pre-42-server-connection-id", + "schemaVersion": "1.13", + "runOnRequirements": [ + { + "maxServerVersion": "4.0.99" + } + ], + "createEntities": [ + { + "client": { + "id": "client", + "observeLogMessages": { + "command": "debug" + } + } + }, + { + "database": { + "id": "database", + "client": "client", + "databaseName": "logging-server-connection-id-tests" + } + }, + { + "collection": { + "id": "collection", + "database": "database", + "collectionName": "logging-tests-collection" + } + } + ], + "initialData": [ + { + "databaseName": "logging-server-connection-id-tests", + "collectionName": "logging-tests-collection", + "documents": [] + } + ], + "tests": [ + { + "description": "command log messages do not include server connection id", + "operations": [ + { + "name": "insertOne", + "object": "collection", + "arguments": { + "document": { + "x": 1 + } + } + }, + { + "name": "find", + "object": "collection", + "arguments": { + "filter": { + "$or": true + } + }, + "expectError": { + "isError": true + } + } + ], + "expectLogMessages": [ + { + "client": "client", + "messages": [ + { + "level": "debug", + "component": "command", + "data": { + "message": "Command started", + "commandName": "insert", + "serverConnectionId": { + "$$exists": false + } + } + }, + { + "level": "debug", + "component": "command", + "data": { + "message": "Command succeeded", + "commandName": "insert", + "serverConnectionId": { + "$$exists": false + } + } + }, + { + "level": "debug", + "component": "command", + "data": { + "message": "Command started", + "commandName": "find", + "serverConnectionId": { + "$$exists": false + } + } + }, + { + "level": "debug", + "component": "command", + "data": { + "message": "Command failed", + "commandName": "find", + "serverConnectionId": { + "$$exists": false + } + } + } + ] + } + ] + } + ] +} diff --git a/src/libmongoc/tests/json/command-logging-and-monitoring/logging/redacted-commands.json b/src/libmongoc/tests/json/command-logging-and-monitoring/logging/redacted-commands.json new file mode 100644 index 00000000000..43b9ff74f29 --- /dev/null +++ b/src/libmongoc/tests/json/command-logging-and-monitoring/logging/redacted-commands.json @@ -0,0 +1,1438 @@ +{ + "description": "redacted-commands", + "schemaVersion": "1.13", + "runOnRequirements": [ + { + "minServerVersion": "5.0", + "auth": false + } + ], + "createEntities": [ + { + "client": { + "id": "client", + "useMultipleMongoses": false, + "observeLogMessages": { + "command": "debug" + } + } + }, + { + "client": { + "id": "failPointClient", + "useMultipleMongoses": false + } + }, + { + "database": { + "id": "database", + "client": "client", + "databaseName": "logging-redaction-tests" + } + } + ], + "tests": [ + { + "description": "authenticate command and resulting server-generated error are redacted", + "operations": [ + { + "name": "runCommand", + "object": "database", + "arguments": { + "commandName": "authenticate", + "command": { + "authenticate": 1, + "mechanism": "MONGODB-X509", + "user": "CN=myName,OU=myOrgUnit,O=myOrg,L=myLocality,ST=myState,C=myCountry", + "db": "$external" + } + }, + "expectError": { + "isClientError": false + } + } + ], + "expectLogMessages": [ + { + "client": "client", + "messages": [ + { + "level": "debug", + "component": "command", + "data": { + "message": "Command started", + "databaseName": "logging-redaction-tests", + "commandName": "authenticate", + "command": { + "$$matchAsDocument": {} + } + } + }, + { + "level": "debug", + "component": "command", + "failureIsRedacted": true, + "data": { + "message": "Command failed", + "commandName": "authenticate", + "failure": { + "$$exists": true + } + } + } + ] + } + ] + }, + { + "description": "network error in response to authenticate is not redacted", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 1 + }, + "data": { + "failCommands": [ + "authenticate" + ], + "closeConnection": true + } + } + } + }, + { + "name": "runCommand", + "object": "database", + "arguments": { + "commandName": "authenticate", + "command": { + "authenticate": 1, + "mechanism": "MONGODB-X509", + "user": "CN=myName,OU=myOrgUnit,O=myOrg,L=myLocality,ST=myState,C=myCountry" + } + }, + "expectError": { + "isClientError": true + } + } + ], + "expectLogMessages": [ + { + "client": "client", + "messages": [ + { + "level": "debug", + "component": "command", + "data": { + "message": "Command started", + "databaseName": "logging-redaction-tests", + "commandName": "authenticate", + "command": { + "$$matchAsDocument": {} + } + } + }, + { + "level": "debug", + "component": "command", + "failureIsRedacted": false, + "data": { + "message": "Command failed", + "commandName": "authenticate", + "failure": { + "$$exists": true + } + } + } + ] + } + ] + }, + { + "description": "saslStart command and resulting server-generated error are redacted", + "operations": [ + { + "name": "runCommand", + "object": "database", + "arguments": { + "commandName": "saslStart", + "command": { + "saslStart": 1, + "payload": "definitely-invalid-payload", + "db": "admin" + } + }, + "expectError": { + "isClientError": false + } + } + ], + "expectLogMessages": [ + { + "client": "client", + "messages": [ + { + "level": "debug", + "component": "command", + "data": { + "message": "Command started", + "databaseName": "logging-redaction-tests", + "commandName": "saslStart", + "command": { + "$$matchAsDocument": {} + } + } + }, + { + "level": "debug", + "component": "command", + "failureIsRedacted": true, + "data": { + "message": "Command failed", + "commandName": "saslStart", + "failure": { + "$$exists": true + } + } + } + ] + } + ] + }, + { + "description": "network error in response to saslStart is not redacted", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 1 + }, + "data": { + "failCommands": [ + "saslStart" + ], + "closeConnection": true + } + } + } + }, + { + "name": "runCommand", + "object": "database", + "arguments": { + "commandName": "saslStart", + "command": { + "saslStart": 1, + "payload": "ZmFrZXNhc2xwYXlsb2Fk", + "mechanism": "MONGODB-X509" + } + }, + "expectError": { + "isClientError": true + } + } + ], + "expectLogMessages": [ + { + "client": "client", + "messages": [ + { + "level": "debug", + "component": "command", + "data": { + "message": "Command started", + "databaseName": "logging-redaction-tests", + "commandName": "saslStart", + "command": { + "$$matchAsDocument": {} + } + } + }, + { + "level": "debug", + "component": "command", + "failureIsRedacted": false, + "data": { + "message": "Command failed", + "commandName": "saslStart", + "failure": { + "$$exists": true + } + } + } + ] + } + ] + }, + { + "description": "saslContinue command and resulting server-generated error are redacted", + "operations": [ + { + "name": "runCommand", + "object": "database", + "arguments": { + "commandName": "saslContinue", + "command": { + "saslContinue": 1, + "conversationId": 0, + "payload": "definitely-invalid-payload" + } + }, + "expectError": { + "isClientError": false + } + } + ], + "expectLogMessages": [ + { + "client": "client", + "messages": [ + { + "level": "debug", + "component": "command", + "data": { + "message": "Command started", + "databaseName": "logging-redaction-tests", + "commandName": "saslContinue", + "command": { + "$$matchAsDocument": {} + } + } + }, + { + "level": "debug", + "component": "command", + "failureIsRedacted": true, + "data": { + "message": "Command failed", + "commandName": "saslContinue", + "failure": { + "$$exists": true + } + } + } + ] + } + ] + }, + { + "description": "network error in response to saslContinue is not redacted", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 1 + }, + "data": { + "failCommands": [ + "saslContinue" + ], + "closeConnection": true + } + } + } + }, + { + "name": "runCommand", + "object": "database", + "arguments": { + "commandName": "saslContinue", + "command": { + "saslContinue": 1, + "conversationId": 0, + "payload": "ZmFrZXNhc2xwYXlsb2Fk" + } + }, + "expectError": { + "isClientError": true + } + } + ], + "expectLogMessages": [ + { + "client": "client", + "messages": [ + { + "level": "debug", + "component": "command", + "data": { + "message": "Command started", + "databaseName": "logging-redaction-tests", + "commandName": "saslContinue", + "command": { + "$$matchAsDocument": {} + } + } + }, + { + "level": "debug", + "component": "command", + "failureIsRedacted": false, + "data": { + "message": "Command failed", + "commandName": "saslContinue", + "failure": { + "$$exists": true + } + } + } + ] + } + ] + }, + { + "description": "getnonce command and server reply are redacted", + "runOnRequirements": [ + { + "maxServerVersion": "6.1.99" + } + ], + "operations": [ + { + "name": "runCommand", + "object": "database", + "arguments": { + "commandName": "getnonce", + "command": { + "getnonce": 1 + } + } + } + ], + "expectLogMessages": [ + { + "client": "client", + "messages": [ + { + "level": "debug", + "component": "command", + "data": { + "message": "Command started", + "databaseName": "logging-redaction-tests", + "commandName": "getnonce", + "command": { + "$$matchAsDocument": {} + } + } + }, + { + "level": "debug", + "component": "command", + "data": { + "message": "Command succeeded", + "commandName": "getnonce", + "reply": { + "$$matchAsDocument": {} + } + } + } + ] + } + ] + }, + { + "description": "network error in response to getnonce is not redacted", + "runOnRequirements": [ + { + "maxServerVersion": "6.1.99" + } + ], + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 1 + }, + "data": { + "failCommands": [ + "getnonce" + ], + "closeConnection": true + } + } + } + }, + { + "name": "runCommand", + "object": "database", + "arguments": { + "commandName": "getnonce", + "command": { + "getnonce": 1 + } + }, + "expectError": { + "isClientError": true + } + } + ], + "expectLogMessages": [ + { + "client": "client", + "messages": [ + { + "level": "debug", + "component": "command", + "data": { + "message": "Command started", + "databaseName": "logging-redaction-tests", + "commandName": "getnonce", + "command": { + "$$matchAsDocument": {} + } + } + }, + { + "level": "debug", + "component": "command", + "failureIsRedacted": false, + "data": { + "message": "Command failed", + "commandName": "getnonce", + "failure": { + "$$exists": true + } + } + } + ] + } + ] + }, + { + "description": "createUser command and resulting server-generated error are redacted", + "operations": [ + { + "name": "runCommand", + "object": "database", + "arguments": { + "commandName": "createUser", + "command": { + "createUser": "private", + "pwd": {}, + "roles": [] + } + }, + "expectError": { + "isClientError": false + } + } + ], + "expectLogMessages": [ + { + "client": "client", + "messages": [ + { + "level": "debug", + "component": "command", + "data": { + "message": "Command started", + "databaseName": "logging-redaction-tests", + "commandName": "createUser", + "command": { + "$$matchAsDocument": {} + } + } + }, + { + "level": "debug", + "component": "command", + "failureIsRedacted": true, + "data": { + "message": "Command failed", + "commandName": "createUser", + "failure": { + "$$exists": true + } + } + } + ] + } + ] + }, + { + "description": "network error in response to createUser is not redacted", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 1 + }, + "data": { + "failCommands": [ + "createUser" + ], + "closeConnection": true + } + } + } + }, + { + "name": "runCommand", + "object": "database", + "arguments": { + "commandName": "createUser", + "command": { + "createUser": "private", + "pwd": "pwd", + "roles": [] + } + }, + "expectError": { + "isClientError": true + } + } + ], + "expectLogMessages": [ + { + "client": "client", + "messages": [ + { + "level": "debug", + "component": "command", + "data": { + "message": "Command started", + "databaseName": "logging-redaction-tests", + "commandName": "createUser", + "command": { + "$$matchAsDocument": {} + } + } + }, + { + "level": "debug", + "component": "command", + "failureIsRedacted": false, + "data": { + "message": "Command failed", + "commandName": "createUser", + "failure": { + "$$exists": true + } + } + } + ] + } + ] + }, + { + "description": "updateUser command and resulting server-generated error are redacted", + "operations": [ + { + "name": "runCommand", + "object": "database", + "arguments": { + "commandName": "updateUser", + "command": { + "updateUser": "private", + "pwd": {}, + "roles": [] + } + }, + "expectError": { + "isClientError": false + } + } + ], + "expectLogMessages": [ + { + "client": "client", + "messages": [ + { + "level": "debug", + "component": "command", + "data": { + "message": "Command started", + "databaseName": "logging-redaction-tests", + "commandName": "updateUser", + "command": { + "$$matchAsDocument": {} + } + } + }, + { + "level": "debug", + "component": "command", + "failureIsRedacted": true, + "data": { + "message": "Command failed", + "commandName": "updateUser", + "failure": { + "$$exists": true + } + } + } + ] + } + ] + }, + { + "description": "network error in response to updateUser is not redacted", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 1 + }, + "data": { + "failCommands": [ + "updateUser" + ], + "closeConnection": true + } + } + } + }, + { + "name": "runCommand", + "object": "database", + "arguments": { + "commandName": "updateUser", + "command": { + "updateUser": "private", + "pwd": "pwd", + "roles": [] + } + }, + "expectError": { + "isClientError": true + } + } + ], + "expectLogMessages": [ + { + "client": "client", + "messages": [ + { + "level": "debug", + "component": "command", + "data": { + "message": "Command started", + "databaseName": "logging-redaction-tests", + "commandName": "updateUser", + "command": { + "$$matchAsDocument": {} + } + } + }, + { + "level": "debug", + "component": "command", + "failureIsRedacted": false, + "data": { + "message": "Command failed", + "commandName": "updateUser", + "failure": { + "$$exists": true + } + } + } + ] + } + ] + }, + { + "description": "copydbgetnonce command and resulting server-generated error are redacted", + "runOnRequirements": [ + { + "maxServerVersion": "3.6.99" + } + ], + "operations": [ + { + "name": "runCommand", + "object": "database", + "arguments": { + "commandName": "copydbgetnonce", + "command": { + "copydbgetnonce": "private" + } + }, + "expectError": { + "isClientError": false + } + } + ], + "expectLogMessages": [ + { + "client": "client", + "messages": [ + { + "level": "debug", + "component": "command", + "data": { + "message": "Command started", + "databaseName": "logging-redaction-tests", + "commandName": "copydbgetnonce", + "command": { + "$$matchAsDocument": {} + } + } + }, + { + "level": "debug", + "component": "command", + "failureIsRedacted": true, + "data": { + "message": "Command failed", + "commandName": "copydbgetnonce", + "failure": { + "$$exists": true + } + } + } + ] + } + ] + }, + { + "description": "network error in response to copydbgetnonce is not redacted", + "runOnRequirements": [ + { + "maxServerVersion": "3.6.99" + } + ], + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 1 + }, + "data": { + "failCommands": [ + "copydbgetnonce" + ], + "closeConnection": true + } + } + } + }, + { + "name": "runCommand", + "object": "database", + "arguments": { + "commandName": "copydbgetnonce", + "command": { + "copydbgetnonce": "private" + } + }, + "expectError": { + "isClientError": true + } + } + ], + "expectLogMessages": [ + { + "client": "client", + "messages": [ + { + "level": "debug", + "component": "command", + "data": { + "message": "Command started", + "databaseName": "logging-redaction-tests", + "commandName": "copydbgetnonce", + "command": { + "$$matchAsDocument": {} + } + } + }, + { + "level": "debug", + "component": "command", + "failureIsRedacted": false, + "data": { + "message": "Command failed", + "commandName": "copydbgetnonce", + "failure": { + "$$exists": true + } + } + } + ] + } + ] + }, + { + "description": "copydbsaslstart command and resulting server-generated error are redacted", + "runOnRequirements": [ + { + "maxServerVersion": "4.0.99" + } + ], + "operations": [ + { + "name": "runCommand", + "object": "database", + "arguments": { + "commandName": "copydbsaslstart", + "command": { + "copydbsaslstart": "private" + } + }, + "expectError": { + "isClientError": false + } + } + ], + "expectLogMessages": [ + { + "client": "client", + "messages": [ + { + "level": "debug", + "component": "command", + "data": { + "message": "Command started", + "databaseName": "logging-redaction-tests", + "commandName": "copydbsaslstart", + "command": { + "$$matchAsDocument": {} + } + } + }, + { + "level": "debug", + "component": "command", + "failureIsRedacted": true, + "data": { + "message": "Command failed", + "commandName": "copydbsaslstart", + "failure": { + "$$exists": true + } + } + } + ] + } + ] + }, + { + "description": "network error in response to copydbsaslstart is not redacted", + "runOnRequirements": [ + { + "maxServerVersion": "4.0.99" + } + ], + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 1 + }, + "data": { + "failCommands": [ + "copydbsaslstart" + ], + "closeConnection": true + } + } + } + }, + { + "name": "runCommand", + "object": "database", + "arguments": { + "commandName": "copydbsaslstart", + "command": { + "copydbsaslstart": "private" + } + }, + "expectError": { + "isClientError": true + } + } + ], + "expectLogMessages": [ + { + "client": "client", + "messages": [ + { + "level": "debug", + "component": "command", + "data": { + "message": "Command started", + "databaseName": "logging-redaction-tests", + "commandName": "copydbgetnonce", + "command": { + "$$matchAsDocument": {} + } + } + }, + { + "level": "debug", + "component": "command", + "failureIsRedacted": false, + "data": { + "message": "Command failed", + "commandName": "copydbgetnonce", + "failure": { + "$$exists": true + } + } + } + ] + } + ] + }, + { + "description": "copydb command and resulting server-generated error are redacted", + "runOnRequirements": [ + { + "maxServerVersion": "4.0.99" + } + ], + "operations": [ + { + "name": "runCommand", + "object": "database", + "arguments": { + "commandName": "copydb", + "command": { + "copydb": "private" + } + }, + "expectError": { + "isClientError": false + } + } + ], + "expectLogMessages": [ + { + "client": "client", + "messages": [ + { + "level": "debug", + "component": "command", + "data": { + "message": "Command started", + "databaseName": "logging-redaction-tests", + "commandName": "copydb", + "command": { + "$$matchAsDocument": {} + } + } + }, + { + "level": "debug", + "component": "command", + "failureIsRedacted": true, + "data": { + "message": "Command failed", + "commandName": "copydb", + "failure": { + "$$exists": true + } + } + } + ] + } + ] + }, + { + "description": "network error in response to copydb is not redacted", + "runOnRequirements": [ + { + "maxServerVersion": "4.0.99" + } + ], + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 1 + }, + "data": { + "failCommands": [ + "copydb" + ], + "closeConnection": true + } + } + } + }, + { + "name": "runCommand", + "object": "database", + "arguments": { + "commandName": "copydb", + "command": { + "copydb": "private" + } + }, + "expectError": { + "isClientError": true + } + } + ], + "expectLogMessages": [ + { + "client": "client", + "messages": [ + { + "level": "debug", + "component": "command", + "data": { + "message": "Command started", + "databaseName": "logging-redaction-tests", + "commandName": "copydb", + "command": { + "$$matchAsDocument": {} + } + } + }, + { + "level": "debug", + "component": "command", + "failureIsRedacted": false, + "data": { + "message": "Command failed", + "commandName": "copydb", + "failure": { + "$$exists": true + } + } + } + ] + } + ] + }, + { + "description": "hello with speculative authenticate command and server reply are redacted", + "runOnRequirements": [ + { + "minServerVersion": "4.9" + } + ], + "operations": [ + { + "name": "runCommand", + "object": "database", + "arguments": { + "commandName": "hello", + "command": { + "hello": 1, + "speculativeAuthenticate": { + "saslStart": 1 + } + } + } + } + ], + "expectLogMessages": [ + { + "client": "client", + "messages": [ + { + "level": "debug", + "component": "command", + "data": { + "message": "Command started", + "databaseName": "logging-redaction-tests", + "commandName": "hello", + "command": { + "$$matchAsDocument": {} + } + } + }, + { + "level": "debug", + "component": "command", + "data": { + "message": "Command succeeded", + "commandName": "hello", + "reply": { + "$$matchAsDocument": {} + } + } + } + ] + } + ] + }, + { + "description": "legacy hello with speculative authenticate command and server reply are redacted", + "operations": [ + { + "name": "runCommand", + "object": "database", + "arguments": { + "commandName": "ismaster", + "command": { + "ismaster": 1, + "speculativeAuthenticate": { + "saslStart": 1 + } + } + } + }, + { + "name": "runCommand", + "object": "database", + "arguments": { + "commandName": "isMaster", + "command": { + "isMaster": 1, + "speculativeAuthenticate": { + "saslStart": 1 + } + } + } + } + ], + "expectLogMessages": [ + { + "client": "client", + "messages": [ + { + "level": "debug", + "component": "command", + "data": { + "message": "Command started", + "databaseName": "logging-redaction-tests", + "commandName": "ismaster", + "command": { + "$$matchAsDocument": {} + } + } + }, + { + "level": "debug", + "component": "command", + "data": { + "message": "Command succeeded", + "commandName": "ismaster", + "reply": { + "$$matchAsDocument": {} + } + } + }, + { + "level": "debug", + "component": "command", + "data": { + "message": "Command started", + "databaseName": "logging-redaction-tests", + "commandName": "isMaster", + "command": { + "$$matchAsDocument": {} + } + } + }, + { + "level": "debug", + "component": "command", + "data": { + "message": "Command succeeded", + "commandName": "isMaster", + "reply": { + "$$matchAsDocument": {} + } + } + } + ] + } + ] + }, + { + "description": "hello without speculative authenticate command and server reply are not redacted", + "runOnRequirements": [ + { + "minServerVersion": "4.9" + } + ], + "operations": [ + { + "name": "runCommand", + "object": "database", + "arguments": { + "commandName": "hello", + "command": { + "hello": 1 + } + } + } + ], + "expectLogMessages": [ + { + "client": "client", + "messages": [ + { + "level": "debug", + "component": "command", + "data": { + "message": "Command started", + "databaseName": "logging-redaction-tests", + "commandName": "hello", + "command": { + "$$matchAsDocument": { + "$$matchAsRoot": { + "hello": 1 + } + } + } + } + }, + { + "level": "debug", + "component": "command", + "data": { + "message": "Command succeeded", + "commandName": "hello", + "reply": { + "$$matchAsDocument": { + "$$matchAsRoot": { + "ok": 1, + "isWritablePrimary": true + } + } + } + } + } + ] + } + ] + }, + { + "description": "legacy hello without speculative authenticate command and server reply are not redacted", + "operations": [ + { + "name": "runCommand", + "object": "database", + "arguments": { + "commandName": "ismaster", + "command": { + "ismaster": 1 + } + } + }, + { + "name": "runCommand", + "object": "database", + "arguments": { + "commandName": "isMaster", + "command": { + "isMaster": 1 + } + } + } + ], + "expectLogMessages": [ + { + "client": "client", + "messages": [ + { + "level": "debug", + "component": "command", + "data": { + "message": "Command started", + "databaseName": "logging-redaction-tests", + "commandName": "ismaster", + "command": { + "$$matchAsDocument": { + "$$matchAsRoot": { + "ismaster": 1 + } + } + } + } + }, + { + "level": "debug", + "component": "command", + "data": { + "message": "Command succeeded", + "commandName": "ismaster", + "reply": { + "$$matchAsDocument": { + "$$matchAsRoot": { + "ok": 1, + "ismaster": true + } + } + } + } + }, + { + "level": "debug", + "component": "command", + "data": { + "message": "Command started", + "databaseName": "logging-redaction-tests", + "commandName": "isMaster", + "command": { + "$$matchAsDocument": { + "$$matchAsRoot": { + "isMaster": 1 + } + } + } + } + }, + { + "level": "debug", + "component": "command", + "data": { + "message": "Command succeeded", + "commandName": "isMaster", + "reply": { + "$$matchAsDocument": { + "$$matchAsRoot": { + "ok": 1, + "ismaster": true + } + } + } + } + } + ] + } + ] + } + ] +} diff --git a/src/libmongoc/tests/json/command-logging-and-monitoring/logging/server-connection-id.json b/src/libmongoc/tests/json/command-logging-and-monitoring/logging/server-connection-id.json new file mode 100644 index 00000000000..abbbbc74421 --- /dev/null +++ b/src/libmongoc/tests/json/command-logging-and-monitoring/logging/server-connection-id.json @@ -0,0 +1,131 @@ +{ + "description": "server-connection-id", + "schemaVersion": "1.13", + "runOnRequirements": [ + { + "minServerVersion": "4.2" + } + ], + "createEntities": [ + { + "client": { + "id": "client", + "observeLogMessages": { + "command": "debug" + } + } + }, + { + "database": { + "id": "database", + "client": "client", + "databaseName": "logging-server-connection-id-tests" + } + }, + { + "collection": { + "id": "collection", + "database": "database", + "collectionName": "logging-tests-collection" + } + } + ], + "initialData": [ + { + "databaseName": "logging-server-connection-id-tests", + "collectionName": "logging-tests-collection", + "documents": [] + } + ], + "tests": [ + { + "description": "command log messages include server connection id", + "operations": [ + { + "name": "insertOne", + "object": "collection", + "arguments": { + "document": { + "x": 1 + } + } + }, + { + "name": "find", + "object": "collection", + "arguments": { + "filter": { + "$or": true + } + }, + "expectError": { + "isError": true + } + } + ], + "expectLogMessages": [ + { + "client": "client", + "messages": [ + { + "level": "debug", + "component": "command", + "data": { + "message": "Command started", + "commandName": "insert", + "serverConnectionId": { + "$$type": [ + "int", + "long" + ] + } + } + }, + { + "level": "debug", + "component": "command", + "data": { + "message": "Command succeeded", + "commandName": "insert", + "serverConnectionId": { + "$$type": [ + "int", + "long" + ] + } + } + }, + { + "level": "debug", + "component": "command", + "data": { + "message": "Command started", + "commandName": "find", + "serverConnectionId": { + "$$type": [ + "int", + "long" + ] + } + } + }, + { + "level": "debug", + "component": "command", + "data": { + "message": "Command failed", + "commandName": "find", + "serverConnectionId": { + "$$type": [ + "int", + "long" + ] + } + } + } + ] + } + ] + } + ] +} diff --git a/src/libmongoc/tests/json/command-logging-and-monitoring/logging/service-id.json b/src/libmongoc/tests/json/command-logging-and-monitoring/logging/service-id.json new file mode 100644 index 00000000000..ea39d612315 --- /dev/null +++ b/src/libmongoc/tests/json/command-logging-and-monitoring/logging/service-id.json @@ -0,0 +1,207 @@ +{ + "description": "service-id", + "schemaVersion": "1.13", + "createEntities": [ + { + "client": { + "id": "client", + "observeLogMessages": { + "command": "debug" + } + } + }, + { + "database": { + "id": "database", + "client": "client", + "databaseName": "logging-server-connection-id-tests" + } + }, + { + "collection": { + "id": "collection", + "database": "database", + "collectionName": "logging-tests-collection" + } + } + ], + "initialData": [ + { + "databaseName": "logging-server-connection-id-tests", + "collectionName": "logging-tests-collection", + "documents": [] + } + ], + "tests": [ + { + "description": "command log messages include serviceId when in LB mode", + "runOnRequirements": [ + { + "topologies": [ + "load-balanced" + ] + } + ], + "operations": [ + { + "name": "insertOne", + "object": "collection", + "arguments": { + "document": { + "x": 1 + } + } + }, + { + "name": "find", + "object": "collection", + "arguments": { + "filter": { + "$or": true + } + }, + "expectError": { + "isError": true + } + } + ], + "expectLogMessages": [ + { + "client": "client", + "messages": [ + { + "level": "debug", + "component": "command", + "data": { + "message": "Command started", + "commandName": "insert", + "serviceId": { + "$$type": "string" + } + } + }, + { + "level": "debug", + "component": "command", + "data": { + "message": "Command succeeded", + "commandName": "insert", + "serviceId": { + "$$type": "string" + } + } + }, + { + "level": "debug", + "component": "command", + "data": { + "message": "Command started", + "commandName": "find", + "serviceId": { + "$$type": "string" + } + } + }, + { + "level": "debug", + "component": "command", + "data": { + "message": "Command failed", + "commandName": "find", + "serviceId": { + "$$type": "string" + } + } + } + ] + } + ] + }, + { + "description": "command log messages omit serviceId when not in LB mode", + "runOnRequirements": [ + { + "topologies": [ + "single", + "replicaset", + "sharded" + ] + } + ], + "operations": [ + { + "name": "insertOne", + "object": "collection", + "arguments": { + "document": { + "x": 1 + } + } + }, + { + "name": "find", + "object": "collection", + "arguments": { + "filter": { + "$or": true + } + }, + "expectError": { + "isError": true + } + } + ], + "expectLogMessages": [ + { + "client": "client", + "messages": [ + { + "level": "debug", + "component": "command", + "data": { + "message": "Command started", + "commandName": "insert", + "serviceId": { + "$$exists": false + } + } + }, + { + "level": "debug", + "component": "command", + "data": { + "message": "Command succeeded", + "commandName": "insert", + "serviceId": { + "$$exists": false + } + } + }, + { + "level": "debug", + "component": "command", + "data": { + "message": "Command started", + "commandName": "find", + "serviceId": { + "$$exists": false + } + } + }, + { + "level": "debug", + "component": "command", + "data": { + "message": "Command failed", + "commandName": "find", + "serviceId": { + "$$exists": false + } + } + } + ] + } + ] + } + ] +} diff --git a/src/libmongoc/tests/json/command-logging-and-monitoring/logging/unacknowledged-write.json b/src/libmongoc/tests/json/command-logging-and-monitoring/logging/unacknowledged-write.json new file mode 100644 index 00000000000..0d33c020d54 --- /dev/null +++ b/src/libmongoc/tests/json/command-logging-and-monitoring/logging/unacknowledged-write.json @@ -0,0 +1,151 @@ +{ + "description": "unacknowledged-write", + "schemaVersion": "1.16", + "createEntities": [ + { + "client": { + "id": "client", + "useMultipleMongoses": false, + "observeLogMessages": { + "command": "debug" + } + } + }, + { + "database": { + "id": "database", + "client": "client", + "databaseName": "logging-tests" + } + }, + { + "collection": { + "id": "collection", + "database": "database", + "collectionName": "logging-tests-collection", + "collectionOptions": { + "writeConcern": { + "w": 0 + } + } + } + } + ], + "initialData": [ + { + "collectionName": "logging-tests-collection", + "databaseName": "logging-tests", + "documents": [ + { + "_id": 1 + } + ] + } + ], + "tests": [ + { + "description": "An unacknowledged write generates a succeeded log message with ok: 1 reply", + "operations": [ + { + "name": "insertOne", + "object": "collection", + "arguments": { + "document": { + "_id": 2 + } + } + }, + { + "name": "find", + "object": "collection", + "arguments": { + "filter": {} + }, + "expectResult": [ + { + "_id": 1 + }, + { + "_id": 2 + } + ] + } + ], + "expectLogMessages": [ + { + "client": "client", + "ignoreExtraMessages": true, + "messages": [ + { + "level": "debug", + "component": "command", + "data": { + "message": "Command started", + "databaseName": "logging-tests", + "commandName": "insert", + "command": { + "$$matchAsDocument": { + "$$matchAsRoot": { + "insert": "logging-tests-collection", + "$db": "logging-tests" + } + } + }, + "requestId": { + "$$type": [ + "int", + "long" + ] + }, + "serverHost": { + "$$type": "string" + }, + "serverPort": { + "$$type": [ + "int", + "long" + ] + } + } + }, + { + "level": "debug", + "component": "command", + "data": { + "message": "Command succeeded", + "commandName": "insert", + "reply": { + "$$matchAsDocument": { + "ok": 1 + } + }, + "requestId": { + "$$type": [ + "int", + "long" + ] + }, + "serverHost": { + "$$type": "string" + }, + "serverPort": { + "$$type": [ + "int", + "long" + ] + }, + "durationMS": { + "$$type": [ + "double", + "int", + "long" + ] + } + } + } + ] + } + ] + } + ] +} diff --git a/src/libmongoc/tests/json/command-logging-and-monitoring/monitoring/bulkWrite.json b/src/libmongoc/tests/json/command-logging-and-monitoring/monitoring/bulkWrite.json new file mode 100644 index 00000000000..49c728442e0 --- /dev/null +++ b/src/libmongoc/tests/json/command-logging-and-monitoring/monitoring/bulkWrite.json @@ -0,0 +1,154 @@ +{ + "description": "bulkWrite", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client", + "observeEvents": [ + "commandStartedEvent", + "commandSucceededEvent", + "commandFailedEvent" + ] + } + }, + { + "database": { + "id": "database", + "client": "client", + "databaseName": "command-monitoring-tests" + } + }, + { + "collection": { + "id": "collection", + "database": "database", + "collectionName": "test" + } + } + ], + "initialData": [ + { + "collectionName": "test", + "databaseName": "command-monitoring-tests", + "documents": [ + { + "_id": 1, + "x": 11 + }, + { + "_id": 2, + "x": 22 + }, + { + "_id": 3, + "x": 33 + } + ] + } + ], + "tests": [ + { + "description": "A successful mixed bulk write", + "operations": [ + { + "name": "bulkWrite", + "object": "collection", + "arguments": { + "requests": [ + { + "insertOne": { + "document": { + "_id": 4, + "x": 44 + } + } + }, + { + "updateOne": { + "filter": { + "_id": 3 + }, + "update": { + "$set": { + "x": 333 + } + } + } + } + ] + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "command": { + "insert": "test", + "documents": [ + { + "_id": 4, + "x": 44 + } + ], + "ordered": true + }, + "commandName": "insert", + "databaseName": "command-monitoring-tests" + } + }, + { + "commandSucceededEvent": { + "reply": { + "ok": 1, + "n": 1 + }, + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "command": { + "update": "test", + "updates": [ + { + "q": { + "_id": 3 + }, + "u": { + "$set": { + "x": 333 + } + }, + "upsert": { + "$$unsetOrMatches": false + }, + "multi": { + "$$unsetOrMatches": false + } + } + ], + "ordered": true + }, + "commandName": "update", + "databaseName": "command-monitoring-tests" + } + }, + { + "commandSucceededEvent": { + "reply": { + "ok": 1, + "n": 1 + }, + "commandName": "update" + } + } + ] + } + ] + } + ] +} diff --git a/src/libmongoc/tests/json/command-logging-and-monitoring/monitoring/command.json b/src/libmongoc/tests/json/command-logging-and-monitoring/monitoring/command.json new file mode 100644 index 00000000000..c28af95fed2 --- /dev/null +++ b/src/libmongoc/tests/json/command-logging-and-monitoring/monitoring/command.json @@ -0,0 +1,83 @@ +{ + "description": "command", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client", + "observeEvents": [ + "commandStartedEvent", + "commandSucceededEvent", + "commandFailedEvent" + ] + } + }, + { + "database": { + "id": "database", + "client": "client", + "databaseName": "command-monitoring-tests" + } + }, + { + "collection": { + "id": "collection", + "database": "database", + "collectionName": "test" + } + } + ], + "initialData": [ + { + "collectionName": "test", + "databaseName": "command-monitoring-tests", + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + ], + "tests": [ + { + "description": "A successful command", + "operations": [ + { + "name": "runCommand", + "object": "database", + "arguments": { + "command": { + "ping": 1 + }, + "commandName": "ping" + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "command": { + "ping": 1 + }, + "commandName": "ping", + "databaseName": "command-monitoring-tests" + } + }, + { + "commandSucceededEvent": { + "reply": { + "ok": 1 + }, + "commandName": "ping" + } + } + ] + } + ] + } + ] +} diff --git a/src/libmongoc/tests/json/command-logging-and-monitoring/monitoring/deleteMany.json b/src/libmongoc/tests/json/command-logging-and-monitoring/monitoring/deleteMany.json new file mode 100644 index 00000000000..78ebad1f98c --- /dev/null +++ b/src/libmongoc/tests/json/command-logging-and-monitoring/monitoring/deleteMany.json @@ -0,0 +1,162 @@ +{ + "description": "deleteMany", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client", + "observeEvents": [ + "commandStartedEvent", + "commandSucceededEvent", + "commandFailedEvent" + ] + } + }, + { + "database": { + "id": "database", + "client": "client", + "databaseName": "command-monitoring-tests" + } + }, + { + "collection": { + "id": "collection", + "database": "database", + "collectionName": "test" + } + } + ], + "initialData": [ + { + "collectionName": "test", + "databaseName": "command-monitoring-tests", + "documents": [ + { + "_id": 1, + "x": 11 + }, + { + "_id": 2, + "x": 22 + }, + { + "_id": 3, + "x": 33 + } + ] + } + ], + "tests": [ + { + "description": "A successful deleteMany", + "operations": [ + { + "name": "deleteMany", + "object": "collection", + "arguments": { + "filter": { + "_id": { + "$gt": 1 + } + } + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "command": { + "delete": "test", + "deletes": [ + { + "q": { + "_id": { + "$gt": 1 + } + }, + "limit": 0 + } + ], + "ordered": true + }, + "commandName": "delete", + "databaseName": "command-monitoring-tests" + } + }, + { + "commandSucceededEvent": { + "reply": { + "ok": 1, + "n": 2 + }, + "commandName": "delete" + } + } + ] + } + ] + }, + { + "description": "A successful deleteMany with write errors", + "operations": [ + { + "name": "deleteMany", + "object": "collection", + "arguments": { + "filter": { + "_id": { + "$unsupported": 1 + } + } + }, + "expectError": { + "isClientError": false + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "command": { + "delete": "test", + "deletes": [ + { + "q": { + "_id": { + "$unsupported": 1 + } + }, + "limit": 0 + } + ], + "ordered": true + }, + "commandName": "delete", + "databaseName": "command-monitoring-tests" + } + }, + { + "commandSucceededEvent": { + "reply": { + "ok": 1, + "n": 0, + "writeErrors": { + "$$type": "array" + } + }, + "commandName": "delete" + } + } + ] + } + ] + } + ] +} diff --git a/src/libmongoc/tests/json/command-logging-and-monitoring/monitoring/deleteOne.json b/src/libmongoc/tests/json/command-logging-and-monitoring/monitoring/deleteOne.json new file mode 100644 index 00000000000..2420794fe50 --- /dev/null +++ b/src/libmongoc/tests/json/command-logging-and-monitoring/monitoring/deleteOne.json @@ -0,0 +1,162 @@ +{ + "description": "deleteOne", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client", + "observeEvents": [ + "commandStartedEvent", + "commandSucceededEvent", + "commandFailedEvent" + ] + } + }, + { + "database": { + "id": "database", + "client": "client", + "databaseName": "command-monitoring-tests" + } + }, + { + "collection": { + "id": "collection", + "database": "database", + "collectionName": "test" + } + } + ], + "initialData": [ + { + "collectionName": "test", + "databaseName": "command-monitoring-tests", + "documents": [ + { + "_id": 1, + "x": 11 + }, + { + "_id": 2, + "x": 22 + }, + { + "_id": 3, + "x": 33 + } + ] + } + ], + "tests": [ + { + "description": "A successful deleteOne", + "operations": [ + { + "name": "deleteOne", + "object": "collection", + "arguments": { + "filter": { + "_id": { + "$gt": 1 + } + } + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "command": { + "delete": "test", + "deletes": [ + { + "q": { + "_id": { + "$gt": 1 + } + }, + "limit": 1 + } + ], + "ordered": true + }, + "commandName": "delete", + "databaseName": "command-monitoring-tests" + } + }, + { + "commandSucceededEvent": { + "reply": { + "ok": 1, + "n": 1 + }, + "commandName": "delete" + } + } + ] + } + ] + }, + { + "description": "A successful deleteOne with write errors", + "operations": [ + { + "name": "deleteOne", + "object": "collection", + "arguments": { + "filter": { + "_id": { + "$unsupported": 1 + } + } + }, + "expectError": { + "isClientError": false + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "command": { + "delete": "test", + "deletes": [ + { + "q": { + "_id": { + "$unsupported": 1 + } + }, + "limit": 1 + } + ], + "ordered": true + }, + "commandName": "delete", + "databaseName": "command-monitoring-tests" + } + }, + { + "commandSucceededEvent": { + "reply": { + "ok": 1, + "n": 0, + "writeErrors": { + "$$type": "array" + } + }, + "commandName": "delete" + } + } + ] + } + ] + } + ] +} diff --git a/src/libmongoc/tests/json/command-logging-and-monitoring/monitoring/find.json b/src/libmongoc/tests/json/command-logging-and-monitoring/monitoring/find.json new file mode 100644 index 00000000000..bc9668499b3 --- /dev/null +++ b/src/libmongoc/tests/json/command-logging-and-monitoring/monitoring/find.json @@ -0,0 +1,558 @@ +{ + "description": "find", + "schemaVersion": "1.15", + "createEntities": [ + { + "client": { + "id": "client", + "observeEvents": [ + "commandStartedEvent", + "commandSucceededEvent", + "commandFailedEvent" + ] + } + }, + { + "database": { + "id": "database", + "client": "client", + "databaseName": "command-monitoring-tests" + } + }, + { + "collection": { + "id": "collection", + "database": "database", + "collectionName": "test" + } + } + ], + "_yamlAnchors": { + "namespace": "command-monitoring-tests.test" + }, + "initialData": [ + { + "collectionName": "test", + "databaseName": "command-monitoring-tests", + "documents": [ + { + "_id": 1, + "x": 11 + }, + { + "_id": 2, + "x": 22 + }, + { + "_id": 3, + "x": 33 + }, + { + "_id": 4, + "x": 44 + }, + { + "_id": 5, + "x": 55 + } + ] + } + ], + "tests": [ + { + "description": "A successful find with no options", + "operations": [ + { + "name": "find", + "object": "collection", + "arguments": { + "filter": { + "_id": 1 + } + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "command": { + "find": "test", + "filter": { + "_id": 1 + } + }, + "commandName": "find", + "databaseName": "command-monitoring-tests" + } + }, + { + "commandSucceededEvent": { + "reply": { + "ok": 1, + "cursor": { + "id": 0, + "ns": "command-monitoring-tests.test", + "firstBatch": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + "commandName": "find", + "databaseName": "command-monitoring-tests" + } + } + ] + } + ] + }, + { + "description": "A successful find with options", + "operations": [ + { + "name": "find", + "object": "collection", + "arguments": { + "filter": { + "_id": { + "$gt": 1 + } + }, + "sort": { + "x": -1 + }, + "projection": { + "_id": 0, + "x": 1 + }, + "skip": 2, + "comment": "test", + "hint": { + "_id": 1 + }, + "max": { + "_id": 6 + }, + "maxTimeMS": 6000, + "min": { + "_id": 0 + } + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "command": { + "find": "test", + "filter": { + "_id": { + "$gt": 1 + } + }, + "sort": { + "x": -1 + }, + "projection": { + "_id": 0, + "x": 1 + }, + "skip": 2, + "comment": "test", + "hint": { + "_id": 1 + }, + "max": { + "_id": 6 + }, + "maxTimeMS": 6000, + "min": { + "_id": 0 + } + }, + "commandName": "find", + "databaseName": "command-monitoring-tests" + } + }, + { + "commandSucceededEvent": { + "reply": { + "ok": 1, + "cursor": { + "id": 0, + "ns": "command-monitoring-tests.test", + "firstBatch": [ + { + "x": 33 + }, + { + "x": 22 + } + ] + } + }, + "commandName": "find", + "databaseName": "command-monitoring-tests" + } + } + ] + } + ] + }, + { + "description": "A successful find with showRecordId and returnKey", + "operations": [ + { + "name": "find", + "object": "collection", + "arguments": { + "filter": {}, + "sort": { + "_id": 1 + }, + "showRecordId": true, + "returnKey": true + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "command": { + "find": "test", + "showRecordId": true, + "returnKey": true + }, + "commandName": "find", + "databaseName": "command-monitoring-tests" + } + }, + { + "commandSucceededEvent": { + "reply": { + "ok": 1, + "cursor": { + "id": 0, + "ns": "command-monitoring-tests.test", + "firstBatch": [ + { + "_id": 1 + }, + { + "_id": 2 + }, + { + "_id": 3 + }, + { + "_id": 4 + }, + { + "_id": 5 + } + ] + } + }, + "commandName": "find", + "databaseName": "command-monitoring-tests" + } + } + ] + } + ] + }, + { + "description": "A successful find with a getMore", + "operations": [ + { + "name": "find", + "object": "collection", + "arguments": { + "filter": { + "_id": { + "$gte": 1 + } + }, + "sort": { + "_id": 1 + }, + "batchSize": 3 + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "command": { + "find": "test", + "filter": { + "_id": { + "$gte": 1 + } + }, + "sort": { + "_id": 1 + }, + "batchSize": 3 + }, + "commandName": "find", + "databaseName": "command-monitoring-tests" + } + }, + { + "commandSucceededEvent": { + "reply": { + "ok": 1, + "cursor": { + "id": { + "$$type": [ + "int", + "long" + ] + }, + "ns": "command-monitoring-tests.test", + "firstBatch": [ + { + "_id": 1, + "x": 11 + }, + { + "_id": 2, + "x": 22 + }, + { + "_id": 3, + "x": 33 + } + ] + } + }, + "commandName": "find", + "databaseName": "command-monitoring-tests" + } + }, + { + "commandStartedEvent": { + "command": { + "getMore": { + "$$type": [ + "int", + "long" + ] + }, + "collection": "test", + "batchSize": 3 + }, + "commandName": "getMore", + "databaseName": "command-monitoring-tests" + } + }, + { + "commandSucceededEvent": { + "reply": { + "ok": 1, + "cursor": { + "id": 0, + "ns": "command-monitoring-tests.test", + "nextBatch": [ + { + "_id": 4, + "x": 44 + }, + { + "_id": 5, + "x": 55 + } + ] + } + }, + "commandName": "getMore", + "databaseName": "command-monitoring-tests" + } + } + ] + } + ] + }, + { + "description": "A successful find event with a getmore and the server kills the cursor (<= 4.4)", + "runOnRequirements": [ + { + "minServerVersion": "3.1", + "maxServerVersion": "4.4.99", + "topologies": [ + "single", + "replicaset" + ] + } + ], + "operations": [ + { + "name": "find", + "object": "collection", + "arguments": { + "filter": { + "_id": { + "$gte": 1 + } + }, + "sort": { + "_id": 1 + }, + "batchSize": 3, + "limit": 4 + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "command": { + "find": "test", + "filter": { + "_id": { + "$gte": 1 + } + }, + "sort": { + "_id": 1 + }, + "batchSize": 3, + "limit": 4 + }, + "commandName": "find", + "databaseName": "command-monitoring-tests" + } + }, + { + "commandSucceededEvent": { + "reply": { + "ok": 1, + "cursor": { + "id": { + "$$type": [ + "int", + "long" + ] + }, + "ns": "command-monitoring-tests.test", + "firstBatch": [ + { + "_id": 1, + "x": 11 + }, + { + "_id": 2, + "x": 22 + }, + { + "_id": 3, + "x": 33 + } + ] + } + }, + "commandName": "find", + "databaseName": "command-monitoring-tests" + } + }, + { + "commandStartedEvent": { + "command": { + "getMore": { + "$$type": [ + "int", + "long" + ] + }, + "collection": "test", + "batchSize": 1 + }, + "commandName": "getMore", + "databaseName": "command-monitoring-tests" + } + }, + { + "commandSucceededEvent": { + "reply": { + "ok": 1, + "cursor": { + "id": 0, + "ns": "command-monitoring-tests.test", + "nextBatch": [ + { + "_id": 4, + "x": 44 + } + ] + } + }, + "commandName": "getMore", + "databaseName": "command-monitoring-tests" + } + } + ] + } + ] + }, + { + "description": "A failed find event", + "operations": [ + { + "name": "find", + "object": "collection", + "arguments": { + "filter": { + "$or": true + } + }, + "expectError": { + "isClientError": false + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "command": { + "find": "test", + "filter": { + "$or": true + } + }, + "commandName": "find", + "databaseName": "command-monitoring-tests" + } + }, + { + "commandFailedEvent": { + "commandName": "find", + "databaseName": "command-monitoring-tests" + } + } + ] + } + ] + } + ] +} diff --git a/src/libmongoc/tests/json/command-logging-and-monitoring/monitoring/insertMany.json b/src/libmongoc/tests/json/command-logging-and-monitoring/monitoring/insertMany.json new file mode 100644 index 00000000000..a80a218c67a --- /dev/null +++ b/src/libmongoc/tests/json/command-logging-and-monitoring/monitoring/insertMany.json @@ -0,0 +1,148 @@ +{ + "description": "insertMany", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client", + "observeEvents": [ + "commandStartedEvent", + "commandSucceededEvent", + "commandFailedEvent" + ] + } + }, + { + "database": { + "id": "database", + "client": "client", + "databaseName": "command-monitoring-tests" + } + }, + { + "collection": { + "id": "collection", + "database": "database", + "collectionName": "test" + } + } + ], + "initialData": [ + { + "collectionName": "test", + "databaseName": "command-monitoring-tests", + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + ], + "tests": [ + { + "description": "A successful insertMany", + "operations": [ + { + "name": "insertMany", + "object": "collection", + "arguments": { + "documents": [ + { + "_id": 2, + "x": 22 + } + ] + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "command": { + "insert": "test", + "documents": [ + { + "_id": 2, + "x": 22 + } + ], + "ordered": true + }, + "commandName": "insert", + "databaseName": "command-monitoring-tests" + } + }, + { + "commandSucceededEvent": { + "reply": { + "ok": 1, + "n": 1 + }, + "commandName": "insert" + } + } + ] + } + ] + }, + { + "description": "A successful insertMany with write errors", + "operations": [ + { + "name": "insertMany", + "object": "collection", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + }, + "expectError": { + "isClientError": false + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "command": { + "insert": "test", + "documents": [ + { + "_id": 1, + "x": 11 + } + ], + "ordered": true + }, + "commandName": "insert", + "databaseName": "command-monitoring-tests" + } + }, + { + "commandSucceededEvent": { + "reply": { + "ok": 1, + "n": 0, + "writeErrors": { + "$$type": "array" + } + }, + "commandName": "insert" + } + } + ] + } + ] + } + ] +} diff --git a/src/libmongoc/tests/json/command-logging-and-monitoring/monitoring/insertOne.json b/src/libmongoc/tests/json/command-logging-and-monitoring/monitoring/insertOne.json new file mode 100644 index 00000000000..6ff732e41be --- /dev/null +++ b/src/libmongoc/tests/json/command-logging-and-monitoring/monitoring/insertOne.json @@ -0,0 +1,144 @@ +{ + "description": "insertOne", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client", + "observeEvents": [ + "commandStartedEvent", + "commandSucceededEvent", + "commandFailedEvent" + ] + } + }, + { + "database": { + "id": "database", + "client": "client", + "databaseName": "command-monitoring-tests" + } + }, + { + "collection": { + "id": "collection", + "database": "database", + "collectionName": "test" + } + } + ], + "initialData": [ + { + "collectionName": "test", + "databaseName": "command-monitoring-tests", + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + ], + "tests": [ + { + "description": "A successful insertOne", + "operations": [ + { + "name": "insertOne", + "object": "collection", + "arguments": { + "document": { + "_id": 2, + "x": 22 + } + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "command": { + "insert": "test", + "documents": [ + { + "_id": 2, + "x": 22 + } + ], + "ordered": true + }, + "commandName": "insert", + "databaseName": "command-monitoring-tests" + } + }, + { + "commandSucceededEvent": { + "reply": { + "ok": 1, + "n": 1 + }, + "commandName": "insert" + } + } + ] + } + ] + }, + { + "description": "A successful insertOne with write errors", + "operations": [ + { + "name": "insertOne", + "object": "collection", + "arguments": { + "document": { + "_id": 1, + "x": 11 + } + }, + "expectError": { + "isClientError": false + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "command": { + "insert": "test", + "documents": [ + { + "_id": 1, + "x": 11 + } + ], + "ordered": true + }, + "commandName": "insert", + "databaseName": "command-monitoring-tests" + } + }, + { + "commandSucceededEvent": { + "reply": { + "ok": 1, + "n": 0, + "writeErrors": { + "$$type": "array" + } + }, + "commandName": "insert" + } + } + ] + } + ] + } + ] +} diff --git a/src/libmongoc/tests/json/command-logging-and-monitoring/monitoring/pre-42-server-connection-id.json b/src/libmongoc/tests/json/command-logging-and-monitoring/monitoring/pre-42-server-connection-id.json new file mode 100644 index 00000000000..141fbe584f7 --- /dev/null +++ b/src/libmongoc/tests/json/command-logging-and-monitoring/monitoring/pre-42-server-connection-id.json @@ -0,0 +1,101 @@ +{ + "description": "pre-42-server-connection-id", + "schemaVersion": "1.6", + "runOnRequirements": [ + { + "maxServerVersion": "4.0.99" + } + ], + "createEntities": [ + { + "client": { + "id": "client", + "observeEvents": [ + "commandStartedEvent", + "commandSucceededEvent", + "commandFailedEvent" + ] + } + }, + { + "database": { + "id": "database", + "client": "client", + "databaseName": "server-connection-id-tests" + } + }, + { + "collection": { + "id": "collection", + "database": "database", + "collectionName": "coll" + } + } + ], + "initialData": [ + { + "databaseName": "server-connection-id-tests", + "collectionName": "coll", + "documents": [] + } + ], + "tests": [ + { + "description": "command events do not include server connection id", + "operations": [ + { + "name": "insertOne", + "object": "collection", + "arguments": { + "document": { + "x": 1 + } + } + }, + { + "name": "find", + "object": "collection", + "arguments": { + "filter": { + "$or": true + } + }, + "expectError": { + "isError": true + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "insert", + "hasServerConnectionId": false + } + }, + { + "commandSucceededEvent": { + "commandName": "insert", + "hasServerConnectionId": false + } + }, + { + "commandStartedEvent": { + "commandName": "find", + "hasServerConnectionId": false + } + }, + { + "commandFailedEvent": { + "commandName": "find", + "hasServerConnectionId": false + } + } + ] + } + ] + } + ] +} diff --git a/src/libmongoc/tests/json/command-logging-and-monitoring/monitoring/redacted-commands.json b/src/libmongoc/tests/json/command-logging-and-monitoring/monitoring/redacted-commands.json new file mode 100644 index 00000000000..4302ba89004 --- /dev/null +++ b/src/libmongoc/tests/json/command-logging-and-monitoring/monitoring/redacted-commands.json @@ -0,0 +1,679 @@ +{ + "description": "redacted-commands", + "schemaVersion": "1.5", + "runOnRequirements": [ + { + "minServerVersion": "5.0", + "auth": false + } + ], + "createEntities": [ + { + "client": { + "id": "client", + "observeEvents": [ + "commandStartedEvent", + "commandSucceededEvent" + ], + "observeSensitiveCommands": true + } + }, + { + "database": { + "id": "database", + "client": "client", + "databaseName": "command-monitoring-tests" + } + } + ], + "tests": [ + { + "description": "authenticate", + "operations": [ + { + "name": "runCommand", + "object": "database", + "arguments": { + "commandName": "authenticate", + "command": { + "authenticate": 1, + "mechanism": "MONGODB-X509", + "user": "CN=myName,OU=myOrgUnit,O=myOrg,L=myLocality,ST=myState,C=myCountry", + "db": "$external" + } + }, + "expectError": { + "isError": true + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "authenticate", + "command": { + "authenticate": { + "$$exists": false + }, + "mechanism": { + "$$exists": false + }, + "user": { + "$$exists": false + }, + "db": { + "$$exists": false + } + } + } + } + ] + } + ] + }, + { + "description": "saslStart", + "operations": [ + { + "name": "runCommand", + "object": "database", + "arguments": { + "commandName": "saslStart", + "command": { + "saslStart": 1, + "payload": "definitely-invalid-payload", + "db": "admin" + } + }, + "expectError": { + "isError": true + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "saslStart", + "command": { + "saslStart": { + "$$exists": false + }, + "payload": { + "$$exists": false + }, + "db": { + "$$exists": false + } + } + } + } + ] + } + ] + }, + { + "description": "saslContinue", + "operations": [ + { + "name": "runCommand", + "object": "database", + "arguments": { + "commandName": "saslContinue", + "command": { + "saslContinue": 1, + "conversationId": 0, + "payload": "definitely-invalid-payload" + } + }, + "expectError": { + "isError": true + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "saslContinue", + "command": { + "saslContinue": { + "$$exists": false + }, + "conversationId": { + "$$exists": false + }, + "payload": { + "$$exists": false + } + } + } + } + ] + } + ] + }, + { + "description": "getnonce", + "runOnRequirements": [ + { + "maxServerVersion": "6.1.99" + } + ], + "operations": [ + { + "name": "runCommand", + "object": "database", + "arguments": { + "commandName": "getnonce", + "command": { + "getnonce": 1 + } + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "getnonce", + "command": { + "getnonce": { + "$$exists": false + } + } + } + }, + { + "commandSucceededEvent": { + "commandName": "getnonce", + "reply": { + "ok": { + "$$exists": false + }, + "nonce": { + "$$exists": false + } + } + } + } + ] + } + ] + }, + { + "description": "createUser", + "operations": [ + { + "name": "runCommand", + "object": "database", + "arguments": { + "commandName": "createUser", + "command": { + "createUser": "private", + "pwd": {}, + "roles": [] + } + }, + "expectError": { + "isError": true + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "createUser", + "command": { + "createUser": { + "$$exists": false + }, + "pwd": { + "$$exists": false + }, + "roles": { + "$$exists": false + } + } + } + } + ] + } + ] + }, + { + "description": "updateUser", + "operations": [ + { + "name": "runCommand", + "object": "database", + "arguments": { + "commandName": "updateUser", + "command": { + "updateUser": "private", + "pwd": {}, + "roles": [] + } + }, + "expectError": { + "isError": true + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "updateUser", + "command": { + "updateUser": { + "$$exists": false + }, + "pwd": { + "$$exists": false + }, + "roles": { + "$$exists": false + } + } + } + } + ] + } + ] + }, + { + "description": "copydbgetnonce", + "runOnRequirements": [ + { + "maxServerVersion": "3.6.99" + } + ], + "operations": [ + { + "name": "runCommand", + "object": "database", + "arguments": { + "commandName": "copydbgetnonce", + "command": { + "copydbgetnonce": "private" + } + }, + "expectError": { + "isError": true + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "copydbgetnonce", + "command": { + "copydbgetnonce": { + "$$exists": false + } + } + } + } + ] + } + ] + }, + { + "description": "copydbsaslstart", + "runOnRequirements": [ + { + "maxServerVersion": "4.0.99" + } + ], + "operations": [ + { + "name": "runCommand", + "object": "database", + "arguments": { + "commandName": "copydbsaslstart", + "command": { + "copydbsaslstart": "private" + } + }, + "expectError": { + "isError": true + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "copydbsaslstart", + "command": { + "copydbsaslstart": { + "$$exists": false + } + } + } + } + ] + } + ] + }, + { + "description": "copydb", + "runOnRequirements": [ + { + "maxServerVersion": "4.0.99" + } + ], + "operations": [ + { + "name": "runCommand", + "object": "database", + "arguments": { + "commandName": "copydb", + "command": { + "copydb": "private" + } + }, + "expectError": { + "isError": true + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "copydb", + "command": { + "copydb": { + "$$exists": false + } + } + } + } + ] + } + ] + }, + { + "description": "hello with speculative authenticate", + "runOnRequirements": [ + { + "minServerVersion": "4.9" + } + ], + "operations": [ + { + "name": "runCommand", + "object": "database", + "arguments": { + "commandName": "hello", + "command": { + "hello": 1, + "speculativeAuthenticate": { + "saslStart": 1 + } + } + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "hello", + "command": { + "hello": { + "$$exists": false + }, + "speculativeAuthenticate": { + "$$exists": false + } + } + } + }, + { + "commandSucceededEvent": { + "commandName": "hello", + "reply": { + "isWritablePrimary": { + "$$exists": false + }, + "speculativeAuthenticate": { + "$$exists": false + } + } + } + } + ] + } + ] + }, + { + "description": "legacy hello with speculative authenticate", + "operations": [ + { + "name": "runCommand", + "object": "database", + "arguments": { + "commandName": "ismaster", + "command": { + "ismaster": 1, + "speculativeAuthenticate": { + "saslStart": 1 + } + } + } + }, + { + "name": "runCommand", + "object": "database", + "arguments": { + "commandName": "isMaster", + "command": { + "isMaster": 1, + "speculativeAuthenticate": { + "saslStart": 1 + } + } + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "ismaster", + "command": { + "ismaster": { + "$$exists": false + }, + "speculativeAuthenticate": { + "$$exists": false + } + } + } + }, + { + "commandSucceededEvent": { + "commandName": "ismaster", + "reply": { + "ismaster": { + "$$exists": false + }, + "speculativeAuthenticate": { + "$$exists": false + } + } + } + }, + { + "commandStartedEvent": { + "commandName": "isMaster", + "command": { + "isMaster": { + "$$exists": false + }, + "speculativeAuthenticate": { + "$$exists": false + } + } + } + }, + { + "commandSucceededEvent": { + "commandName": "isMaster", + "reply": { + "ismaster": { + "$$exists": false + }, + "speculativeAuthenticate": { + "$$exists": false + } + } + } + } + ] + } + ] + }, + { + "description": "hello without speculative authenticate is not redacted", + "runOnRequirements": [ + { + "minServerVersion": "4.9" + } + ], + "operations": [ + { + "name": "runCommand", + "object": "database", + "arguments": { + "commandName": "hello", + "command": { + "hello": 1 + } + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "hello", + "command": { + "hello": 1 + } + } + }, + { + "commandSucceededEvent": { + "commandName": "hello", + "reply": { + "isWritablePrimary": { + "$$exists": true + } + } + } + } + ] + } + ] + }, + { + "description": "legacy hello without speculative authenticate is not redacted", + "operations": [ + { + "name": "runCommand", + "object": "database", + "arguments": { + "commandName": "ismaster", + "command": { + "ismaster": 1 + } + } + }, + { + "name": "runCommand", + "object": "database", + "arguments": { + "commandName": "isMaster", + "command": { + "isMaster": 1 + } + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "ismaster", + "command": { + "ismaster": 1 + } + } + }, + { + "commandSucceededEvent": { + "commandName": "ismaster", + "reply": { + "ismaster": { + "$$exists": true + } + } + } + }, + { + "commandStartedEvent": { + "commandName": "isMaster", + "command": { + "isMaster": 1 + } + } + }, + { + "commandSucceededEvent": { + "commandName": "isMaster", + "reply": { + "ismaster": { + "$$exists": true + } + } + } + } + ] + } + ] + } + ] +} diff --git a/src/libmongoc/tests/json/command-logging-and-monitoring/monitoring/server-connection-id.json b/src/libmongoc/tests/json/command-logging-and-monitoring/monitoring/server-connection-id.json new file mode 100644 index 00000000000..a8f27637fc0 --- /dev/null +++ b/src/libmongoc/tests/json/command-logging-and-monitoring/monitoring/server-connection-id.json @@ -0,0 +1,101 @@ +{ + "description": "server-connection-id", + "schemaVersion": "1.6", + "runOnRequirements": [ + { + "minServerVersion": "4.2" + } + ], + "createEntities": [ + { + "client": { + "id": "client", + "observeEvents": [ + "commandStartedEvent", + "commandSucceededEvent", + "commandFailedEvent" + ] + } + }, + { + "database": { + "id": "database", + "client": "client", + "databaseName": "server-connection-id-tests" + } + }, + { + "collection": { + "id": "collection", + "database": "database", + "collectionName": "coll" + } + } + ], + "initialData": [ + { + "databaseName": "server-connection-id-tests", + "collectionName": "coll", + "documents": [] + } + ], + "tests": [ + { + "description": "command events include server connection id", + "operations": [ + { + "name": "insertOne", + "object": "collection", + "arguments": { + "document": { + "x": 1 + } + } + }, + { + "name": "find", + "object": "collection", + "arguments": { + "filter": { + "$or": true + } + }, + "expectError": { + "isError": true + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "insert", + "hasServerConnectionId": true + } + }, + { + "commandSucceededEvent": { + "commandName": "insert", + "hasServerConnectionId": true + } + }, + { + "commandStartedEvent": { + "commandName": "find", + "hasServerConnectionId": true + } + }, + { + "commandFailedEvent": { + "commandName": "find", + "hasServerConnectionId": true + } + } + ] + } + ] + } + ] +} diff --git a/src/libmongoc/tests/json/command-logging-and-monitoring/monitoring/unacknowledgedBulkWrite.json b/src/libmongoc/tests/json/command-logging-and-monitoring/monitoring/unacknowledgedBulkWrite.json new file mode 100644 index 00000000000..78ddde767ff --- /dev/null +++ b/src/libmongoc/tests/json/command-logging-and-monitoring/monitoring/unacknowledgedBulkWrite.json @@ -0,0 +1,117 @@ +{ + "description": "unacknowledgedBulkWrite", + "schemaVersion": "1.7", + "createEntities": [ + { + "client": { + "id": "client", + "useMultipleMongoses": false, + "observeEvents": [ + "commandStartedEvent", + "commandSucceededEvent", + "commandFailedEvent" + ] + } + }, + { + "database": { + "id": "database", + "client": "client", + "databaseName": "command-monitoring-tests" + } + }, + { + "collection": { + "id": "collection", + "database": "database", + "collectionName": "test", + "collectionOptions": { + "writeConcern": { + "w": 0 + } + } + } + } + ], + "initialData": [ + { + "collectionName": "test", + "databaseName": "command-monitoring-tests", + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + ], + "tests": [ + { + "description": "A successful unordered bulk write with an unacknowledged write concern", + "operations": [ + { + "name": "bulkWrite", + "object": "collection", + "arguments": { + "requests": [ + { + "insertOne": { + "document": { + "_id": "unorderedBulkWriteInsertW0", + "x": 44 + } + } + } + ], + "ordered": false + } + }, + { + "name": "find", + "object": "collection", + "arguments": { + "filter": {} + } + } + ], + "expectEvents": [ + { + "client": "client", + "ignoreExtraEvents": true, + "events": [ + { + "commandStartedEvent": { + "command": { + "insert": "test", + "documents": [ + { + "_id": "unorderedBulkWriteInsertW0", + "x": 44 + } + ], + "ordered": false, + "writeConcern": { + "w": 0 + } + }, + "commandName": "insert", + "databaseName": "command-monitoring-tests" + } + }, + { + "commandSucceededEvent": { + "reply": { + "ok": 1, + "n": { + "$$exists": false + } + }, + "commandName": "insert" + } + } + ] + } + ] + } + ] +} diff --git a/src/libmongoc/tests/json/command-logging-and-monitoring/monitoring/updateMany.json b/src/libmongoc/tests/json/command-logging-and-monitoring/monitoring/updateMany.json new file mode 100644 index 00000000000..b15434226c8 --- /dev/null +++ b/src/libmongoc/tests/json/command-logging-and-monitoring/monitoring/updateMany.json @@ -0,0 +1,188 @@ +{ + "description": "updateMany", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client", + "observeEvents": [ + "commandStartedEvent", + "commandSucceededEvent", + "commandFailedEvent" + ] + } + }, + { + "database": { + "id": "database", + "client": "client", + "databaseName": "command-monitoring-tests" + } + }, + { + "collection": { + "id": "collection", + "database": "database", + "collectionName": "test" + } + } + ], + "initialData": [ + { + "collectionName": "test", + "databaseName": "command-monitoring-tests", + "documents": [ + { + "_id": 1, + "x": 11 + }, + { + "_id": 2, + "x": 22 + }, + { + "_id": 3, + "x": 33 + } + ] + } + ], + "tests": [ + { + "description": "A successful updateMany", + "operations": [ + { + "name": "updateMany", + "object": "collection", + "arguments": { + "filter": { + "_id": { + "$gt": 1 + } + }, + "update": { + "$inc": { + "x": 1 + } + } + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "command": { + "update": "test", + "updates": [ + { + "q": { + "_id": { + "$gt": 1 + } + }, + "u": { + "$inc": { + "x": 1 + } + }, + "upsert": { + "$$unsetOrMatches": false + }, + "multi": true + } + ], + "ordered": true + }, + "commandName": "update", + "databaseName": "command-monitoring-tests" + } + }, + { + "commandSucceededEvent": { + "reply": { + "ok": 1, + "n": 2 + }, + "commandName": "update" + } + } + ] + } + ] + }, + { + "description": "A successful updateMany with write errors", + "operations": [ + { + "name": "updateMany", + "object": "collection", + "arguments": { + "filter": { + "_id": { + "$gt": 1 + } + }, + "update": { + "$unsupported": { + "x": 1 + } + } + }, + "expectError": { + "isClientError": false + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "command": { + "update": "test", + "updates": [ + { + "q": { + "_id": { + "$gt": 1 + } + }, + "u": { + "$unsupported": { + "x": 1 + } + }, + "upsert": { + "$$unsetOrMatches": false + }, + "multi": true + } + ], + "ordered": true + }, + "commandName": "update", + "databaseName": "command-monitoring-tests" + } + }, + { + "commandSucceededEvent": { + "reply": { + "ok": 1, + "n": 0, + "writeErrors": { + "$$type": "array" + } + }, + "commandName": "update" + } + } + ] + } + ] + } + ] +} diff --git a/src/libmongoc/tests/json/command-logging-and-monitoring/monitoring/updateOne.json b/src/libmongoc/tests/json/command-logging-and-monitoring/monitoring/updateOne.json new file mode 100644 index 00000000000..a0ae99e88db --- /dev/null +++ b/src/libmongoc/tests/json/command-logging-and-monitoring/monitoring/updateOne.json @@ -0,0 +1,260 @@ +{ + "description": "updateOne", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client", + "observeEvents": [ + "commandStartedEvent", + "commandSucceededEvent", + "commandFailedEvent" + ] + } + }, + { + "database": { + "id": "database", + "client": "client", + "databaseName": "command-monitoring-tests" + } + }, + { + "collection": { + "id": "collection", + "database": "database", + "collectionName": "test" + } + } + ], + "initialData": [ + { + "collectionName": "test", + "databaseName": "command-monitoring-tests", + "documents": [ + { + "_id": 1, + "x": 11 + }, + { + "_id": 2, + "x": 22 + }, + { + "_id": 3, + "x": 33 + } + ] + } + ], + "tests": [ + { + "description": "A successful updateOne", + "operations": [ + { + "name": "updateOne", + "object": "collection", + "arguments": { + "filter": { + "_id": { + "$gt": 1 + } + }, + "update": { + "$inc": { + "x": 1 + } + } + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "command": { + "update": "test", + "updates": [ + { + "q": { + "_id": { + "$gt": 1 + } + }, + "u": { + "$inc": { + "x": 1 + } + }, + "upsert": { + "$$unsetOrMatches": false + }, + "multi": { + "$$unsetOrMatches": false + } + } + ], + "ordered": true + }, + "commandName": "update", + "databaseName": "command-monitoring-tests" + } + }, + { + "commandSucceededEvent": { + "reply": { + "ok": 1, + "n": 1 + }, + "commandName": "update" + } + } + ] + } + ] + }, + { + "description": "A successful updateOne with upsert where the upserted id is not an ObjectId", + "operations": [ + { + "name": "updateOne", + "object": "collection", + "arguments": { + "filter": { + "_id": 4 + }, + "update": { + "$inc": { + "x": 1 + } + }, + "upsert": true + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "command": { + "update": "test", + "updates": [ + { + "q": { + "_id": 4 + }, + "u": { + "$inc": { + "x": 1 + } + }, + "upsert": true, + "multi": { + "$$unsetOrMatches": false + } + } + ], + "ordered": true + }, + "commandName": "update", + "databaseName": "command-monitoring-tests" + } + }, + { + "commandSucceededEvent": { + "reply": { + "ok": 1, + "n": 1, + "upserted": [ + { + "index": 0, + "_id": 4 + } + ] + }, + "commandName": "update" + } + } + ] + } + ] + }, + { + "description": "A successful updateOne with write errors", + "operations": [ + { + "name": "updateOne", + "object": "collection", + "arguments": { + "filter": { + "_id": { + "$gt": 1 + } + }, + "update": { + "$unsupported": { + "x": 1 + } + } + }, + "expectError": { + "isClientError": false + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "command": { + "update": "test", + "updates": [ + { + "q": { + "_id": { + "$gt": 1 + } + }, + "u": { + "$unsupported": { + "x": 1 + } + }, + "upsert": { + "$$unsetOrMatches": false + }, + "multi": { + "$$unsetOrMatches": false + } + } + ], + "ordered": true + }, + "commandName": "update", + "databaseName": "command-monitoring-tests" + } + }, + { + "commandSucceededEvent": { + "reply": { + "ok": 1, + "n": 0, + "writeErrors": { + "$$type": "array" + } + }, + "commandName": "update" + } + } + ] + } + ] + } + ] +} diff --git a/src/libmongoc/tests/json/command-logging-and-monitoring/monitoring/writeConcernError.json b/src/libmongoc/tests/json/command-logging-and-monitoring/monitoring/writeConcernError.json new file mode 100644 index 00000000000..455e5422b72 --- /dev/null +++ b/src/libmongoc/tests/json/command-logging-and-monitoring/monitoring/writeConcernError.json @@ -0,0 +1,155 @@ +{ + "description": "writeConcernError", + "schemaVersion": "1.4", + "runOnRequirements": [ + { + "minServerVersion": "4.3.1", + "topologies": [ + "replicaset" + ], + "serverless": "forbid" + } + ], + "createEntities": [ + { + "client": { + "id": "client", + "observeEvents": [ + "commandStartedEvent", + "commandSucceededEvent", + "commandFailedEvent" + ] + } + }, + { + "database": { + "id": "database", + "client": "client", + "databaseName": "command-monitoring-tests" + } + }, + { + "collection": { + "id": "collection", + "database": "database", + "collectionName": "test" + } + } + ], + "initialData": [ + { + "collectionName": "test", + "databaseName": "command-monitoring-tests", + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + ], + "tests": [ + { + "description": "A retryable write with write concern errors publishes success event", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 1 + }, + "data": { + "failCommands": [ + "insert" + ], + "errorLabels": [ + "RetryableWriteError" + ], + "writeConcernError": { + "code": 91 + } + } + } + } + }, + { + "name": "insertOne", + "object": "collection", + "arguments": { + "document": { + "_id": 2, + "x": 22 + } + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "command": { + "insert": "test", + "documents": [ + { + "_id": 2, + "x": 22 + } + ], + "ordered": true + }, + "commandName": "insert", + "databaseName": "command-monitoring-tests" + } + }, + { + "commandSucceededEvent": { + "reply": { + "ok": 1, + "n": 1, + "errorLabels": [ + "RetryableWriteError" + ], + "writeConcernError": { + "code": 91 + } + }, + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "command": { + "insert": "test", + "documents": [ + { + "_id": 2, + "x": 22 + } + ], + "ordered": true + }, + "commandName": "insert", + "databaseName": "command-monitoring-tests" + } + }, + { + "commandSucceededEvent": { + "reply": { + "ok": 1, + "n": 1 + }, + "commandName": "insert" + } + } + ] + } + ] + } + ] +} From 4edaa2bfc9576c7a14a1bb9fe1038473e57fcf0d Mon Sep 17 00:00:00 2001 From: Micah Scott Date: Mon, 11 Nov 2024 14:38:45 -0800 Subject: [PATCH 040/139] note about CLAM migration --- src/libmongoc/tests/test-mongoc-command-monitoring.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libmongoc/tests/test-mongoc-command-monitoring.c b/src/libmongoc/tests/test-mongoc-command-monitoring.c index ce1b1b9f9c3..8f42a568cd6 100644 --- a/src/libmongoc/tests/test-mongoc-command-monitoring.c +++ b/src/libmongoc/tests/test-mongoc-command-monitoring.c @@ -97,6 +97,7 @@ test_command_monitoring_cb (void *scenario) static void test_all_spec_tests (TestSuite *suite) { + // Newer versions of the 'unified' tests have migrated to connection-logging-and-monitoring run_unified_tests (suite, JSON_DIR, "command_monitoring/unified"); install_json_test_suite (suite, JSON_DIR, "command_monitoring/legacy", &test_command_monitoring_cb); } From ce6fc875b6a862283cbb36eb2436b96f31d9c9d9 Mon Sep 17 00:00:00 2001 From: Micah Scott Date: Mon, 11 Nov 2024 14:41:30 -0800 Subject: [PATCH 041/139] Try to run all the CLAM unified tests --- src/libmongoc/tests/unified/runner.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libmongoc/tests/unified/runner.c b/src/libmongoc/tests/unified/runner.c index 78b9c777c1f..0eb12f1b86e 100644 --- a/src/libmongoc/tests/unified/runner.c +++ b/src/libmongoc/tests/unified/runner.c @@ -1723,5 +1723,5 @@ test_install_unified (TestSuite *suite) run_unified_tests (suite, JSON_DIR, "index-management"); - run_unified_tests (suite, JSON_DIR, "command-logging-and-monitoring/monitoring"); + run_unified_tests (suite, JSON_DIR, "command-logging-and-monitoring"); } From afc20f050ec9cf75f0b049e9372d527faede0164 Mon Sep 17 00:00:00 2001 From: Micah Scott Date: Mon, 11 Nov 2024 17:56:08 -0800 Subject: [PATCH 042/139] per-component level settings, eager one-time log init --- src/libmongoc/src/mongoc/mongoc-init.c | 3 + .../mongoc/mongoc-structured-log-private.h | 3 + .../src/mongoc/mongoc-structured-log.c | 200 +++++++++++------- .../src/mongoc/mongoc-structured-log.h | 61 +++++- 4 files changed, 181 insertions(+), 86 deletions(-) diff --git a/src/libmongoc/src/mongoc/mongoc-init.c b/src/libmongoc/src/mongoc/mongoc-init.c index 4e72f0ec436..ca1f08ae2dc 100644 --- a/src/libmongoc/src/mongoc/mongoc-init.c +++ b/src/libmongoc/src/mongoc/mongoc-init.c @@ -19,6 +19,7 @@ #include "mongoc-config.h" #include "mongoc-counters-private.h" +#include "mongoc-structured-log-private.h" #include "mongoc-init.h" #include "mongoc-handshake-private.h" @@ -96,6 +97,8 @@ mongoc_cyrus_mutex_free (void *mutex) static BSON_ONCE_FUN (_mongoc_do_init) { + _mongoc_structured_log_init (); + #ifdef MONGOC_ENABLE_SASL_CYRUS int status; #endif diff --git a/src/libmongoc/src/mongoc/mongoc-structured-log-private.h b/src/libmongoc/src/mongoc/mongoc-structured-log-private.h index ed9489d9add..0622bb6dc33 100644 --- a/src/libmongoc/src/mongoc/mongoc-structured-log-private.h +++ b/src/libmongoc/src/mongoc/mongoc-structured-log-private.h @@ -143,6 +143,9 @@ struct mongoc_structured_log_entry_t { const mongoc_structured_log_builder_stage_t *builder; // Required }; +void +_mongoc_structured_log_init (void); + char * mongoc_structured_log_document_to_json (const bson_t *document, size_t *length); diff --git a/src/libmongoc/src/mongoc/mongoc-structured-log.c b/src/libmongoc/src/mongoc/mongoc-structured-log.c index f4398db762e..e08bbee3446 100644 --- a/src/libmongoc/src/mongoc/mongoc-structured-log.c +++ b/src/libmongoc/src/mongoc/mongoc-structured-log.c @@ -18,22 +18,24 @@ #include "mongoc-structured-log-private.h" #include "mongoc-thread-private.h" #include "mongoc-util-private.h" +#include "common-atomic-private.h" static void mongoc_structured_log_default_handler (const mongoc_structured_log_entry_t *entry, void *user_data); -static bson_once_t once = BSON_ONCE_INIT; -static bson_mutex_t gStructuredLogMutex; -static mongoc_structured_log_func_t gStructuredLogger = mongoc_structured_log_default_handler; -static void *gStructuredLoggerData; -static FILE *log_stream; +#define STRUCTURED_LOG_COMPONENT_TABLE_SIZE (1 + (size_t) MONGOC_STRUCTURED_LOG_COMPONENT_CONNECTION) -static BSON_ONCE_FUN (_mongoc_ensure_mutex_once) -{ - bson_mutex_init (&gStructuredLogMutex); - - BSON_ONCE_RETURN; -} +static struct { + mongoc_structured_log_func_t func; + void *user_data; + bson_mutex_t func_mutex; // Mutex prevents func reentrancy, ensures atomic updates to (func, user_data) + FILE *stream; + int32_t max_document_length; + int component_level_table[STRUCTURED_LOG_COMPONENT_TABLE_SIZE]; // Really mongoc_structured_log_level_t; int typed to + // support atomic fetch +} gStructuredLog = { + .func = mongoc_structured_log_default_handler, +}; bson_t * mongoc_structured_log_entry_message_as_bson (const mongoc_structured_log_entry_t *entry) @@ -62,37 +64,35 @@ mongoc_structured_log_entry_get_component (const mongoc_structured_log_entry_t * void mongoc_structured_log_set_handler (mongoc_structured_log_func_t log_func, void *user_data) { - bson_once (&once, &_mongoc_ensure_mutex_once); - - bson_mutex_lock (&gStructuredLogMutex); - gStructuredLogger = log_func; - gStructuredLoggerData = user_data; - bson_mutex_unlock (&gStructuredLogMutex); + bson_mutex_lock (&gStructuredLog.func_mutex); + mcommon_atomic_ptr_exchange ((void *) &gStructuredLog.func, (void *) log_func, mcommon_memory_order_relaxed); + gStructuredLog.user_data = user_data; + bson_mutex_unlock (&gStructuredLog.func_mutex); } bool _mongoc_structured_log_should_log (const mongoc_structured_log_envelope_t *envelope) { - // @todo Implement early-out settings for limiting max log level - BSON_UNUSED (envelope); - // Don't take mutex, no need for atomicity. - // This should be a low cost early-out when logging is disabled. - return gStructuredLogger != NULL; + if (!mcommon_atomic_ptr_fetch ((void *) &gStructuredLog.func, mcommon_memory_order_relaxed)) { + return false; + } + + mongoc_structured_log_component_t component = envelope->component; + mongoc_structured_log_level_t level = envelope->level; + BSON_ASSERT (component >= 0 && component < STRUCTURED_LOG_COMPONENT_TABLE_SIZE); + return level <= + mcommon_atomic_int_fetch (&gStructuredLog.component_level_table[component], mcommon_memory_order_relaxed); } void _mongoc_structured_log_with_entry (const mongoc_structured_log_entry_t *entry) { - bson_once (&once, &_mongoc_ensure_mutex_once); - bson_mutex_lock (&gStructuredLogMutex); - - if (!gStructuredLogger) { - bson_mutex_unlock (&gStructuredLogMutex); - return; + bson_mutex_lock (&gStructuredLog.func_mutex); + mongoc_structured_log_func_t func = gStructuredLog.func; + if (func) { + func (entry, gStructuredLog.user_data); } - - gStructuredLogger (entry, gStructuredLoggerData); - bson_mutex_unlock (&gStructuredLogMutex); + bson_mutex_unlock (&gStructuredLog.func_mutex); } static mongoc_structured_log_level_t @@ -126,91 +126,125 @@ _mongoc_structured_log_get_log_level_from_env (const char *variable) } } -static mongoc_structured_log_level_t -_mongoc_structured_log_get_log_level (mongoc_structured_log_component_t component) +const char * +mongoc_structured_log_get_level_name (mongoc_structured_log_level_t level) { - switch (component) { - case MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND: - return _mongoc_structured_log_get_log_level_from_env ("MONGODB_LOGGING_COMMAND"); - case MONGOC_STRUCTURED_LOG_COMPONENT_CONNECTION: - return _mongoc_structured_log_get_log_level_from_env ("MONGODB_LOGGING_CONNECTION"); - case MONGOC_STRUCTURED_LOG_COMPONENT_SDAM: - return _mongoc_structured_log_get_log_level_from_env ("MONGODB_LOGGING_SDAM"); - case MONGOC_STRUCTURED_LOG_COMPONENT_SERVER_SELECTION: - return _mongoc_structured_log_get_log_level_from_env ("MONGODB_LOGGING_SERVER_SELECTION"); - default: - MONGOC_ERROR ("Requesting log level for unsupported component %d", (int) component); - exit (EXIT_FAILURE); + static const char *table[] = { + "emergency", "alert", "critical", "error", "warning", "notice", "info", "debug", "trace"}; + if (level >= 0 && level < (sizeof table / sizeof table[0])) { + return table[level]; } + return NULL; } -static void -_mongoc_structured_log_initialize_stream (void) +const char * +mongoc_structured_log_get_component_name (mongoc_structured_log_component_t component) +{ + static const char *table[] = {"command", "sdam", "server selection", "connection"}; + if (component >= 0 && component < (sizeof table / sizeof table[0])) { + return table[component]; + } + return NULL; +} + +static int32_t +_mongoc_structured_log_get_max_document_length_from_env (void) +{ + const char *max_length_str = getenv ("MONGODB_LOGGING_MAX_DOCUMENT_LENGTH"); + + if (!max_length_str) { + return MONGOC_STRUCTURED_LOG_DEFAULT_MAX_DOCUMENT_LENGTH; + } + + if (!strcmp (max_length_str, "unlimited")) { + return BSON_MAX_LEN_UNLIMITED; + } + + return strtoul (max_length_str, NULL, 10); +} + +void +mongoc_structured_log_set_max_level_for_component (mongoc_structured_log_component_t component, + mongoc_structured_log_level_t level) +{ + BSON_ASSERT (level >= MONGOC_STRUCTURED_LOG_LEVEL_EMERGENCY); + BSON_ASSERT (level <= MONGOC_STRUCTURED_LOG_LEVEL_TRACE); + BSON_ASSERT (component >= 0 && component < STRUCTURED_LOG_COMPONENT_TABLE_SIZE); + mcommon_atomic_int_exchange (&gStructuredLog.component_level_table[component], level, mcommon_memory_order_relaxed); +} + +void +mongoc_structured_log_set_max_level_for_all_components (mongoc_structured_log_level_t level) +{ + for (size_t component = 0; component < STRUCTURED_LOG_COMPONENT_TABLE_SIZE; component++) { + mongoc_structured_log_set_max_level_for_component (component, level); + } +} + +void +_mongoc_structured_log_init (void) +{ + bson_mutex_init (&gStructuredLog.func_mutex); + gStructuredLog.max_document_length = _mongoc_structured_log_get_max_document_length_from_env (); + + mongoc_structured_log_set_max_level_for_component ( + MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND, + _mongoc_structured_log_get_log_level_from_env ("MONGODB_LOGGING_COMMAND")); + mongoc_structured_log_set_max_level_for_component ( + MONGOC_STRUCTURED_LOG_COMPONENT_CONNECTION, + _mongoc_structured_log_get_log_level_from_env ("MONGODB_LOGGING_CONNECTION")); + mongoc_structured_log_set_max_level_for_component ( + MONGOC_STRUCTURED_LOG_COMPONENT_SDAM, _mongoc_structured_log_get_log_level_from_env ("MONGODB_LOGGING_SDAM")); + mongoc_structured_log_set_max_level_for_component ( + MONGOC_STRUCTURED_LOG_COMPONENT_SERVER_SELECTION, + _mongoc_structured_log_get_log_level_from_env ("MONGODB_LOGGING_SERVER_SELECTION")); +} + +static FILE * +_mongoc_structured_log_open_stream (void) { const char *log_target = getenv ("MONGODB_LOGGING_PATH"); bool log_to_stderr = !log_target || !strcmp (log_target, "stderr"); - - log_stream = log_to_stderr ? stderr : fopen (log_target, "a"); + FILE *log_stream = log_to_stderr ? stderr : fopen (log_target, "a"); if (!log_stream) { MONGOC_ERROR ("Cannot open log file %s for writing", log_target); exit (EXIT_FAILURE); } + return log_stream; } static FILE * _mongoc_structured_log_get_stream (void) { - if (!log_stream) { - _mongoc_structured_log_initialize_stream (); + FILE *log_stream = gStructuredLog.stream; + if (log_stream) { + return log_stream; } - + log_stream = _mongoc_structured_log_open_stream (); + gStructuredLog.stream = log_stream; return log_stream; } static void mongoc_structured_log_default_handler (const mongoc_structured_log_entry_t *entry, void *user_data) { - // @todo This really needs a cache, we shouldn't be parsing env vars for each should_log check - mongoc_structured_log_level_t log_level = - _mongoc_structured_log_get_log_level (mongoc_structured_log_entry_get_component (entry)); - - if (log_level < mongoc_structured_log_entry_get_level (entry)) { - return; - } - bson_t *bson_message = mongoc_structured_log_entry_message_as_bson (entry); char *json_message = bson_as_relaxed_extended_json (bson_message, NULL); fprintf (_mongoc_structured_log_get_stream (), - "Structured log: %d, %d, %s\n", - (int) mongoc_structured_log_entry_get_level (entry), - (int) mongoc_structured_log_entry_get_component (entry), + "mongoc_structured_log -- %s -- %s -- %s\n", + mongoc_structured_log_get_level_name (mongoc_structured_log_entry_get_level (entry)), + mongoc_structured_log_get_component_name (mongoc_structured_log_entry_get_component (entry)), json_message); bson_free (json_message); bson_destroy (bson_message); } -static int32_t -mongoc_structured_log_get_max_length (void) -{ - const char *max_length_str = getenv ("MONGODB_LOGGING_MAX_DOCUMENT_LENGTH"); - - if (!max_length_str) { - return MONGOC_STRUCTURED_LOG_DEFAULT_MAX_DOCUMENT_LENGTH; - } - - if (!strcmp (max_length_str, "unlimited")) { - return BSON_MAX_LEN_UNLIMITED; - } - - return strtoul (max_length_str, NULL, 10); -} - char * mongoc_structured_log_document_to_json (const bson_t *document, size_t *length) { - bson_json_opts_t *opts = bson_json_opts_new (BSON_JSON_MODE_CANONICAL, mongoc_structured_log_get_max_length ()); + bson_json_opts_t *opts = bson_json_opts_new (BSON_JSON_MODE_CANONICAL, gStructuredLog.max_document_length); char *json = bson_as_json_with_opts (document, length, opts); bson_json_opts_destroy (opts); return json; @@ -219,8 +253,10 @@ mongoc_structured_log_document_to_json (const bson_t *document, size_t *length) void mongoc_structured_log_get_handler (mongoc_structured_log_func_t *log_func, void **user_data) { - *log_func = gStructuredLogger; - *user_data = gStructuredLoggerData; + bson_mutex_lock (&gStructuredLog.func_mutex); + *log_func = gStructuredLog.func; + *user_data = gStructuredLog.user_data; + bson_mutex_unlock (&gStructuredLog.func_mutex); } const mongoc_structured_log_builder_stage_t * diff --git a/src/libmongoc/src/mongoc/mongoc-structured-log.h b/src/libmongoc/src/mongoc/mongoc-structured-log.h index 5ab156ad499..ef848459fdd 100644 --- a/src/libmongoc/src/mongoc/mongoc-structured-log.h +++ b/src/libmongoc/src/mongoc/mongoc-structured-log.h @@ -38,10 +38,10 @@ typedef enum { } mongoc_structured_log_level_t; typedef enum { - MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND, - MONGOC_STRUCTURED_LOG_COMPONENT_SDAM, - MONGOC_STRUCTURED_LOG_COMPONENT_SERVER_SELECTION, - MONGOC_STRUCTURED_LOG_COMPONENT_CONNECTION, + MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND = 0, + MONGOC_STRUCTURED_LOG_COMPONENT_SDAM = 1, + MONGOC_STRUCTURED_LOG_COMPONENT_SERVER_SELECTION = 2, + MONGOC_STRUCTURED_LOG_COMPONENT_CONNECTION = 3, } mongoc_structured_log_component_t; typedef struct mongoc_structured_log_entry_t mongoc_structured_log_entry_t; @@ -71,6 +71,33 @@ typedef void (*mongoc_structured_log_func_t) (const mongoc_structured_log_entry_ MONGOC_EXPORT (void) mongoc_structured_log_set_handler (mongoc_structured_log_func_t log_func, void *user_data); +/** + * mongoc_structured_log_set_max_level_for_component: + * @level: Maximum log level for this component. + * @component: A logging component to set the level limit for. + * + * Sets the maximum log level per-component. Only log messages at or below + * this severity level will be passed to mongoc_structured_log_func_t. + * + * By default, each component's log level comes from the environment + * variables `MONGOC_LOGGING_` captured during mongoc_init(). + */ +MONGOC_EXPORT (void) +mongoc_structured_log_set_max_level_for_component (mongoc_structured_log_component_t component, + mongoc_structured_log_level_t level); + +/** + * mongoc_structured_log_set_max_level_for_all_components: + * @level: Maximum log level for all components. + * + * Sets all per-component maximum log levels to the same value. + * Only log messages at or below this severity level will be passed to + * mongoc_structured_log_func_t. Effective even for logging components not + * known at compile-time. + */ +MONGOC_EXPORT (void) +mongoc_structured_log_set_max_level_for_all_components (mongoc_structured_log_level_t level); + /** * mongoc_structured_log_entry_message_as_bson: * @entry: A log entry to extract the message from. @@ -99,6 +126,32 @@ mongoc_structured_log_entry_get_level (const mongoc_structured_log_entry_t *entr MONGOC_EXPORT (mongoc_structured_log_component_t) mongoc_structured_log_entry_get_component (const mongoc_structured_log_entry_t *entry); +/** + * mongoc_structured_log_get_level_name: + * @level: A log level code. + * + * Returns the name for a log level code as a constant string that does + * not need to be deallocated, or NULL if the level has no known name. + * + * Strings are lower-case words: + * "emergency", "alert", "critical", "error", "warning", "notice", "info", "debug", "trace" + */ +MONGOC_EXPORT (const char *) +mongoc_structured_log_get_level_name (mongoc_structured_log_level_t level); + +/** + * mongoc_structured_log_get_component_name: + * @level: A log level code. + * + * Returns the short name for a component as a constant string that does + * not need to be deallocated, or NULL if the level has no known name. + * + * Strings are lower-case words or phrases: + * "command", "sdam", "server selection", "connection" + */ +MONGOC_EXPORT (const char *) +mongoc_structured_log_get_component_name (mongoc_structured_log_component_t component); + BSON_END_DECLS #endif /* MONGOC_STRUCTURED_LOG_H */ From 9f9df9865452e91eed2f1b852e321b3d52e67971 Mon Sep 17 00:00:00 2001 From: Micah Scott Date: Mon, 11 Nov 2024 20:17:26 -0800 Subject: [PATCH 043/139] Use _bsonDSL_mapMacro in mongoc_structured_log --- src/libmongoc/src/mongoc/mongoc-client.c | 52 ++++----- src/libmongoc/src/mongoc/mongoc-cluster.c | 44 +++----- .../src/mongoc/mongoc-cursor-legacy.c | 16 +-- src/libmongoc/src/mongoc/mongoc-cursor.c | 66 +++++------ .../mongoc/mongoc-structured-log-private.h | 104 +++++++++--------- .../src/mongoc/mongoc-structured-log.c | 2 +- .../tests/test-mongoc-structured-log.c | 52 ++++----- 7 files changed, 140 insertions(+), 196 deletions(-) diff --git a/src/libmongoc/src/mongoc/mongoc-client.c b/src/libmongoc/src/mongoc/mongoc-client.c index fb95495cfd7..53ca42823c0 100644 --- a/src/libmongoc/src/mongoc/mongoc-client.c +++ b/src/libmongoc/src/mongoc/mongoc-client.c @@ -2170,16 +2170,12 @@ _mongoc_client_monitor_op_killcursors (mongoc_cluster_t *cluster, MONGOC_STRUCTURED_LOG_LEVEL_INFO, MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND, "Command started", - MONGOC_STRUCTURED_LOG_INT32 ("requestId", cluster->request_id), - MONGOC_STRUCTURED_LOG_SERVER_DESCRIPTION (server_stream->sd, - (MONGOC_STRUCTURED_LOG_SERVER_DESCRIPTION_SERVER_HOST | - MONGOC_STRUCTURED_LOG_SERVER_DESCRIPTION_SERVER_PORT | - MONGOC_STRUCTURED_LOG_SERVER_DESCRIPTION_SERVER_CONNECTION_ID | - MONGOC_STRUCTURED_LOG_SERVER_DESCRIPTION_SERVICE_ID)), - MONGOC_STRUCTURED_LOG_UTF8 ("databaseName", db), - MONGOC_STRUCTURED_LOG_UTF8 ("commandName", "killCursors"), - MONGOC_STRUCTURED_LOG_INT64 ("operationId", operation_id), - MONGOC_STRUCTURED_LOG_BSON_AS_JSON ("command", &doc)); + int32 ("requestId", cluster->request_id), + server_description (server_stream->sd, SERVER_HOST, SERVER_PORT, SERVER_CONNECTION_ID, SERVICE_ID), + utf8 ("databaseName", db), + utf8 ("commandName", "killCursors"), + int64 ("operationId", operation_id), + bson_as_json ("command", &doc)); if (!client->apm_callbacks.started) { bson_destroy (&doc); @@ -2235,17 +2231,13 @@ _mongoc_client_monitor_op_killcursors_succeeded (mongoc_cluster_t *cluster, MONGOC_STRUCTURED_LOG_LEVEL_INFO, MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND, "Command succeeded", - MONGOC_STRUCTURED_LOG_INT32 ("requestId", cluster->request_id), - MONGOC_STRUCTURED_LOG_SERVER_DESCRIPTION (server_stream->sd, - (MONGOC_STRUCTURED_LOG_SERVER_DESCRIPTION_SERVER_HOST | - MONGOC_STRUCTURED_LOG_SERVER_DESCRIPTION_SERVER_PORT | - MONGOC_STRUCTURED_LOG_SERVER_DESCRIPTION_SERVER_CONNECTION_ID | - MONGOC_STRUCTURED_LOG_SERVER_DESCRIPTION_SERVICE_ID)), - MONGOC_STRUCTURED_LOG_UTF8 ("databaseName", db), - MONGOC_STRUCTURED_LOG_UTF8 ("commandName", "killCursors"), - MONGOC_STRUCTURED_LOG_INT64 ("operationId", operation_id), - MONGOC_STRUCTURED_LOG_INT64 ("durationMS", duration), - MONGOC_STRUCTURED_LOG_BSON_AS_JSON ("reply", &doc)); + int32 ("requestId", cluster->request_id), + server_description (server_stream->sd, SERVER_HOST, SERVER_PORT, SERVER_CONNECTION_ID, SERVICE_ID), + utf8 ("databaseName", db), + utf8 ("commandName", "killCursors"), + int64 ("operationId", operation_id), + int64 ("durationMS", duration), + bson_as_json ("reply", &doc)); if (!client->apm_callbacks.succeeded) { bson_destroy (&doc); @@ -2297,17 +2289,13 @@ _mongoc_client_monitor_op_killcursors_failed (mongoc_cluster_t *cluster, MONGOC_STRUCTURED_LOG_LEVEL_INFO, MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND, "Command failed", - MONGOC_STRUCTURED_LOG_INT32 ("requestId", cluster->request_id), - MONGOC_STRUCTURED_LOG_SERVER_DESCRIPTION (server_stream->sd, - (MONGOC_STRUCTURED_LOG_SERVER_DESCRIPTION_SERVER_HOST | - MONGOC_STRUCTURED_LOG_SERVER_DESCRIPTION_SERVER_PORT | - MONGOC_STRUCTURED_LOG_SERVER_DESCRIPTION_SERVER_CONNECTION_ID | - MONGOC_STRUCTURED_LOG_SERVER_DESCRIPTION_SERVICE_ID)), - MONGOC_STRUCTURED_LOG_UTF8 ("databaseName", db), - MONGOC_STRUCTURED_LOG_UTF8 ("commandName", "killCursors"), - MONGOC_STRUCTURED_LOG_INT64 ("operationId", operation_id), - MONGOC_STRUCTURED_LOG_INT64 ("durationMS", duration), - MONGOC_STRUCTURED_LOG_BSON_AS_JSON ("failure", &doc)); + int32 ("requestId", cluster->request_id), + server_description (server_stream->sd, SERVER_HOST, SERVER_PORT, SERVER_CONNECTION_ID, SERVICE_ID), + utf8 ("databaseName", db), + utf8 ("commandName", "killCursors"), + int64 ("operationId", operation_id), + int64 ("durationMS", duration), + bson_as_json ("failure", &doc)); if (!client->apm_callbacks.failed) { bson_destroy (&doc); diff --git a/src/libmongoc/src/mongoc/mongoc-cluster.c b/src/libmongoc/src/mongoc/mongoc-cluster.c index 79109a0da67..a7a3c96eb14 100644 --- a/src/libmongoc/src/mongoc/mongoc-cluster.c +++ b/src/libmongoc/src/mongoc/mongoc-cluster.c @@ -537,15 +537,9 @@ mongoc_cluster_run_command_monitored (mongoc_cluster_t *cluster, mongoc_cmd_t *c MONGOC_STRUCTURED_LOG_LEVEL_INFO, MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND, "Command started", - MONGOC_STRUCTURED_LOG_INT32 ("requestId", request_id), - MONGOC_STRUCTURED_LOG_SERVER_DESCRIPTION (server_stream->sd, - (MONGOC_STRUCTURED_LOG_SERVER_DESCRIPTION_SERVER_HOST | - MONGOC_STRUCTURED_LOG_SERVER_DESCRIPTION_SERVER_PORT | - MONGOC_STRUCTURED_LOG_SERVER_DESCRIPTION_SERVER_CONNECTION_ID | - MONGOC_STRUCTURED_LOG_SERVER_DESCRIPTION_SERVICE_ID)), - MONGOC_STRUCTURED_LOG_CMD (cmd, - (MONGOC_STRUCTURED_LOG_CMD_DATABASE_NAME | MONGOC_STRUCTURED_LOG_CMD_COMMAND_NAME | - MONGOC_STRUCTURED_LOG_CMD_OPERATION_ID | MONGOC_STRUCTURED_LOG_CMD_COMMAND))); + int32 ("requestId", request_id), + server_description (server_stream->sd, SERVER_HOST, SERVER_PORT, SERVER_CONNECTION_ID, SERVICE_ID), + cmd (cmd, DATABASE_NAME, COMMAND_NAME, OPERATION_ID, COMMAND)); if (callbacks->started) { mongoc_apm_command_started_init_with_cmd ( @@ -574,17 +568,11 @@ mongoc_cluster_run_command_monitored (mongoc_cluster_t *cluster, mongoc_cmd_t *c MONGOC_STRUCTURED_LOG_LEVEL_INFO, MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND, "Command succeeded", - MONGOC_STRUCTURED_LOG_INT32 ("requestId", request_id), - MONGOC_STRUCTURED_LOG_SERVER_DESCRIPTION (server_stream->sd, - (MONGOC_STRUCTURED_LOG_SERVER_DESCRIPTION_SERVER_HOST | - MONGOC_STRUCTURED_LOG_SERVER_DESCRIPTION_SERVER_PORT | - MONGOC_STRUCTURED_LOG_SERVER_DESCRIPTION_SERVER_CONNECTION_ID | - MONGOC_STRUCTURED_LOG_SERVER_DESCRIPTION_SERVICE_ID)), - MONGOC_STRUCTURED_LOG_CMD (cmd, - (MONGOC_STRUCTURED_LOG_CMD_DATABASE_NAME | MONGOC_STRUCTURED_LOG_CMD_COMMAND_NAME | - MONGOC_STRUCTURED_LOG_CMD_OPERATION_ID)), - MONGOC_STRUCTURED_LOG_INT64 ("durationMS", duration), - MONGOC_STRUCTURED_LOG_BSON_AS_JSON ("reply", cmd->is_acknowledged ? reply : &fake_reply)); + int32 ("requestId", request_id), + server_description (server_stream->sd, SERVER_HOST, SERVER_PORT, SERVER_CONNECTION_ID, SERVICE_ID), + cmd (cmd, DATABASE_NAME, COMMAND_NAME, OPERATION_ID), + int64 ("durationMS", duration), + bson_as_json ("reply", cmd->is_acknowledged ? reply : &fake_reply)); if (callbacks->succeeded) { mongoc_apm_command_succeeded_init (&succeeded_event, @@ -613,17 +601,11 @@ mongoc_cluster_run_command_monitored (mongoc_cluster_t *cluster, mongoc_cmd_t *c MONGOC_STRUCTURED_LOG_LEVEL_INFO, MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND, "Command failed", - MONGOC_STRUCTURED_LOG_INT32 ("requestId", request_id), - MONGOC_STRUCTURED_LOG_SERVER_DESCRIPTION (server_stream->sd, - (MONGOC_STRUCTURED_LOG_SERVER_DESCRIPTION_SERVER_HOST | - MONGOC_STRUCTURED_LOG_SERVER_DESCRIPTION_SERVER_PORT | - MONGOC_STRUCTURED_LOG_SERVER_DESCRIPTION_SERVER_CONNECTION_ID | - MONGOC_STRUCTURED_LOG_SERVER_DESCRIPTION_SERVICE_ID)), - MONGOC_STRUCTURED_LOG_CMD (cmd, - (MONGOC_STRUCTURED_LOG_CMD_DATABASE_NAME | MONGOC_STRUCTURED_LOG_CMD_COMMAND_NAME | - MONGOC_STRUCTURED_LOG_CMD_OPERATION_ID)), - MONGOC_STRUCTURED_LOG_INT64 ("durationMS", duration), - MONGOC_STRUCTURED_LOG_BSON_AS_JSON ("failure", reply)); + int32 ("requestId", request_id), + server_description (server_stream->sd, SERVER_HOST, SERVER_PORT, SERVER_CONNECTION_ID, SERVICE_ID), + cmd (cmd, DATABASE_NAME, COMMAND_NAME, OPERATION_ID), + int64 ("durationMS", duration), + bson_as_json ("failure", reply)); if (callbacks->failed) { mongoc_apm_command_failed_init (&failed_event, diff --git a/src/libmongoc/src/mongoc/mongoc-cursor-legacy.c b/src/libmongoc/src/mongoc/mongoc-cursor-legacy.c index 57142fad712..041f39c1f78 100644 --- a/src/libmongoc/src/mongoc/mongoc-cursor-legacy.c +++ b/src/libmongoc/src/mongoc/mongoc-cursor-legacy.c @@ -53,16 +53,12 @@ _mongoc_cursor_monitor_legacy_get_more (mongoc_cursor_t *cursor, mongoc_server_s MONGOC_STRUCTURED_LOG_LEVEL_INFO, MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND, "Command started", - MONGOC_STRUCTURED_LOG_INT32 ("requestId", client->cluster.request_id), - MONGOC_STRUCTURED_LOG_SERVER_DESCRIPTION (server_stream->sd, - (MONGOC_STRUCTURED_LOG_SERVER_DESCRIPTION_SERVER_HOST | - MONGOC_STRUCTURED_LOG_SERVER_DESCRIPTION_SERVER_PORT | - MONGOC_STRUCTURED_LOG_SERVER_DESCRIPTION_SERVER_CONNECTION_ID | - MONGOC_STRUCTURED_LOG_SERVER_DESCRIPTION_SERVICE_ID)), - MONGOC_STRUCTURED_LOG_UTF8_N ("databaseName", cursor->ns, cursor->dblen), - MONGOC_STRUCTURED_LOG_UTF8 ("commandName", "getMore"), - MONGOC_STRUCTURED_LOG_INT64 ("operationId", cursor->operation_id), - MONGOC_STRUCTURED_LOG_BSON_AS_JSON ("command", &doc)); + int32 ("requestId", client->cluster.request_id), + server_description (server_stream->sd, SERVER_HOST, SERVER_PORT, SERVER_CONNECTION_ID, SERVICE_ID), + utf8_n ("databaseName", cursor->ns, cursor->dblen), + utf8 ("commandName", "getMore"), + int64 ("operationId", cursor->operation_id), + bson_as_json ("command", &doc)); if (!client->apm_callbacks.started) { /* successful */ diff --git a/src/libmongoc/src/mongoc/mongoc-cursor.c b/src/libmongoc/src/mongoc/mongoc-cursor.c index 097055a92f3..8a582db34b3 100644 --- a/src/libmongoc/src/mongoc/mongoc-cursor.c +++ b/src/libmongoc/src/mongoc/mongoc-cursor.c @@ -650,16 +650,12 @@ _mongoc_cursor_monitor_command (mongoc_cursor_t *cursor, MONGOC_STRUCTURED_LOG_LEVEL_INFO, MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND, "Command started", - MONGOC_STRUCTURED_LOG_INT32 ("requestId", client->cluster.request_id), - MONGOC_STRUCTURED_LOG_SERVER_DESCRIPTION (server_stream->sd, - (MONGOC_STRUCTURED_LOG_SERVER_DESCRIPTION_SERVER_HOST | - MONGOC_STRUCTURED_LOG_SERVER_DESCRIPTION_SERVER_PORT | - MONGOC_STRUCTURED_LOG_SERVER_DESCRIPTION_SERVER_CONNECTION_ID | - MONGOC_STRUCTURED_LOG_SERVER_DESCRIPTION_SERVICE_ID)), - MONGOC_STRUCTURED_LOG_UTF8_N ("databaseName", cursor->ns, cursor->dblen), - MONGOC_STRUCTURED_LOG_UTF8 ("commandName", cmd_name), - MONGOC_STRUCTURED_LOG_INT64 ("operationId", cursor->operation_id), - MONGOC_STRUCTURED_LOG_BSON_AS_JSON ("command", cmd)); + int32 ("requestId", client->cluster.request_id), + server_description (server_stream->sd, SERVER_HOST, SERVER_PORT, SERVER_CONNECTION_ID, SERVICE_ID), + utf8_n ("databaseName", cursor->ns, cursor->dblen), + utf8 ("commandName", cmd_name), + int64 ("operationId", cursor->operation_id), + bson_as_json ("command", cmd)); if (!client->apm_callbacks.started) { /* successful */ @@ -743,21 +739,16 @@ _mongoc_cursor_monitor_succeeded (mongoc_cursor_t *cursor, bson_destroy (&docs_array); - mongoc_structured_log ( - MONGOC_STRUCTURED_LOG_LEVEL_INFO, - MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND, - "Command succeeded", - MONGOC_STRUCTURED_LOG_INT32 ("requestId", client->cluster.request_id), - MONGOC_STRUCTURED_LOG_SERVER_DESCRIPTION (stream->sd, - (MONGOC_STRUCTURED_LOG_SERVER_DESCRIPTION_SERVER_HOST | - MONGOC_STRUCTURED_LOG_SERVER_DESCRIPTION_SERVER_PORT | - MONGOC_STRUCTURED_LOG_SERVER_DESCRIPTION_SERVER_CONNECTION_ID | - MONGOC_STRUCTURED_LOG_SERVER_DESCRIPTION_SERVICE_ID)), - MONGOC_STRUCTURED_LOG_UTF8 ("databaseName", db), - MONGOC_STRUCTURED_LOG_UTF8 ("commandName", cmd_name), - MONGOC_STRUCTURED_LOG_INT64 ("operationId", cursor->operation_id), - MONGOC_STRUCTURED_LOG_INT64 ("durationMS", duration), - MONGOC_STRUCTURED_LOG_BSON_AS_JSON ("reply", &reply)); + mongoc_structured_log (MONGOC_STRUCTURED_LOG_LEVEL_INFO, + MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND, + "Command succeeded", + int32 ("requestId", client->cluster.request_id), + server_description (stream->sd, SERVER_HOST, SERVER_PORT, SERVER_CONNECTION_ID, SERVICE_ID), + utf8 ("databaseName", db), + utf8 ("commandName", cmd_name), + int64 ("operationId", cursor->operation_id), + int64 ("durationMS", duration), + bson_as_json ("reply", &reply)); if (client->apm_callbacks.succeeded) { mongoc_apm_command_succeeded_init (&event, @@ -805,21 +796,16 @@ _mongoc_cursor_monitor_failed (mongoc_cursor_t *cursor, bsonBuildDecl (reply, kv ("ok", int32 (0))); char *db = bson_strndup (cursor->ns, cursor->dblen); - mongoc_structured_log ( - MONGOC_STRUCTURED_LOG_LEVEL_INFO, - MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND, - "Command failed", - MONGOC_STRUCTURED_LOG_INT32 ("requestId", client->cluster.request_id), - MONGOC_STRUCTURED_LOG_SERVER_DESCRIPTION (stream->sd, - (MONGOC_STRUCTURED_LOG_SERVER_DESCRIPTION_SERVER_HOST | - MONGOC_STRUCTURED_LOG_SERVER_DESCRIPTION_SERVER_PORT | - MONGOC_STRUCTURED_LOG_SERVER_DESCRIPTION_SERVER_CONNECTION_ID | - MONGOC_STRUCTURED_LOG_SERVER_DESCRIPTION_SERVICE_ID)), - MONGOC_STRUCTURED_LOG_UTF8 ("databaseName", db), - MONGOC_STRUCTURED_LOG_UTF8 ("commandName", cmd_name), - MONGOC_STRUCTURED_LOG_INT64 ("operationId", cursor->operation_id), - MONGOC_STRUCTURED_LOG_INT64 ("durationMS", duration), - MONGOC_STRUCTURED_LOG_BSON_AS_JSON ("failure", &reply)); + mongoc_structured_log (MONGOC_STRUCTURED_LOG_LEVEL_INFO, + MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND, + "Command failed", + int32 ("requestId", client->cluster.request_id), + server_description (stream->sd, SERVER_HOST, SERVER_PORT, SERVER_CONNECTION_ID, SERVICE_ID), + utf8 ("databaseName", db), + utf8 ("commandName", cmd_name), + int64 ("operationId", cursor->operation_id), + int64 ("durationMS", duration), + bson_as_json ("failure", &reply)); if (client->apm_callbacks.failed) { mongoc_apm_command_failed_init (&event, diff --git a/src/libmongoc/src/mongoc/mongoc-structured-log-private.h b/src/libmongoc/src/mongoc/mongoc-structured-log-private.h index 0622bb6dc33..71787b2aef5 100644 --- a/src/libmongoc/src/mongoc/mongoc-structured-log-private.h +++ b/src/libmongoc/src/mongoc/mongoc-structured-log-private.h @@ -20,6 +20,7 @@ #define MONGOC_STRUCTURED_LOG_PRIVATE_H #include +#include #include "mongoc-structured-log.h" #include "mongoc-cmd-private.h" #include "mongoc-server-description-private.h" @@ -30,68 +31,65 @@ BSON_BEGIN_DECLS #define MONGOC_STRUCTURED_LOG_DEFAULT_MAX_DOCUMENT_LENGTH 1000 #define mongoc_structured_log(_level, _component, ...) \ - _MONGOC_STRUCTURED_LOG (_level, _component, __VA_ARGS__, {.func = NULL}) + _mongoc_structured_log_with_end_of_list (_level, _component, __VA_ARGS__, end_of_list ()) -#define _MONGOC_STRUCTURED_LOG(_level, _component, _message, ...) \ +#define _mongoc_structured_log_with_end_of_list(_level, _component, _message, ...) \ do { \ mongoc_structured_log_entry_t _entry = { \ .envelope.level = (_level), .envelope.component = (_component), .envelope.message = (_message)}; \ if (_mongoc_structured_log_should_log (&_entry.envelope)) { \ - const mongoc_structured_log_builder_stage_t _builder[] = {__VA_ARGS__}; \ + const mongoc_structured_log_builder_stage_t _builder[] = { \ + _mongoc_structured_log_tokens_to_stages (__VA_ARGS__)}; \ _entry.builder = _builder; \ _mongoc_structured_log_with_entry (&_entry); \ } \ } while (0) -#define MONGOC_STRUCTURED_LOG_UTF8(_key_or_null, _value_utf8) \ - { \ - .func = _mongoc_structured_log_append_utf8, .arg1.utf8 = (_key_or_null), .arg2.utf8 = (_value_utf8) \ - } - -#define MONGOC_STRUCTURED_LOG_UTF8_N(_key_literal, _value_utf8, _value_len) \ - MONGOC_STRUCTURED_LOG_UTF8_NN (_key_literal, strlen (_key_literal), _value_utf8, _value_len) - -#define MONGOC_STRUCTURED_LOG_UTF8_NN(_key_or_null, _key_len, _value_utf8, _value_len) \ - {.func = _mongoc_structured_log_append_utf8_n_stage0, .arg1.utf8 = (_key_or_null), .arg2.int32 = (_key_len)}, \ - { \ - .func = _mongoc_structured_log_append_utf8_n_stage1, .arg1.utf8 = (_value_utf8), .arg2.int32 = (_value_len) \ - } - -#define MONGOC_STRUCTURED_LOG_INT32(_key_or_null, _value_int32) \ - { \ - .func = _mongoc_structured_log_append_int32, .arg1.utf8 = (_key_or_null), .arg2.int32 = (_value_int32) \ - } - -#define MONGOC_STRUCTURED_LOG_INT64(_key_or_null, _value_int64) \ - { \ - .func = _mongoc_structured_log_append_int64, .arg1.utf8 = (_key_or_null), .arg2.int64 = (_value_int64) \ - } - -#define MONGOC_STRUCTURED_LOG_BOOL(_key_or_null, _value_bool) \ - { \ - .func = _mongoc_structured_log_append_bool, .arg1.utf8 = (_key_or_null), .arg2.boolean = (_value_bool) \ - } - -#define MONGOC_STRUCTURED_LOG_OID_AS_HEX(_key_or_null, _value_oid) \ - { \ - .func = _mongoc_structured_log_append_oid_as_hex, .arg1.utf8 = (_key_or_null), .arg2.oid = (_value_oid) \ - } - -#define MONGOC_STRUCTURED_LOG_BSON_AS_JSON(_key_or_null, _value_bson) \ - { \ - .func = _mongoc_structured_log_append_bson_as_json, .arg1.utf8 = (_key_or_null), .arg2.bson = (_value_bson) \ - } - -#define MONGOC_STRUCTURED_LOG_CMD(_cmd, _flags) \ - { \ - .func = _mongoc_structured_log_append_cmd, .arg1.cmd = (_cmd), .arg2.cmd_flags = (_flags) \ - } - -#define MONGOC_STRUCTURED_LOG_SERVER_DESCRIPTION(_server_description, _flags) \ - { \ - .func = _mongoc_structured_log_append_server_description, .arg1.server_description = (_server_description), \ - .arg2.server_description_flags = (_flags) \ - } +#define _mongoc_structured_log_tokens_to_stages(...) \ + _bsonDSL_eval (_bsonDSL_mapMacro (_mongoc_structured_log_token_to_stage, ~, __VA_ARGS__)) + +#define _mongoc_structured_log_flag_expr(_action, _constant, _counter) | (_constant##_##_action) + +#define _mongoc_structured_log_token_to_stage(_action, _constant, _counter) _mongoc_structured_log_token_##_action + +#define _mongoc_structured_log_token_end_of_list() {.func = NULL}, + +#define _mongoc_structured_log_token_utf8(_key_or_null, _value_utf8) \ + {.func = _mongoc_structured_log_append_utf8, .arg1.utf8 = (_key_or_null), .arg2.utf8 = (_value_utf8)}, + +#define _mongoc_structured_log_token_utf8_n(_key_literal, _value_utf8, _value_len) \ + _mongoc_structured_log_token_utf8_nn (_key_literal, strlen (_key_literal), _value_utf8, _value_len) + +#define _mongoc_structured_log_token_utf8_nn(_key_or_null, _key_len, _value_utf8, _value_len) \ + {.func = _mongoc_structured_log_append_utf8_n_stage0, .arg1.utf8 = (_key_or_null), .arg2.int32 = (_key_len)}, \ + {.func = _mongoc_structured_log_append_utf8_n_stage1, .arg1.utf8 = (_value_utf8), .arg2.int32 = (_value_len)}, + +#define _mongoc_structured_log_token_int32(_key_or_null, _value_int32) \ + {.func = _mongoc_structured_log_append_int32, .arg1.utf8 = (_key_or_null), .arg2.int32 = (_value_int32)}, + +#define _mongoc_structured_log_token_int64(_key_or_null, _value_int64) \ + {.func = _mongoc_structured_log_append_int64, .arg1.utf8 = (_key_or_null), .arg2.int64 = (_value_int64)}, + +#define _mongoc_structured_log_token_boolean(_key_or_null, _value_boolean) \ + {.func = _mongoc_structured_log_append_boolean, .arg1.utf8 = (_key_or_null), .arg2.boolean = (_value_boolean)}, + +#define _mongoc_structured_log_token_oid_as_hex(_key_or_null, _value_oid) \ + {.func = _mongoc_structured_log_append_oid_as_hex, .arg1.utf8 = (_key_or_null), .arg2.oid = (_value_oid)}, + +#define _mongoc_structured_log_token_bson_as_json(_key_or_null, _value_bson) \ + {.func = _mongoc_structured_log_append_bson_as_json, .arg1.utf8 = (_key_or_null), .arg2.bson = (_value_bson)}, + +#define _mongoc_structured_log_token_cmd(_cmd, ...) \ + {.func = _mongoc_structured_log_append_cmd, \ + .arg1.cmd = (_cmd), \ + .arg2.cmd_flags = \ + (0 _bsonDSL_mapMacro (_mongoc_structured_log_flag_expr, MONGOC_STRUCTURED_LOG_CMD, __VA_ARGS__))}, + +#define _mongoc_structured_log_token_server_description(_server_description, ...) \ + {.func = _mongoc_structured_log_append_server_description, \ + .arg1.server_description = (_server_description), \ + .arg2.server_description_flags = (0 _bsonDSL_mapMacro ( \ + _mongoc_structured_log_flag_expr, MONGOC_STRUCTURED_LOG_SERVER_DESCRIPTION, __VA_ARGS__))}, typedef struct mongoc_structured_log_builder_stage_t mongoc_structured_log_builder_stage_t; @@ -174,7 +172,7 @@ const mongoc_structured_log_builder_stage_t * _mongoc_structured_log_append_int64 (bson_t *bson, const mongoc_structured_log_builder_stage_t *stage); const mongoc_structured_log_builder_stage_t * -_mongoc_structured_log_append_bool (bson_t *bson, const mongoc_structured_log_builder_stage_t *stage); +_mongoc_structured_log_append_boolean (bson_t *bson, const mongoc_structured_log_builder_stage_t *stage); const mongoc_structured_log_builder_stage_t * _mongoc_structured_log_append_oid_as_hex (bson_t *bson, const mongoc_structured_log_builder_stage_t *stage); diff --git a/src/libmongoc/src/mongoc/mongoc-structured-log.c b/src/libmongoc/src/mongoc/mongoc-structured-log.c index e08bbee3446..1e4636de323 100644 --- a/src/libmongoc/src/mongoc/mongoc-structured-log.c +++ b/src/libmongoc/src/mongoc/mongoc-structured-log.c @@ -314,7 +314,7 @@ _mongoc_structured_log_append_int64 (bson_t *bson, const mongoc_structured_log_b } const mongoc_structured_log_builder_stage_t * -_mongoc_structured_log_append_bool (bson_t *bson, const mongoc_structured_log_builder_stage_t *stage) +_mongoc_structured_log_append_boolean (bson_t *bson, const mongoc_structured_log_builder_stage_t *stage) { const char *key_or_null = stage->arg1.utf8; if (key_or_null) { diff --git a/src/libmongoc/tests/test-mongoc-structured-log.c b/src/libmongoc/tests/test-mongoc-structured-log.c index bf79be8ad71..ab3c19f2e07 100644 --- a/src/libmongoc/tests/test-mongoc-structured-log.c +++ b/src/libmongoc/tests/test-mongoc-structured-log.c @@ -118,7 +118,7 @@ test_log_entry_with_extra_data (void) mongoc_structured_log (MONGOC_STRUCTURED_LOG_LEVEL_WARNING, MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND, "Plain log entry", - MONGOC_STRUCTURED_LOG_INT32 ("extra", 1)); + int32 ("extra", 1)); ASSERT_CMPINT (assumption.calls, ==, 1); restore_state (old_state); @@ -210,36 +210,30 @@ test_log_entry_with_all_data_types (void) "Log entry with all data types", // Basic BSON types. // Most support optional values (skip when key is NULL) - MONGOC_STRUCTURED_LOG_UTF8 ("kStr", "string value"), - MONGOC_STRUCTURED_LOG_UTF8 ("kNullStr", NULL), - MONGOC_STRUCTURED_LOG_UTF8 (NULL, NULL), - MONGOC_STRUCTURED_LOG_UTF8_NN ("kStrN1ZZZ", 6, non_terminated_test_string, sizeof non_terminated_test_string), - MONGOC_STRUCTURED_LOG_UTF8_N ("kStrN2", non_terminated_test_string, sizeof non_terminated_test_string), - MONGOC_STRUCTURED_LOG_UTF8_NN ("kNullStrN1ZZZ", 10, NULL, 12345), - MONGOC_STRUCTURED_LOG_UTF8_NN ("kNullStrN2", -1, NULL, 12345), - MONGOC_STRUCTURED_LOG_UTF8_NN (NULL, 999, NULL, 999), - MONGOC_STRUCTURED_LOG_UTF8_N ("kNullStrN3", NULL, 12345), - MONGOC_STRUCTURED_LOG_INT32 ("kInt32", -12345), - MONGOC_STRUCTURED_LOG_INT32 (NULL, 9999), - MONGOC_STRUCTURED_LOG_INT64 ("kInt64", 0x76543210aabbccdd), - MONGOC_STRUCTURED_LOG_INT64 (NULL, -1), - MONGOC_STRUCTURED_LOG_BOOL ("kTrue", true), - MONGOC_STRUCTURED_LOG_BOOL ("kFalse", false), - MONGOC_STRUCTURED_LOG_BOOL (NULL, true), + utf8 ("kStr", "string value"), + utf8 ("kNullStr", NULL), + utf8 (NULL, NULL), + utf8_nn ("kStrN1ZZZ", 6, non_terminated_test_string, sizeof non_terminated_test_string), + utf8_n ("kStrN2", non_terminated_test_string, sizeof non_terminated_test_string), + utf8_nn ("kNullStrN1ZZZ", 10, NULL, 12345), + utf8_nn ("kNullStrN2", -1, NULL, 12345), + utf8_nn (NULL, 999, NULL, 999), + utf8_n ("kNullStrN3", NULL, 12345), + int32 ("kInt32", -12345), + int32 (NULL, 9999), + int64 ("kInt64", 0x76543210aabbccdd), + int64 (NULL, -1), + boolean ("kTrue", true), + boolean ("kFalse", false), + boolean (NULL, true), // Deferred conversions - MONGOC_STRUCTURED_LOG_BSON_AS_JSON ("kJSON", json_doc), - MONGOC_STRUCTURED_LOG_BSON_AS_JSON (NULL, NULL), - MONGOC_STRUCTURED_LOG_OID_AS_HEX ("kOID", &oid), - MONGOC_STRUCTURED_LOG_OID_AS_HEX (NULL, NULL), + bson_as_json ("kJSON", json_doc), + bson_as_json (NULL, NULL), + oid_as_hex ("kOID", &oid), + oid_as_hex (NULL, NULL), // Common structures, with explicit set of keys to include - MONGOC_STRUCTURED_LOG_CMD (&cmd, - (MONGOC_STRUCTURED_LOG_CMD_COMMAND | MONGOC_STRUCTURED_LOG_CMD_DATABASE_NAME | - MONGOC_STRUCTURED_LOG_CMD_COMMAND_NAME | MONGOC_STRUCTURED_LOG_CMD_OPERATION_ID)), - MONGOC_STRUCTURED_LOG_SERVER_DESCRIPTION (&server_description, - (MONGOC_STRUCTURED_LOG_SERVER_DESCRIPTION_SERVER_HOST | - MONGOC_STRUCTURED_LOG_SERVER_DESCRIPTION_SERVER_PORT | - MONGOC_STRUCTURED_LOG_SERVER_DESCRIPTION_SERVER_CONNECTION_ID | - MONGOC_STRUCTURED_LOG_SERVER_DESCRIPTION_SERVICE_ID))); + cmd (&cmd, DATABASE_NAME, COMMAND_NAME, OPERATION_ID, COMMAND), + server_description (&server_description, SERVER_HOST, SERVER_PORT, SERVER_CONNECTION_ID, SERVICE_ID)); ASSERT_CMPINT (assumption.calls, ==, 1); restore_state (old_state); From c7b5248c3263a534914d4d02598be86b61c5de16 Mon Sep 17 00:00:00 2001 From: Micah Scott Date: Mon, 11 Nov 2024 21:22:54 -0800 Subject: [PATCH 044/139] bring naming into agreement with logging spec --- .../src/mongoc/mongoc-structured-log.c | 64 +++++++++++-------- .../src/mongoc/mongoc-structured-log.h | 12 +--- 2 files changed, 39 insertions(+), 37 deletions(-) diff --git a/src/libmongoc/src/mongoc/mongoc-structured-log.c b/src/libmongoc/src/mongoc/mongoc-structured-log.c index 1e4636de323..73823178cf7 100644 --- a/src/libmongoc/src/mongoc/mongoc-structured-log.c +++ b/src/libmongoc/src/mongoc/mongoc-structured-log.c @@ -95,42 +95,43 @@ _mongoc_structured_log_with_entry (const mongoc_structured_log_entry_t *entry) bson_mutex_unlock (&gStructuredLog.func_mutex); } -static mongoc_structured_log_level_t -_mongoc_structured_log_get_log_level_from_env (const char *variable) +static bool +_mongoc_structured_log_get_log_level_from_env (const char *variable, mongoc_structured_log_level_t *out) { const char *level = getenv (variable); if (!level) { - return MONGOC_STRUCTURED_LOG_DEFAULT_LEVEL; + return false; } else if (!strcasecmp (level, "trace")) { - return MONGOC_STRUCTURED_LOG_LEVEL_TRACE; + *out = MONGOC_STRUCTURED_LOG_LEVEL_TRACE; } else if (!strcasecmp (level, "debug")) { - return MONGOC_STRUCTURED_LOG_LEVEL_DEBUG; + *out = MONGOC_STRUCTURED_LOG_LEVEL_DEBUG; } else if (!strcasecmp (level, "info")) { - return MONGOC_STRUCTURED_LOG_LEVEL_INFO; + *out = MONGOC_STRUCTURED_LOG_LEVEL_INFO; } else if (!strcasecmp (level, "notice")) { - return MONGOC_STRUCTURED_LOG_LEVEL_NOTICE; + *out = MONGOC_STRUCTURED_LOG_LEVEL_NOTICE; } else if (!strcasecmp (level, "warn")) { - return MONGOC_STRUCTURED_LOG_LEVEL_WARNING; + *out = MONGOC_STRUCTURED_LOG_LEVEL_WARNING; } else if (!strcasecmp (level, "error")) { - return MONGOC_STRUCTURED_LOG_LEVEL_ERROR; + *out = MONGOC_STRUCTURED_LOG_LEVEL_ERROR; } else if (!strcasecmp (level, "critical")) { - return MONGOC_STRUCTURED_LOG_LEVEL_CRITICAL; + *out = MONGOC_STRUCTURED_LOG_LEVEL_CRITICAL; } else if (!strcasecmp (level, "alert")) { - return MONGOC_STRUCTURED_LOG_LEVEL_ALERT; + *out = MONGOC_STRUCTURED_LOG_LEVEL_ALERT; } else if (!strcasecmp (level, "emergency")) { - return MONGOC_STRUCTURED_LOG_LEVEL_EMERGENCY; + *out = MONGOC_STRUCTURED_LOG_LEVEL_EMERGENCY; } else { MONGOC_ERROR ("Invalid log level %s read for variable %s", level, variable); exit (EXIT_FAILURE); } + return true; } const char * mongoc_structured_log_get_level_name (mongoc_structured_log_level_t level) { static const char *table[] = { - "emergency", "alert", "critical", "error", "warning", "notice", "info", "debug", "trace"}; + "Emergency", "Alert", "Critical", "Error", "Warning", "Notice", "Informational", "Debug", "Trace"}; if (level >= 0 && level < (sizeof table / sizeof table[0])) { return table[level]; } @@ -140,7 +141,7 @@ mongoc_structured_log_get_level_name (mongoc_structured_log_level_t level) const char * mongoc_structured_log_get_component_name (mongoc_structured_log_component_t component) { - static const char *table[] = {"command", "sdam", "server selection", "connection"}; + static const char *table[] = {"command", "topology", "serverSelection", "connection"}; if (component >= 0 && component < (sizeof table / sizeof table[0])) { return table[component]; } @@ -150,7 +151,7 @@ mongoc_structured_log_get_component_name (mongoc_structured_log_component_t comp static int32_t _mongoc_structured_log_get_max_document_length_from_env (void) { - const char *max_length_str = getenv ("MONGODB_LOGGING_MAX_DOCUMENT_LENGTH"); + const char *max_length_str = getenv ("MONGODB_LOG_MAX_DOCUMENT_LENGTH"); if (!max_length_str) { return MONGOC_STRUCTURED_LOG_DEFAULT_MAX_DOCUMENT_LENGTH; @@ -187,23 +188,30 @@ _mongoc_structured_log_init (void) bson_mutex_init (&gStructuredLog.func_mutex); gStructuredLog.max_document_length = _mongoc_structured_log_get_max_document_length_from_env (); - mongoc_structured_log_set_max_level_for_component ( - MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND, - _mongoc_structured_log_get_log_level_from_env ("MONGODB_LOGGING_COMMAND")); - mongoc_structured_log_set_max_level_for_component ( - MONGOC_STRUCTURED_LOG_COMPONENT_CONNECTION, - _mongoc_structured_log_get_log_level_from_env ("MONGODB_LOGGING_CONNECTION")); - mongoc_structured_log_set_max_level_for_component ( - MONGOC_STRUCTURED_LOG_COMPONENT_SDAM, _mongoc_structured_log_get_log_level_from_env ("MONGODB_LOGGING_SDAM")); - mongoc_structured_log_set_max_level_for_component ( - MONGOC_STRUCTURED_LOG_COMPONENT_SERVER_SELECTION, - _mongoc_structured_log_get_log_level_from_env ("MONGODB_LOGGING_SERVER_SELECTION")); + mongoc_structured_log_level_t level; + if (!_mongoc_structured_log_get_log_level_from_env ("MONGODB_LOG_ALL", &level)) { + level = MONGOC_STRUCTURED_LOG_DEFAULT_LEVEL; + } + mongoc_structured_log_set_max_level_for_all_components (level); + + if (_mongoc_structured_log_get_log_level_from_env ("MONGODB_LOG_COMMAND", &level)) { + mongoc_structured_log_set_max_level_for_component (MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND, level); + } + if (_mongoc_structured_log_get_log_level_from_env ("MONGODB_LOG_CONNECTION", &level)) { + mongoc_structured_log_set_max_level_for_component (MONGOC_STRUCTURED_LOG_COMPONENT_CONNECTION, level); + } + if (_mongoc_structured_log_get_log_level_from_env ("MONGODB_LOG_TOPOLOGY", &level)) { + mongoc_structured_log_set_max_level_for_component (MONGOC_STRUCTURED_LOG_COMPONENT_TOPOLOGY, level); + } + if (_mongoc_structured_log_get_log_level_from_env ("MONGODB_LOG_SERVER_SELECTION", &level)) { + mongoc_structured_log_set_max_level_for_component (MONGOC_STRUCTURED_LOG_COMPONENT_SERVER_SELECTION, level); + } } static FILE * _mongoc_structured_log_open_stream (void) { - const char *log_target = getenv ("MONGODB_LOGGING_PATH"); + const char *log_target = getenv ("MONGODB_LOG_PATH"); bool log_to_stderr = !log_target || !strcmp (log_target, "stderr"); FILE *log_stream = log_to_stderr ? stderr : fopen (log_target, "a"); if (!log_stream) { @@ -232,7 +240,7 @@ mongoc_structured_log_default_handler (const mongoc_structured_log_entry_t *entr char *json_message = bson_as_relaxed_extended_json (bson_message, NULL); fprintf (_mongoc_structured_log_get_stream (), - "mongoc_structured_log -- %s -- %s -- %s\n", + "MONGODB_LOG %s %s %s\n", mongoc_structured_log_get_level_name (mongoc_structured_log_entry_get_level (entry)), mongoc_structured_log_get_component_name (mongoc_structured_log_entry_get_component (entry)), json_message); diff --git a/src/libmongoc/src/mongoc/mongoc-structured-log.h b/src/libmongoc/src/mongoc/mongoc-structured-log.h index ef848459fdd..951a54febc2 100644 --- a/src/libmongoc/src/mongoc/mongoc-structured-log.h +++ b/src/libmongoc/src/mongoc/mongoc-structured-log.h @@ -39,7 +39,7 @@ typedef enum { typedef enum { MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND = 0, - MONGOC_STRUCTURED_LOG_COMPONENT_SDAM = 1, + MONGOC_STRUCTURED_LOG_COMPONENT_TOPOLOGY = 1, MONGOC_STRUCTURED_LOG_COMPONENT_SERVER_SELECTION = 2, MONGOC_STRUCTURED_LOG_COMPONENT_CONNECTION = 3, } mongoc_structured_log_component_t; @@ -130,11 +130,8 @@ mongoc_structured_log_entry_get_component (const mongoc_structured_log_entry_t * * mongoc_structured_log_get_level_name: * @level: A log level code. * - * Returns the name for a log level code as a constant string that does + * Returns the canonical name for a log level as a constant string that does * not need to be deallocated, or NULL if the level has no known name. - * - * Strings are lower-case words: - * "emergency", "alert", "critical", "error", "warning", "notice", "info", "debug", "trace" */ MONGOC_EXPORT (const char *) mongoc_structured_log_get_level_name (mongoc_structured_log_level_t level); @@ -143,11 +140,8 @@ mongoc_structured_log_get_level_name (mongoc_structured_log_level_t level); * mongoc_structured_log_get_component_name: * @level: A log level code. * - * Returns the short name for a component as a constant string that does + * Returns the canonical name for a component as a constant string that does * not need to be deallocated, or NULL if the level has no known name. - * - * Strings are lower-case words or phrases: - * "command", "sdam", "server selection", "connection" */ MONGOC_EXPORT (const char *) mongoc_structured_log_get_component_name (mongoc_structured_log_component_t component); From 5fad75139149e6c7ac0b0a964d5771036fd9e43d Mon Sep 17 00:00:00 2001 From: Micah Scott Date: Tue, 12 Nov 2024 08:12:50 -0800 Subject: [PATCH 045/139] public log level getter, refinements to enum type conversions --- .../src/mongoc/mongoc-structured-log.c | 82 +++++++++---------- .../src/mongoc/mongoc-structured-log.h | 11 ++- 2 files changed, 51 insertions(+), 42 deletions(-) diff --git a/src/libmongoc/src/mongoc/mongoc-structured-log.c b/src/libmongoc/src/mongoc/mongoc-structured-log.c index 73823178cf7..e9da6a9144f 100644 --- a/src/libmongoc/src/mongoc/mongoc-structured-log.c +++ b/src/libmongoc/src/mongoc/mongoc-structured-log.c @@ -70,18 +70,49 @@ mongoc_structured_log_set_handler (mongoc_structured_log_func_t log_func, void * bson_mutex_unlock (&gStructuredLog.func_mutex); } +void +mongoc_structured_log_get_handler (mongoc_structured_log_func_t *log_func, void **user_data) +{ + bson_mutex_lock (&gStructuredLog.func_mutex); + *log_func = gStructuredLog.func; + *user_data = gStructuredLog.user_data; + bson_mutex_unlock (&gStructuredLog.func_mutex); +} + +void +mongoc_structured_log_set_max_level_for_component (mongoc_structured_log_component_t component, + mongoc_structured_log_level_t level) +{ + BSON_ASSERT (level >= MONGOC_STRUCTURED_LOG_LEVEL_EMERGENCY && level <= MONGOC_STRUCTURED_LOG_LEVEL_TRACE); + unsigned table_index = (unsigned) component; + BSON_ASSERT (table_index < STRUCTURED_LOG_COMPONENT_TABLE_SIZE); + mcommon_atomic_int_exchange ( + &gStructuredLog.component_level_table[table_index], level, mcommon_memory_order_relaxed); +} + +void +mongoc_structured_log_set_max_level_for_all_components (mongoc_structured_log_level_t level) +{ + for (int component = 0; component < STRUCTURED_LOG_COMPONENT_TABLE_SIZE; component++) { + mongoc_structured_log_set_max_level_for_component ((mongoc_structured_log_component_t) component, level); + } +} + +mongoc_structured_log_level_t +mongoc_structured_log_get_max_level_for_component (mongoc_structured_log_component_t component) +{ + unsigned table_index = (unsigned) component; + BSON_ASSERT (table_index < STRUCTURED_LOG_COMPONENT_TABLE_SIZE); + return mcommon_atomic_int_fetch (&gStructuredLog.component_level_table[table_index], mcommon_memory_order_relaxed); +} + bool _mongoc_structured_log_should_log (const mongoc_structured_log_envelope_t *envelope) { if (!mcommon_atomic_ptr_fetch ((void *) &gStructuredLog.func, mcommon_memory_order_relaxed)) { return false; } - - mongoc_structured_log_component_t component = envelope->component; - mongoc_structured_log_level_t level = envelope->level; - BSON_ASSERT (component >= 0 && component < STRUCTURED_LOG_COMPONENT_TABLE_SIZE); - return level <= - mcommon_atomic_int_fetch (&gStructuredLog.component_level_table[component], mcommon_memory_order_relaxed); + return envelope->level <= mongoc_structured_log_get_max_level_for_component (envelope->component); } void @@ -130,22 +161,18 @@ _mongoc_structured_log_get_log_level_from_env (const char *variable, mongoc_stru const char * mongoc_structured_log_get_level_name (mongoc_structured_log_level_t level) { + unsigned table_index = (unsigned) level; static const char *table[] = { "Emergency", "Alert", "Critical", "Error", "Warning", "Notice", "Informational", "Debug", "Trace"}; - if (level >= 0 && level < (sizeof table / sizeof table[0])) { - return table[level]; - } - return NULL; + return (table_index < (sizeof table / sizeof table[0])) ? table[table_index] : NULL; } const char * mongoc_structured_log_get_component_name (mongoc_structured_log_component_t component) { + unsigned table_index = (unsigned) component; static const char *table[] = {"command", "topology", "serverSelection", "connection"}; - if (component >= 0 && component < (sizeof table / sizeof table[0])) { - return table[component]; - } - return NULL; + return (table_index < (sizeof table / sizeof table[0])) ? table[table_index] : NULL; } static int32_t @@ -164,24 +191,6 @@ _mongoc_structured_log_get_max_document_length_from_env (void) return strtoul (max_length_str, NULL, 10); } -void -mongoc_structured_log_set_max_level_for_component (mongoc_structured_log_component_t component, - mongoc_structured_log_level_t level) -{ - BSON_ASSERT (level >= MONGOC_STRUCTURED_LOG_LEVEL_EMERGENCY); - BSON_ASSERT (level <= MONGOC_STRUCTURED_LOG_LEVEL_TRACE); - BSON_ASSERT (component >= 0 && component < STRUCTURED_LOG_COMPONENT_TABLE_SIZE); - mcommon_atomic_int_exchange (&gStructuredLog.component_level_table[component], level, mcommon_memory_order_relaxed); -} - -void -mongoc_structured_log_set_max_level_for_all_components (mongoc_structured_log_level_t level) -{ - for (size_t component = 0; component < STRUCTURED_LOG_COMPONENT_TABLE_SIZE; component++) { - mongoc_structured_log_set_max_level_for_component (component, level); - } -} - void _mongoc_structured_log_init (void) { @@ -258,15 +267,6 @@ mongoc_structured_log_document_to_json (const bson_t *document, size_t *length) return json; } -void -mongoc_structured_log_get_handler (mongoc_structured_log_func_t *log_func, void **user_data) -{ - bson_mutex_lock (&gStructuredLog.func_mutex); - *log_func = gStructuredLog.func; - *user_data = gStructuredLog.user_data; - bson_mutex_unlock (&gStructuredLog.func_mutex); -} - const mongoc_structured_log_builder_stage_t * _mongoc_structured_log_append_utf8 (bson_t *bson, const mongoc_structured_log_builder_stage_t *stage) { diff --git a/src/libmongoc/src/mongoc/mongoc-structured-log.h b/src/libmongoc/src/mongoc/mongoc-structured-log.h index 951a54febc2..8f6f883be75 100644 --- a/src/libmongoc/src/mongoc/mongoc-structured-log.h +++ b/src/libmongoc/src/mongoc/mongoc-structured-log.h @@ -80,7 +80,7 @@ mongoc_structured_log_set_handler (mongoc_structured_log_func_t log_func, void * * this severity level will be passed to mongoc_structured_log_func_t. * * By default, each component's log level comes from the environment - * variables `MONGOC_LOGGING_` captured during mongoc_init(). + * variables `MONGOC_LOG_` captured during mongoc_init(). */ MONGOC_EXPORT (void) mongoc_structured_log_set_max_level_for_component (mongoc_structured_log_component_t component, @@ -98,6 +98,15 @@ mongoc_structured_log_set_max_level_for_component (mongoc_structured_log_compone MONGOC_EXPORT (void) mongoc_structured_log_set_max_level_for_all_components (mongoc_structured_log_level_t level); +/** + * mongoc_structured_log_get_max_level_for_component: + * @component: A logging component to check the log level limit for. + * + * Returns the current maximum log level for one component. + */ +MONGOC_EXPORT (mongoc_structured_log_level_t) +mongoc_structured_log_get_max_level_for_component (mongoc_structured_log_component_t component); + /** * mongoc_structured_log_entry_message_as_bson: * @entry: A log entry to extract the message from. From 7cf023add77c4c12161d1ef34ca49eef0826e30a Mon Sep 17 00:00:00 2001 From: Micah Scott Date: Tue, 12 Nov 2024 08:33:34 -0800 Subject: [PATCH 046/139] use 'item' instead of 'token', additional comments --- .../mongoc/mongoc-structured-log-private.h | 50 +++++++++++-------- .../src/mongoc/mongoc-structured-log.c | 11 ++-- 2 files changed, 33 insertions(+), 28 deletions(-) diff --git a/src/libmongoc/src/mongoc/mongoc-structured-log-private.h b/src/libmongoc/src/mongoc/mongoc-structured-log-private.h index 71787b2aef5..78b41f7f46a 100644 --- a/src/libmongoc/src/mongoc/mongoc-structured-log-private.h +++ b/src/libmongoc/src/mongoc/mongoc-structured-log-private.h @@ -39,56 +39,56 @@ BSON_BEGIN_DECLS .envelope.level = (_level), .envelope.component = (_component), .envelope.message = (_message)}; \ if (_mongoc_structured_log_should_log (&_entry.envelope)) { \ const mongoc_structured_log_builder_stage_t _builder[] = { \ - _mongoc_structured_log_tokens_to_stages (__VA_ARGS__)}; \ + _mongoc_structured_log_items_to_stages (__VA_ARGS__)}; \ _entry.builder = _builder; \ _mongoc_structured_log_with_entry (&_entry); \ } \ } while (0) -#define _mongoc_structured_log_tokens_to_stages(...) \ - _bsonDSL_eval (_bsonDSL_mapMacro (_mongoc_structured_log_token_to_stage, ~, __VA_ARGS__)) +#define _mongoc_structured_log_items_to_stages(...) \ + _bsonDSL_eval (_bsonDSL_mapMacro (_mongoc_structured_log_item_to_stage, ~, __VA_ARGS__)) #define _mongoc_structured_log_flag_expr(_action, _constant, _counter) | (_constant##_##_action) -#define _mongoc_structured_log_token_to_stage(_action, _constant, _counter) _mongoc_structured_log_token_##_action +#define _mongoc_structured_log_item_to_stage(_action, _constant, _counter) _mongoc_structured_log_item_##_action -#define _mongoc_structured_log_token_end_of_list() {.func = NULL}, +#define _mongoc_structured_log_item_end_of_list() {.func = NULL}, -#define _mongoc_structured_log_token_utf8(_key_or_null, _value_utf8) \ +#define _mongoc_structured_log_item_utf8(_key_or_null, _value_utf8) \ {.func = _mongoc_structured_log_append_utf8, .arg1.utf8 = (_key_or_null), .arg2.utf8 = (_value_utf8)}, -#define _mongoc_structured_log_token_utf8_n(_key_literal, _value_utf8, _value_len) \ - _mongoc_structured_log_token_utf8_nn (_key_literal, strlen (_key_literal), _value_utf8, _value_len) +#define _mongoc_structured_log_item_utf8_n(_key_literal, _value_utf8, _value_len) \ + _mongoc_structured_log_item_utf8_nn (_key_literal, strlen (_key_literal), _value_utf8, _value_len) -#define _mongoc_structured_log_token_utf8_nn(_key_or_null, _key_len, _value_utf8, _value_len) \ +#define _mongoc_structured_log_item_utf8_nn(_key_or_null, _key_len, _value_utf8, _value_len) \ {.func = _mongoc_structured_log_append_utf8_n_stage0, .arg1.utf8 = (_key_or_null), .arg2.int32 = (_key_len)}, \ {.func = _mongoc_structured_log_append_utf8_n_stage1, .arg1.utf8 = (_value_utf8), .arg2.int32 = (_value_len)}, -#define _mongoc_structured_log_token_int32(_key_or_null, _value_int32) \ +#define _mongoc_structured_log_item_int32(_key_or_null, _value_int32) \ {.func = _mongoc_structured_log_append_int32, .arg1.utf8 = (_key_or_null), .arg2.int32 = (_value_int32)}, -#define _mongoc_structured_log_token_int64(_key_or_null, _value_int64) \ +#define _mongoc_structured_log_item_int64(_key_or_null, _value_int64) \ {.func = _mongoc_structured_log_append_int64, .arg1.utf8 = (_key_or_null), .arg2.int64 = (_value_int64)}, -#define _mongoc_structured_log_token_boolean(_key_or_null, _value_boolean) \ +#define _mongoc_structured_log_item_boolean(_key_or_null, _value_boolean) \ {.func = _mongoc_structured_log_append_boolean, .arg1.utf8 = (_key_or_null), .arg2.boolean = (_value_boolean)}, -#define _mongoc_structured_log_token_oid_as_hex(_key_or_null, _value_oid) \ +#define _mongoc_structured_log_item_oid_as_hex(_key_or_null, _value_oid) \ {.func = _mongoc_structured_log_append_oid_as_hex, .arg1.utf8 = (_key_or_null), .arg2.oid = (_value_oid)}, -#define _mongoc_structured_log_token_bson_as_json(_key_or_null, _value_bson) \ +#define _mongoc_structured_log_item_bson_as_json(_key_or_null, _value_bson) \ {.func = _mongoc_structured_log_append_bson_as_json, .arg1.utf8 = (_key_or_null), .arg2.bson = (_value_bson)}, -#define _mongoc_structured_log_token_cmd(_cmd, ...) \ - {.func = _mongoc_structured_log_append_cmd, \ - .arg1.cmd = (_cmd), \ - .arg2.cmd_flags = \ +#define _mongoc_structured_log_item_cmd(_cmd, ...) \ + {.func = _mongoc_structured_log_append_cmd, \ + .arg1.cmd = (_cmd), \ + .arg2.cmd_flags = \ (0 _bsonDSL_mapMacro (_mongoc_structured_log_flag_expr, MONGOC_STRUCTURED_LOG_CMD, __VA_ARGS__))}, -#define _mongoc_structured_log_token_server_description(_server_description, ...) \ - {.func = _mongoc_structured_log_append_server_description, \ - .arg1.server_description = (_server_description), \ - .arg2.server_description_flags = (0 _bsonDSL_mapMacro ( \ +#define _mongoc_structured_log_item_server_description(_server_description, ...) \ + {.func = _mongoc_structured_log_append_server_description, \ + .arg1.server_description = (_server_description), \ + .arg2.server_description_flags = (0 _bsonDSL_mapMacro ( \ _mongoc_structured_log_flag_expr, MONGOC_STRUCTURED_LOG_SERVER_DESCRIPTION, __VA_ARGS__))}, typedef struct mongoc_structured_log_builder_stage_t mongoc_structured_log_builder_stage_t; @@ -111,6 +111,12 @@ typedef enum { } mongoc_structured_log_server_description_flags_t; struct mongoc_structured_log_builder_stage_t { + // Why "stages" instead of a variable size argument list per item? + // This approach keeps function pointers and other types of data + // separated, reducing opportunities for malicious control flow. + // Most items are one stage. Items that need more arguments can use + // multiple consecutive stages, leaving the extra stages' function + // pointers unused and set to placeholder values which can be checked. mongoc_structured_log_builder_func_t func; // NULL sentinel here union { const mongoc_cmd_t *cmd; diff --git a/src/libmongoc/src/mongoc/mongoc-structured-log.c b/src/libmongoc/src/mongoc/mongoc-structured-log.c index e9da6a9144f..c6fb3afe80e 100644 --- a/src/libmongoc/src/mongoc/mongoc-structured-log.c +++ b/src/libmongoc/src/mongoc/mongoc-structured-log.c @@ -26,10 +26,11 @@ mongoc_structured_log_default_handler (const mongoc_structured_log_entry_t *entr #define STRUCTURED_LOG_COMPONENT_TABLE_SIZE (1 + (size_t) MONGOC_STRUCTURED_LOG_COMPONENT_CONNECTION) static struct { + bson_mutex_t func_mutex; // Mutex prevents func reentrancy, ensures atomic updates to (func, user_data) mongoc_structured_log_func_t func; void *user_data; - bson_mutex_t func_mutex; // Mutex prevents func reentrancy, ensures atomic updates to (func, user_data) - FILE *stream; + FILE *stream; // Only used by the default handler + int32_t max_document_length; int component_level_table[STRUCTURED_LOG_COMPONENT_TABLE_SIZE]; // Really mongoc_structured_log_level_t; int typed to // support atomic fetch @@ -109,10 +110,8 @@ mongoc_structured_log_get_max_level_for_component (mongoc_structured_log_compone bool _mongoc_structured_log_should_log (const mongoc_structured_log_envelope_t *envelope) { - if (!mcommon_atomic_ptr_fetch ((void *) &gStructuredLog.func, mcommon_memory_order_relaxed)) { - return false; - } - return envelope->level <= mongoc_structured_log_get_max_level_for_component (envelope->component); + return mcommon_atomic_ptr_fetch ((void *) &gStructuredLog.func, mcommon_memory_order_relaxed) && + envelope->level <= mongoc_structured_log_get_max_level_for_component (envelope->component); } void From 87811ae54d56455711648d278fa13a67293057de Mon Sep 17 00:00:00 2001 From: Micah Scott Date: Tue, 12 Nov 2024 09:18:05 -0800 Subject: [PATCH 047/139] public conversions between log level/component and string --- .../src/mongoc/mongoc-structured-log.c | 68 +++++++++++-------- .../src/mongoc/mongoc-structured-log.h | 20 ++++++ 2 files changed, 60 insertions(+), 28 deletions(-) diff --git a/src/libmongoc/src/mongoc/mongoc-structured-log.c b/src/libmongoc/src/mongoc/mongoc-structured-log.c index c6fb3afe80e..c204dc4802f 100644 --- a/src/libmongoc/src/mongoc/mongoc-structured-log.c +++ b/src/libmongoc/src/mongoc/mongoc-structured-log.c @@ -25,6 +25,11 @@ mongoc_structured_log_default_handler (const mongoc_structured_log_entry_t *entr #define STRUCTURED_LOG_COMPONENT_TABLE_SIZE (1 + (size_t) MONGOC_STRUCTURED_LOG_COMPONENT_CONNECTION) +static const char *gStructuredLogLevelNames[] = { + "Emergency", "Alert", "Critical", "Error", "Warning", "Notice", "Informational", "Debug", "Trace"}; + +static const char *gStructuredLogComponentNames[] = {"command", "topology", "serverSelection", "connection"}; + static struct { bson_mutex_t func_mutex; // Mutex prevents func reentrancy, ensures atomic updates to (func, user_data) mongoc_structured_log_func_t func; @@ -129,49 +134,56 @@ static bool _mongoc_structured_log_get_log_level_from_env (const char *variable, mongoc_structured_log_level_t *out) { const char *level = getenv (variable); - if (!level) { return false; - } else if (!strcasecmp (level, "trace")) { - *out = MONGOC_STRUCTURED_LOG_LEVEL_TRACE; - } else if (!strcasecmp (level, "debug")) { - *out = MONGOC_STRUCTURED_LOG_LEVEL_DEBUG; - } else if (!strcasecmp (level, "info")) { - *out = MONGOC_STRUCTURED_LOG_LEVEL_INFO; - } else if (!strcasecmp (level, "notice")) { - *out = MONGOC_STRUCTURED_LOG_LEVEL_NOTICE; - } else if (!strcasecmp (level, "warn")) { - *out = MONGOC_STRUCTURED_LOG_LEVEL_WARNING; - } else if (!strcasecmp (level, "error")) { - *out = MONGOC_STRUCTURED_LOG_LEVEL_ERROR; - } else if (!strcasecmp (level, "critical")) { - *out = MONGOC_STRUCTURED_LOG_LEVEL_CRITICAL; - } else if (!strcasecmp (level, "alert")) { - *out = MONGOC_STRUCTURED_LOG_LEVEL_ALERT; - } else if (!strcasecmp (level, "emergency")) { - *out = MONGOC_STRUCTURED_LOG_LEVEL_EMERGENCY; - } else { - MONGOC_ERROR ("Invalid log level %s read for variable %s", level, variable); - exit (EXIT_FAILURE); } - return true; + if (mongoc_structured_log_get_named_level (level, out)) { + return true; + } + MONGOC_ERROR ("Invalid log level '%s' read from environment variable %s", level, variable); + exit (EXIT_FAILURE); } const char * mongoc_structured_log_get_level_name (mongoc_structured_log_level_t level) { unsigned table_index = (unsigned) level; - static const char *table[] = { - "Emergency", "Alert", "Critical", "Error", "Warning", "Notice", "Informational", "Debug", "Trace"}; - return (table_index < (sizeof table / sizeof table[0])) ? table[table_index] : NULL; + const size_t table_size = sizeof gStructuredLogLevelNames / sizeof gStructuredLogLevelNames[0]; + return table_index < table_size ? gStructuredLogLevelNames[table_index] : NULL; +} + +bool +mongoc_structured_log_get_named_level (const char *name, mongoc_structured_log_level_t *out) +{ + const size_t table_size = sizeof gStructuredLogLevelNames / sizeof gStructuredLogLevelNames[0]; + for (unsigned table_index = 0; table_index < table_size; table_index++) { + if (!strcasecmp (name, gStructuredLogLevelNames[table_index])) { + *out = (mongoc_structured_log_level_t) table_index; + return true; + } + } + return false; } const char * mongoc_structured_log_get_component_name (mongoc_structured_log_component_t component) { unsigned table_index = (unsigned) component; - static const char *table[] = {"command", "topology", "serverSelection", "connection"}; - return (table_index < (sizeof table / sizeof table[0])) ? table[table_index] : NULL; + const size_t table_size = sizeof gStructuredLogComponentNames / sizeof gStructuredLogComponentNames[0]; + return table_index < table_size ? gStructuredLogComponentNames[table_index] : NULL; +} + +bool +mongoc_structured_log_get_named_component (const char *name, mongoc_structured_log_component_t *out) +{ + const size_t table_size = sizeof gStructuredLogComponentNames / sizeof gStructuredLogComponentNames[0]; + for (unsigned table_index = 0; table_index < table_size; table_index++) { + if (!strcasecmp (name, gStructuredLogComponentNames[table_index])) { + *out = (mongoc_structured_log_component_t) table_index; + return true; + } + } + return false; } static int32_t diff --git a/src/libmongoc/src/mongoc/mongoc-structured-log.h b/src/libmongoc/src/mongoc/mongoc-structured-log.h index 8f6f883be75..a3c7ae5072e 100644 --- a/src/libmongoc/src/mongoc/mongoc-structured-log.h +++ b/src/libmongoc/src/mongoc/mongoc-structured-log.h @@ -145,6 +145,16 @@ mongoc_structured_log_entry_get_component (const mongoc_structured_log_entry_t * MONGOC_EXPORT (const char *) mongoc_structured_log_get_level_name (mongoc_structured_log_level_t level); +/** + * mongoc_structured_log_get_named_level: + * @name: A log level name. + * @out: On success, writes a level here and returns true. Returns false if name is not recognized. + * + * Try to parse the string as a log level name. Case insensitive. + */ +MONGOC_EXPORT (bool) +mongoc_structured_log_get_named_level (const char *name, mongoc_structured_log_level_t *out); + /** * mongoc_structured_log_get_component_name: * @level: A log level code. @@ -155,6 +165,16 @@ mongoc_structured_log_get_level_name (mongoc_structured_log_level_t level); MONGOC_EXPORT (const char *) mongoc_structured_log_get_component_name (mongoc_structured_log_component_t component); +/** + * mongoc_structured_log_get_named_component: + * @name: A log component name. + * @out: On success, writes a component here and returns true. Returns false if name is not recognized. + * + * Try to parse the string as a log component name. Case insensitive. + */ +MONGOC_EXPORT (bool) +mongoc_structured_log_get_named_component (const char *name, mongoc_structured_log_component_t *out); + BSON_END_DECLS #endif /* MONGOC_STRUCTURED_LOG_H */ From 4f54845262e4dd5783604ca1a6b8f78a6f739a50 Mon Sep 17 00:00:00 2001 From: Micah Scott Date: Tue, 12 Nov 2024 09:23:34 -0800 Subject: [PATCH 048/139] nitpick, one item can result in any number of stages --- src/libmongoc/src/mongoc/mongoc-structured-log-private.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libmongoc/src/mongoc/mongoc-structured-log-private.h b/src/libmongoc/src/mongoc/mongoc-structured-log-private.h index 78b41f7f46a..70873799d44 100644 --- a/src/libmongoc/src/mongoc/mongoc-structured-log-private.h +++ b/src/libmongoc/src/mongoc/mongoc-structured-log-private.h @@ -46,11 +46,11 @@ BSON_BEGIN_DECLS } while (0) #define _mongoc_structured_log_items_to_stages(...) \ - _bsonDSL_eval (_bsonDSL_mapMacro (_mongoc_structured_log_item_to_stage, ~, __VA_ARGS__)) + _bsonDSL_eval (_bsonDSL_mapMacro (_mongoc_structured_log_item_to_stages, ~, __VA_ARGS__)) #define _mongoc_structured_log_flag_expr(_action, _constant, _counter) | (_constant##_##_action) -#define _mongoc_structured_log_item_to_stage(_action, _constant, _counter) _mongoc_structured_log_item_##_action +#define _mongoc_structured_log_item_to_stages(_action, _constant, _counter) _mongoc_structured_log_item_##_action #define _mongoc_structured_log_item_end_of_list() {.func = NULL}, From f57e55712a01bdcce7d9f279f8b3e510172024eb Mon Sep 17 00:00:00 2001 From: Micah Scott Date: Tue, 12 Nov 2024 09:47:49 -0800 Subject: [PATCH 049/139] strict parsing for max document length env var --- src/libmongoc/src/mongoc/mongoc-structured-log.c | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/libmongoc/src/mongoc/mongoc-structured-log.c b/src/libmongoc/src/mongoc/mongoc-structured-log.c index c204dc4802f..9fe6a41fe7b 100644 --- a/src/libmongoc/src/mongoc/mongoc-structured-log.c +++ b/src/libmongoc/src/mongoc/mongoc-structured-log.c @@ -189,7 +189,8 @@ mongoc_structured_log_get_named_component (const char *name, mongoc_structured_l static int32_t _mongoc_structured_log_get_max_document_length_from_env (void) { - const char *max_length_str = getenv ("MONGODB_LOG_MAX_DOCUMENT_LENGTH"); + const char *variable = "MONGODB_LOG_MAX_DOCUMENT_LENGTH"; + const char *max_length_str = getenv (variable); if (!max_length_str) { return MONGOC_STRUCTURED_LOG_DEFAULT_MAX_DOCUMENT_LENGTH; @@ -199,7 +200,14 @@ _mongoc_structured_log_get_max_document_length_from_env (void) return BSON_MAX_LEN_UNLIMITED; } - return strtoul (max_length_str, NULL, 10); + char *endptr; + long int_value = strtol (max_length_str, &endptr, 10); + if (int_value >= 0 && endptr != max_length_str && !*endptr) { + return (int32_t) int_value; + } + + MONGOC_ERROR ("Invalid length '%s' read from environment variable %s", max_length_str, variable); + exit (EXIT_FAILURE); } void From ca456b16eadefa99bdacfcd071e95a8a31fdc626 Mon Sep 17 00:00:00 2001 From: Micah Scott Date: Tue, 12 Nov 2024 10:57:44 -0800 Subject: [PATCH 050/139] work in progress, test runner support for log messages --- src/libmongoc/tests/unified/entity-map.c | 77 ++++++++++++++++++++++-- src/libmongoc/tests/unified/entity-map.h | 8 +++ src/libmongoc/tests/unified/runner.c | 47 ++++++++++++++- src/libmongoc/tests/unified/runner.h | 1 + 4 files changed, 125 insertions(+), 8 deletions(-) diff --git a/src/libmongoc/tests/unified/entity-map.c b/src/libmongoc/tests/unified/entity-map.c index 446ae1220b5..c2ae04e351e 100644 --- a/src/libmongoc/tests/unified/entity-map.c +++ b/src/libmongoc/tests/unified/entity-map.c @@ -130,7 +130,7 @@ uri_apply_options (mongoc_uri_t *uri, bson_t *opts, bson_error_t *error) return ret; } -event_t * +static event_t * event_new (const char *type) { event_t *event = NULL; @@ -140,7 +140,7 @@ event_new (const char *type) return event; } -void +static void event_destroy (event_t *event) { if (!event) { @@ -155,6 +155,29 @@ event_destroy (event_t *event) bson_free (event); } +static log_message_t * +log_message_new (const mongoc_structured_log_entry_t *entry) +{ + log_message_t *log_message = NULL; + + log_message = bson_malloc0 (sizeof (log_message_t)); + log_message->component = mongoc_structured_log_entry_get_component (entry); + log_message->level = mongoc_structured_log_entry_get_level (entry); + log_message->message = mongoc_structured_log_entry_message_as_bson (entry); + return log_message; +} + +static void +log_message_destroy (log_message_t *log_message) +{ + if (!log_message) { + return; + } + + bson_destroy (log_message->message); + bson_free (log_message); +} + static entity_t * entity_new (entity_map_t *em, const char *type) { @@ -167,6 +190,16 @@ entity_new (entity_map_t *em, const char *type) return entity; } +static void +structured_log_cb (const mongoc_structured_log_entry_t *entry, void *user_data) +{ + BSON_ASSERT_PARAM (entry); + BSON_ASSERT_PARAM (user_data); + entity_t *entity = (entity_t *) user_data; + log_message_t *log_message = log_message_new (entry); + LL_APPEND (entity->log_messages, log_message); +} + static bool is_sensitive_command (event_t *event) { @@ -219,7 +252,6 @@ should_ignore_event (entity_t *client_entity, event_t *event) return is_sensitive_command (event); } - typedef void *(*apm_func_void_t) (const void *); typedef const bson_t *(*apm_func_bson_t) (const void *); typedef const char *(*apm_func_utf8_t) (const void *); @@ -726,6 +758,29 @@ entity_client_new (entity_map_t *em, bson_t *bson, bson_error_t *error) else (do (test_error ("Unknown event type '%s'", bsonAs (cstr))))))), visitOthers ( errorf (err, "Unexpected field '%s' in storeEventsAsEntities", bson_iter_key (&bsonVisitIter)))))), + // Log messages to observe: + find (key ("observeLogMessages"), + if (not(type (doc)), then (error ("'observeLogMessages' must be a document"))), + do ({ + // Initialize all components to the lowest available level, and install a handler. + mongoc_structured_log_set_max_level_for_all_components (MONGOC_STRUCTURED_LOG_LEVEL_EMERGENCY); + mongoc_structured_log_set_handler (structured_log_cb, entity); + }), + visitEach ( + if (not(type (utf8)), then (error ("Every value in 'observeLogMessages' must be a log level string"))), + do ({ + const char *const component_name = bson_iter_key (&bsonVisitIter); + mongoc_structured_log_component_t component; + if (!mongoc_structured_log_get_named_component (component_name, &component)) { + test_error ("Unknown log component '%s' given in 'observeLogMessages'", component_name); + } + const char *const level_name = bson_iter_utf8 (&bsonVisitIter, NULL); + mongoc_structured_log_level_t level; + if (!mongoc_structured_log_get_named_level (level_name, &level)) { + test_error ("Unknown log level '%s' given in 'observeLogMessages'", component_name); + } + mongoc_structured_log_set_max_level_for_component (component, level); + }))), visitOthers ( dupPath (errpath), errorf (err, "At [%s]: Unknown key '%s' given in entity options", errpath, bson_iter_key (&bsonVisitIter)))); @@ -1703,7 +1758,6 @@ static void entity_destroy (entity_t *entity) { event_t *event = NULL; - event_t *tmp = NULL; if (!entity) { return; @@ -1715,6 +1769,7 @@ entity_destroy (entity_t *entity) mongoc_client_t *client = NULL; client = (mongoc_client_t *) entity->value; + mongoc_structured_log_set_handler(NULL, NULL); mongoc_client_destroy (client); } else if (0 == strcmp ("clientEncryption", entity->type)) { mongoc_client_encryption_t *ce = NULL; @@ -1773,9 +1828,19 @@ entity_destroy (entity_t *entity) test_error ("Attempting to destroy unrecognized entity type: %s, id: %s", entity->type, entity->id); } - LL_FOREACH_SAFE (entity->events, event, tmp) { - event_destroy (event); + event_t *tmp; + LL_FOREACH_SAFE (entity->events, event, tmp) + { + event_destroy (event); + } + } + { + log_message_t *log_message, *tmp; + LL_FOREACH_SAFE (entity->log_messages, log_message, tmp) + { + log_message_destroy (log_message); + } } _mongoc_array_destroy (&entity->observe_events); diff --git a/src/libmongoc/tests/unified/entity-map.h b/src/libmongoc/tests/unified/entity-map.h index 56035640774..bdc2d323105 100644 --- a/src/libmongoc/tests/unified/entity-map.h +++ b/src/libmongoc/tests/unified/entity-map.h @@ -34,6 +34,13 @@ typedef struct _event_t { struct _event_t *next; } event_t; +typedef struct _log_message_t { + struct _log_message_t *next; + mongoc_structured_log_component_t component; + mongoc_structured_log_level_t level; + bson_t *message; +} log_message_t; + typedef struct _observe_event_t { const char *type; // Non-owning. Type of event to observe. } observe_event_t; @@ -51,6 +58,7 @@ typedef struct _entity_t { bool *observe_sensitive_commands; struct _entity_t *next; event_t *events; + log_message_t *log_messages; struct _entity_map_t *entity_map; // Parent entity map. mongoc_array_t observe_events; // observe_event_t [N]. mongoc_array_t store_events; // store_event_t [N]. diff --git a/src/libmongoc/tests/unified/runner.c b/src/libmongoc/tests/unified/runner.c index 0eb12f1b86e..d312c50e3f2 100644 --- a/src/libmongoc/tests/unified/runner.c +++ b/src/libmongoc/tests/unified/runner.c @@ -450,6 +450,7 @@ test_new (test_file_t *test_file, bson_t *bson) bson_parser_utf8_optional (parser, "skipReason", &test->skip_reason); bson_parser_array (parser, "operations", &test->operations); bson_parser_array_optional (parser, "expectEvents", &test->expect_events); + bson_parser_array_optional (parser, "expectLogMessages", &test->expect_log_messages); bson_parser_array_optional (parser, "outcome", &test->outcome); bson_parser_parse_or_assert (parser, bson); bson_parser_destroy (parser); @@ -471,6 +472,7 @@ test_destroy (test_t *test) entity_map_destroy (test->entity_map); bson_destroy (test->outcome); bson_destroy (test->expect_events); + bson_destroy (test->expect_log_messages); bson_destroy (test->operations); bson_destroy (test->run_on_requirements); bson_free (test->description); @@ -1174,6 +1176,15 @@ test_check_expected_events_for_client (test_t *test, bson_t *expected_events_for return ret; } +static bool +test_check_expected_log_messages_for_client (test_t *test, + bson_t *expected_log_messages_for_client, + bson_error_t *error) +{ + // @todo + return true; +} + static bool test_check_expected_events (test_t *test, bson_error_t *error) { @@ -1190,7 +1201,7 @@ test_check_expected_events (test_t *test, bson_error_t *error) bson_t expected_events_for_client; bson_iter_bson (&iter, &expected_events_for_client); if (!test_check_expected_events_for_client (test, &expected_events_for_client, error)) { - test_diagnostics_error_info ("checking expectations: %s", tmp_json (&expected_events_for_client)); + test_diagnostics_error_info ("checking expected events: %s", tmp_json (&expected_events_for_client)); goto done; } } @@ -1201,6 +1212,33 @@ test_check_expected_events (test_t *test, bson_error_t *error) return ret; } +static bool +test_check_expected_log_messages (test_t *test, bson_error_t *error) +{ + bool ret = false; + bson_iter_t iter; + + if (!test->expect_log_messages) { + ret = true; + goto done; + } + + BSON_FOREACH (test->expect_log_messages, iter) + { + bson_t expected_log_messages_for_client; + bson_iter_bson (&iter, &expected_log_messages_for_client); + if (!test_check_expected_log_messages_for_client (test, &expected_log_messages_for_client, error)) { + test_diagnostics_error_info ("checking expected log messages: %s", + tmp_json (&expected_log_messages_for_client)); + goto done; + } + } + + ret = true; +done: + return ret; +} + static bool test_check_outcome_collection (test_t *test, bson_t *collection_data, bson_error_t *error) { @@ -1607,7 +1645,12 @@ test_run (test_t *test, bson_error_t *error) entity_map_disable_event_listeners (test->entity_map); if (!test_check_expected_events (test, error)) { - test_diagnostics_error_info ("%s", "checking expectations"); + test_diagnostics_error_info ("%s", "checking expected events"); + goto done; + } + + if (!test_check_expected_log_messages (test, error)) { + test_diagnostics_error_info ("%s", "checking expected log messages"); goto done; } diff --git a/src/libmongoc/tests/unified/runner.h b/src/libmongoc/tests/unified/runner.h index 62cc3a7c7f0..ea28db919ab 100644 --- a/src/libmongoc/tests/unified/runner.h +++ b/src/libmongoc/tests/unified/runner.h @@ -58,6 +58,7 @@ typedef struct { char *skip_reason; bson_t *operations; bson_t *expect_events; + bson_t *expect_log_messages; bson_t *outcome; entity_map_t *entity_map; failpoint_t *failpoints; From 0dbcafefc3b54ff9ae40da320c0218db566dcfdf Mon Sep 17 00:00:00 2001 From: Micah Scott Date: Tue, 12 Nov 2024 12:21:52 -0800 Subject: [PATCH 051/139] clear global structured log handler when any client is destroyed --- src/libmongoc/tests/unified/entity-map.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libmongoc/tests/unified/entity-map.c b/src/libmongoc/tests/unified/entity-map.c index c2ae04e351e..54c7e20fa68 100644 --- a/src/libmongoc/tests/unified/entity-map.c +++ b/src/libmongoc/tests/unified/entity-map.c @@ -1769,7 +1769,7 @@ entity_destroy (entity_t *entity) mongoc_client_t *client = NULL; client = (mongoc_client_t *) entity->value; - mongoc_structured_log_set_handler(NULL, NULL); + mongoc_structured_log_set_handler (NULL, NULL); mongoc_client_destroy (client); } else if (0 == strcmp ("clientEncryption", entity->type)) { mongoc_client_encryption_t *ce = NULL; From b189cb5ea90c0e1c427d9663902b9a5b16cd608b Mon Sep 17 00:00:00 2001 From: Micah Scott Date: Tue, 12 Nov 2024 12:28:40 -0800 Subject: [PATCH 052/139] first pass at createEntities unified test op --- src/libmongoc/tests/unified/operation.c | 35 ++++++++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/src/libmongoc/tests/unified/operation.c b/src/libmongoc/tests/unified/operation.c index 1fbcdca7707..cec480f4077 100644 --- a/src/libmongoc/tests/unified/operation.c +++ b/src/libmongoc/tests/unified/operation.c @@ -3784,6 +3784,37 @@ operation_updateSearchIndex (test_t *test, operation_t *op, result_t *result, bs return ret; } +static bool +operation_create_entities (test_t *test, operation_t *op, result_t *result, bson_error_t *error) +{ + bool ret = false; + bson_parser_t *bp = bson_parser_new (); + bson_t *entities = NULL; + bson_iter_t entity_iter; + + bson_parser_array_optional (bp, "entities", &entities); + if (!bson_parser_parse (bp, op->arguments, error)) { + goto done; + } + + BSON_FOREACH (entities, entity_iter) + { + bson_t entity; + bson_iter_bson (&entity_iter, &entity); + bool create_ret = entity_map_create (test->entity_map, &entity, error); + bson_destroy (&entity); + if (!create_ret) { + goto done; + } + } + + result_from_ok (result); + ret = true; +done: + bson_parser_destroy_with_parsed_fields (bp); + return ret; +} + typedef struct { const char *op; bool (*fn) (test_t *, operation_t *, result_t *, bson_error_t *); @@ -3877,13 +3908,15 @@ operation_run (test_t *test, bson_t *op_bson, bson_error_t *error) {"download", operation_download}, {"upload", operation_upload}, - /* Session operations. */ + /* Session operations */ {"endSession", operation_end_session}, {"startTransaction", operation_start_transaction}, {"commitTransaction", operation_commit_transaction}, {"withTransaction", operation_with_transaction}, {"abortTransaction", operation_abort_transaction}, + /* Entity operations */ + {"createEntities", operation_create_entities}, }; bool ret = false; From 7ddda2dc387dae0ab2ae8bb2185381951773a607 Mon Sep 17 00:00:00 2001 From: Micah Scott Date: Tue, 12 Nov 2024 14:10:46 -0800 Subject: [PATCH 053/139] remove misleading initializers The "just_false" initializer here is particularly bad, as had it been effective at all it would result in free()'ing a stack address. None of these initializers are needed: bson_parse zeroes its outputs early, when the parameters are added to the parser. --- src/libmongoc/tests/unified/runner.c | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/src/libmongoc/tests/unified/runner.c b/src/libmongoc/tests/unified/runner.c index d312c50e3f2..81657a7715f 100644 --- a/src/libmongoc/tests/unified/runner.c +++ b/src/libmongoc/tests/unified/runner.c @@ -1089,17 +1089,16 @@ static bool test_check_expected_events_for_client (test_t *test, bson_t *expected_events_for_client, bson_error_t *error) { bool ret = false; - bson_parser_t *bp = NULL; - char *client_id = NULL; - bson_t *expected_events = NULL; - bool just_false = false; - bool *ignore_extra_events = &just_false; + bson_parser_t *bp; + char *client_id; + bson_t *expected_events; + bool *ignore_extra_events; entity_t *entity = NULL; bson_iter_t iter; - event_t *eiter = NULL; + event_t *eiter; uint32_t expected_num_events; - uint32_t actual_num_events = 0; - char *event_type = NULL; + uint32_t actual_num_events; + char *event_type; bp = bson_parser_new (); bson_parser_utf8 (bp, "client", &client_id); From 0e18fc06d38f2793ae87263b0dac455fa9b758f1 Mon Sep 17 00:00:00 2001 From: Micah Scott Date: Tue, 12 Nov 2024 15:29:49 -0800 Subject: [PATCH 054/139] Implement test_check_expected_log_messages_for_client --- src/libmongoc/tests/unified/runner.c | 157 +++++++++++++++++++++++++-- 1 file changed, 148 insertions(+), 9 deletions(-) diff --git a/src/libmongoc/tests/unified/runner.c b/src/libmongoc/tests/unified/runner.c index 81657a7715f..64a1949979e 100644 --- a/src/libmongoc/tests/unified/runner.c +++ b/src/libmongoc/tests/unified/runner.c @@ -1175,15 +1175,6 @@ test_check_expected_events_for_client (test_t *test, bson_t *expected_events_for return ret; } -static bool -test_check_expected_log_messages_for_client (test_t *test, - bson_t *expected_log_messages_for_client, - bson_error_t *error) -{ - // @todo - return true; -} - static bool test_check_expected_events (test_t *test, bson_error_t *error) { @@ -1205,9 +1196,157 @@ test_check_expected_events (test_t *test, bson_error_t *error) } } + ret = true; +done: + return ret; +} + +static bool +test_check_log_message (test_t *test, bson_t *expected, log_message_t *actual, bson_error_t *error) +{ + bool ret = false; + + bson_parser_t *bp = bson_parser_new (); + char *expected_level_str; + char *expected_component_str; + bool *failure_is_redacted; + bson_t *expected_message_doc; + bson_parser_utf8 (bp, "level", &expected_level_str); + bson_parser_utf8 (bp, "component", &expected_component_str); + bson_parser_bool_optional (bp, "failureIsRedacted", &failure_is_redacted); + bson_parser_doc (bp, "data", &expected_message_doc); + if (!bson_parser_parse (bp, expected, error)) { + goto done; + } + + const char *actual_level_str = mongoc_structured_log_get_level_name (actual->level); + if (0 != bson_strcasecmp (expected_level_str, actual_level_str)) { + test_set_error (error, "expected log level: %s, but got: %s", expected_level_str, actual_level_str); + goto done; + } + + const char *actual_component_str = mongoc_structured_log_get_component_name (actual->component); + if (0 != bson_strcasecmp (expected_component_str, actual_component_str)) { + test_set_error (error, "expected log component: %s, but got: %s", expected_component_str, actual_component_str); + goto done; + } + + // @todo + BSON_ASSERT (failure_is_redacted == 0 || *failure_is_redacted == false); + + if (!bson_equal (expected_message_doc, actual->message)) { + char *expected_message_str = bson_as_relaxed_extended_json (expected_message_doc, NULL); + char *actual_message_str = bson_as_relaxed_extended_json (actual->message, NULL); + test_set_error (error, "expected log message: %s, but got: %s", expected_message_str, actual_message_str); + bson_free (expected_message_str); + bson_free (actual_message_str); + goto done; + } ret = true; done: + bson_parser_destroy_with_parsed_fields (bp); + return ret; +} + +static bool +test_log_message_should_be_ignored (test_t *test, + log_message_t *message, + bson_t *optional_ignore_list, + bson_error_t *error) +{ + if (optional_ignore_list) { + bson_iter_t iter; + BSON_FOREACH (optional_ignore_list, iter) + { + bson_t expected; + bson_iter_bson (&iter, &expected); + bool is_match = test_check_log_message (test, &expected, message, error); + bson_destroy (&expected); + if (is_match) { + return true; + } + } + } + return false; +} + +static bool +test_check_expected_log_messages_for_client (test_t *test, + bson_t *expected_log_messages_for_client, + bson_error_t *error) +{ + bool ret = false; + + bson_parser_t *bp = bson_parser_new (); + char *client_id; + bson_t *expected_messages; + bson_t *ignore_messages; + bool *ignore_extra_messages; + bson_parser_utf8 (bp, "client", &client_id); + bson_parser_array (bp, "messages", &expected_messages); + bson_parser_array_optional (bp, "ignoreMessages", &ignore_messages); + bson_parser_bool_optional (bp, "ignoreExtraMessages", &ignore_extra_messages); + if (!bson_parser_parse (bp, expected_log_messages_for_client, error)) { + goto done; + } + + entity_t *entity = entity_map_get (test->entity_map, client_id, error); + if (0 != strcmp (entity->type, "client")) { + test_set_error (error, "expected entity %s to be client, got: %s", entity->id, entity->type); + goto done; + } + + log_message_t *actual_message_iter = entity->log_messages; + bson_iter_t expected_message_iter; + bool expected_message_iter_ok = + bson_iter_init (&expected_message_iter, expected_messages) && bson_iter_next (&expected_message_iter); + + while (actual_message_iter || expected_message_iter_ok) { + if (actual_message_iter && + test_log_message_should_be_ignored (test, actual_message_iter, ignore_messages, error)) { + actual_message_iter = actual_message_iter->next; + continue; + } + if (!actual_message_iter) { + bson_t expected_message; + bson_iter_bson (&expected_message_iter, &expected_message); + test_diagnostics_error_info ("missing expected log message: %s", tmp_json (&expected_message)); + test_set_error (error, "additional log messages expected beyond those collected"); + bson_destroy (&expected_message); + goto done; + } + if (!expected_message_iter_ok) { + if (ignore_extra_messages && *ignore_extra_messages) { + break; + } else { + test_diagnostics_error_info ("extra log message: %s", tmp_json (actual_message_iter->message)); + test_set_error (error, "unexpected extra log messages"); + goto done; + } + } + bson_t expected_message; + bson_iter_bson (&expected_message_iter, &expected_message); + bool is_match = test_check_log_message (test, &expected_message, actual_message_iter, error); + if (!is_match) { + test_diagnostics_error_info ("expected log message: %s\nactual log message: %s, %s, %s", + tmp_json (&expected_message), + mongoc_structured_log_get_level_name (actual_message_iter->level), + mongoc_structured_log_get_component_name (actual_message_iter->component), + tmp_json (actual_message_iter->message)); + test_set_error (error, "log message does not match expected"); + } + bson_destroy (&expected_message); + if (!is_match) { + goto done; + } + actual_message_iter = actual_message_iter->next; + expected_message_iter_ok = bson_iter_next (&expected_message_iter); + } + + ret = true; +done: + bson_parser_destroy_with_parsed_fields (bp); return ret; } From 0b0ea30de94b9d92b9f7a2b6a5fe91eb0a4eca02 Mon Sep 17 00:00:00 2001 From: Micah Scott Date: Tue, 12 Nov 2024 15:34:55 -0800 Subject: [PATCH 055/139] spec says command logging is at debug level, not info --- src/libmongoc/src/mongoc/mongoc-client.c | 6 +++--- src/libmongoc/src/mongoc/mongoc-cluster.c | 6 +++--- src/libmongoc/src/mongoc/mongoc-cursor-legacy.c | 2 +- src/libmongoc/src/mongoc/mongoc-cursor.c | 6 +++--- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/libmongoc/src/mongoc/mongoc-client.c b/src/libmongoc/src/mongoc/mongoc-client.c index 53ca42823c0..e8b69ac20ae 100644 --- a/src/libmongoc/src/mongoc/mongoc-client.c +++ b/src/libmongoc/src/mongoc/mongoc-client.c @@ -2167,7 +2167,7 @@ _mongoc_client_monitor_op_killcursors (mongoc_cluster_t *cluster, _mongoc_client_prepare_killcursors_command (cursor_id, collection, &doc); mongoc_structured_log ( - MONGOC_STRUCTURED_LOG_LEVEL_INFO, + MONGOC_STRUCTURED_LOG_LEVEL_DEBUG, MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND, "Command started", int32 ("requestId", cluster->request_id), @@ -2228,7 +2228,7 @@ _mongoc_client_monitor_op_killcursors_succeeded (mongoc_cluster_t *cluster, bson_append_array_builder_end (&doc, cursors_unknown); mongoc_structured_log ( - MONGOC_STRUCTURED_LOG_LEVEL_INFO, + MONGOC_STRUCTURED_LOG_LEVEL_DEBUG, MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND, "Command succeeded", int32 ("requestId", cluster->request_id), @@ -2286,7 +2286,7 @@ _mongoc_client_monitor_op_killcursors_failed (mongoc_cluster_t *cluster, bson_append_int32 (&doc, "ok", 2, 0); mongoc_structured_log ( - MONGOC_STRUCTURED_LOG_LEVEL_INFO, + MONGOC_STRUCTURED_LOG_LEVEL_DEBUG, MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND, "Command failed", int32 ("requestId", cluster->request_id), diff --git a/src/libmongoc/src/mongoc/mongoc-cluster.c b/src/libmongoc/src/mongoc/mongoc-cluster.c index a7a3c96eb14..7c4518e4bdd 100644 --- a/src/libmongoc/src/mongoc/mongoc-cluster.c +++ b/src/libmongoc/src/mongoc/mongoc-cluster.c @@ -534,7 +534,7 @@ mongoc_cluster_run_command_monitored (mongoc_cluster_t *cluster, mongoc_cmd_t *c } mongoc_structured_log ( - MONGOC_STRUCTURED_LOG_LEVEL_INFO, + MONGOC_STRUCTURED_LOG_LEVEL_DEBUG, MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND, "Command started", int32 ("requestId", request_id), @@ -565,7 +565,7 @@ mongoc_cluster_run_command_monitored (mongoc_cluster_t *cluster, mongoc_cmd_t *c } mongoc_structured_log ( - MONGOC_STRUCTURED_LOG_LEVEL_INFO, + MONGOC_STRUCTURED_LOG_LEVEL_DEBUG, MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND, "Command succeeded", int32 ("requestId", request_id), @@ -598,7 +598,7 @@ mongoc_cluster_run_command_monitored (mongoc_cluster_t *cluster, mongoc_cmd_t *c int64_t duration = bson_get_monotonic_time () - started; mongoc_structured_log ( - MONGOC_STRUCTURED_LOG_LEVEL_INFO, + MONGOC_STRUCTURED_LOG_LEVEL_DEBUG, MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND, "Command failed", int32 ("requestId", request_id), diff --git a/src/libmongoc/src/mongoc/mongoc-cursor-legacy.c b/src/libmongoc/src/mongoc/mongoc-cursor-legacy.c index 041f39c1f78..658f1348965 100644 --- a/src/libmongoc/src/mongoc/mongoc-cursor-legacy.c +++ b/src/libmongoc/src/mongoc/mongoc-cursor-legacy.c @@ -50,7 +50,7 @@ _mongoc_cursor_monitor_legacy_get_more (mongoc_cursor_t *cursor, mongoc_server_s _mongoc_cursor_prepare_getmore_command (cursor, &doc); mongoc_structured_log ( - MONGOC_STRUCTURED_LOG_LEVEL_INFO, + MONGOC_STRUCTURED_LOG_LEVEL_DEBUG, MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND, "Command started", int32 ("requestId", client->cluster.request_id), diff --git a/src/libmongoc/src/mongoc/mongoc-cursor.c b/src/libmongoc/src/mongoc/mongoc-cursor.c index 8a582db34b3..a76582e0799 100644 --- a/src/libmongoc/src/mongoc/mongoc-cursor.c +++ b/src/libmongoc/src/mongoc/mongoc-cursor.c @@ -647,7 +647,7 @@ _mongoc_cursor_monitor_command (mongoc_cursor_t *cursor, client = cursor->client; mongoc_structured_log ( - MONGOC_STRUCTURED_LOG_LEVEL_INFO, + MONGOC_STRUCTURED_LOG_LEVEL_DEBUG, MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND, "Command started", int32 ("requestId", client->cluster.request_id), @@ -739,7 +739,7 @@ _mongoc_cursor_monitor_succeeded (mongoc_cursor_t *cursor, bson_destroy (&docs_array); - mongoc_structured_log (MONGOC_STRUCTURED_LOG_LEVEL_INFO, + mongoc_structured_log (MONGOC_STRUCTURED_LOG_LEVEL_DEBUG, MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND, "Command succeeded", int32 ("requestId", client->cluster.request_id), @@ -796,7 +796,7 @@ _mongoc_cursor_monitor_failed (mongoc_cursor_t *cursor, bsonBuildDecl (reply, kv ("ok", int32 (0))); char *db = bson_strndup (cursor->ns, cursor->dblen); - mongoc_structured_log (MONGOC_STRUCTURED_LOG_LEVEL_INFO, + mongoc_structured_log (MONGOC_STRUCTURED_LOG_LEVEL_DEBUG, MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND, "Command failed", int32 ("requestId", client->cluster.request_id), From 271233d78bb3e5885256e2c68237256481903ae5 Mon Sep 17 00:00:00 2001 From: Micah Scott Date: Tue, 12 Nov 2024 16:44:27 -0800 Subject: [PATCH 056/139] Test comparison wants bson_match, not exact equality --- src/libmongoc/tests/unified/runner.c | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/src/libmongoc/tests/unified/runner.c b/src/libmongoc/tests/unified/runner.c index 64a1949979e..cea720b628e 100644 --- a/src/libmongoc/tests/unified/runner.c +++ b/src/libmongoc/tests/unified/runner.c @@ -1231,15 +1231,21 @@ test_check_log_message (test_t *test, bson_t *expected, log_message_t *actual, b goto done; } - // @todo - BSON_ASSERT (failure_is_redacted == 0 || *failure_is_redacted == false); - - if (!bson_equal (expected_message_doc, actual->message)) { - char *expected_message_str = bson_as_relaxed_extended_json (expected_message_doc, NULL); - char *actual_message_str = bson_as_relaxed_extended_json (actual->message, NULL); - test_set_error (error, "expected log message: %s, but got: %s", expected_message_str, actual_message_str); - bson_free (expected_message_str); - bson_free (actual_message_str); + // @todo; failureIsRedacted needs implementing + BSON_ASSERT (!failure_is_redacted); + + bson_val_t *expected_val = bson_val_from_bson (expected_message_doc); + bson_val_t *actual_val = bson_val_from_bson (actual->message); + bool is_match = bson_match (expected_val, actual_val, false, error); + bson_val_destroy (actual_val); + bson_val_destroy (expected_val); + + if (!is_match) { + char *expected_str = bson_as_relaxed_extended_json (expected_message_doc, NULL); + char *actual_str = bson_as_relaxed_extended_json (actual->message, NULL); + test_set_error (error, "expected log message: %s, but got: %s", expected_str, actual_str); + bson_free (actual_str); + bson_free (expected_str); goto done; } From a883c3f3f87dc156fa56754d01ed230946c830cb Mon Sep 17 00:00:00 2001 From: Micah Scott Date: Tue, 12 Nov 2024 17:26:50 -0800 Subject: [PATCH 057/139] recursive log suppression internal to test-libmongoc --- src/libmongoc/tests/test-libmongoc.c | 30 ++++++++++++++++++++++++ src/libmongoc/tests/test-libmongoc.h | 2 ++ src/libmongoc/tests/unified/entity-map.c | 8 ++++--- 3 files changed, 37 insertions(+), 3 deletions(-) diff --git a/src/libmongoc/tests/test-libmongoc.c b/src/libmongoc/tests/test-libmongoc.c index 0ba5bdc70f4..5f7e60e1853 100644 --- a/src/libmongoc/tests/test-libmongoc.c +++ b/src/libmongoc/tests/test-libmongoc.c @@ -31,6 +31,7 @@ #include "test-conveniences.h" #include "test-libmongoc.h" #include +#include #ifdef BSON_HAVE_STRINGS_H #include @@ -58,11 +59,36 @@ typedef struct { static bson_mutex_t captured_logs_mutex; static mongoc_array_t captured_logs; static bool capturing_logs; +static int suppress_structured_logs_counter_atomic; #ifdef MONGOC_ENABLE_SSL static mongoc_ssl_opt_t gSSLOptions; #endif +bool +test_is_suppressing_structured_logs (void) +{ + int c = mcommon_atomic_int_fetch (&suppress_structured_logs_counter_atomic, mcommon_memory_order_seq_cst); + BSON_ASSERT (c >= 0); + return c > 0; +} + +// Ignore logs generated by test-internal operations. Can be nested. Structured and traditional logs. +static void +test_begin_suppressing_structured_logs (void) +{ + mcommon_atomic_int_fetch_add (&suppress_structured_logs_counter_atomic, 1, mcommon_memory_order_seq_cst); +} + +static void +test_end_suppressing_structured_logs (void) +{ + int previous = + mcommon_atomic_int_fetch_sub (&suppress_structured_logs_counter_atomic, 1, mcommon_memory_order_seq_cst); + BSON_ASSERT (previous > 0); +} + + static log_entry_t * log_entry_create (mongoc_log_level_t level, const char *msg) { @@ -842,6 +868,8 @@ call_hello_with_host_and_port (const char *host_and_port, bson_t *reply) mongoc_client_t *client; bson_error_t error; + test_begin_suppressing_structured_logs (); + if (test_framework_get_user_password (&user, &password)) { uri_str = bson_strdup_printf ( "mongodb://%s:%s@%s%s", user, password, host_and_port, test_framework_get_ssl () ? "/?ssl=true" : ""); @@ -888,6 +916,7 @@ call_hello_with_host_and_port (const char *host_and_port, bson_t *reply) mongoc_client_destroy (client); mongoc_uri_destroy (uri); bson_free (uri_str); + test_end_suppressing_structured_logs (); } /* @@ -2566,6 +2595,7 @@ test_libmongoc_destroy (TestSuite *suite) TestSuite_Destroy (suite); capture_logs (false); /* clear entries */ _mongoc_array_destroy (&captured_logs); + BSON_ASSERT (!test_is_suppressing_structured_logs ()); mongoc_cleanup (); } diff --git a/src/libmongoc/tests/test-libmongoc.h b/src/libmongoc/tests/test-libmongoc.h index 5f9ecfe4122..575fe59cb57 100644 --- a/src/libmongoc/tests/test-libmongoc.h +++ b/src/libmongoc/tests/test-libmongoc.h @@ -36,6 +36,8 @@ char * gen_collection_name (const char *prefix); mongoc_collection_t * get_test_collection (mongoc_client_t *client, const char *prefix); +bool +test_is_suppressing_structured_logs (void); void capture_logs (bool capture); void diff --git a/src/libmongoc/tests/unified/entity-map.c b/src/libmongoc/tests/unified/entity-map.c index 54c7e20fa68..92369ca206f 100644 --- a/src/libmongoc/tests/unified/entity-map.c +++ b/src/libmongoc/tests/unified/entity-map.c @@ -195,9 +195,11 @@ structured_log_cb (const mongoc_structured_log_entry_t *entry, void *user_data) { BSON_ASSERT_PARAM (entry); BSON_ASSERT_PARAM (user_data); - entity_t *entity = (entity_t *) user_data; - log_message_t *log_message = log_message_new (entry); - LL_APPEND (entity->log_messages, log_message); + if (!test_is_suppressing_structured_logs ()) { + entity_t *entity = (entity_t *) user_data; + log_message_t *log_message = log_message_new (entry); + LL_APPEND (entity->log_messages, log_message); + } } static bool From 2a1f9a4ba3ff06c77a62ba063effa2919526d0f6 Mon Sep 17 00:00:00 2001 From: Micah Scott Date: Tue, 12 Nov 2024 17:39:29 -0800 Subject: [PATCH 058/139] we must omit all-zero serviceId from logs --- src/libmongoc/src/mongoc/mongoc-structured-log.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/libmongoc/src/mongoc/mongoc-structured-log.c b/src/libmongoc/src/mongoc/mongoc-structured-log.c index 9fe6a41fe7b..b10cbc3e606 100644 --- a/src/libmongoc/src/mongoc/mongoc-structured-log.c +++ b/src/libmongoc/src/mongoc/mongoc-structured-log.c @@ -434,9 +434,12 @@ _mongoc_structured_log_append_server_description (bson_t *bson, const mongoc_str BSON_APPEND_INT64 (bson, "serverConnectionId", sd->server_connection_id); } if (flags & MONGOC_STRUCTURED_LOG_SERVER_DESCRIPTION_SERVICE_ID) { - char str[25]; - bson_oid_to_string (&sd->service_id, str); - BSON_APPEND_UTF8 (bson, "serviceId", str); + static const bson_oid_t oid_zero; + if (!bson_oid_equal (&sd->service_id, &oid_zero)) { + char str[25]; + bson_oid_to_string (&sd->service_id, str); + BSON_APPEND_UTF8 (bson, "serviceId", str); + } } return stage + 1; } From d54dd315869e099248e2332a6142d30ccb4919be Mon Sep 17 00:00:00 2001 From: Micah Scott Date: Tue, 12 Nov 2024 18:47:49 -0800 Subject: [PATCH 059/139] stub implementation for waitForEvent with CMAP ignored --- src/libmongoc/tests/unified/operation.c | 61 +++++++++++++++++++++++-- src/libmongoc/tests/unified/runner.c | 18 ++++++++ src/libmongoc/tests/unified/runner.h | 4 ++ 3 files changed, 80 insertions(+), 3 deletions(-) diff --git a/src/libmongoc/tests/unified/operation.c b/src/libmongoc/tests/unified/operation.c index cec480f4077..a9e429fd8aa 100644 --- a/src/libmongoc/tests/unified/operation.c +++ b/src/libmongoc/tests/unified/operation.c @@ -3788,15 +3788,15 @@ static bool operation_create_entities (test_t *test, operation_t *op, result_t *result, bson_error_t *error) { bool ret = false; - bson_parser_t *bp = bson_parser_new (); - bson_t *entities = NULL; - bson_iter_t entity_iter; + bson_parser_t *bp = bson_parser_new (); + bson_t *entities; bson_parser_array_optional (bp, "entities", &entities); if (!bson_parser_parse (bp, op->arguments, error)) { goto done; } + bson_iter_t entity_iter; BSON_FOREACH (entities, entity_iter) { bson_t entity; @@ -3815,6 +3815,60 @@ operation_create_entities (test_t *test, operation_t *op, result_t *result, bson return ret; } +static bool +operation_wait_for_event (test_t *test, operation_t *op, result_t *result, bson_error_t *error) +{ + bool ret = false; + int64_t start_time = bson_get_monotonic_time (); + + bson_parser_t *bp = bson_parser_new (); + char *client_id; + bson_t *expected_event; + int64_t *expected_count; + bson_parser_utf8 (bp, "client", &client_id); + bson_parser_doc (bp, "event", &expected_event); + bson_parser_int (bp, "count", &expected_count); + if (!bson_parser_parse (bp, op->arguments, error)) { + goto done; + } + + entity_t *client = entity_map_get (test->entity_map, client_id, error); + if (!client) { + goto done; + } + + // @todo CDRIVER-3525 test support for CMAP events once supported by libmongoc + bson_iter_t iter; + bson_iter_init (&iter, expected_event); + bson_iter_next (&iter); + const char *expected_event_type = bson_iter_key (&iter); + if (is_unsupported_event_type (expected_event_type)) { + MONGOC_DEBUG ("Skipping wait for unsupported event type '%s'", expected_event_type); + result_from_ok (result); + ret = true; + goto done; + } + + while (true) { + int64_t count; + if (!test_count_matching_events_for_client (test, client, expected_event, error, &count)) { + goto done; + } + if (count >= *expected_count) { + break; + } + ASSERT_CMPINT64 (bson_get_monotonic_time () - start_time, <, (int64_t) 10 * 1000 * 1000); + // @todo waiting not implemented + BSON_ASSERT (false); + } + + result_from_ok (result); + ret = true; +done: + bson_parser_destroy_with_parsed_fields (bp); + return ret; +} + typedef struct { const char *op; bool (*fn) (test_t *, operation_t *, result_t *, bson_error_t *); @@ -3833,6 +3887,7 @@ operation_run (test_t *test, bson_t *op_bson, bson_error_t *error) {"listDatabases", operation_list_databases}, {"listDatabaseNames", operation_list_database_names}, {"clientBulkWrite", operation_client_bulkwrite}, + {"waitForEvent", operation_wait_for_event}, /* ClientEncryption operations */ {"createDataKey", operation_create_datakey}, diff --git a/src/libmongoc/tests/unified/runner.c b/src/libmongoc/tests/unified/runner.c index cea720b628e..859f3ce13d3 100644 --- a/src/libmongoc/tests/unified/runner.c +++ b/src/libmongoc/tests/unified/runner.c @@ -1085,6 +1085,24 @@ test_check_event (test_t *test, bson_t *expected, event_t *actual, bson_error_t return ret; } +bool +test_count_matching_events_for_client ( + test_t *test, entity_t *client, bson_t *expected_event, bson_error_t *error, int64_t *count_out) +{ + int64_t count = 0; + + event_t *eiter; + LL_FOREACH (client->events, eiter) + { + if (test_check_event (test, expected_event, eiter, error)) { + count++; + } + } + + *count_out = count; + return true; +} + static bool test_check_expected_events_for_client (test_t *test, bson_t *expected_events_for_client, bson_error_t *error) { diff --git a/src/libmongoc/tests/unified/runner.h b/src/libmongoc/tests/unified/runner.h index ea28db919ab..a6e9ee7fae6 100644 --- a/src/libmongoc/tests/unified/runner.h +++ b/src/libmongoc/tests/unified/runner.h @@ -69,6 +69,10 @@ typedef struct { void register_failpoint (test_t *test, char *failpoint, char *client_id, uint32_t server_id); +bool +test_count_matching_events_for_client ( + test_t *test, entity_t *client, bson_t *expected_event, bson_error_t *error, int64_t *count_out); + /* Run a directory of test files through the unified test runner. */ void run_unified_tests (TestSuite *suite, const char *base, const char *subdir); From 690f10d901e5472bf9c5783485c82a687ee7210d Mon Sep 17 00:00:00 2001 From: Micah Scott Date: Wed, 13 Nov 2024 10:22:49 -0800 Subject: [PATCH 060/139] centralize the concept of all-zero oid in common-oid-private --- src/common/src/common-oid-private.h | 36 ++++++++++++++++ src/common/src/common-oid.c | 33 +++++++++++++++ src/common/tests/test-common-oid.c | 42 +++++++++++++++++++ src/libmongoc/CMakeLists.txt | 2 + src/libmongoc/src/mongoc/mongoc-apm.c | 9 ++-- src/libmongoc/src/mongoc/mongoc-cluster.c | 5 ++- .../mongoc-server-description-private.h | 4 +- .../src/mongoc/mongoc-server-description.c | 18 +++----- .../src/mongoc/mongoc-structured-log.c | 4 +- .../mongoc-topology-description-private.h | 2 +- .../src/mongoc/mongoc-topology-private.h | 4 +- src/libmongoc/src/mongoc/mongoc-topology.c | 3 +- src/libmongoc/tests/json-test-operations.c | 3 +- src/libmongoc/tests/json-test.c | 5 ++- src/libmongoc/tests/test-libmongoc-main.c | 1 + src/libmongoc/tests/test-mongoc-client.c | 5 ++- src/libmongoc/tests/test-mongoc-cluster.c | 7 ++-- src/libmongoc/tests/test-mongoc-exhaust.c | 3 +- src/libmongoc/tests/test-mongoc-sdam.c | 3 +- .../tests/test-mongoc-topology-description.c | 11 ++--- src/libmongoc/tests/test-mongoc-topology.c | 8 ++-- src/libmongoc/tests/unified/runner.c | 7 ++-- 22 files changed, 163 insertions(+), 52 deletions(-) create mode 100644 src/common/src/common-oid-private.h create mode 100644 src/common/src/common-oid.c create mode 100644 src/common/tests/test-common-oid.c diff --git a/src/common/src/common-oid-private.h b/src/common/src/common-oid-private.h new file mode 100644 index 00000000000..b6121545475 --- /dev/null +++ b/src/common/src/common-oid-private.h @@ -0,0 +1,36 @@ +/* + * Copyright 2009-present MongoDB, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "common-prelude.h" + +#ifndef MONGO_C_DRIVER_COMMON_OID_PRIVATE_H +#define MONGO_C_DRIVER_COMMON_OID_PRIVATE_H + +#include + +BSON_BEGIN_DECLS + +extern const bson_oid_t kZeroObjectId; + +void +mcommon_oid_set_zero (bson_oid_t *oid); + +bool +mcommon_oid_is_zero (const bson_oid_t *oid); + +BSON_END_DECLS + +#endif /* MONGO_C_DRIVER_COMMON_OID_PRIVATE_H */ diff --git a/src/common/src/common-oid.c b/src/common/src/common-oid.c new file mode 100644 index 00000000000..687f47fe577 --- /dev/null +++ b/src/common/src/common-oid.c @@ -0,0 +1,33 @@ +/* + * Copyright 2009-present MongoDB, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +const bson_oid_t kZeroObjectId = {{0}}; + +void +mcommon_oid_set_zero (bson_oid_t *oid) +{ + BSON_ASSERT (oid); + memset (oid, 0, sizeof *oid); +} + +bool +mcommon_oid_is_zero (const bson_oid_t *oid) +{ + BSON_ASSERT (oid); + return bson_oid_equal_unsafe (oid, &kZeroObjectId); +} diff --git a/src/common/tests/test-common-oid.c b/src/common/tests/test-common-oid.c new file mode 100644 index 00000000000..c17566f01ba --- /dev/null +++ b/src/common/tests/test-common-oid.c @@ -0,0 +1,42 @@ +/* + * Copyright 2009-present MongoDB, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "TestSuite.h" + +#include + +static void +test_mcommon_oid_zero (void) +{ + bson_oid_t oid; + bson_oid_init_from_string (&oid, "000000000000000000000000"); + BSON_ASSERT (true == bson_oid_equal (&oid, &kZeroObjectId)); + BSON_ASSERT (true == mcommon_oid_is_zero (&oid)); + bson_oid_init_from_string (&oid, "010000000000000000000000"); + BSON_ASSERT (false == mcommon_oid_is_zero (&oid)); + bson_oid_init_from_string (&oid, "000000000000000000000001"); + BSON_ASSERT (false == mcommon_oid_is_zero (&oid)); + bson_oid_init_from_string (&oid, "ffffffffffffffffffffffff"); + BSON_ASSERT (false == mcommon_oid_is_zero (&oid)); + mcommon_oid_set_zero (&oid); + BSON_ASSERT (true == mcommon_oid_is_zero (&oid)); +} + +void +test_mcommon_oid_install (TestSuite *suite) +{ + TestSuite_Add (suite, "/mcommon/oid/zero", test_mcommon_oid_zero); +} diff --git a/src/libmongoc/CMakeLists.txt b/src/libmongoc/CMakeLists.txt index c03a368e6dd..b2c707e2355 100644 --- a/src/libmongoc/CMakeLists.txt +++ b/src/libmongoc/CMakeLists.txt @@ -664,6 +664,7 @@ set (MONGOC_SOURCES ${mongo-c-driver_SOURCE_DIR}/src/common/src/common-atomic.c ${mongo-c-driver_SOURCE_DIR}/src/common/src/common-b64.c ${mongo-c-driver_SOURCE_DIR}/src/common/src/common-md5.c + ${mongo-c-driver_SOURCE_DIR}/src/common/src/common-oid.c ${mongo-c-driver_SOURCE_DIR}/src/common/src/common-thread.c ${PROJECT_SOURCE_DIR}/src/mongoc/mongoc-crypto.c @@ -1032,6 +1033,7 @@ endif () set (test-libmongoc-sources ${mongo-c-driver_SOURCE_DIR}/src/common/tests/test-common-atomic.c ${mongo-c-driver_SOURCE_DIR}/src/common/tests/test-common-cmp.c + ${mongo-c-driver_SOURCE_DIR}/src/common/tests/test-common-oid.c ${mongo-c-driver_SOURCE_DIR}/src/libbson/tests/corpus-test.c ${mongo-c-driver_SOURCE_DIR}/src/libbson/tests/corpus-test.h ${mongo-c-driver_SOURCE_DIR}/src/libbson/tests/test-atomic.c diff --git a/src/libmongoc/src/mongoc/mongoc-apm.c b/src/libmongoc/src/mongoc/mongoc-apm.c index 221d225b181..ef693d62903 100644 --- a/src/libmongoc/src/mongoc/mongoc-apm.c +++ b/src/libmongoc/src/mongoc/mongoc-apm.c @@ -18,8 +18,7 @@ #include "mongoc-apm-private.h" #include "mongoc-cmd-private.h" #include "mongoc-handshake-private.h" - -static bson_oid_t kObjectIdZero = {{0}}; +#include /* * An Application Performance Management (APM) implementation, complying with @@ -370,7 +369,7 @@ mongoc_apm_command_started_get_server_id (const mongoc_apm_command_started_t *ev const bson_oid_t * mongoc_apm_command_started_get_service_id (const mongoc_apm_command_started_t *event) { - if (0 == bson_oid_compare (&event->service_id, &kObjectIdZero)) { + if (mcommon_oid_is_zero (&event->service_id)) { /* serviceId is unset. */ return NULL; } @@ -466,7 +465,7 @@ mongoc_apm_command_succeeded_get_server_id (const mongoc_apm_command_succeeded_t const bson_oid_t * mongoc_apm_command_succeeded_get_service_id (const mongoc_apm_command_succeeded_t *event) { - if (0 == bson_oid_compare (&event->service_id, &kObjectIdZero)) { + if (mcommon_oid_is_zero (&event->service_id)) { /* serviceId is unset. */ return NULL; } @@ -562,7 +561,7 @@ mongoc_apm_command_failed_get_server_id (const mongoc_apm_command_failed_t *even const bson_oid_t * mongoc_apm_command_failed_get_service_id (const mongoc_apm_command_failed_t *event) { - if (0 == bson_oid_compare (&event->service_id, &kObjectIdZero)) { + if (mcommon_oid_is_zero (&event->service_id)) { /* serviceId is unset. */ return NULL; } diff --git a/src/libmongoc/src/mongoc/mongoc-cluster.c b/src/libmongoc/src/mongoc/mongoc-cluster.c index 7c4518e4bdd..302a3284d85 100644 --- a/src/libmongoc/src/mongoc/mongoc-cluster.c +++ b/src/libmongoc/src/mongoc/mongoc-cluster.c @@ -60,6 +60,7 @@ #include #include +#include #include @@ -2041,9 +2042,9 @@ _mongoc_cluster_stream_for_server (mongoc_cluster_t *cluster, mongoc_topology_description_invalidate_server (tdmod.new_td, server_id, err_ptr); mongoc_cluster_disconnect_node (cluster, server_id); /* This is not load balanced mode, so there are no service IDs associated - * with connections. Pass kZeroServiceId to clear the entire connection + * with connections. Pass kZeroObjectId to clear the entire connection * pool to this server. */ - _mongoc_topology_description_clear_connection_pool (tdmod.new_td, server_id, &kZeroServiceId); + _mongoc_topology_description_clear_connection_pool (tdmod.new_td, server_id, &kZeroObjectId); if (!topology->single_threaded) { _mongoc_topology_background_monitoring_cancel_check (topology, server_id); diff --git a/src/libmongoc/src/mongoc/mongoc-server-description-private.h b/src/libmongoc/src/mongoc/mongoc-server-description-private.h index 452d8c3186b..17391404181 100644 --- a/src/libmongoc/src/mongoc/mongoc-server-description-private.h +++ b/src/libmongoc/src/mongoc/mongoc-server-description-private.h @@ -122,7 +122,7 @@ struct _mongoc_server_description_t { /* _generation_map_ stores all generations for all service IDs associated * with this server. _generation_map_ is only accessed on the server * description for monitoring. In non-load-balanced mode, there are no - * service IDs. The only server generation is mapped from kZeroServiceID */ + * service IDs. The only server generation is mapped from kZeroObjectId */ mongoc_generation_map_t *_generation_map_; bson_oid_t service_id; int64_t server_connection_id; @@ -218,8 +218,6 @@ mongoc_server_description_topology_version_cmp (const bson_t *tv1, const bson_t void mongoc_server_description_set_topology_version (mongoc_server_description_t *sd, const bson_t *tv); -extern const bson_oid_t kZeroServiceId; - bool mongoc_server_description_has_service_id (const mongoc_server_description_t *description); diff --git a/src/libmongoc/src/mongoc/mongoc-server-description.c b/src/libmongoc/src/mongoc/mongoc-server-description.c index 6fb0ef40e9f..5fd94e36992 100644 --- a/src/libmongoc/src/mongoc/mongoc-server-description.c +++ b/src/libmongoc/src/mongoc/mongoc-server-description.c @@ -27,15 +27,12 @@ #include #include +#include #include #define ALPHA 0.2 -static bson_oid_t kObjectIdZero = {{0}}; - -const bson_oid_t kZeroServiceId = {{0}}; - static bool _match_tag_set (const mongoc_server_description_t *sd, bson_iter_t *tag_set_iter); @@ -96,8 +93,8 @@ mongoc_server_description_reset (mongoc_server_description_t *sd) sd->me = NULL; sd->current_primary = NULL; sd->set_version = MONGOC_NO_SET_VERSION; - bson_oid_copy_unsafe (&kObjectIdZero, &sd->election_id); - bson_oid_copy_unsafe (&kObjectIdZero, &sd->service_id); + mcommon_oid_set_zero (&sd->election_id); + mcommon_oid_set_zero (&sd->service_id); sd->server_connection_id = MONGOC_NO_SERVER_CONNECTION_ID; } @@ -259,7 +256,7 @@ mongoc_server_description_has_set_version (const mongoc_server_description_t *de bool mongoc_server_description_has_election_id (const mongoc_server_description_t *description) { - return 0 != bson_oid_compare (&description->election_id, &kObjectIdZero); + return !mcommon_oid_is_zero (&description->election_id); } /* @@ -462,7 +459,7 @@ mongoc_server_description_set_election_id (mongoc_server_description_t *descript if (election_id) { bson_oid_copy_unsafe (election_id, &description->election_id); } else { - bson_oid_copy_unsafe (&kObjectIdZero, &description->election_id); + mcommon_oid_set_zero (&description->election_id); } } @@ -1222,8 +1219,5 @@ mongoc_server_description_set_topology_version (mongoc_server_description_t *sd, bool mongoc_server_description_has_service_id (const mongoc_server_description_t *description) { - if (0 == bson_oid_compare (&description->service_id, &kZeroServiceId)) { - return false; - } - return true; + return !mcommon_oid_is_zero (&description->service_id); } diff --git a/src/libmongoc/src/mongoc/mongoc-structured-log.c b/src/libmongoc/src/mongoc/mongoc-structured-log.c index b10cbc3e606..1a2a073bc09 100644 --- a/src/libmongoc/src/mongoc/mongoc-structured-log.c +++ b/src/libmongoc/src/mongoc/mongoc-structured-log.c @@ -19,6 +19,7 @@ #include "mongoc-thread-private.h" #include "mongoc-util-private.h" #include "common-atomic-private.h" +#include "common-oid-private.h" static void mongoc_structured_log_default_handler (const mongoc_structured_log_entry_t *entry, void *user_data); @@ -434,8 +435,7 @@ _mongoc_structured_log_append_server_description (bson_t *bson, const mongoc_str BSON_APPEND_INT64 (bson, "serverConnectionId", sd->server_connection_id); } if (flags & MONGOC_STRUCTURED_LOG_SERVER_DESCRIPTION_SERVICE_ID) { - static const bson_oid_t oid_zero; - if (!bson_oid_equal (&sd->service_id, &oid_zero)) { + if (!mcommon_oid_is_zero (&sd->service_id)) { char str[25]; bson_oid_to_string (&sd->service_id, str); BSON_APPEND_UTF8 (bson, "serviceId", str); diff --git a/src/libmongoc/src/mongoc/mongoc-topology-description-private.h b/src/libmongoc/src/mongoc/mongoc-topology-description-private.h index 870635707f5..12c5c125b86 100644 --- a/src/libmongoc/src/mongoc/mongoc-topology-description-private.h +++ b/src/libmongoc/src/mongoc/mongoc-topology-description-private.h @@ -165,7 +165,7 @@ mongoc_topology_description_reconcile (mongoc_topology_description_t *td, mongoc * @param td The topology description that will be updated. * @param server_id The ID of the server to invalidate. * @param service_id A service ID for load-balanced deployments. Use - * kZeroServiceID if not applicable. + * kZeroObjectId if not applicable. * * @note Not applicable to single-threaded clients, which only maintain a * single connection per server and therefore have no connection pool. diff --git a/src/libmongoc/src/mongoc/mongoc-topology-private.h b/src/libmongoc/src/mongoc/mongoc-topology-private.h index a94b5b76788..e38956ed815 100644 --- a/src/libmongoc/src/mongoc/mongoc-topology-private.h +++ b/src/libmongoc/src/mongoc/mongoc-topology-private.h @@ -411,7 +411,7 @@ typedef enum { * @param generation The generation of the server description the caller was * using. * @param service_id A service ID for a load-balanced deployment. If not - * applicable, pass kZeroServiceID. + * applicable, pass kZeroObjectId. * @return true If the topology was updated and the pool was cleared. * @return false If no modifications were made and the error was ignored. * @@ -470,7 +470,7 @@ _mongoc_topology_get_srv_polling_rescan_interval_ms (mongoc_topology_t const *to * @param td The topology that contains the server * @param server_id The ID of the server to inspect * @param service_id The service ID of the connection if applicable, or - * kZeroServiceID. + * kZeroObjectId. * @returns uint32_t A generation counter for the given server, or zero if the * server does not exist in the topology. */ diff --git a/src/libmongoc/src/mongoc/mongoc-topology.c b/src/libmongoc/src/mongoc/mongoc-topology.c index be86f365cd8..b4125bcbb03 100644 --- a/src/libmongoc/src/mongoc/mongoc-topology.c +++ b/src/libmongoc/src/mongoc/mongoc-topology.c @@ -38,6 +38,7 @@ #include #include #include +#include static void _topology_collect_errors (const mongoc_topology_description_t *topology, bson_error_t *error_out); @@ -172,7 +173,7 @@ _mongoc_topology_scanner_cb ( /* Server monitoring: When a server check fails due to a network error * (including a network timeout), the client MUST clear its connection * pool for the server */ - _mongoc_topology_description_clear_connection_pool (td, id, &kZeroServiceId); + _mongoc_topology_description_clear_connection_pool (td, id, &kZeroObjectId); } /* Server Discovery and Monitoring Spec: "Once a server is connected, the diff --git a/src/libmongoc/tests/json-test-operations.c b/src/libmongoc/tests/json-test-operations.c index 92e80ded567..d080d8543a0 100644 --- a/src/libmongoc/tests/json-test-operations.c +++ b/src/libmongoc/tests/json-test-operations.c @@ -27,6 +27,7 @@ #include "mongoc/mongoc-topology-private.h" #include "mongoc/mongoc-uri-private.h" #include "mongoc/mongoc-util-private.h" +#include #include "json-test-operations.h" #include "json-test.h" @@ -2009,7 +2010,7 @@ _get_total_pool_cleared_event (json_test_ctx_t *ctx) const mongoc_server_description_t *sd; sd = mongoc_set_get_item_const (mc_tpld_servers_const (td.ptr), i); - total += mc_tpl_sd_get_generation (sd, &kZeroServiceId); + total += mc_tpl_sd_get_generation (sd, &kZeroObjectId); } mc_tpld_drop_ref (&td); return total; diff --git a/src/libmongoc/tests/json-test.c b/src/libmongoc/tests/json-test.c index 75d96cb5250..bf4d413d2f5 100644 --- a/src/libmongoc/tests/json-test.c +++ b/src/libmongoc/tests/json-test.c @@ -21,6 +21,7 @@ #include "mongoc/mongoc-util-private.h" #include "mongoc/mongoc-uri-private.h" #include "mongoc/mongoc-client-side-encryption.h" +#include #include "json-test.h" #include "json-test-operations.h" @@ -265,7 +266,7 @@ process_sdam_test_hello_responses (bson_t *phase, mongoc_topology_t *topology) generation = bson_iter_int32 (&app_error_field_iter); } else { /* Default to the current generation. */ - generation = mc_tpl_sd_get_generation (sd, &kZeroServiceId); + generation = mc_tpl_sd_get_generation (sd, &kZeroObjectId); } BSON_ASSERT (bson_iter_init_find (&app_error_field_iter, &app_error, "maxWireVersion")); @@ -304,7 +305,7 @@ process_sdam_test_hello_responses (bson_t *phase, mongoc_topology_t *topology) memset (&err, 0, sizeof (bson_error_t)); _mongoc_topology_handle_app_error ( - topology, sd->id, handshake_complete, type, &response, &err, max_wire_version, generation, &kZeroServiceId); + topology, sd->id, handshake_complete, type, &response, &err, max_wire_version, generation, &kZeroObjectId); mc_tpld_drop_ref (&td); } } diff --git a/src/libmongoc/tests/test-libmongoc-main.c b/src/libmongoc/tests/test-libmongoc-main.c index 921145caf01..20b25c40cfa 100644 --- a/src/libmongoc/tests/test-libmongoc-main.c +++ b/src/libmongoc/tests/test-libmongoc-main.c @@ -48,6 +48,7 @@ main (int argc, char *argv[]) TEST_INSTALL (test_bson_cmp_install); TEST_INSTALL (test_mcommon_cmp_install); TEST_INSTALL (test_mcommon_atomic_install); + TEST_INSTALL (test_mcommon_oid_install); /* libmongoc */ diff --git a/src/libmongoc/tests/test-mongoc-client.c b/src/libmongoc/tests/test-mongoc-client.c index 6bedc8e10c5..29270e9257a 100644 --- a/src/libmongoc/tests/test-mongoc-client.c +++ b/src/libmongoc/tests/test-mongoc-client.c @@ -24,6 +24,7 @@ #include "mock_server/mock-server.h" #include "mock_server/mock-rs.h" #include // BEGIN_IGNORE_DEPRECATIONS +#include #ifdef BSON_HAVE_STRINGS_H @@ -3717,7 +3718,7 @@ test_mongoc_client_recv_network_error (void) /* The server should be a standalone. */ sd = mongoc_topology_description_server_by_id_const (mc_tpld_unsafe_get_const (client->topology), 1, &error); ASSERT_OR_PRINT (sd, error); - generation = mc_tpl_sd_get_generation (sd, &kZeroServiceId); + generation = mc_tpl_sd_get_generation (sd, &kZeroObjectId); BSON_ASSERT (sd->type == MONGOC_SERVER_STANDALONE); mock_server_destroy (server); @@ -3733,7 +3734,7 @@ test_mongoc_client_recv_network_error (void) td = mc_tpld_take_ref (client->topology); sd = mongoc_topology_description_server_by_id_const (td.ptr, 1, &error); ASSERT_OR_PRINT (sd, error); - ASSERT_CMPINT (mc_tpl_sd_get_generation (sd, &kZeroServiceId), ==, generation + 1); + ASSERT_CMPINT (mc_tpl_sd_get_generation (sd, &kZeroObjectId), ==, generation + 1); BSON_ASSERT (sd->type == MONGOC_SERVER_UNKNOWN); mongoc_client_destroy (client); diff --git a/src/libmongoc/tests/test-mongoc-cluster.c b/src/libmongoc/tests/test-mongoc-cluster.c index de74d49f9af..c3a929dcf2d 100644 --- a/src/libmongoc/tests/test-mongoc-cluster.c +++ b/src/libmongoc/tests/test-mongoc-cluster.c @@ -5,6 +5,7 @@ #include "mongoc/mongoc-client-pool-private.h" #include "mongoc/mongoc-topology-background-monitoring-private.h" #include "mongoc/mongoc-uri-private.h" +#include #include "mock_server/mock-server.h" #include "mock_server/future.h" @@ -1669,8 +1670,7 @@ test_cluster_stream_invalidation_single (void) ASSERT_OR_PRINT (stream, error); BSON_ASSERT (mongoc_cluster_stream_valid (&client->cluster, stream)); tdmod = mc_tpld_modify_begin (client->topology); - _mongoc_topology_description_clear_connection_pool ( - tdmod.new_td, mongoc_server_description_id (sd), &kZeroServiceId); + _mongoc_topology_description_clear_connection_pool (tdmod.new_td, mongoc_server_description_id (sd), &kZeroObjectId); mc_tpld_modify_commit (tdmod); BSON_ASSERT (!mongoc_cluster_stream_valid (&client->cluster, stream)); mongoc_server_stream_cleanup (stream); @@ -1720,8 +1720,7 @@ test_cluster_stream_invalidation_pooled (void) ASSERT_OR_PRINT (stream, error); BSON_ASSERT (mongoc_cluster_stream_valid (&client->cluster, stream)); tdmod = mc_tpld_modify_begin (client->topology); - _mongoc_topology_description_clear_connection_pool ( - tdmod.new_td, mongoc_server_description_id (sd), &kZeroServiceId); + _mongoc_topology_description_clear_connection_pool (tdmod.new_td, mongoc_server_description_id (sd), &kZeroObjectId); mc_tpld_modify_commit (tdmod); BSON_ASSERT (!mongoc_cluster_stream_valid (&client->cluster, stream)); mongoc_server_stream_cleanup (stream); diff --git a/src/libmongoc/tests/test-mongoc-exhaust.c b/src/libmongoc/tests/test-mongoc-exhaust.c index b6645dcf3e6..dcdff3a9fdc 100644 --- a/src/libmongoc/tests/test-mongoc-exhaust.c +++ b/src/libmongoc/tests/test-mongoc-exhaust.c @@ -13,6 +13,7 @@ #include "mock_server/future-functions.h" #include "mock_server/mock-server.h" #include // BEGIN_IGNORE_DEPRECATIONS +#include #undef MONGOC_LOG_DOMAIN @@ -81,7 +82,7 @@ get_generation (mongoc_client_t *client, mongoc_cursor_t *cursor) sd = mongoc_topology_description_server_by_id_const (td.ptr, server_id, &error); ASSERT_OR_PRINT (sd, error); - generation = mc_tpl_sd_get_generation (sd, &kZeroServiceId); + generation = mc_tpl_sd_get_generation (sd, &kZeroObjectId); mc_tpld_drop_ref (&td); return generation; diff --git a/src/libmongoc/tests/test-mongoc-sdam.c b/src/libmongoc/tests/test-mongoc-sdam.c index 87e41d02047..3b26f682b30 100644 --- a/src/libmongoc/tests/test-mongoc-sdam.c +++ b/src/libmongoc/tests/test-mongoc-sdam.c @@ -1,6 +1,7 @@ #include #include +#include #include "json-test.h" @@ -84,7 +85,7 @@ _topology_has_description (const mongoc_topology_description_t *topology, bson_t BSON_ASSERT (bson_iter_recurse (&server_iter, &iter)); BSON_ASSERT (bson_iter_find (&iter, "generation") && BSON_ITER_HOLDS_INT32 (&iter)); expected_generation = bson_iter_int32 (&iter); - ASSERT_CMPINT32 (expected_generation, ==, mc_tpl_sd_get_generation (sd, &kZeroServiceId)); + ASSERT_CMPINT32 (expected_generation, ==, mc_tpl_sd_get_generation (sd, &kZeroObjectId)); } else if (strcmp ("logicalSessionTimeoutMinutes", bson_iter_key (&server_iter)) == 0) { if (BSON_ITER_HOLDS_NULL (&server_iter)) { if (sd->session_timeout_minutes != MONGOC_NO_SESSIONS) { diff --git a/src/libmongoc/tests/test-mongoc-topology-description.c b/src/libmongoc/tests/test-mongoc-topology-description.c index 4351416a8f1..ca1e403e7c2 100644 --- a/src/libmongoc/tests/test-mongoc-topology-description.c +++ b/src/libmongoc/tests/test-mongoc-topology-description.c @@ -2,6 +2,7 @@ #include "mongoc/mongoc-set-private.h" #include "mongoc/mongoc-client-pool-private.h" #include "mongoc/mongoc-client-private.h" +#include #include "TestSuite.h" #include "test-libmongoc.h" @@ -258,11 +259,11 @@ test_topology_pool_clear (void) topology = mongoc_topology_new (uri, true); tdmod = mc_tpld_modify_begin (topology); - ASSERT_CMPUINT32 (0, ==, _mongoc_topology_get_connection_pool_generation (tdmod.new_td, 1, &kZeroServiceId)); - ASSERT_CMPUINT32 (0, ==, _mongoc_topology_get_connection_pool_generation (tdmod.new_td, 2, &kZeroServiceId)); - _mongoc_topology_description_clear_connection_pool (tdmod.new_td, 1, &kZeroServiceId); - ASSERT_CMPUINT32 (1, ==, _mongoc_topology_get_connection_pool_generation (tdmod.new_td, 1, &kZeroServiceId)); - ASSERT_CMPUINT32 (0, ==, _mongoc_topology_get_connection_pool_generation (tdmod.new_td, 2, &kZeroServiceId)); + ASSERT_CMPUINT32 (0, ==, _mongoc_topology_get_connection_pool_generation (tdmod.new_td, 1, &kZeroObjectId)); + ASSERT_CMPUINT32 (0, ==, _mongoc_topology_get_connection_pool_generation (tdmod.new_td, 2, &kZeroObjectId)); + _mongoc_topology_description_clear_connection_pool (tdmod.new_td, 1, &kZeroObjectId); + ASSERT_CMPUINT32 (1, ==, _mongoc_topology_get_connection_pool_generation (tdmod.new_td, 1, &kZeroObjectId)); + ASSERT_CMPUINT32 (0, ==, _mongoc_topology_get_connection_pool_generation (tdmod.new_td, 2, &kZeroObjectId)); mongoc_uri_destroy (uri); mc_tpld_modify_drop (tdmod); diff --git a/src/libmongoc/tests/test-mongoc-topology.c b/src/libmongoc/tests/test-mongoc-topology.c index f60485306cd..0e38db11e1e 100644 --- a/src/libmongoc/tests/test-mongoc-topology.c +++ b/src/libmongoc/tests/test-mongoc-topology.c @@ -1,6 +1,7 @@ #include #include #include +#include #include "mongoc/mongoc-client-private.h" #include "mongoc/mongoc-server-api-private.h" @@ -595,12 +596,11 @@ test_invalid_cluster_node (void *ctx) ASSERT_OR_PRINT (sd, error); /* Both generations match, and are the first generation. */ ASSERT_CMPINT32 (cluster_node->handshake_sd->generation, ==, 0); - ASSERT_CMPINT32 (mc_tpl_sd_get_generation (sd, &kZeroServiceId), ==, 0); + ASSERT_CMPINT32 (mc_tpl_sd_get_generation (sd, &kZeroObjectId), ==, 0); /* update the server's generation, simulating a connection pool clearing */ tdmod = mc_tpld_modify_begin (client->topology); - mc_tpl_sd_increment_generation (mongoc_topology_description_server_by_id (tdmod.new_td, id, &error), - &kZeroServiceId); + mc_tpl_sd_increment_generation (mongoc_topology_description_server_by_id (tdmod.new_td, id, &error), &kZeroObjectId); mc_tpld_modify_commit (tdmod); /* cluster discards node and creates new one with the current generation */ @@ -656,7 +656,7 @@ test_max_wire_version_race_condition (void *ctx) tdmod = mc_tpld_modify_begin (client->topology); sd = mongoc_set_get (mc_tpld_servers (tdmod.new_td), id); BSON_ASSERT (sd); - mc_tpl_sd_increment_generation (sd, &kZeroServiceId); + mc_tpl_sd_increment_generation (sd, &kZeroObjectId); mongoc_server_description_reset (sd); mc_tpld_modify_commit (tdmod); diff --git a/src/libmongoc/tests/unified/runner.c b/src/libmongoc/tests/unified/runner.c index 859f3ce13d3..0bc72dd4afa 100644 --- a/src/libmongoc/tests/unified/runner.c +++ b/src/libmongoc/tests/unified/runner.c @@ -26,6 +26,7 @@ #include "util.h" #include #include +#include typedef struct { const char *file_description; @@ -1052,11 +1053,9 @@ test_check_event (test_t *test, bson_t *expected, event_t *actual, bson_error_t } if (expected_has_service_id) { - char oid_str[25] = {0}; - bool has_service_id = false; - + bool has_service_id = !mcommon_oid_is_zero (&actual->service_id); + char oid_str[25]; bson_oid_to_string (&actual->service_id, oid_str); - has_service_id = 0 != bson_oid_compare (&actual->service_id, &kZeroServiceId); if (*expected_has_service_id && !has_service_id) { test_error ("expected serviceId, but got none"); From 506bf05a4223010b5d3a17cbe0b00e8f82b1a6b4 Mon Sep 17 00:00:00 2001 From: Micah Scott Date: Wed, 13 Nov 2024 12:37:55 -0800 Subject: [PATCH 061/139] Implement $$matchAsDocument and $$matchAsRoot in bson_match --- src/libmongoc/tests/bsonutil/bson-match.c | 276 +++++++++++++++------- src/libmongoc/tests/bsonutil/bson-match.h | 21 +- src/libmongoc/tests/unified/entity-map.c | 37 ++- 3 files changed, 226 insertions(+), 108 deletions(-) diff --git a/src/libmongoc/tests/bsonutil/bson-match.c b/src/libmongoc/tests/bsonutil/bson-match.c index 269cef20fe3..cac7b95e26e 100644 --- a/src/libmongoc/tests/bsonutil/bson-match.c +++ b/src/libmongoc/tests/bsonutil/bson-match.c @@ -23,7 +23,7 @@ typedef struct _special_functor_t { special_fn fn; - void *ctx; + void *user_data; char *keyword; struct _special_functor_t *next; } special_functor_t; @@ -32,7 +32,7 @@ struct _bson_matcher_t { special_functor_t *specials; }; -#define MATCH_ERR(format, ...) test_set_error (error, "match error at path: '%s': " format, path, __VA_ARGS__) +#define MATCH_ERR(format, ...) test_set_error (error, "match error at path: '%s': " format, context->path, __VA_ARGS__) static char * get_first_key (const bson_t *bson) @@ -62,18 +62,16 @@ is_special_match (const bson_t *bson) /* implements $$placeholder */ static bool -special_placeholder (bson_matcher_t *matcher, +special_placeholder (const bson_matcher_context_t *context, const bson_t *assertion, const bson_val_t *actual, - void *ctx, - const char *path, + void *user_data, bson_error_t *error) { - BSON_UNUSED (matcher); + BSON_UNUSED (context); BSON_UNUSED (assertion); BSON_UNUSED (actual); - BSON_UNUSED (ctx); - BSON_UNUSED (path); + BSON_UNUSED (user_data); BSON_UNUSED (error); /* Nothing to do (not an operator, just a reserved key value). The meaning @@ -83,19 +81,18 @@ special_placeholder (bson_matcher_t *matcher, /* implements $$exists */ static bool -special_exists (bson_matcher_t *matcher, +special_exists (const bson_matcher_context_t *context, const bson_t *assertion, const bson_val_t *actual, - void *ctx, - const char *path, + void *user_data, bson_error_t *error) { bool ret = false; bson_iter_t iter; bool should_exist; - BSON_UNUSED (matcher); - BSON_UNUSED (ctx); + BSON_UNUSED (context); + BSON_UNUSED (user_data); bson_iter_init (&iter, assertion); BSON_ASSERT (bson_iter_next (&iter)); @@ -122,18 +119,17 @@ special_exists (bson_matcher_t *matcher, /* implements $$type */ static bool -special_type (bson_matcher_t *matcher, +special_type (const bson_matcher_context_t *context, const bson_t *assertion, const bson_val_t *actual, - void *ctx, - const char *path, + void *user_data, bson_error_t *error) { bool ret = false; bson_iter_t iter; - BSON_UNUSED (matcher); - BSON_UNUSED (ctx); + BSON_UNUSED (context); + BSON_UNUSED (user_data); bson_iter_init (&iter, assertion); BSON_ASSERT (bson_iter_next (&iter)); @@ -187,18 +183,17 @@ special_type (bson_matcher_t *matcher, /* implements $$unsetOrMatches */ static bool -special_unset_or_matches (bson_matcher_t *matcher, +special_unset_or_matches (const bson_matcher_context_t *context, const bson_t *assertion, const bson_val_t *actual, - void *ctx, - const char *path, + void *user_data, bson_error_t *error) { bool ret = false; bson_iter_t iter; bson_val_t *expected = NULL; - BSON_UNUSED (ctx); + BSON_UNUSED (user_data); bson_iter_init (&iter, assertion); BSON_ASSERT (bson_iter_next (&iter)); @@ -209,7 +204,7 @@ special_unset_or_matches (bson_matcher_t *matcher, goto done; } - if (!bson_matcher_match (matcher, expected, actual, path, false, error)) { + if (!bson_matcher_match (context, expected, actual, error)) { goto done; } @@ -221,11 +216,10 @@ special_unset_or_matches (bson_matcher_t *matcher, /* implements $$matchesHexBytes */ static bool -special_matches_hex_bytes (bson_matcher_t *matcher, +special_matches_hex_bytes (const bson_matcher_context_t *context, const bson_t *assertion, const bson_val_t *actual, - void *ctx, - const char *path, + void *user_data, bson_error_t *error) { bool ret = false; @@ -237,8 +231,8 @@ special_matches_hex_bytes (bson_matcher_t *matcher, char *actual_bytes_string = NULL; bson_iter_t iter; - BSON_UNUSED (matcher); - BSON_UNUSED (ctx); + BSON_UNUSED (context); + BSON_UNUSED (user_data); bson_iter_init (&iter, assertion); BSON_ASSERT (bson_iter_next (&iter)); @@ -288,14 +282,101 @@ special_matches_hex_bytes (bson_matcher_t *matcher, bson_free (actual_bytes_string); ret = true; - goto done; done: return ret; } +/* implements $$matchAsDocument */ static bool -evaluate_special ( - bson_matcher_t *matcher, const bson_t *assertion, const bson_val_t *actual, const char *path, bson_error_t *error) +special_match_as_document (const bson_matcher_context_t *context, + const bson_t *assertion, + const bson_val_t *actual, + void *user_data, + bson_error_t *error) +{ + bool ret = false; + bson_t actual_as_bson; + BSON_UNUSED (user_data); + + bson_iter_t iter; + bson_iter_init (&iter, assertion); + BSON_ASSERT (bson_iter_next (&iter)); + if (!BSON_ITER_HOLDS_DOCUMENT (&iter)) { + MATCH_ERR ("%s", "$$matchAsDocument does not contain a document"); + goto done; + } + + if (!actual) { + MATCH_ERR ("%s", "does not exist but should"); + goto done; + } + + if (bson_val_type (actual) != BSON_TYPE_UTF8) { + MATCH_ERR ("%s", "value type is not utf8"); + goto done; + } + const char *actual_json = bson_val_to_utf8 (actual); + if (!bson_init_from_json (&actual_as_bson, actual_json, -1, error)) { + MATCH_ERR ("%s", "value can't be parsed as JSON"); + goto done; + } + + bson_val_t *expected_val = bson_val_from_iter (&iter); + bson_val_t *actual_val = bson_val_from_bson (&actual_as_bson); + ret = bson_matcher_match (context, expected_val, actual_val, error); + bson_val_destroy (actual_val); + bson_val_destroy (expected_val); + +done: + bson_destroy (&actual_as_bson); + return ret; +} + +/* implements $$matchAsRoot */ +static bool +special_match_as_root (const bson_matcher_context_t *context, + const bson_t *assertion, + const bson_val_t *actual, + void *user_data, + bson_error_t *error) +{ + bool ret = false; + BSON_UNUSED (user_data); + + bson_iter_t iter; + bson_iter_init (&iter, assertion); + BSON_ASSERT (bson_iter_next (&iter)); + if (!BSON_ITER_HOLDS_DOCUMENT (&iter)) { + MATCH_ERR ("%s", "$$matchAsRoot does not contain a document"); + goto done; + } + + if (!actual) { + MATCH_ERR ("%s", "does not exist but should"); + goto done; + } + + if (bson_val_type (actual) != BSON_TYPE_DOCUMENT) { + MATCH_ERR ("%s", "value is not a document"); + goto done; + } + + bson_matcher_context_t as_root_context = *context; + as_root_context.is_root = true; + + bson_val_t *expected_val = bson_val_from_iter (&iter); + ret = bson_matcher_match (&as_root_context, expected_val, actual, error); + bson_val_destroy (expected_val); + +done: + return ret; +} + +static bool +evaluate_special (const bson_matcher_context_t *context, + const bson_t *assertion, + const bson_val_t *actual, + bson_error_t *error) { bson_iter_t iter; const char *assertion_key; @@ -305,10 +386,10 @@ evaluate_special ( BSON_ASSERT (bson_iter_next (&iter)); assertion_key = bson_iter_key (&iter); - LL_FOREACH (matcher->specials, special_iter) + LL_FOREACH (context->matcher->specials, special_iter) { if (0 == strcmp (assertion_key, special_iter->keyword)) { - return special_iter->fn (matcher, assertion, actual, special_iter->ctx, path, error); + return special_iter->fn (context, assertion, actual, special_iter->user_data, error); } } @@ -327,12 +408,14 @@ bson_matcher_new (void) bson_matcher_add_special (matcher, "$$type", special_type, NULL); bson_matcher_add_special (matcher, "$$unsetOrMatches", special_unset_or_matches, NULL); bson_matcher_add_special (matcher, "$$matchesHexBytes", special_matches_hex_bytes, NULL); + bson_matcher_add_special (matcher, "$$matchAsDocument", special_match_as_document, NULL); + bson_matcher_add_special (matcher, "$$matchAsRoot", special_match_as_root, NULL); return matcher; } /* Add a hook function for matching a special $$ operator */ void -bson_matcher_add_special (bson_matcher_t *matcher, const char *keyword, special_fn special, void *ctx) +bson_matcher_add_special (bson_matcher_t *matcher, const char *keyword, special_fn special, void *user_data) { special_functor_t *functor; @@ -343,7 +426,7 @@ bson_matcher_add_special (bson_matcher_t *matcher, const char *keyword, special_ functor = bson_malloc (sizeof (special_functor_t)); functor->keyword = bson_strdup (keyword); functor->fn = special; - functor->ctx = ctx; + functor->user_data = user_data; LL_PREPEND (matcher->specials, functor); } @@ -365,15 +448,12 @@ bson_matcher_destroy (bson_matcher_t *matcher) } bool -bson_matcher_match (bson_matcher_t *matcher, +bson_matcher_match (const bson_matcher_context_t *context, const bson_val_t *expected, const bson_val_t *actual, - const char *path, - bool array_of_root_docs, bson_error_t *error) { bool ret = false; - bool is_root = (0 == strcmp (path, "")); if (bson_val_type (expected) == BSON_TYPE_DOCUMENT) { bson_iter_t expected_iter; @@ -384,7 +464,7 @@ bson_matcher_match (bson_matcher_t *matcher, /* handle special operators (e.g. $$type) */ if (is_special_match (expected_bson)) { - ret = evaluate_special (matcher, expected_bson, actual, path, error); + ret = evaluate_special (context, expected_bson, actual, error); goto done; } @@ -401,7 +481,6 @@ bson_matcher_match (bson_matcher_t *matcher, bson_val_t *expected_val = NULL; bson_val_t *actual_val = NULL; bson_iter_t actual_iter; - char *path_child = NULL; key = bson_iter_key (&expected_iter); expected_val = bson_val_from_iter (&expected_iter); @@ -412,9 +491,10 @@ bson_matcher_match (bson_matcher_t *matcher, if (bson_val_type (expected_val) == BSON_TYPE_DOCUMENT && is_special_match (bson_val_to_document (expected_val))) { - bool special_ret; - path_child = bson_strdup_printf ("%s.%s", path, key); - special_ret = evaluate_special (matcher, bson_val_to_document (expected_val), actual_val, path, error); + char *path_child = bson_strdup_printf ("%s.%s", context->path, key); + bson_matcher_context_t special_context = {.matcher = context->matcher, .path = path_child}; + bool special_ret = + evaluate_special (&special_context, bson_val_to_document (expected_val), actual_val, error); bson_free (path_child); bson_val_destroy (expected_val); bson_val_destroy (actual_val); @@ -431,16 +511,18 @@ bson_matcher_match (bson_matcher_t *matcher, goto done; } - path_child = bson_strdup_printf ("%s.%s", path, key); - if (!bson_matcher_match (matcher, expected_val, actual_val, path_child, false, error)) { - bson_val_destroy (expected_val); - bson_val_destroy (actual_val); - bson_free (path_child); - goto done; - } + char *path_child = bson_strdup_printf ("%s.%s", context->path, key); + bson_matcher_context_t document_child_context = { + .matcher = context->matcher, + .path = path_child, + }; + bool document_child_ret = bson_matcher_match (&document_child_context, expected_val, actual_val, error); bson_val_destroy (expected_val); bson_val_destroy (actual_val); bson_free (path_child); + if (!document_child_ret) { + goto done; + } } expected_keys = bson_count_keys (expected_bson); @@ -451,8 +533,9 @@ bson_matcher_match (bson_matcher_t *matcher, * not present in the expected document."" * * This logic must also handle the case where `expected` is one of any - * number of root documents within an array (i.e. cursor result). */ - if (!(is_root || array_of_root_docs) && expected_keys < actual_keys) { + * number of root documents within an array (i.e. cursor result); see + * array_child_context below. */ + if (!context->is_root && expected_keys < actual_keys) { MATCH_ERR ("expected %" PRIu32 " keys in document, got: %" PRIu32, expected_keys, actual_keys); goto done; } @@ -465,7 +548,6 @@ bson_matcher_match (bson_matcher_t *matcher, bson_iter_t expected_iter; const bson_t *expected_bson = bson_val_to_array (expected); const bson_t *actual_bson = NULL; - char *path_child = NULL; uint32_t expected_keys = bson_count_keys (expected_bson); uint32_t actual_keys; @@ -499,17 +581,19 @@ bson_matcher_match (bson_matcher_t *matcher, actual_val = bson_val_from_iter (&actual_iter); - path_child = bson_strdup_printf ("%s.%s", path, key); - if (!bson_matcher_match ( - matcher, expected_val, actual_val, path_child, (is_root && array_of_root_docs), error)) { - bson_val_destroy (expected_val); - bson_val_destroy (actual_val); - bson_free (path_child); - goto done; - } + char *path_child = bson_strdup_printf ("%s.%s", context->path, key); + bson_matcher_context_t array_child_context = { + .matcher = context->matcher, + .path = path_child, + .is_root = context->is_root && context->array_of_root_docs, + }; + bool array_child_ret = bson_matcher_match (&array_child_context, expected_val, actual_val, error); bson_val_destroy (expected_val); bson_val_destroy (actual_val); bson_free (path_child); + if (!array_child_ret) { + goto done; + } } ret = true; goto done; @@ -522,7 +606,7 @@ bson_matcher_match (bson_matcher_t *matcher, ret = true; done: - if (!ret && is_root) { + if (!ret && context->is_root) { /* Append the error with more context at the root match. */ bson_error_t tmp_error; @@ -541,9 +625,14 @@ bson_matcher_match (bson_matcher_t *matcher, bool bson_match (const bson_val_t *expected, const bson_val_t *actual, bool array_of_root_docs, bson_error_t *error) { - bson_matcher_t *matcher = bson_matcher_new (); - bool matched = bson_matcher_match (matcher, expected, actual, "", array_of_root_docs, error); - bson_matcher_destroy (matcher); + bson_matcher_context_t root_context = { + .matcher = bson_matcher_new (), + .path = "", + .is_root = true, + .array_of_root_docs = array_of_root_docs, + }; + bool matched = bson_matcher_match (&root_context, expected, actual, error); + bson_matcher_destroy (root_context.matcher); return matched; } @@ -557,22 +646,45 @@ typedef struct { static void test_match (void) { - testcase_t tests[] = {{"int32 ==", "{'a': 1}", "{'a': 1}", true}, - {"int32 !=", "{'a': 1}", "{'a': 0}", false}, - {"$$exists", "{'a': {'$$exists': true}}", "{'a': 0}", true}, - {"$$exists fail", "{'a': {'$$exists': true}}", "{'b': 0}", false}, - {"does not $$exists", "{'a': {'$$exists': false}}", "{'b': 0}", true}, - {"$$unsetOrMatches match", "{'a': {'$$unsetOrMatches': 1}}", "{'a': 1}", true}, - {"$$unsetOrMatches unset", "{'a': {'$$unsetOrMatches': 1}}", "{}", true}, - {"$$unsetOrMatches mismatch", "{'a': {'$$unsetOrMatches': 'abc'}}", "{'a': 1}", false}, - {"$$type match", "{'a': {'$$type': 'string'}}", "{'a': 'abc'}", true}, - {"$$type mismatch", "{'a': {'$$type': 'string'}}", "{'a': 1}", false}, - {"$$type array match", "{'a': {'$$type': ['string', 'int']}}", "{'a': 1}", true}, - {"$$type array mismatch", "{'a': {'$$type': ['string', 'int']}}", "{'a': 1.2}", false}, - {"extra keys in root ok", "{'a': 1}", "{'a': 1, 'b': 2}", true}, - {"extra keys in subdoc not ok", "{'a': {'b': 1}}", "{'a': {'b': 1, 'c': 2}}", false}, - {"numeric type mismatch is ok", "{'a': 1}", "{'a': 1.0}", true}, - {"comparing number to string is an error", "{'a': 1}", "{'a': 'foo'}", false}}; + testcase_t tests[] = { + {"int32 ==", "{'a': 1}", "{'a': 1}", true}, + {"int32 !=", "{'a': 1}", "{'a': 0}", false}, + {"$$exists", "{'a': {'$$exists': true}}", "{'a': 0}", true}, + {"$$exists fail", "{'a': {'$$exists': true}}", "{'b': 0}", false}, + {"does not $$exists", "{'a': {'$$exists': false}}", "{'b': 0}", true}, + {"$$unsetOrMatches match", "{'a': {'$$unsetOrMatches': 1}}", "{'a': 1}", true}, + {"$$unsetOrMatches unset", "{'a': {'$$unsetOrMatches': 1}}", "{}", true}, + {"$$unsetOrMatches mismatch", "{'a': {'$$unsetOrMatches': 'abc'}}", "{'a': 1}", false}, + {"$$type match", "{'a': {'$$type': 'string'}}", "{'a': 'abc'}", true}, + {"$$type mismatch", "{'a': {'$$type': 'string'}}", "{'a': 1}", false}, + {"$$type array match", "{'a': {'$$type': ['string', 'int']}}", "{'a': 1}", true}, + {"$$type array mismatch", "{'a': {'$$type': ['string', 'int']}}", "{'a': 1.2}", false}, + {"extra keys in root ok", "{'a': 1}", "{'a': 1, 'b': 2}", true}, + {"extra keys in subdoc not ok", "{'a': {'b': 1}}", "{'a': {'b': 1, 'c': 2}}", false}, + {"extra keys in subdoc allowed explicitly", + "{'a': {'$$matchAsRoot': {'b': 1}}}", + "{'a': {'b': 1, 'c': 2}}", + true}, + {"missing key in matchAsRoot subdoc", + "{'a': {'$$matchAsRoot': {'b': 1, 'd': 1}}}", + "{'a': {'b': 1, 'c': 2}}", + false}, + {"$$matchAsDocument match", "{'a': {'$$matchAsDocument': {'b': 'abc'}}}", "{'a': '{\\'b\\':\\'abc\\'}'}", true}, + {"$$matchAsDocument mismatch", + "{'a': {'$$matchAsDocument': {'b': 'abc'}}}", + "{'a': '{\\'b\\':\\'xyz\\'}'}", + false}, + {"$$matchAsDocument parse error", "{'a': {'$$matchAsDocument': {'b': 'abc'}}}", "{'a': 'nope'}", false}, + {"$$matchAsDocument extra keys not ok", + "{'a': {'$$matchAsDocument': {'b': 'abc'}}}", + "{'a': '{\\'b\\':\\'abc\\',\\'c\\':1}'}", + false}, + {"$$matchAsDocument and $$matchAsRoot, extra keys ok", + "{'a': {'$$matchAsDocument': {'$$matchAsRoot': {'b': 'abc'}}}}", + "{'a': '{\\'b\\':\\'abc\\',\\'c\\':1}'}", + true}, + {"numeric type mismatch is ok", "{'a': 1}", "{'a': 1.0}", true}, + {"comparing number to string is an error", "{'a': 1}", "{'a': 'foo'}", false}}; int i; for (i = 0; i < sizeof (tests) / sizeof (testcase_t); i++) { diff --git a/src/libmongoc/tests/bsonutil/bson-match.h b/src/libmongoc/tests/bsonutil/bson-match.h index 703154ca5d3..3f9aa2639d6 100644 --- a/src/libmongoc/tests/bsonutil/bson-match.h +++ b/src/libmongoc/tests/bsonutil/bson-match.h @@ -30,13 +30,22 @@ typedef struct _bson_matcher_t bson_matcher_t; bson_matcher_t * bson_matcher_new (void); -typedef bool (*special_fn) (bson_matcher_t *matcher, +/* Current state of an ongoing match operation, of interest to custom operators. + * Each recursion level has its own instance on the stack. */ +typedef struct bson_matcher_context_t { + bson_matcher_t *matcher; + const char *path; + bool is_root; + bool array_of_root_docs; +} bson_matcher_context_t; + +typedef bool (*special_fn) (const bson_matcher_context_t *context, const bson_t *assertion, const bson_val_t *actual, - void *ctx, - const char *path, + void *user_data, bson_error_t *error); + /* Adds a handler function for matching a special $$ operator. * * Example: @@ -45,14 +54,12 @@ typedef bool (*special_fn) (bson_matcher_t *matcher, * expectation. */ void -bson_matcher_add_special (bson_matcher_t *matcher, const char *keyword, special_fn special, void *ctx); +bson_matcher_add_special (bson_matcher_t *matcher, const char *keyword, special_fn special, void *user_data); bool -bson_matcher_match (bson_matcher_t *matcher, +bson_matcher_match (const bson_matcher_context_t *context, const bson_val_t *expected, const bson_val_t *actual, - const char *path, - bool array_of_root_docs, bson_error_t *error); void diff --git a/src/libmongoc/tests/unified/entity-map.c b/src/libmongoc/tests/unified/entity-map.c index 92369ca206f..39b83633141 100644 --- a/src/libmongoc/tests/unified/entity-map.c +++ b/src/libmongoc/tests/unified/entity-map.c @@ -2117,18 +2117,17 @@ entity_map_add_size_t (entity_map_t *em, const char *id, size_t *value, bson_err /* implement $$sessionLsid */ static bool -special_session_lsid (bson_matcher_t *matcher, +special_session_lsid (const bson_matcher_context_t *context, const bson_t *assertion, const bson_val_t *actual, - void *ctx, - const char *path, + void *user_data, bson_error_t *error) { bool ret = false; const char *id; bson_val_t *session_val = NULL; bson_t *lsid = NULL; - entity_map_t *em = (entity_map_t *) ctx; + entity_map_t *em = (entity_map_t *) user_data; bson_iter_t iter; bson_iter_init (&iter, assertion); @@ -2146,11 +2145,10 @@ special_session_lsid (bson_matcher_t *matcher, } session_val = bson_val_from_bson (lsid); - if (!bson_matcher_match (matcher, session_val, actual, path, false, error)) { + if (!bson_matcher_match (context, session_val, actual, error)) { goto done; } - ret = true; done: bson_val_destroy (session_val); @@ -2159,16 +2157,15 @@ special_session_lsid (bson_matcher_t *matcher, /* implement $$matchesEntity */ bool -special_matches_entity (bson_matcher_t *matcher, +special_matches_entity (const bson_matcher_context_t *context, const bson_t *assertion, const bson_val_t *actual, - void *ctx, - const char *path, + void *user_data, bson_error_t *error) { bool ret = false; bson_iter_t iter; - entity_map_t *em = (entity_map_t *) ctx; + entity_map_t *em = (entity_map_t *) user_data; bson_val_t *entity_val = NULL; const char *id; @@ -2186,7 +2183,7 @@ special_matches_entity (bson_matcher_t *matcher, goto done; } - if (!bson_matcher_match (matcher, entity_val, actual, path, false, error)) { + if (!bson_matcher_match (context, entity_val, actual, error)) { goto done; } @@ -2199,14 +2196,16 @@ bool entity_map_match ( entity_map_t *em, const bson_val_t *expected, const bson_val_t *actual, bool array_of_root_docs, bson_error_t *error) { - bson_matcher_t *matcher; - bool ret; - - matcher = bson_matcher_new (); - bson_matcher_add_special (matcher, "$$sessionLsid", special_session_lsid, em); - bson_matcher_add_special (matcher, "$$matchesEntity", special_matches_entity, em); - ret = bson_matcher_match (matcher, expected, actual, "", array_of_root_docs, error); - bson_matcher_destroy (matcher); + bson_matcher_context_t root_context = { + .matcher = bson_matcher_new (), + .path = "", + .is_root = true, + .array_of_root_docs = array_of_root_docs, + }; + bson_matcher_add_special (root_context.matcher, "$$sessionLsid", special_session_lsid, em); + bson_matcher_add_special (root_context.matcher, "$$matchesEntity", special_matches_entity, em); + bool ret = bson_matcher_match (&root_context, expected, actual, error); + bson_matcher_destroy (root_context.matcher); return ret; } From 76649aafdb944669734d5032e938b796ffbe2536 Mon Sep 17 00:00:00 2001 From: Micah Scott Date: Wed, 13 Nov 2024 12:59:41 -0800 Subject: [PATCH 062/139] add driver-connection-id CLAM test to skips for lack of CMAP --- src/libmongoc/tests/unified/runner.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libmongoc/tests/unified/runner.c b/src/libmongoc/tests/unified/runner.c index 0bc72dd4afa..617bf193761 100644 --- a/src/libmongoc/tests/unified/runner.c +++ b/src/libmongoc/tests/unified/runner.c @@ -50,6 +50,7 @@ skipped_unified_test_t SKIPPED_TESTS[] = { {"assertNumberConnectionsCheckedOut", SKIP_ALL_TESTS}, {"entity-client-cmap-events", SKIP_ALL_TESTS}, {"expectedEventsForClient-eventType", SKIP_ALL_TESTS}, + {"driver-connection-id", SKIP_ALL_TESTS}, // CDRIVER-4115: listCollections does not support batchSize. {"cursors are correctly pinned to connections for load-balanced clusters", "listCollections pins the cursor to a connection"}, // CDRIVER-4116: listIndexes does not support batchSize. From 68ac6d0cca37cfda6e261580e0babd48d6e480cd Mon Sep 17 00:00:00 2001 From: Micah Scott Date: Wed, 13 Nov 2024 16:38:24 -0800 Subject: [PATCH 063/139] Structured log support for redacted replies and failures --- src/libmongoc/src/mongoc/mongoc-client.c | 2 +- src/libmongoc/src/mongoc/mongoc-cluster.c | 16 +- src/libmongoc/src/mongoc/mongoc-cursor.c | 2 +- .../mongoc/mongoc-structured-log-private.h | 17 + .../src/mongoc/mongoc-structured-log.c | 117 ++++++- .../tests/test-mongoc-structured-log.c | 294 ++++++++++++++---- src/libmongoc/tests/unified/runner.c | 83 ++++- 7 files changed, 437 insertions(+), 94 deletions(-) diff --git a/src/libmongoc/src/mongoc/mongoc-client.c b/src/libmongoc/src/mongoc/mongoc-client.c index e8b69ac20ae..16a8ca2157f 100644 --- a/src/libmongoc/src/mongoc/mongoc-client.c +++ b/src/libmongoc/src/mongoc/mongoc-client.c @@ -2237,7 +2237,7 @@ _mongoc_client_monitor_op_killcursors_succeeded (mongoc_cluster_t *cluster, utf8 ("commandName", "killCursors"), int64 ("operationId", operation_id), int64 ("durationMS", duration), - bson_as_json ("reply", &doc)); + cmd_reply ("killCursors", &doc)); if (!client->apm_callbacks.succeeded) { bson_destroy (&doc); diff --git a/src/libmongoc/src/mongoc/mongoc-cluster.c b/src/libmongoc/src/mongoc/mongoc-cluster.c index 302a3284d85..fb549a06a0f 100644 --- a/src/libmongoc/src/mongoc/mongoc-cluster.c +++ b/src/libmongoc/src/mongoc/mongoc-cluster.c @@ -510,7 +510,7 @@ mongoc_cluster_run_command_monitored (mongoc_cluster_t *cluster, mongoc_cmd_t *c bson_t encrypted = BSON_INITIALIZER; bson_t decrypted = BSON_INITIALIZER; mongoc_cmd_t encrypted_cmd; - bool is_redacted = false; + bool is_redacted_by_apm = false; server_stream = cmd->server_stream; server_id = server_stream->sd->id; @@ -544,7 +544,7 @@ mongoc_cluster_run_command_monitored (mongoc_cluster_t *cluster, mongoc_cmd_t *c if (callbacks->started) { mongoc_apm_command_started_init_with_cmd ( - &started_event, cmd, request_id, &is_redacted, cluster->client->apm_context); + &started_event, cmd, request_id, &is_redacted_by_apm, cluster->client->apm_context); callbacks->started (&started_event); mongoc_apm_command_started_cleanup (&started_event); @@ -570,10 +570,10 @@ mongoc_cluster_run_command_monitored (mongoc_cluster_t *cluster, mongoc_cmd_t *c MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND, "Command succeeded", int32 ("requestId", request_id), + int64 ("durationMS", duration), server_description (server_stream->sd, SERVER_HOST, SERVER_PORT, SERVER_CONNECTION_ID, SERVICE_ID), cmd (cmd, DATABASE_NAME, COMMAND_NAME, OPERATION_ID), - int64 ("durationMS", duration), - bson_as_json ("reply", cmd->is_acknowledged ? reply : &fake_reply)); + cmd_reply (cmd->command_name, cmd->is_acknowledged ? reply : &fake_reply)); if (callbacks->succeeded) { mongoc_apm_command_succeeded_init (&succeeded_event, @@ -587,7 +587,7 @@ mongoc_cluster_run_command_monitored (mongoc_cluster_t *cluster, mongoc_cmd_t *c server_id, &server_stream->sd->service_id, server_stream->sd->server_connection_id, - is_redacted, + is_redacted_by_apm, cluster->client->apm_context); callbacks->succeeded (&succeeded_event); @@ -603,10 +603,10 @@ mongoc_cluster_run_command_monitored (mongoc_cluster_t *cluster, mongoc_cmd_t *c MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND, "Command failed", int32 ("requestId", request_id), + int64 ("durationMS", duration), server_description (server_stream->sd, SERVER_HOST, SERVER_PORT, SERVER_CONNECTION_ID, SERVICE_ID), cmd (cmd, DATABASE_NAME, COMMAND_NAME, OPERATION_ID), - int64 ("durationMS", duration), - bson_as_json ("failure", reply)); + cmd_failure (cmd->command_name, reply, error)); if (callbacks->failed) { mongoc_apm_command_failed_init (&failed_event, @@ -621,7 +621,7 @@ mongoc_cluster_run_command_monitored (mongoc_cluster_t *cluster, mongoc_cmd_t *c server_id, &server_stream->sd->service_id, server_stream->sd->server_connection_id, - is_redacted, + is_redacted_by_apm, cluster->client->apm_context); callbacks->failed (&failed_event); diff --git a/src/libmongoc/src/mongoc/mongoc-cursor.c b/src/libmongoc/src/mongoc/mongoc-cursor.c index a76582e0799..3f1ca5849dc 100644 --- a/src/libmongoc/src/mongoc/mongoc-cursor.c +++ b/src/libmongoc/src/mongoc/mongoc-cursor.c @@ -748,7 +748,7 @@ _mongoc_cursor_monitor_succeeded (mongoc_cursor_t *cursor, utf8 ("commandName", cmd_name), int64 ("operationId", cursor->operation_id), int64 ("durationMS", duration), - bson_as_json ("reply", &reply)); + cmd_reply (cmd_name, &reply)); if (client->apm_callbacks.succeeded) { mongoc_apm_command_succeeded_init (&event, diff --git a/src/libmongoc/src/mongoc/mongoc-structured-log-private.h b/src/libmongoc/src/mongoc/mongoc-structured-log-private.h index 70873799d44..667e08b772e 100644 --- a/src/libmongoc/src/mongoc/mongoc-structured-log-private.h +++ b/src/libmongoc/src/mongoc/mongoc-structured-log-private.h @@ -85,6 +85,13 @@ BSON_BEGIN_DECLS .arg2.cmd_flags = \ (0 _bsonDSL_mapMacro (_mongoc_structured_log_flag_expr, MONGOC_STRUCTURED_LOG_CMD, __VA_ARGS__))}, +#define _mongoc_structured_log_item_cmd_reply(_cmd_name, _reply_bson) \ + {.func = _mongoc_structured_log_append_cmd_reply, .arg1.utf8 = (_cmd_name), .arg2.bson = (_reply_bson)}, + +#define _mongoc_structured_log_item_cmd_failure(_cmd_name, _reply_bson, _error) \ + {.func = _mongoc_structured_log_append_cmd_failure_stage0, .arg1.utf8 = (_cmd_name), .arg2.bson = (_reply_bson)}, \ + {.func = _mongoc_structured_log_append_cmd_failure_stage1, .arg1.error = (_error)}, + #define _mongoc_structured_log_item_server_description(_server_description, ...) \ {.func = _mongoc_structured_log_append_server_description, \ .arg1.server_description = (_server_description), \ @@ -119,6 +126,7 @@ struct mongoc_structured_log_builder_stage_t { // pointers unused and set to placeholder values which can be checked. mongoc_structured_log_builder_func_t func; // NULL sentinel here union { + const bson_error_t *error; const mongoc_cmd_t *cmd; const mongoc_server_description_t *server_description; const void *utf8; @@ -189,6 +197,15 @@ _mongoc_structured_log_append_bson_as_json (bson_t *bson, const mongoc_structure const mongoc_structured_log_builder_stage_t * _mongoc_structured_log_append_cmd (bson_t *bson, const mongoc_structured_log_builder_stage_t *stage); +const mongoc_structured_log_builder_stage_t * +_mongoc_structured_log_append_cmd_reply (bson_t *bson, const mongoc_structured_log_builder_stage_t *stage); + +const mongoc_structured_log_builder_stage_t * +_mongoc_structured_log_append_cmd_failure_stage0 (bson_t *bson, const mongoc_structured_log_builder_stage_t *stage); + +const mongoc_structured_log_builder_stage_t * +_mongoc_structured_log_append_cmd_failure_stage1 (bson_t *bson, const mongoc_structured_log_builder_stage_t *stage); + const mongoc_structured_log_builder_stage_t * _mongoc_structured_log_append_server_description (bson_t *bson, const mongoc_structured_log_builder_stage_t *stage); diff --git a/src/libmongoc/src/mongoc/mongoc-structured-log.c b/src/libmongoc/src/mongoc/mongoc-structured-log.c index 1a2a073bc09..1589d0529d3 100644 --- a/src/libmongoc/src/mongoc/mongoc-structured-log.c +++ b/src/libmongoc/src/mongoc/mongoc-structured-log.c @@ -18,6 +18,7 @@ #include "mongoc-structured-log-private.h" #include "mongoc-thread-private.h" #include "mongoc-util-private.h" +#include "mongoc-apm-private.h" #include "common-atomic-private.h" #include "common-oid-private.h" @@ -384,6 +385,8 @@ _mongoc_structured_log_append_cmd (bson_t *bson, const mongoc_structured_log_bui const mongoc_cmd_t *cmd = stage->arg1.cmd; const mongoc_structured_log_cmd_flags_t flags = stage->arg2.cmd_flags; + BSON_ASSERT (cmd); + if (flags & MONGOC_STRUCTURED_LOG_CMD_DATABASE_NAME) { BSON_APPEND_UTF8 (bson, "databaseName", cmd->db_name); } @@ -394,37 +397,123 @@ _mongoc_structured_log_append_cmd (bson_t *bson, const mongoc_structured_log_bui BSON_APPEND_INT64 (bson, "operationId", cmd->operation_id); } if (flags & MONGOC_STRUCTURED_LOG_CMD_COMMAND) { - bson_t *command_copy = NULL; - - // @todo This is a performance bottleneck, we shouldn't be copying - // a potentially large command to serialize a potentially very - // small part of it. We should be outputting JSON, constrained - // by length limit, while visiting borrowed references to each - // command attribute and each payload. CDRIVER-4814 - if (cmd->payloads_count > 0) { - command_copy = bson_copy (cmd->command); - _mongoc_cmd_append_payload_as_array (cmd, command_copy); + if (mongoc_apm_is_sensitive_command_message (cmd->command_name, cmd->command)) { + BSON_APPEND_UTF8 (bson, "command", "{}"); + } else { + bson_t *command_copy = NULL; + + if (cmd->payloads_count > 0) { + // @todo This is a performance bottleneck, we shouldn't be copying + // a potentially large command to serialize a potentially very + // small part of it. We should be appending JSON to a single buffer + // for all nesting levels, constrained by length limit, while visiting + // borrowed references to each command attribute and each payload. CDRIVER-4814 + command_copy = bson_copy (cmd->command); + _mongoc_cmd_append_payload_as_array (cmd, command_copy); + } + + size_t json_length; + char *json = mongoc_structured_log_document_to_json (command_copy ? command_copy : cmd->command, &json_length); + if (json) { + const char *key = "command"; + bson_append_utf8 (bson, key, strlen (key), json, json_length); + bson_free (json); + } + + bson_destroy (command_copy); } + } + return stage + 1; +} + +const mongoc_structured_log_builder_stage_t * +_mongoc_structured_log_append_cmd_reply (bson_t *bson, const mongoc_structured_log_builder_stage_t *stage) +{ + const char *cmd_name = stage->arg1.utf8; + const bson_t *reply = stage->arg2.bson; + BSON_ASSERT (cmd_name); + BSON_ASSERT (reply); + + if (mongoc_apm_is_sensitive_command_message (cmd_name, reply)) { + BSON_APPEND_UTF8 (bson, "reply", "{}"); + } else { size_t json_length; - char *json = mongoc_structured_log_document_to_json (command_copy ? command_copy : cmd->command, &json_length); + char *json = mongoc_structured_log_document_to_json (reply, &json_length); if (json) { - const char *key = "command"; + const char *key = "reply"; bson_append_utf8 (bson, key, strlen (key), json, json_length); bson_free (json); } - - bson_destroy (command_copy); } return stage + 1; } +const mongoc_structured_log_builder_stage_t * +_mongoc_structured_log_append_cmd_failure_stage0 (bson_t *bson, const mongoc_structured_log_builder_stage_t *stage) +{ + BSON_ASSERT (stage[1].func == _mongoc_structured_log_append_cmd_failure_stage1); + const char *cmd_name = stage[0].arg1.utf8; + const bson_t *reply = stage[0].arg2.bson; + const bson_error_t *error = stage[1].arg1.error; + + BSON_ASSERT (cmd_name); + BSON_ASSERT (reply); + BSON_ASSERT (error); + + bool is_server_side = error->domain == MONGOC_ERROR_SERVER || error->domain == MONGOC_ERROR_WRITE_CONCERN_ERROR; + if (is_server_side) { + bool is_sensitive = mongoc_apm_is_sensitive_command_message (cmd_name, reply); + + if (is_sensitive) { + // Redacted server-side message, must be a document with at most 'code', 'codeName', 'errorLabels' + bson_t failure; + bson_iter_t iter; + BSON_ASSERT (BSON_APPEND_DOCUMENT_BEGIN (bson, "failure", &failure)); + bson_iter_init (&iter, reply); + while (bson_iter_next (&iter)) { + const char *key = bson_iter_key (&iter); + if (!strcmp (key, "code") || !strcmp (key, "codeName") || !strcmp (key, "errorLabels")) { + bson_append_iter (&failure, key, bson_iter_key_len (&iter), &iter); + } + } + BSON_ASSERT (bson_append_document_end (bson, &failure)); + + } else { + // Non-redacted server side message, pass through + BSON_APPEND_DOCUMENT (bson, "failure", reply); + } + + } else { + // Client-side errors converted directly from bson_error_t, never redacted + bson_t failure; + BSON_ASSERT (BSON_APPEND_DOCUMENT_BEGIN (bson, "failure", &failure)); + BSON_APPEND_INT32 (&failure, "code", error->code); + BSON_APPEND_INT32 (&failure, "domain", error->domain); + BSON_APPEND_UTF8 (&failure, "message", error->message); + BSON_ASSERT (bson_append_document_end (bson, &failure)); + } + return stage + 2; +} + +const mongoc_structured_log_builder_stage_t * +_mongoc_structured_log_append_cmd_failure_stage1 (bson_t *bson, const mongoc_structured_log_builder_stage_t *stage) +{ + // Never called, marks the second stage in a two-stage cmd_failure + BSON_UNUSED (bson); + BSON_UNUSED (stage); + BSON_ASSERT (false); + return NULL; +} + const mongoc_structured_log_builder_stage_t * _mongoc_structured_log_append_server_description (bson_t *bson, const mongoc_structured_log_builder_stage_t *stage) { const mongoc_server_description_t *sd = stage->arg1.server_description; const mongoc_structured_log_server_description_flags_t flags = stage->arg2.server_description_flags; + BSON_ASSERT (sd); + if (flags & MONGOC_STRUCTURED_LOG_SERVER_DESCRIPTION_SERVER_HOST) { BSON_APPEND_UTF8 (bson, "serverHost", sd->host.host); } diff --git a/src/libmongoc/tests/test-mongoc-structured-log.c b/src/libmongoc/tests/test-mongoc-structured-log.c index ab3c19f2e07..2328a61a909 100644 --- a/src/libmongoc/tests/test-mongoc-structured-log.c +++ b/src/libmongoc/tests/test-mongoc-structured-log.c @@ -80,7 +80,7 @@ structured_log_func (const mongoc_structured_log_entry_t *entry, void *user_data } void -test_plain_log_entry (void) +test_structured_log_plain (void) { struct log_assumption assumption = { .expected_envelope.level = MONGOC_STRUCTURED_LOG_LEVEL_WARNING, @@ -102,7 +102,7 @@ test_plain_log_entry (void) } void -test_log_entry_with_extra_data (void) +test_structured_log_with_extra_data (void) { struct log_assumption assumption = { .expected_envelope.level = MONGOC_STRUCTURED_LOG_LEVEL_WARNING, @@ -126,7 +126,7 @@ test_log_entry_with_extra_data (void) } void -test_log_entry_with_all_data_types (void) +test_structured_log_basic_data_types (void) { const char non_terminated_test_string[] = {0, 1, 2, 3, 'a', '\\'}; bson_t *bson_str_n = bson_new (); @@ -136,9 +136,9 @@ test_log_entry_with_all_data_types (void) struct log_assumption assumption = { .expected_envelope.level = MONGOC_STRUCTURED_LOG_LEVEL_WARNING, .expected_envelope.component = MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND, - .expected_envelope.message = "Log entry with all data types", + .expected_envelope.message = "Log entry with all basic data types", .expected_bson = BCON_NEW ("message", - BCON_UTF8 ("Log entry with all data types"), + BCON_UTF8 ("Log entry with all basic data types"), "kStr", BCON_UTF8 ("string value"), "kNullStr", @@ -157,11 +157,181 @@ test_log_entry_with_all_data_types (void) "kTrue", BCON_BOOL (true), "kFalse", - BCON_BOOL (false), - "kJSON", - BCON_UTF8 ("{ \"k\" : \"v\" }"), + BCON_BOOL (false)), + .expected_calls = 1, + }; + + structured_log_state old_state = save_state (); + mongoc_structured_log_set_handler (structured_log_func, &assumption); + + mongoc_structured_log (MONGOC_STRUCTURED_LOG_LEVEL_WARNING, + MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND, + "Log entry with all basic data types", + utf8 ("kStr", "string value"), + utf8 ("kNullStr", NULL), + utf8 (NULL, NULL), + utf8_nn ("kStrN1ZZZ", 6, non_terminated_test_string, sizeof non_terminated_test_string), + utf8_n ("kStrN2", non_terminated_test_string, sizeof non_terminated_test_string), + utf8_nn ("kNullStrN1ZZZ", 10, NULL, 12345), + utf8_nn ("kNullStrN2", -1, NULL, 12345), + utf8_nn (NULL, 999, NULL, 999), + utf8_n ("kNullStrN3", NULL, 12345), + int32 ("kInt32", -12345), + int32 (NULL, 9999), + int64 ("kInt64", 0x76543210aabbccdd), + int64 (NULL, -1), + boolean ("kTrue", true), + boolean ("kFalse", false), + boolean (NULL, true)); + + ASSERT_CMPINT (assumption.calls, ==, 1); + restore_state (old_state); + bson_destroy (assumption.expected_bson); + bson_destroy (bson_str_n); +} + +void +test_structured_log_json (void) +{ + struct log_assumption assumption = { + .expected_envelope.level = MONGOC_STRUCTURED_LOG_LEVEL_WARNING, + .expected_envelope.component = MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND, + .expected_envelope.message = "Log entry with deferred BSON-to-JSON", + .expected_bson = BCON_NEW ( + "message", BCON_UTF8 ("Log entry with deferred BSON-to-JSON"), "kJSON", BCON_UTF8 ("{ \"k\" : \"v\" }")), + .expected_calls = 1, + }; + + bson_t *json_doc = BCON_NEW ("k", BCON_UTF8 ("v")); + + structured_log_state old_state = save_state (); + mongoc_structured_log_set_handler (structured_log_func, &assumption); + + mongoc_structured_log (MONGOC_STRUCTURED_LOG_LEVEL_WARNING, + MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND, + "Log entry with deferred BSON-to-JSON", + bson_as_json ("kJSON", json_doc), + bson_as_json (NULL, NULL)); + + ASSERT_CMPINT (assumption.calls, ==, 1); + restore_state (old_state); + bson_destroy (assumption.expected_bson); + bson_destroy (json_doc); +} + +void +test_structured_log_oid (void) +{ + struct log_assumption assumption = { + .expected_envelope.level = MONGOC_STRUCTURED_LOG_LEVEL_WARNING, + .expected_envelope.component = MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND, + .expected_envelope.message = "Log entry with deferred OID-to-hex conversion", + .expected_bson = BCON_NEW ("message", + BCON_UTF8 ("Log entry with deferred OID-to-hex conversion"), "kOID", - BCON_UTF8 ("112233445566778899aabbcc"), + BCON_UTF8 ("112233445566778899aabbcc")), + .expected_calls = 1, + }; + + bson_oid_t oid; + bson_oid_init_from_string (&oid, "112233445566778899aabbcc"); + + structured_log_state old_state = save_state (); + mongoc_structured_log_set_handler (structured_log_func, &assumption); + + mongoc_structured_log (MONGOC_STRUCTURED_LOG_LEVEL_WARNING, + MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND, + "Log entry with deferred OID-to-hex conversion", + oid_as_hex ("kOID", &oid), + oid_as_hex (NULL, NULL)); + + ASSERT_CMPINT (assumption.calls, ==, 1); + restore_state (old_state); + bson_destroy (assumption.expected_bson); +} + +void +test_structured_log_server_description (void) +{ + struct log_assumption assumption = { + .expected_envelope.level = MONGOC_STRUCTURED_LOG_LEVEL_WARNING, + .expected_envelope.component = MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND, + .expected_envelope.message = "Log entry with server description", + .expected_bson = BCON_NEW ("message", + BCON_UTF8 ("Log entry with server description"), + "serverHost", + BCON_UTF8 ("db1.example.com"), + "serverHost", + BCON_UTF8 ("db2.example.com"), + "serverPort", + BCON_INT32 (2340), + "serverConnectionId", + BCON_INT64 (0x3deeff00112233f0), + "serverHost", + BCON_UTF8 ("db1.example.com"), + "serverPort", + BCON_INT32 (2340), + "serverHost", + BCON_UTF8 ("db1.example.com"), + "serverPort", + BCON_INT32 (2340), + "serverConnectionId", + BCON_INT64 (0x3deeff00112233f0), + "serviceId", + BCON_UTF8 ("2233445566778899aabbccdd"), + "serverHost", + BCON_UTF8 ("db2.example.com"), + "serverPort", + BCON_INT32 (2341), + "serverConnectionId", + BCON_INT64 (0x3deeff00112233f1)), + .expected_calls = 1, + }; + + mongoc_server_description_t server_description_1 = { + .host.host = "db1.example.com", + .host.port = 2340, + .server_connection_id = 0x3deeff00112233f0, + }; + bson_oid_init_from_string (&server_description_1.service_id, "2233445566778899aabbccdd"); + + mongoc_server_description_t server_description_2 = { + .host.host = "db2.example.com", + .host.port = 2341, + .server_connection_id = 0x3deeff00112233f1, + .service_id = {{0}}, + }; + + structured_log_state old_state = save_state (); + mongoc_structured_log_set_handler (structured_log_func, &assumption); + + mongoc_structured_log ( + MONGOC_STRUCTURED_LOG_LEVEL_WARNING, + MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND, + "Log entry with server description", + server_description (&server_description_1, SERVER_HOST), + server_description (&server_description_2, SERVICE_ID), + server_description (&server_description_2, SERVER_HOST), + server_description (&server_description_1, SERVER_PORT), + server_description (&server_description_1, SERVER_CONNECTION_ID), + server_description (&server_description_1, SERVER_HOST, SERVER_PORT), + server_description (&server_description_1, SERVER_HOST, SERVER_PORT, SERVER_CONNECTION_ID, SERVICE_ID), + server_description (&server_description_2, SERVER_HOST, SERVER_PORT, SERVER_CONNECTION_ID, SERVICE_ID)); + + ASSERT_CMPINT (assumption.calls, ==, 1); + restore_state (old_state); + bson_destroy (assumption.expected_bson); +} + +void +test_structured_log_command (void) +{ + struct log_assumption assumption = { + .expected_envelope.level = MONGOC_STRUCTURED_LOG_LEVEL_WARNING, + .expected_envelope.component = MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND, + .expected_envelope.message = "Log entry with command and reply fields", + .expected_bson = BCON_NEW ("message", + BCON_UTF8 ("Log entry with command and reply fields"), "databaseName", BCON_UTF8 ("Some database"), "commandName", @@ -170,22 +340,47 @@ test_log_entry_with_all_data_types (void) BCON_INT64 (0x12345678eeff0011), "command", BCON_UTF8 ("{ \"c\" : \"d\" }"), - "serverHost", - BCON_UTF8 ("db.example.com"), - "serverPort", - BCON_INT32 (2345), - "serverConnectionId", - BCON_INT64 (0x3deeff0011223345), - "serviceId", - BCON_UTF8 ("2233445566778899aabbccdd")), + "reply", // Un-redacted successful reply + BCON_UTF8 ("{ \"r\" : \"s\", \"code\" : { \"$numberInt\" : \"1\" } }"), + "reply", // Redacted successful reply + BCON_UTF8 ("{}"), + "failure", // Un-redacted server side error + "{", + "r", + BCON_UTF8 ("s"), + "code", + BCON_INT32 (1), + "}", + "failure", // Redacted server side error + "{", + "code", + BCON_INT32 (1), + "}", + "failure", // Client side error + "{", + "code", + BCON_INT32 (123), + "domain", + BCON_INT32 (456), + "message", + BCON_UTF8 ("oh no"), + "}"), .expected_calls = 1, }; - bson_t *json_doc = BCON_NEW ("k", BCON_UTF8 ("v")); bson_t *cmd_doc = BCON_NEW ("c", BCON_UTF8 ("d")); + bson_t *reply_doc = BCON_NEW ("r", BCON_UTF8 ("s"), "code", BCON_INT32 (1)); - bson_oid_t oid; - bson_oid_init_from_string (&oid, "112233445566778899aabbcc"); + const bson_error_t server_error = { + .domain = MONGOC_ERROR_SERVER, + .code = 99, + .message = "unused", + }; + const bson_error_t client_error = { + .domain = 456, + .code = 123, + .message = "oh no", + }; mongoc_cmd_t cmd = { .db_name = "Some database", @@ -194,59 +389,34 @@ test_log_entry_with_all_data_types (void) .command = cmd_doc, }; - mongoc_server_description_t server_description = { - .host.host = "db.example.com", - .host.port = 2345, - .server_connection_id = 0x3deeff0011223345, - }; - bson_oid_init_from_string (&server_description.service_id, "2233445566778899aabbccdd"); - structured_log_state old_state = save_state (); mongoc_structured_log_set_handler (structured_log_func, &assumption); - mongoc_structured_log ( - MONGOC_STRUCTURED_LOG_LEVEL_WARNING, - MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND, - "Log entry with all data types", - // Basic BSON types. - // Most support optional values (skip when key is NULL) - utf8 ("kStr", "string value"), - utf8 ("kNullStr", NULL), - utf8 (NULL, NULL), - utf8_nn ("kStrN1ZZZ", 6, non_terminated_test_string, sizeof non_terminated_test_string), - utf8_n ("kStrN2", non_terminated_test_string, sizeof non_terminated_test_string), - utf8_nn ("kNullStrN1ZZZ", 10, NULL, 12345), - utf8_nn ("kNullStrN2", -1, NULL, 12345), - utf8_nn (NULL, 999, NULL, 999), - utf8_n ("kNullStrN3", NULL, 12345), - int32 ("kInt32", -12345), - int32 (NULL, 9999), - int64 ("kInt64", 0x76543210aabbccdd), - int64 (NULL, -1), - boolean ("kTrue", true), - boolean ("kFalse", false), - boolean (NULL, true), - // Deferred conversions - bson_as_json ("kJSON", json_doc), - bson_as_json (NULL, NULL), - oid_as_hex ("kOID", &oid), - oid_as_hex (NULL, NULL), - // Common structures, with explicit set of keys to include - cmd (&cmd, DATABASE_NAME, COMMAND_NAME, OPERATION_ID, COMMAND), - server_description (&server_description, SERVER_HOST, SERVER_PORT, SERVER_CONNECTION_ID, SERVICE_ID)); + mongoc_structured_log (MONGOC_STRUCTURED_LOG_LEVEL_WARNING, + MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND, + "Log entry with command and reply fields", + cmd (&cmd, DATABASE_NAME, COMMAND_NAME, OPERATION_ID, COMMAND), + cmd_reply ("ping", reply_doc), + cmd_reply ("authenticate", reply_doc), + cmd_failure ("ping", reply_doc, &server_error), + cmd_failure ("authenticate", reply_doc, &server_error), + cmd_failure ("authenticate", reply_doc, &client_error)); ASSERT_CMPINT (assumption.calls, ==, 1); restore_state (old_state); bson_destroy (assumption.expected_bson); - bson_destroy (json_doc); bson_destroy (cmd_doc); - bson_destroy (bson_str_n); + bson_destroy (reply_doc); } void test_structured_log_install (TestSuite *suite) { - TestSuite_Add (suite, "/structured_log/plain", test_plain_log_entry); - TestSuite_Add (suite, "/structured_log/with_extra_data", test_log_entry_with_extra_data); - TestSuite_Add (suite, "/structured_log/with_all_data_types", test_log_entry_with_all_data_types); + TestSuite_Add (suite, "/structured_log/plain", test_structured_log_plain); + TestSuite_Add (suite, "/structured_log/with_extra_data", test_structured_log_with_extra_data); + TestSuite_Add (suite, "/structured_log/basic_data_types", test_structured_log_basic_data_types); + TestSuite_Add (suite, "/structured_log/json", test_structured_log_json); + TestSuite_Add (suite, "/structured_log/oid", test_structured_log_oid); + TestSuite_Add (suite, "/structured_log/server_description", test_structured_log_server_description); + TestSuite_Add (suite, "/structured_log/command", test_structured_log_command); } diff --git a/src/libmongoc/tests/unified/runner.c b/src/libmongoc/tests/unified/runner.c index 617bf193761..194c2332498 100644 --- a/src/libmongoc/tests/unified/runner.c +++ b/src/libmongoc/tests/unified/runner.c @@ -1219,6 +1219,59 @@ test_check_expected_events (test_t *test, bson_error_t *error) return ret; } +static bool +check_failure_is_redacted (const bson_iter_t *failure_iter, bson_error_t *error) +{ + if (BSON_ITER_HOLDS_UTF8 (failure_iter)) { + test_diagnostics_error_info ("%s", "expected redacted 'failure', found string message (not allowed)"); + return false; + } + if (!BSON_ITER_HOLDS_DOCUMENT (failure_iter)) { + test_diagnostics_error_info ("%s", "expected redacted 'failure' document, found unexpected type"); + return false; + } + + bson_t failure; + bson_iter_bson (failure_iter, &failure); + + bson_parser_t *bp = bson_parser_new (); + int64_t *failure_code; + char *failure_code_name; + bson_t *failure_error_labels; + bson_parser_int_optional (bp, "code", &failure_code); + bson_parser_utf8_optional (bp, "codeName", &failure_code_name); + bson_parser_array_optional (bp, "errorLabels", &failure_error_labels); + bool parse_result = bson_parser_parse (bp, &failure, error); + bson_parser_destroy_with_parsed_fields (bp); + + bson_destroy (&failure); + return parse_result; +} + +static bool +check_failure_is_detailed (const bson_iter_t *failure_iter, bson_error_t *error) +{ + if (BSON_ITER_HOLDS_UTF8 (failure_iter)) { + // Strings are fine, that's enough proof that the failure was not redacted + return true; + } + if (!BSON_ITER_HOLDS_DOCUMENT (failure_iter)) { + test_diagnostics_error_info ("%s", "expected non-redacted 'failure' document, found unexpected type"); + return false; + } + + // Look for keys that indicate an un-redacted message + bson_iter_t child; + BSON_ASSERT (bson_iter_recurse (failure_iter, &child)); + while (bson_iter_next (&child)) { + const char *key = bson_iter_key (&child); + if (!strcmp (key, "message") || !strcmp (key, "details")) { + return true; + } + } + return false; +} + static bool test_check_log_message (test_t *test, bson_t *expected, log_message_t *actual, bson_error_t *error) { @@ -1249,21 +1302,35 @@ test_check_log_message (test_t *test, bson_t *expected, log_message_t *actual, b goto done; } - // @todo; failureIsRedacted needs implementing - BSON_ASSERT (!failure_is_redacted); + if (failure_is_redacted) { + bson_iter_t failure_iter; + if (!bson_iter_init_find (&failure_iter, actual->message, "failure")) { + test_set_error (error, "expected log 'failure' to exist"); + goto done; + }; + if (*failure_is_redacted) { + if (!check_failure_is_redacted (&failure_iter, error)) { + test_diagnostics_error_info ("actual log message: %s", tmp_json (actual->message)); + test_set_error (error, "expected log 'failure' to be redacted"); + goto done; + } + } else { + if (!check_failure_is_detailed (&failure_iter, error)) { + test_diagnostics_error_info ("actual log message: %s", tmp_json (actual->message)); + test_set_error (error, "expected a complete un-redacted 'failure'"); + goto done; + } + } + } bson_val_t *expected_val = bson_val_from_bson (expected_message_doc); bson_val_t *actual_val = bson_val_from_bson (actual->message); bool is_match = bson_match (expected_val, actual_val, false, error); bson_val_destroy (actual_val); bson_val_destroy (expected_val); - if (!is_match) { - char *expected_str = bson_as_relaxed_extended_json (expected_message_doc, NULL); - char *actual_str = bson_as_relaxed_extended_json (actual->message, NULL); - test_set_error (error, "expected log message: %s, but got: %s", expected_str, actual_str); - bson_free (actual_str); - bson_free (expected_str); + test_set_error ( + error, "expected log message: %s, but got: %s", tmp_json (expected_message_doc), tmp_json (actual->message)); goto done; } From ad7a333ee48481613c09a4c307f900abbff98e35 Mon Sep 17 00:00:00 2001 From: Micah Scott Date: Wed, 13 Nov 2024 17:17:48 -0800 Subject: [PATCH 064/139] trivial improvements to test-mongoc-structured-log --- src/libmongoc/tests/test-mongoc-structured-log.c | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/libmongoc/tests/test-mongoc-structured-log.c b/src/libmongoc/tests/test-mongoc-structured-log.c index 2328a61a909..c3f127366f8 100644 --- a/src/libmongoc/tests/test-mongoc-structured-log.c +++ b/src/libmongoc/tests/test-mongoc-structured-log.c @@ -102,13 +102,13 @@ test_structured_log_plain (void) } void -test_structured_log_with_extra_data (void) +test_structured_log_plain_with_extra_data (void) { struct log_assumption assumption = { .expected_envelope.level = MONGOC_STRUCTURED_LOG_LEVEL_WARNING, .expected_envelope.component = MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND, - .expected_envelope.message = "Plain log entry", - .expected_bson = BCON_NEW ("message", BCON_UTF8 ("Plain log entry"), "extra", BCON_INT32 (1)), + .expected_envelope.message = "Plain log entry with extra data", + .expected_bson = BCON_NEW ("message", BCON_UTF8 ("Plain log entry with extra data"), "extra", BCON_INT32 (1)), .expected_calls = 1, }; @@ -117,7 +117,7 @@ test_structured_log_with_extra_data (void) mongoc_structured_log (MONGOC_STRUCTURED_LOG_LEVEL_WARNING, MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND, - "Plain log entry", + "Plain log entry with extra data", int32 ("extra", 1)); ASSERT_CMPINT (assumption.calls, ==, 1); @@ -332,6 +332,8 @@ test_structured_log_command (void) .expected_envelope.message = "Log entry with command and reply fields", .expected_bson = BCON_NEW ("message", BCON_UTF8 ("Log entry with command and reply fields"), + "commandName", + BCON_UTF8 ("Not a command"), "databaseName", BCON_UTF8 ("Some database"), "commandName", @@ -395,6 +397,7 @@ test_structured_log_command (void) mongoc_structured_log (MONGOC_STRUCTURED_LOG_LEVEL_WARNING, MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND, "Log entry with command and reply fields", + cmd (&cmd, COMMAND_NAME), cmd (&cmd, DATABASE_NAME, COMMAND_NAME, OPERATION_ID, COMMAND), cmd_reply ("ping", reply_doc), cmd_reply ("authenticate", reply_doc), @@ -413,7 +416,7 @@ void test_structured_log_install (TestSuite *suite) { TestSuite_Add (suite, "/structured_log/plain", test_structured_log_plain); - TestSuite_Add (suite, "/structured_log/with_extra_data", test_structured_log_with_extra_data); + TestSuite_Add (suite, "/structured_log/plain_with_extra_data", test_structured_log_plain_with_extra_data); TestSuite_Add (suite, "/structured_log/basic_data_types", test_structured_log_basic_data_types); TestSuite_Add (suite, "/structured_log/json", test_structured_log_json); TestSuite_Add (suite, "/structured_log/oid", test_structured_log_oid); From fe299f5e12705319a7e2a4f5ece863f6f6cbbb08 Mon Sep 17 00:00:00 2001 From: Micah Scott Date: Wed, 13 Nov 2024 18:11:25 -0800 Subject: [PATCH 065/139] centralize structured logging about durations --- src/libmongoc/src/mongoc/mongoc-client.c | 4 +- src/libmongoc/src/mongoc/mongoc-cluster.c | 4 +- src/libmongoc/src/mongoc/mongoc-cursor.c | 4 +- .../mongoc/mongoc-structured-log-private.h | 5 +++ .../tests/test-mongoc-structured-log.c | 40 +++++++++++++++++++ 5 files changed, 51 insertions(+), 6 deletions(-) diff --git a/src/libmongoc/src/mongoc/mongoc-client.c b/src/libmongoc/src/mongoc/mongoc-client.c index 16a8ca2157f..d2af94c6a62 100644 --- a/src/libmongoc/src/mongoc/mongoc-client.c +++ b/src/libmongoc/src/mongoc/mongoc-client.c @@ -2236,7 +2236,7 @@ _mongoc_client_monitor_op_killcursors_succeeded (mongoc_cluster_t *cluster, utf8 ("databaseName", db), utf8 ("commandName", "killCursors"), int64 ("operationId", operation_id), - int64 ("durationMS", duration), + monotonic_time_duration (duration), cmd_reply ("killCursors", &doc)); if (!client->apm_callbacks.succeeded) { @@ -2294,7 +2294,7 @@ _mongoc_client_monitor_op_killcursors_failed (mongoc_cluster_t *cluster, utf8 ("databaseName", db), utf8 ("commandName", "killCursors"), int64 ("operationId", operation_id), - int64 ("durationMS", duration), + monotonic_time_duration (duration), bson_as_json ("failure", &doc)); if (!client->apm_callbacks.failed) { diff --git a/src/libmongoc/src/mongoc/mongoc-cluster.c b/src/libmongoc/src/mongoc/mongoc-cluster.c index fb549a06a0f..4ced0532b8d 100644 --- a/src/libmongoc/src/mongoc/mongoc-cluster.c +++ b/src/libmongoc/src/mongoc/mongoc-cluster.c @@ -570,7 +570,7 @@ mongoc_cluster_run_command_monitored (mongoc_cluster_t *cluster, mongoc_cmd_t *c MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND, "Command succeeded", int32 ("requestId", request_id), - int64 ("durationMS", duration), + monotonic_time_duration (duration), server_description (server_stream->sd, SERVER_HOST, SERVER_PORT, SERVER_CONNECTION_ID, SERVICE_ID), cmd (cmd, DATABASE_NAME, COMMAND_NAME, OPERATION_ID), cmd_reply (cmd->command_name, cmd->is_acknowledged ? reply : &fake_reply)); @@ -603,7 +603,7 @@ mongoc_cluster_run_command_monitored (mongoc_cluster_t *cluster, mongoc_cmd_t *c MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND, "Command failed", int32 ("requestId", request_id), - int64 ("durationMS", duration), + monotonic_time_duration (duration), server_description (server_stream->sd, SERVER_HOST, SERVER_PORT, SERVER_CONNECTION_ID, SERVICE_ID), cmd (cmd, DATABASE_NAME, COMMAND_NAME, OPERATION_ID), cmd_failure (cmd->command_name, reply, error)); diff --git a/src/libmongoc/src/mongoc/mongoc-cursor.c b/src/libmongoc/src/mongoc/mongoc-cursor.c index 3f1ca5849dc..55d72ac0dbb 100644 --- a/src/libmongoc/src/mongoc/mongoc-cursor.c +++ b/src/libmongoc/src/mongoc/mongoc-cursor.c @@ -747,7 +747,7 @@ _mongoc_cursor_monitor_succeeded (mongoc_cursor_t *cursor, utf8 ("databaseName", db), utf8 ("commandName", cmd_name), int64 ("operationId", cursor->operation_id), - int64 ("durationMS", duration), + monotonic_time_duration (duration), cmd_reply (cmd_name, &reply)); if (client->apm_callbacks.succeeded) { @@ -804,7 +804,7 @@ _mongoc_cursor_monitor_failed (mongoc_cursor_t *cursor, utf8 ("databaseName", db), utf8 ("commandName", cmd_name), int64 ("operationId", cursor->operation_id), - int64 ("durationMS", duration), + monotonic_time_duration (duration), bson_as_json ("failure", &reply)); if (client->apm_callbacks.failed) { diff --git a/src/libmongoc/src/mongoc/mongoc-structured-log-private.h b/src/libmongoc/src/mongoc/mongoc-structured-log-private.h index 667e08b772e..a550f0afb3a 100644 --- a/src/libmongoc/src/mongoc/mongoc-structured-log-private.h +++ b/src/libmongoc/src/mongoc/mongoc-structured-log-private.h @@ -98,6 +98,11 @@ BSON_BEGIN_DECLS .arg2.server_description_flags = (0 _bsonDSL_mapMacro ( \ _mongoc_structured_log_flag_expr, MONGOC_STRUCTURED_LOG_SERVER_DESCRIPTION, __VA_ARGS__))}, +// @todo Is the (CLAM) spec asking for only the highest resolution available, or that plus milliseconds? +#define _mongoc_structured_log_item_monotonic_time_duration(_duration) \ + _mongoc_structured_log_item_int32 ("durationMS", (int32_t) ((_duration) / 1000)) \ + _mongoc_structured_log_item_int64 ("durationMicros", (_duration)) + typedef struct mongoc_structured_log_builder_stage_t mongoc_structured_log_builder_stage_t; typedef const mongoc_structured_log_builder_stage_t *(*mongoc_structured_log_builder_func_t) ( diff --git a/src/libmongoc/tests/test-mongoc-structured-log.c b/src/libmongoc/tests/test-mongoc-structured-log.c index c3f127366f8..27a82dd0c4c 100644 --- a/src/libmongoc/tests/test-mongoc-structured-log.c +++ b/src/libmongoc/tests/test-mongoc-structured-log.c @@ -412,6 +412,45 @@ test_structured_log_command (void) bson_destroy (reply_doc); } +void +test_structured_log_duration (void) +{ + struct log_assumption assumption = { + .expected_envelope.level = MONGOC_STRUCTURED_LOG_LEVEL_WARNING, + .expected_envelope.component = MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND, + .expected_envelope.message = "Log entry with duration", + .expected_bson = BCON_NEW ("message", + BCON_UTF8 ("Log entry with duration"), + "durationMS", + BCON_INT32 (1), + "durationMicros", + BCON_INT64 (1999), + "durationMS", + BCON_INT32 (0), + "durationMicros", + BCON_INT64 (10), + "durationMS", + BCON_INT32 (10000000), + "durationMicros", + BCON_INT64 (10000000999)), + .expected_calls = 1, + }; + + structured_log_state old_state = save_state (); + mongoc_structured_log_set_handler (structured_log_func, &assumption); + + mongoc_structured_log (MONGOC_STRUCTURED_LOG_LEVEL_WARNING, + MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND, + "Log entry with duration", + monotonic_time_duration (1999), + monotonic_time_duration (10), + monotonic_time_duration (10000000999)); + + ASSERT_CMPINT (assumption.calls, ==, 1); + restore_state (old_state); + bson_destroy (assumption.expected_bson); +} + void test_structured_log_install (TestSuite *suite) { @@ -422,4 +461,5 @@ test_structured_log_install (TestSuite *suite) TestSuite_Add (suite, "/structured_log/oid", test_structured_log_oid); TestSuite_Add (suite, "/structured_log/server_description", test_structured_log_server_description); TestSuite_Add (suite, "/structured_log/command", test_structured_log_command); + TestSuite_Add (suite, "/structured_log/duration", test_structured_log_duration); } From 1b99bbe8fc4edc38357ef6119e18416ae40fa3cb Mon Sep 17 00:00:00 2001 From: Micah Scott Date: Wed, 13 Nov 2024 18:22:14 -0800 Subject: [PATCH 066/139] mode for struct log bson_as_json should be relaxed --- src/libmongoc/src/mongoc/mongoc-structured-log.c | 2 +- src/libmongoc/tests/test-mongoc-structured-log.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libmongoc/src/mongoc/mongoc-structured-log.c b/src/libmongoc/src/mongoc/mongoc-structured-log.c index 1589d0529d3..7b2395e1fd2 100644 --- a/src/libmongoc/src/mongoc/mongoc-structured-log.c +++ b/src/libmongoc/src/mongoc/mongoc-structured-log.c @@ -282,7 +282,7 @@ mongoc_structured_log_default_handler (const mongoc_structured_log_entry_t *entr char * mongoc_structured_log_document_to_json (const bson_t *document, size_t *length) { - bson_json_opts_t *opts = bson_json_opts_new (BSON_JSON_MODE_CANONICAL, gStructuredLog.max_document_length); + bson_json_opts_t *opts = bson_json_opts_new (BSON_JSON_MODE_RELAXED, gStructuredLog.max_document_length); char *json = bson_as_json_with_opts (document, length, opts); bson_json_opts_destroy (opts); return json; diff --git a/src/libmongoc/tests/test-mongoc-structured-log.c b/src/libmongoc/tests/test-mongoc-structured-log.c index 27a82dd0c4c..47f8e3ea380 100644 --- a/src/libmongoc/tests/test-mongoc-structured-log.c +++ b/src/libmongoc/tests/test-mongoc-structured-log.c @@ -343,7 +343,7 @@ test_structured_log_command (void) "command", BCON_UTF8 ("{ \"c\" : \"d\" }"), "reply", // Un-redacted successful reply - BCON_UTF8 ("{ \"r\" : \"s\", \"code\" : { \"$numberInt\" : \"1\" } }"), + BCON_UTF8 ("{ \"r\" : \"s\", \"code\" : 1 }"), "reply", // Redacted successful reply BCON_UTF8 ("{}"), "failure", // Un-redacted server side error From b597b81848edd4c8dc03f0e5fe27220fc2471610 Mon Sep 17 00:00:00 2001 From: Micah Scott Date: Thu, 14 Nov 2024 09:25:25 -0800 Subject: [PATCH 067/139] suppress test runner capture of structured logs in cleanup and failpoints --- src/libmongoc/tests/test-libmongoc.c | 4 ++-- src/libmongoc/tests/test-libmongoc.h | 4 ++++ src/libmongoc/tests/unified/operation.c | 4 ++++ src/libmongoc/tests/unified/runner.c | 6 ++++++ 4 files changed, 16 insertions(+), 2 deletions(-) diff --git a/src/libmongoc/tests/test-libmongoc.c b/src/libmongoc/tests/test-libmongoc.c index 5f7e60e1853..194e40cc1de 100644 --- a/src/libmongoc/tests/test-libmongoc.c +++ b/src/libmongoc/tests/test-libmongoc.c @@ -74,13 +74,13 @@ test_is_suppressing_structured_logs (void) } // Ignore logs generated by test-internal operations. Can be nested. Structured and traditional logs. -static void +void test_begin_suppressing_structured_logs (void) { mcommon_atomic_int_fetch_add (&suppress_structured_logs_counter_atomic, 1, mcommon_memory_order_seq_cst); } -static void +void test_end_suppressing_structured_logs (void) { int previous = diff --git a/src/libmongoc/tests/test-libmongoc.h b/src/libmongoc/tests/test-libmongoc.h index 575fe59cb57..3c33b79f301 100644 --- a/src/libmongoc/tests/test-libmongoc.h +++ b/src/libmongoc/tests/test-libmongoc.h @@ -39,6 +39,10 @@ get_test_collection (mongoc_client_t *client, const char *prefix); bool test_is_suppressing_structured_logs (void); void +test_begin_suppressing_structured_logs (void); +void +test_end_suppressing_structured_logs (void); +void capture_logs (bool capture); void clear_captured_logs (void); diff --git a/src/libmongoc/tests/unified/operation.c b/src/libmongoc/tests/unified/operation.c index a9e429fd8aa..d39e5b4ca34 100644 --- a/src/libmongoc/tests/unified/operation.c +++ b/src/libmongoc/tests/unified/operation.c @@ -2505,7 +2505,9 @@ operation_failpoint (test_t *test, operation_t *op, result_t *result, bson_error rp = mongoc_read_prefs_new (MONGOC_READ_PRIMARY); bson_destroy (&op_reply); + test_begin_suppressing_structured_logs (); mongoc_client_command_simple (client, "admin", failpoint, rp, &op_reply, &op_error); + test_end_suppressing_structured_logs (); result_from_val_and_reply (result, NULL /* value */, &op_reply, &op_error); /* Add failpoint to list of test_t's known failpoints */ @@ -2563,7 +2565,9 @@ operation_targeted_failpoint (test_t *test, operation_t *op, result_t *result, b rp = mongoc_read_prefs_new (MONGOC_READ_PRIMARY); bson_destroy (&op_reply); + test_begin_suppressing_structured_logs (); mongoc_client_command_simple_with_server_id (client, "admin", failpoint, rp, server_id, &op_reply, &op_error); + test_end_suppressing_structured_logs (); result_from_val_and_reply (result, NULL /* value */, &op_reply, &op_error); /* Add failpoint to list of test_t's known failpoints */ diff --git a/src/libmongoc/tests/unified/runner.c b/src/libmongoc/tests/unified/runner.c index 194c2332498..94b425a8c1c 100644 --- a/src/libmongoc/tests/unified/runner.c +++ b/src/libmongoc/tests/unified/runner.c @@ -142,6 +142,8 @@ cleanup_failpoints (test_t *test, bson_error_t *error) failpoint_t *iter = NULL; mongoc_read_prefs_t *rp = NULL; + test_begin_suppressing_structured_logs (); + rp = mongoc_read_prefs_new (MONGOC_READ_PRIMARY_PREFERRED); LL_FOREACH (test->failpoints, iter) @@ -172,6 +174,7 @@ cleanup_failpoints (test_t *test, bson_error_t *error) ret = true; done: mongoc_read_prefs_destroy (rp); + test_end_suppressing_structured_logs (); return ret; } @@ -271,6 +274,8 @@ test_runner_terminate_open_transactions (test_runner_t *test_runner, bson_error_ bool cmd_ret = false; bson_error_t cmd_error = {0}; + test_begin_suppressing_structured_logs (); + if (test_framework_getenv_bool ("MONGOC_TEST_ATLAS")) { // Not applicable when running as test-atlas-executor. return true; @@ -323,6 +328,7 @@ test_runner_terminate_open_transactions (test_runner_t *test_runner, bson_error_ ret = true; done: + test_end_suppressing_structured_logs (); return ret; } From 27c336f5019c18bb33d89fcef0aa5e39e6dd1b61 Mon Sep 17 00:00:00 2001 From: Micah Scott Date: Thu, 14 Nov 2024 10:41:46 -0800 Subject: [PATCH 068/139] Additional redaction support, doc comments for log macros --- src/libmongoc/src/mongoc/mongoc-client.c | 2 +- src/libmongoc/src/mongoc/mongoc-cluster.c | 4 +- src/libmongoc/src/mongoc/mongoc-cursor.c | 2 +- .../mongoc/mongoc-structured-log-private.h | 175 +++++++++++++++++- .../src/mongoc/mongoc-structured-log.c | 120 +++++++++--- .../tests/test-mongoc-structured-log.c | 43 +++-- 6 files changed, 298 insertions(+), 48 deletions(-) diff --git a/src/libmongoc/src/mongoc/mongoc-client.c b/src/libmongoc/src/mongoc/mongoc-client.c index d2af94c6a62..8d0bf509958 100644 --- a/src/libmongoc/src/mongoc/mongoc-client.c +++ b/src/libmongoc/src/mongoc/mongoc-client.c @@ -2237,7 +2237,7 @@ _mongoc_client_monitor_op_killcursors_succeeded (mongoc_cluster_t *cluster, utf8 ("commandName", "killCursors"), int64 ("operationId", operation_id), monotonic_time_duration (duration), - cmd_reply ("killCursors", &doc)); + cmd_name_reply ("killCursors", &doc)); if (!client->apm_callbacks.succeeded) { bson_destroy (&doc); diff --git a/src/libmongoc/src/mongoc/mongoc-cluster.c b/src/libmongoc/src/mongoc/mongoc-cluster.c index 4ced0532b8d..3a27f09de47 100644 --- a/src/libmongoc/src/mongoc/mongoc-cluster.c +++ b/src/libmongoc/src/mongoc/mongoc-cluster.c @@ -573,7 +573,7 @@ mongoc_cluster_run_command_monitored (mongoc_cluster_t *cluster, mongoc_cmd_t *c monotonic_time_duration (duration), server_description (server_stream->sd, SERVER_HOST, SERVER_PORT, SERVER_CONNECTION_ID, SERVICE_ID), cmd (cmd, DATABASE_NAME, COMMAND_NAME, OPERATION_ID), - cmd_reply (cmd->command_name, cmd->is_acknowledged ? reply : &fake_reply)); + cmd_reply (cmd, cmd->is_acknowledged ? reply : &fake_reply)); if (callbacks->succeeded) { mongoc_apm_command_succeeded_init (&succeeded_event, @@ -606,7 +606,7 @@ mongoc_cluster_run_command_monitored (mongoc_cluster_t *cluster, mongoc_cmd_t *c monotonic_time_duration (duration), server_description (server_stream->sd, SERVER_HOST, SERVER_PORT, SERVER_CONNECTION_ID, SERVICE_ID), cmd (cmd, DATABASE_NAME, COMMAND_NAME, OPERATION_ID), - cmd_failure (cmd->command_name, reply, error)); + cmd_failure (cmd, reply, error)); if (callbacks->failed) { mongoc_apm_command_failed_init (&failed_event, diff --git a/src/libmongoc/src/mongoc/mongoc-cursor.c b/src/libmongoc/src/mongoc/mongoc-cursor.c index 55d72ac0dbb..734b178ed95 100644 --- a/src/libmongoc/src/mongoc/mongoc-cursor.c +++ b/src/libmongoc/src/mongoc/mongoc-cursor.c @@ -748,7 +748,7 @@ _mongoc_cursor_monitor_succeeded (mongoc_cursor_t *cursor, utf8 ("commandName", cmd_name), int64 ("operationId", cursor->operation_id), monotonic_time_duration (duration), - cmd_reply (cmd_name, &reply)); + cmd_name_reply (cmd_name, &reply)); if (client->apm_callbacks.succeeded) { mongoc_apm_command_succeeded_init (&event, diff --git a/src/libmongoc/src/mongoc/mongoc-structured-log-private.h b/src/libmongoc/src/mongoc/mongoc-structured-log-private.h index a550f0afb3a..8cfdd7358fe 100644 --- a/src/libmongoc/src/mongoc/mongoc-structured-log-private.h +++ b/src/libmongoc/src/mongoc/mongoc-structured-log-private.h @@ -30,6 +30,27 @@ BSON_BEGIN_DECLS #define MONGOC_STRUCTURED_LOG_DEFAULT_LEVEL MONGOC_STRUCTURED_LOG_LEVEL_WARNING #define MONGOC_STRUCTURED_LOG_DEFAULT_MAX_DOCUMENT_LENGTH 1000 +/** + * @def mognoc_structured_log(level, component, message, ...) + * @brief Write to the libmongoc structured log. + * + * @param level Log level, as a mongoc_structured_log_level_t expression + * @param component Log component, as a mongoc_structured_log_component_t expression + * @param message Log message, as a const char* expression + * @param ... Optional list of log 'items' that specify additional information to include. + * + * The level, component, and message expressions are always evaluated. + * Any expressions in the optional items list are only evaluated if the log + * hasn't been disabled by a component's maximum log level setting or by + * unsetting the global structured log handler. + * + * Each log 'item' may represent a deferred operation that has minimal cost + * unless mongoc_structured_log_entry_message_as_bson is actually invoked. + * + * Calls implementation functions _mongoc_structured_log_should_log() before + * building the table of item information and _mongoc_structured_log_with_entry() + * once the table is built. + */ #define mongoc_structured_log(_level, _component, ...) \ _mongoc_structured_log_with_end_of_list (_level, _component, __VA_ARGS__, end_of_list ()) @@ -54,51 +75,184 @@ BSON_BEGIN_DECLS #define _mongoc_structured_log_item_end_of_list() {.func = NULL}, +/** + * @def utf8(key, value) + * @brief Structured log item, referencing a NUL-terminated utf8 string value. + * + * @param key Key as a const char * expression, or NULL to skip this item. + * @param value UTF8 value as a const char * expression, or NULL for a null value. + */ #define _mongoc_structured_log_item_utf8(_key_or_null, _value_utf8) \ {.func = _mongoc_structured_log_append_utf8, .arg1.utf8 = (_key_or_null), .arg2.utf8 = (_value_utf8)}, +/** + * @def utf8_n(key, value, value_len) + * @brief Structured log item, referencing a utf8 string value with explicit length. + * + * @param key Document key as a NULL-terminated const char * expression, or NULL to skip this item. + * @param value UTF8 value as a const char * expression, or NULL for a null value. May have embedded NUL bytes. + * @param value_len UTF8 value length in bytes, as an int32_t expression. + */ #define _mongoc_structured_log_item_utf8_n(_key_literal, _value_utf8, _value_len) \ _mongoc_structured_log_item_utf8_nn (_key_literal, strlen (_key_literal), _value_utf8, _value_len) +/** + * @def utf8_nn(key, key_len, value, value_len) + * @brief Structured log item, referencing a utf8 string with explicit key and value lengths. + * + * @param key Key as a NULL-terminated const char * expression, or NULL to skip this item. + * @param value UTF8 value as a const char * expression, or NULL for a null value. May have embedded NUL bytes. + * @param value_len UTF8 value length in bytes, as an int32_t expression. + */ #define _mongoc_structured_log_item_utf8_nn(_key_or_null, _key_len, _value_utf8, _value_len) \ {.func = _mongoc_structured_log_append_utf8_n_stage0, .arg1.utf8 = (_key_or_null), .arg2.int32 = (_key_len)}, \ {.func = _mongoc_structured_log_append_utf8_n_stage1, .arg1.utf8 = (_value_utf8), .arg2.int32 = (_value_len)}, +/** + * @def int32(key, value) + * @brief Structured log item, 32-bit integer + * + * @param key Key as a NULL-terminated const char * expression, or NULL to skip this item. + * @param value Value as an int32_t expression. + */ #define _mongoc_structured_log_item_int32(_key_or_null, _value_int32) \ {.func = _mongoc_structured_log_append_int32, .arg1.utf8 = (_key_or_null), .arg2.int32 = (_value_int32)}, +/** + * @def int64(key, value) + * @brief Structured log item, 64-bit integer + * + * @param key Key as a NULL-terminated const char * expression, or NULL to skip this item. + * @param value Value as an int64_t expression. + */ #define _mongoc_structured_log_item_int64(_key_or_null, _value_int64) \ {.func = _mongoc_structured_log_append_int64, .arg1.utf8 = (_key_or_null), .arg2.int64 = (_value_int64)}, +/** + * @def boolean(key, value) + * @brief Structured log item, boolean + * + * @param key Key as a NULL-terminated const char * expression, or NULL to skip this item. + * @param value Value as a bool expression. + */ #define _mongoc_structured_log_item_boolean(_key_or_null, _value_boolean) \ {.func = _mongoc_structured_log_append_boolean, .arg1.utf8 = (_key_or_null), .arg2.boolean = (_value_boolean)}, +/** + * @def oid_as_hex(key, value) + * @brief Structured log item, bson_oid_t converted to a hex string + * + * @param key Key as a NULL-terminated const char * expression, or NULL to skip this item. + * @param value OID to convert as a const bson_oid_t * expression, or NULL for a null value. + */ #define _mongoc_structured_log_item_oid_as_hex(_key_or_null, _value_oid) \ {.func = _mongoc_structured_log_append_oid_as_hex, .arg1.utf8 = (_key_or_null), .arg2.oid = (_value_oid)}, +/** + * @def bson_as_json(key, value) + * @brief Structured log item, bson_t converted to a JSON string + * + * Always uses relaxed extended JSON format, and the current applicable + * maximum document length for structured logging. + * + * @param key Key as a NULL-terminated const char * expression, or NULL to skip this item. + * @param value BSON to convert as a const bson_t * expression, or NULL for a null value. + */ #define _mongoc_structured_log_item_bson_as_json(_key_or_null, _value_bson) \ {.func = _mongoc_structured_log_append_bson_as_json, .arg1.utf8 = (_key_or_null), .arg2.bson = (_value_bson)}, +/** + * @def cmd(cmd, ...) + * @brief Structured log item, mongoc_cmd_t fields with automatic redaction + * + * @param cmd Borrowed command reference, as a const mongo_cmd_t * expression. Required. + * @param ... Fields to include. Order is not significant. Any of: COMMAND, DATABASE_NAME, COMMAND_NAME, OPERATION_ID. + * */ #define _mongoc_structured_log_item_cmd(_cmd, ...) \ {.func = _mongoc_structured_log_append_cmd, \ .arg1.cmd = (_cmd), \ .arg2.cmd_flags = \ (0 _bsonDSL_mapMacro (_mongoc_structured_log_flag_expr, MONGOC_STRUCTURED_LOG_CMD, __VA_ARGS__))}, -#define _mongoc_structured_log_item_cmd_reply(_cmd_name, _reply_bson) \ - {.func = _mongoc_structured_log_append_cmd_reply, .arg1.utf8 = (_cmd_name), .arg2.bson = (_reply_bson)}, +/** + * @def cmd_reply(cmd, reply) + * @brief Structured log item, command reply for mongoc_cmd_t with automatic redaction + * + * @param cmd Borrowed command reference, as a const mongo_cmd_t * expression. Required. + * @param reply Borrowed reference to reply document, as a const bson_t * expression. Required. + */ +#define _mongoc_structured_log_item_cmd_reply(_cmd, _reply_bson) \ + {.func = _mongoc_structured_log_append_cmd_reply, .arg1.cmd = (_cmd), .arg2.bson = (_reply_bson)}, -#define _mongoc_structured_log_item_cmd_failure(_cmd_name, _reply_bson, _error) \ - {.func = _mongoc_structured_log_append_cmd_failure_stage0, .arg1.utf8 = (_cmd_name), .arg2.bson = (_reply_bson)}, \ +/** + * @def cmd_name_reply(cmd_name, reply) + * @brief Structured log item, reply for named command with automatic redaction + * + * For cases where a mongo_cmd_t is not available; makes redaction decisions based + * on command name but not body, so it's unsuitable for the "hello" reply. + * + * @param cmd Command name as a const char * expression. Required. + * @param reply Borrowed reference to reply document, as a const bson_t * expression. Required. + */ +#define _mongoc_structured_log_item_cmd_name_reply(_cmd_name, _reply_bson) \ + {.func = _mongoc_structured_log_append_cmd_name_reply, .arg1.utf8 = (_cmd_name), .arg2.bson = (_reply_bson)}, + +/** + * @def cmd_failure(cmd, reply, error) + * @brief Structured log item, failure for mongoc_cmd_t with automatic redaction + * + * The 'error' is examined to determine whether this is a client-side or server-side failure. + * The command's name and body may influence the reply's redaction. + * + * @param cmd Borrowed command reference, as a const mongo_cmd_t * expression. Required. + * @param reply Borrowed reference to reply document, as a const bson_t * expression. Required. + * @param error Borrowed reference to a libmongoc error, as a const bson_error_t * expression. Required. + */ +#define _mongoc_structured_log_item_cmd_failure(_cmd, _reply_bson, _error) \ + {.func = _mongoc_structured_log_append_cmd_failure_stage0, .arg1.cmd = (_cmd), .arg2.bson = (_reply_bson)}, \ {.func = _mongoc_structured_log_append_cmd_failure_stage1, .arg1.error = (_error)}, +/** + * @def cmd_name_failure(cmd_name, reply, error) + * @brief Structured log item, failure for named command with automatic redaction + * + * For cases where a mongo_cmd_t is not available; makes redaction decisions based + * on command name but not body, so it's unsuitable for "hello" errors. + * + * The 'error' is examined to determine whether this is a client-side or server-side failure. + * The command's name and body may influence the reply's redaction. + * + * @param cmd Command name as a const char * expression. Required. + * @param reply Borrowed reference to reply document, as a const bson_t * expression. Required. + * @param error Borrowed reference to a libmongoc error, as a const bson_error_t * expression. Required. + */ +#define _mongoc_structured_log_item_cmd_name_failure(_cmd_name, _reply_bson, _error) \ + {.func = _mongoc_structured_log_append_cmd_name_failure_stage0, \ + .arg1.utf8 = (_cmd_name), \ + .arg2.bson = (_reply_bson)}, \ + {.func = _mongoc_structured_log_append_cmd_name_failure_stage1, .arg1.error = (_error)}, + +/** + * @def server_description(sd, ...) + * @brief Structured log item, mongoc_server_description_t fields + * + * @param cmd Borrowed server description reference, as a const mongoc_server_description_t * expression. Required. + * @param ... Fields to include. Order is not significant. Any of: SERVER_HOST, SERVER_PORT, SERVER_CONNECTION_ID, + * SERVICE_ID. + * */ #define _mongoc_structured_log_item_server_description(_server_description, ...) \ {.func = _mongoc_structured_log_append_server_description, \ .arg1.server_description = (_server_description), \ .arg2.server_description_flags = (0 _bsonDSL_mapMacro ( \ _mongoc_structured_log_flag_expr, MONGOC_STRUCTURED_LOG_SERVER_DESCRIPTION, __VA_ARGS__))}, -// @todo Is the (CLAM) spec asking for only the highest resolution available, or that plus milliseconds? +/** + * @def monotonic_time_duration(duration) + * @brief Structured log item, standard format for a duration in monotonic time. + * @param duration Duration in microseconds, as an int64_t expression. + * + * @todo Is the (CLAM) spec asking for only the highest resolution available, or that plus milliseconds? + * */ #define _mongoc_structured_log_item_monotonic_time_duration(_duration) \ _mongoc_structured_log_item_int32 ("durationMS", (int32_t) ((_duration) / 1000)) \ _mongoc_structured_log_item_int64 ("durationMicros", (_duration)) @@ -205,12 +359,23 @@ _mongoc_structured_log_append_cmd (bson_t *bson, const mongoc_structured_log_bui const mongoc_structured_log_builder_stage_t * _mongoc_structured_log_append_cmd_reply (bson_t *bson, const mongoc_structured_log_builder_stage_t *stage); +const mongoc_structured_log_builder_stage_t * +_mongoc_structured_log_append_cmd_name_reply (bson_t *bson, const mongoc_structured_log_builder_stage_t *stage); + const mongoc_structured_log_builder_stage_t * _mongoc_structured_log_append_cmd_failure_stage0 (bson_t *bson, const mongoc_structured_log_builder_stage_t *stage); const mongoc_structured_log_builder_stage_t * _mongoc_structured_log_append_cmd_failure_stage1 (bson_t *bson, const mongoc_structured_log_builder_stage_t *stage); +const mongoc_structured_log_builder_stage_t * +_mongoc_structured_log_append_cmd_name_failure_stage0 (bson_t *bson, + const mongoc_structured_log_builder_stage_t *stage); + +const mongoc_structured_log_builder_stage_t * +_mongoc_structured_log_append_cmd_name_failure_stage1 (bson_t *bson, + const mongoc_structured_log_builder_stage_t *stage); + const mongoc_structured_log_builder_stage_t * _mongoc_structured_log_append_server_description (bson_t *bson, const mongoc_structured_log_builder_stage_t *stage); diff --git a/src/libmongoc/src/mongoc/mongoc-structured-log.c b/src/libmongoc/src/mongoc/mongoc-structured-log.c index 7b2395e1fd2..450ce92eacd 100644 --- a/src/libmongoc/src/mongoc/mongoc-structured-log.c +++ b/src/libmongoc/src/mongoc/mongoc-structured-log.c @@ -356,10 +356,15 @@ const mongoc_structured_log_builder_stage_t * _mongoc_structured_log_append_oid_as_hex (bson_t *bson, const mongoc_structured_log_builder_stage_t *stage) { const char *key_or_null = stage->arg1.utf8; + const bson_oid_t *oid_or_null = stage->arg2.oid; if (key_or_null) { - char str[25]; - bson_oid_to_string (stage->arg2.oid, str); - bson_append_utf8 (bson, key_or_null, -1, str, 24); + if (oid_or_null) { + char str[25]; + bson_oid_to_string (oid_or_null, str); + bson_append_utf8 (bson, key_or_null, -1, str, 24); + } else { + bson_append_null (bson, key_or_null, -1); + } } return stage + 1; } @@ -368,12 +373,17 @@ const mongoc_structured_log_builder_stage_t * _mongoc_structured_log_append_bson_as_json (bson_t *bson, const mongoc_structured_log_builder_stage_t *stage) { const char *key_or_null = stage->arg1.utf8; + const bson_t *bson_or_null = stage->arg2.bson; if (key_or_null) { - size_t json_length; - char *json = mongoc_structured_log_document_to_json (stage->arg2.bson, &json_length); - if (json) { - bson_append_utf8 (bson, key_or_null, -1, json, json_length); - bson_free (json); + if (bson_or_null) { + size_t json_length; + char *json = mongoc_structured_log_document_to_json (bson_or_null, &json_length); + if (json) { + bson_append_utf8 (bson, key_or_null, -1, json, json_length); + bson_free (json); + } + } else { + bson_append_null (bson, key_or_null, -1); } } return stage + 1; @@ -426,16 +436,10 @@ _mongoc_structured_log_append_cmd (bson_t *bson, const mongoc_structured_log_bui return stage + 1; } -const mongoc_structured_log_builder_stage_t * -_mongoc_structured_log_append_cmd_reply (bson_t *bson, const mongoc_structured_log_builder_stage_t *stage) +static void +_mongoc_structured_log_append_redacted_cmd_reply (bson_t *bson, bool is_sensitive, const bson_t *reply) { - const char *cmd_name = stage->arg1.utf8; - const bson_t *reply = stage->arg2.bson; - - BSON_ASSERT (cmd_name); - BSON_ASSERT (reply); - - if (mongoc_apm_is_sensitive_command_message (cmd_name, reply)) { + if (is_sensitive) { BSON_APPEND_UTF8 (bson, "reply", "{}"); } else { size_t json_length; @@ -446,25 +450,45 @@ _mongoc_structured_log_append_cmd_reply (bson_t *bson, const mongoc_structured_l bson_free (json); } } +} + +const mongoc_structured_log_builder_stage_t * +_mongoc_structured_log_append_cmd_reply (bson_t *bson, const mongoc_structured_log_builder_stage_t *stage) +{ + const mongoc_cmd_t *cmd = stage->arg1.cmd; + const bson_t *reply = stage->arg2.bson; + + BSON_ASSERT (cmd); + BSON_ASSERT (reply); + + bool is_sensitive = mongoc_apm_is_sensitive_command_message (cmd->command_name, cmd->command) || + mongoc_apm_is_sensitive_command_message (cmd->command_name, reply); + _mongoc_structured_log_append_redacted_cmd_reply (bson, is_sensitive, reply); return stage + 1; } const mongoc_structured_log_builder_stage_t * -_mongoc_structured_log_append_cmd_failure_stage0 (bson_t *bson, const mongoc_structured_log_builder_stage_t *stage) +_mongoc_structured_log_append_cmd_name_reply (bson_t *bson, const mongoc_structured_log_builder_stage_t *stage) { - BSON_ASSERT (stage[1].func == _mongoc_structured_log_append_cmd_failure_stage1); - const char *cmd_name = stage[0].arg1.utf8; - const bson_t *reply = stage[0].arg2.bson; - const bson_error_t *error = stage[1].arg1.error; + const char *cmd_name = stage->arg1.utf8; + const bson_t *reply = stage->arg2.bson; BSON_ASSERT (cmd_name); BSON_ASSERT (reply); - BSON_ASSERT (error); + bool is_sensitive = mongoc_apm_is_sensitive_command_message (cmd_name, reply); + _mongoc_structured_log_append_redacted_cmd_reply (bson, is_sensitive, reply); + return stage + 1; +} + +static void +_mongoc_structured_log_append_redacted_cmd_failure (bson_t *bson, + bool is_sensitive, + const bson_t *reply, + const bson_error_t *error) +{ bool is_server_side = error->domain == MONGOC_ERROR_SERVER || error->domain == MONGOC_ERROR_WRITE_CONCERN_ERROR; if (is_server_side) { - bool is_sensitive = mongoc_apm_is_sensitive_command_message (cmd_name, reply); - if (is_sensitive) { // Redacted server-side message, must be a document with at most 'code', 'codeName', 'errorLabels' bson_t failure; @@ -478,12 +502,10 @@ _mongoc_structured_log_append_cmd_failure_stage0 (bson_t *bson, const mongoc_str } } BSON_ASSERT (bson_append_document_end (bson, &failure)); - } else { // Non-redacted server side message, pass through BSON_APPEND_DOCUMENT (bson, "failure", reply); } - } else { // Client-side errors converted directly from bson_error_t, never redacted bson_t failure; @@ -493,6 +515,23 @@ _mongoc_structured_log_append_cmd_failure_stage0 (bson_t *bson, const mongoc_str BSON_APPEND_UTF8 (&failure, "message", error->message); BSON_ASSERT (bson_append_document_end (bson, &failure)); } +} + +const mongoc_structured_log_builder_stage_t * +_mongoc_structured_log_append_cmd_failure_stage0 (bson_t *bson, const mongoc_structured_log_builder_stage_t *stage) +{ + BSON_ASSERT (stage[1].func == _mongoc_structured_log_append_cmd_failure_stage1); + const mongoc_cmd_t *cmd = stage[0].arg1.cmd; + const bson_t *reply = stage[0].arg2.bson; + const bson_error_t *error = stage[1].arg1.error; + + BSON_ASSERT (cmd); + BSON_ASSERT (reply); + BSON_ASSERT (error); + + bool is_sensitive = mongoc_apm_is_sensitive_command_message (cmd->command_name, cmd->command) || + mongoc_apm_is_sensitive_command_message (cmd->command_name, reply); + _mongoc_structured_log_append_redacted_cmd_failure (bson, is_sensitive, reply, error); return stage + 2; } @@ -506,6 +545,33 @@ _mongoc_structured_log_append_cmd_failure_stage1 (bson_t *bson, const mongoc_str return NULL; } +const mongoc_structured_log_builder_stage_t * +_mongoc_structured_log_append_cmd_name_failure_stage0 (bson_t *bson, const mongoc_structured_log_builder_stage_t *stage) +{ + BSON_ASSERT (stage[1].func == _mongoc_structured_log_append_cmd_name_failure_stage1); + const char *cmd_name = stage[0].arg1.utf8; + const bson_t *reply = stage[0].arg2.bson; + const bson_error_t *error = stage[1].arg1.error; + + BSON_ASSERT (cmd_name); + BSON_ASSERT (reply); + BSON_ASSERT (error); + + bool is_sensitive = mongoc_apm_is_sensitive_command_message (cmd_name, reply); + _mongoc_structured_log_append_redacted_cmd_failure (bson, is_sensitive, reply, error); + return stage + 2; +} + +const mongoc_structured_log_builder_stage_t * +_mongoc_structured_log_append_cmd_name_failure_stage1 (bson_t *bson, const mongoc_structured_log_builder_stage_t *stage) +{ + // Never called, marks the second stage in a two-stage cmd_name_failure + BSON_UNUSED (bson); + BSON_UNUSED (stage); + BSON_ASSERT (false); + return NULL; +} + const mongoc_structured_log_builder_stage_t * _mongoc_structured_log_append_server_description (bson_t *bson, const mongoc_structured_log_builder_stage_t *stage) { diff --git a/src/libmongoc/tests/test-mongoc-structured-log.c b/src/libmongoc/tests/test-mongoc-structured-log.c index 47f8e3ea380..fa0742abd32 100644 --- a/src/libmongoc/tests/test-mongoc-structured-log.c +++ b/src/libmongoc/tests/test-mongoc-structured-log.c @@ -197,8 +197,12 @@ test_structured_log_json (void) .expected_envelope.level = MONGOC_STRUCTURED_LOG_LEVEL_WARNING, .expected_envelope.component = MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND, .expected_envelope.message = "Log entry with deferred BSON-to-JSON", - .expected_bson = BCON_NEW ( - "message", BCON_UTF8 ("Log entry with deferred BSON-to-JSON"), "kJSON", BCON_UTF8 ("{ \"k\" : \"v\" }")), + .expected_bson = BCON_NEW ("message", + BCON_UTF8 ("Log entry with deferred BSON-to-JSON"), + "kJSON", + BCON_UTF8 ("{ \"k\" : \"v\" }"), + "kNull", + BCON_NULL), .expected_calls = 1, }; @@ -211,6 +215,7 @@ test_structured_log_json (void) MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND, "Log entry with deferred BSON-to-JSON", bson_as_json ("kJSON", json_doc), + bson_as_json ("kNull", NULL), bson_as_json (NULL, NULL)); ASSERT_CMPINT (assumption.calls, ==, 1); @@ -229,7 +234,9 @@ test_structured_log_oid (void) .expected_bson = BCON_NEW ("message", BCON_UTF8 ("Log entry with deferred OID-to-hex conversion"), "kOID", - BCON_UTF8 ("112233445566778899aabbcc")), + BCON_UTF8 ("112233445566778899aabbcc"), + "kNull", + BCON_NULL), .expected_calls = 1, }; @@ -243,6 +250,7 @@ test_structured_log_oid (void) MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND, "Log entry with deferred OID-to-hex conversion", oid_as_hex ("kOID", &oid), + oid_as_hex ("kNull", NULL), oid_as_hex (NULL, NULL)); ASSERT_CMPINT (assumption.calls, ==, 1); @@ -342,18 +350,27 @@ test_structured_log_command (void) BCON_INT64 (0x12345678eeff0011), "command", BCON_UTF8 ("{ \"c\" : \"d\" }"), - "reply", // Un-redacted successful reply + "reply", // Un-redacted successful reply (not-a-command) + BCON_UTF8 ("{ \"r\" : \"s\", \"code\" : 1 }"), + "reply", // Un-redacted successful reply (ping) BCON_UTF8 ("{ \"r\" : \"s\", \"code\" : 1 }"), - "reply", // Redacted successful reply + "reply", // Redacted successful reply (auth) BCON_UTF8 ("{}"), - "failure", // Un-redacted server side error + "failure", // Un-redacted server side error (not-a-command) + "{", + "r", + BCON_UTF8 ("s"), + "code", + BCON_INT32 (1), + "}", + "failure", // Un-redacted server side error (ping) "{", "r", BCON_UTF8 ("s"), "code", BCON_INT32 (1), "}", - "failure", // Redacted server side error + "failure", // Redacted server side error (auth) "{", "code", BCON_INT32 (1), @@ -399,11 +416,13 @@ test_structured_log_command (void) "Log entry with command and reply fields", cmd (&cmd, COMMAND_NAME), cmd (&cmd, DATABASE_NAME, COMMAND_NAME, OPERATION_ID, COMMAND), - cmd_reply ("ping", reply_doc), - cmd_reply ("authenticate", reply_doc), - cmd_failure ("ping", reply_doc, &server_error), - cmd_failure ("authenticate", reply_doc, &server_error), - cmd_failure ("authenticate", reply_doc, &client_error)); + cmd_reply (&cmd, reply_doc), + cmd_name_reply ("ping", reply_doc), + cmd_name_reply ("authenticate", reply_doc), + cmd_failure (&cmd, reply_doc, &server_error), + cmd_name_failure ("ping", reply_doc, &server_error), + cmd_name_failure ("authenticate", reply_doc, &server_error), + cmd_name_failure ("authenticate", reply_doc, &client_error)); ASSERT_CMPINT (assumption.calls, ==, 1); restore_state (old_state); From d7f570ee228ef43388a3d507e01f933148635042 Mon Sep 17 00:00:00 2001 From: Micah Scott Date: Thu, 14 Nov 2024 17:36:33 -0800 Subject: [PATCH 069/139] waitForEvent can poll, no need to get fancy --- src/libmongoc/tests/unified/operation.c | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/libmongoc/tests/unified/operation.c b/src/libmongoc/tests/unified/operation.c index d39e5b4ca34..d5c9be53870 100644 --- a/src/libmongoc/tests/unified/operation.c +++ b/src/libmongoc/tests/unified/operation.c @@ -3819,6 +3819,9 @@ operation_create_entities (test_t *test, operation_t *op, result_t *result, bson return ret; } +#define WAIT_FOR_EVENT_TIMEOUT_MS (10 * 1000) // Specified by test runner spec +#define WAIT_FOR_EVENT_TICK_MS 10 // Same tick size as used in non-unified json test + static bool operation_wait_for_event (test_t *test, operation_t *op, result_t *result, bson_error_t *error) { @@ -3861,9 +3864,12 @@ operation_wait_for_event (test_t *test, operation_t *op, result_t *result, bson_ if (count >= *expected_count) { break; } - ASSERT_CMPINT64 (bson_get_monotonic_time () - start_time, <, (int64_t) 10 * 1000 * 1000); - // @todo waiting not implemented - BSON_ASSERT (false); + ASSERT_CMPINT64 (bson_get_monotonic_time () - start_time, <, (int64_t) WAIT_FOR_EVENT_TIMEOUT_MS * 1000); + + // If any events are coming, they will be from a different thread. + // To keep the tests simple, this polls until match or timeout. + // (Same approach used by json-test-operations, outside the unified tests) + _mongoc_usleep (WAIT_FOR_EVENT_TICK_MS * 1000); } result_from_ok (result); From 46b2ca7d028ad55c58d6c2fa42908e98c9e46ece Mon Sep 17 00:00:00 2001 From: Micah Scott Date: Fri, 15 Nov 2024 14:26:39 -0800 Subject: [PATCH 070/139] bson-dsl oid() values --- src/common/src/bson-dsl.md | 6 ++++++ src/common/src/common-bson-dsl-private.h | 7 +++++++ 2 files changed, 13 insertions(+) diff --git a/src/common/src/bson-dsl.md b/src/common/src/bson-dsl.md index 678570eb307..209d70225ee 100644 --- a/src/common/src/bson-dsl.md +++ b/src/common/src/bson-dsl.md @@ -262,6 +262,12 @@ Generate a UTF-8 value from the given null-terminated character array beginning at `zstr`. +#### `oid(const bson_oid_t* oid)` + +Generate an ObjectId value from the given C expression that evaluates to +a bson_oid_t pointer. + + #### `utf8_w_len(const char* str, int len)` Generate a UTF-8 value from the character array beginning at `str`, with length diff --git a/src/common/src/common-bson-dsl-private.h b/src/common/src/common-bson-dsl-private.h index 9331dd04c1d..a628acd8087 100644 --- a/src/common/src/common-bson-dsl-private.h +++ b/src/common/src/common-bson-dsl-private.h @@ -246,6 +246,13 @@ BSON_IF_GNU_LIKE (_Pragma ("GCC diagnostic ignored \"-Wshadow\"")) #define _bsonArrayOperation_boolean(X) _bsonArrayAppendValue (boolean (X)) #define _bsonValueOperation_boolean(b) _bsonValueOperation_bool (b) +#define _bsonValueOperation_oid(o) \ + if (!bson_append_oid (_bsonBuildAppendArgs, (o))) { \ + bsonBuildError = "Error while appending oid(" _bsonDSL_str (o) ")"; \ + } else \ + ((void) 0) +#define _bsonArrayOperation_oid(X) _bsonArrayAppendValue (oid (X)) + #define _bsonValueOperation_null \ if (!bson_append_null (_bsonBuildAppendArgs)) { \ bsonBuildError = "Error while appending a null"; \ From 78525810c115a77fb7f9c0911d6856e5796a21bb Mon Sep 17 00:00:00 2001 From: Micah Scott Date: Fri, 15 Nov 2024 15:24:05 -0800 Subject: [PATCH 071/139] unified tests: serverDescriptionChangedEvent, and refactor event serialization The command-logging-and-monitoring tests use the serverDescriptionChanged event. In implementing a new event type, it made sense to refactor the implementation for event storage and serialization. The serialization from observeEvents and storeEventsAsEntities unnecessarily diverged into a late-serialization and an early-serialization style. This decides on early serialization, opting to build a canonical bson representation as soon as possible and extract fields from it later as necessary for comparisons. This has fewer opportunities for optimization in some cases, but here the extra simplicity is a good match for the test runner environment. --- src/libmongoc/tests/unified/entity-map.c | 560 ++++++++--------------- src/libmongoc/tests/unified/entity-map.h | 10 +- src/libmongoc/tests/unified/operation.c | 33 +- src/libmongoc/tests/unified/runner.c | 123 +++-- 4 files changed, 295 insertions(+), 431 deletions(-) diff --git a/src/libmongoc/tests/unified/entity-map.c b/src/libmongoc/tests/unified/entity-map.c index 39b83633141..13a31ef293f 100644 --- a/src/libmongoc/tests/unified/entity-map.c +++ b/src/libmongoc/tests/unified/entity-map.c @@ -42,6 +42,8 @@ struct _entity_findcursor_t { mongoc_cursor_t *cursor; }; +typedef void (*event_serialize_func_t) (bson_t *bson, const void *event); + static void entity_destroy (entity_t *entity); @@ -131,12 +133,22 @@ uri_apply_options (mongoc_uri_t *uri, bson_t *opts, bson_error_t *error) } static event_t * -event_new (const char *type) +event_new (const char *type, bson_t *serialized, bool is_sensitive_command) { - event_t *event = NULL; + BSON_ASSERT_PARAM (type); + BSON_ASSERT_PARAM (serialized); + + const int64_t usecs = usecs_since_epoch (); + const double secs = (double) usecs / 1000000.0; + + // Append required common fields + BSON_APPEND_UTF8 (serialized, "name", type); + BSON_APPEND_DOUBLE (serialized, "observedAt", secs); - event = bson_malloc0 (sizeof (event_t)); - event->type = bson_strdup (type); + event_t *event = bson_malloc0 (sizeof *event); + event->type = type; // Borrowed + event->serialized = serialized; // Takes ownership + event->is_sensitive_command = is_sensitive_command; return event; } @@ -147,11 +159,7 @@ event_destroy (event_t *event) return; } - bson_free (event->command_name); - bson_free (event->database_name); - bson_destroy (event->command); - bson_destroy (event->reply); - bson_free (event->type); + bson_destroy (event->serialized); bson_free (event); } @@ -202,28 +210,27 @@ structured_log_cb (const mongoc_structured_log_entry_t *entry, void *user_data) } } -static bool -is_sensitive_command (event_t *event) -{ - const bson_t *body = event->reply ? event->reply : event->command; - BSON_ASSERT (body); - return mongoc_apm_is_sensitive_command_message (event->command_name, body); -} - bool -should_ignore_event (entity_t *client_entity, event_t *event) +should_observe_event (entity_t *client_entity, event_t *event) { - bson_iter_t iter; - - if (0 == strcmp (event->command_name, "configureFailPoint")) { - return true; - } + { + bson_iter_t event_iter; + const char *event_command_name = bson_iter_init_find (&event_iter, event->serialized, "commandName") + ? bson_iter_utf8 (&event_iter, NULL) + : NULL; + if (event_command_name) { + if (0 == strcmp (event_command_name, "configureFailPoint")) { + return false; + } - if (client_entity->ignore_command_monitoring_events) { - BSON_FOREACH (client_entity->ignore_command_monitoring_events, iter) - { - if (0 == strcmp (event->command_name, bson_iter_utf8 (&iter, NULL))) { - return true; + if (client_entity->ignore_command_monitoring_events) { + bson_iter_t ignore_iter; + BSON_FOREACH (client_entity->ignore_command_monitoring_events, ignore_iter) + { + if (0 == strcmp (event_command_name, bson_iter_utf8 (&ignore_iter, NULL))) { + return false; + } + } } } } @@ -242,341 +249,140 @@ should_ignore_event (entity_t *client_entity, event_t *event) } if (!is_observed) { - return true; + return false; } } - if (client_entity->observe_sensitive_commands && *client_entity->observe_sensitive_commands) { - return false; - } - - /* Sensitive commands need to be ignored */ - return is_sensitive_command (event); -} - -typedef void *(*apm_func_void_t) (const void *); -typedef const bson_t *(*apm_func_bson_t) (const void *); -typedef const char *(*apm_func_utf8_t) (const void *); -typedef int64_t (*apm_func_int64_t) (const void *); -typedef const bson_oid_t *(*apm_func_bson_oid_t) (const void *); -typedef int32_t (*apm_func_int32_t) (const void *); -typedef const mongoc_host_list_t *(*apm_func_host_list_t) (const void *); -typedef void (*apm_func_serialize_t) (bson_t *, const void *); - -typedef struct command_callback_funcs_t { - apm_func_void_t get_context; - apm_func_bson_t get_command; - apm_func_bson_t get_reply; - apm_func_utf8_t get_command_name; - apm_func_utf8_t get_database_name; - apm_func_int64_t get_request_id; - apm_func_int64_t get_operation_id; - apm_func_bson_oid_t get_service_id; - apm_func_host_list_t get_host; - apm_func_int64_t get_server_connection_id; - apm_func_serialize_t serialize; -} command_callback_funcs_t; - -#define _apm_func(kind, fn) (mongoc_apm_command_##kind##_##fn##_cb) -#define _apm_func_defn(kind, fn, type) \ - static type mongoc_apm_command_##kind##_##fn##_cb (const void *command) \ - { \ - return (mongoc_apm_command_##kind##_##fn) (command); \ - } \ - struct _force_semicolon - -_apm_func_defn (started, get_context, void *); -_apm_func_defn (started, get_command, const bson_t *); -_apm_func_defn (started, get_command_name, const char *); -_apm_func_defn (started, get_database_name, const char *); -_apm_func_defn (started, get_request_id, int64_t); -_apm_func_defn (started, get_operation_id, int64_t); -_apm_func_defn (started, get_service_id, const bson_oid_t *); -_apm_func_defn (started, get_host, const mongoc_host_list_t *); -_apm_func_defn (started, get_server_connection_id_int64, int64_t); - -_apm_func_defn (failed, get_context, void *); -_apm_func_defn (failed, get_reply, const bson_t *); -_apm_func_defn (failed, get_command_name, const char *); -_apm_func_defn (failed, get_database_name, const char *); -_apm_func_defn (failed, get_request_id, int64_t); -_apm_func_defn (failed, get_operation_id, int64_t); -_apm_func_defn (failed, get_service_id, const bson_oid_t *); -_apm_func_defn (failed, get_host, const mongoc_host_list_t *); -_apm_func_defn (failed, get_server_connection_id_int64, int64_t); - -_apm_func_defn (succeeded, get_context, void *); -_apm_func_defn (succeeded, get_reply, const bson_t *); -_apm_func_defn (succeeded, get_command_name, const char *); -_apm_func_defn (succeeded, get_database_name, const char *); -_apm_func_defn (succeeded, get_request_id, int64_t); -_apm_func_defn (succeeded, get_operation_id, int64_t); -_apm_func_defn (succeeded, get_service_id, const bson_oid_t *); -_apm_func_defn (succeeded, get_host, const mongoc_host_list_t *); -_apm_func_defn (succeeded, get_server_connection_id_int64, int64_t); - -#undef _apm_func_defn - - -static void -observe_event (entity_t *entity, command_callback_funcs_t funcs, const char *type, const void *apm_command) -{ - BSON_ASSERT_PARAM (type); - BSON_ASSERT_PARAM (apm_command); - - BSON_ASSERT (funcs.get_context); - event_t *const event = event_new (type); - - if (funcs.get_command) { - event->command = bson_copy (funcs.get_command (apm_command)); - } - - if (funcs.get_reply) { - event->reply = bson_copy (funcs.get_reply (apm_command)); - } - - BSON_ASSERT (funcs.get_command_name); - event->command_name = bson_strdup (funcs.get_command_name (apm_command)); - - if (funcs.get_database_name) { - event->database_name = bson_strdup (funcs.get_database_name (apm_command)); - } - - BSON_ASSERT (funcs.get_service_id); - const bson_oid_t *const service_id = funcs.get_service_id (apm_command); - if (service_id) { - bson_oid_copy (service_id, &event->service_id); - } - - BSON_ASSERT (funcs.get_server_connection_id); - event->server_connection_id = funcs.get_server_connection_id (apm_command); - - if (should_ignore_event (entity, event)) { - event_destroy (event); - return; - } - - LL_APPEND (entity->events, event); + // Sensitive command events are only observed if explicitly requested + return !event->is_sensitive_command || + (client_entity->observe_sensitive_commands && *client_entity->observe_sensitive_commands); } - static void -store_event_serialize_started (bson_t *doc, const void *apm_command_vp) +event_store_or_destroy (entity_t *entity, event_t *event) { - // Spec: The test runner MAY omit the command field for CommandStartedEvent - // and reply field for CommandSucceededEvent. - // BSON_APPEND_DOCUMENT ( - // doc, "command", mongoc_apm_command_started_get_command (apm_command)); - - const mongoc_apm_command_started_t *const apm_command = apm_command_vp; - - BSON_APPEND_UTF8 (doc, "databaseName", mongoc_apm_command_started_get_database_name (apm_command)); - - BSON_APPEND_UTF8 (doc, "commandName", mongoc_apm_command_started_get_command_name (apm_command)); - - BSON_APPEND_INT64 (doc, "requestId", mongoc_apm_command_started_get_request_id (apm_command)); - - BSON_APPEND_INT64 (doc, "operationId", mongoc_apm_command_started_get_operation_id (apm_command)); - - BSON_APPEND_UTF8 (doc, "connectionId", mongoc_apm_command_started_get_host (apm_command)->host_and_port); + BSON_ASSERT_PARAM (entity); + BSON_ASSERT_PARAM (event); // Takes ownership - BSON_APPEND_INT64 ( - doc, "serverConnectionId", mongoc_apm_command_started_get_server_connection_id_int64 (apm_command)); + BSON_ASSERT (entity->entity_map); + entity_map_t *const em = entity->entity_map; + // Make additional copies as requested by storeEventsAsEntities { - const bson_oid_t *const service_id = mongoc_apm_command_started_get_service_id (apm_command); - - if (service_id) { - BSON_APPEND_OID (doc, "serviceId", service_id); + store_event_t *const begin = (store_event_t *) entity->store_events.data; + store_event_t *const end = begin + entity->store_events.len; + bson_error_t error = {0}; + for (store_event_t *iter = begin; iter != end; ++iter) { + if (bson_strcasecmp (iter->type, event->type) == 0) { + mongoc_array_t *arr = entity_map_get_bson_array (em, iter->entity_id, &error); + ASSERT_OR_PRINT (arr, error); + bson_t *serialized_copy = bson_copy (event->serialized); + _mongoc_array_append_val (arr, serialized_copy); // Transfer ownership. + } } } -} - - -static void -store_event_serialize_failed (bson_t *doc, const void *apm_command_vp) -{ - const mongoc_apm_command_failed_t *const apm_command = apm_command_vp; - - BSON_APPEND_INT64 (doc, "duration", mongoc_apm_command_failed_get_duration (apm_command)); - - BSON_APPEND_UTF8 (doc, "commandName", mongoc_apm_command_failed_get_command_name (apm_command)); - - BSON_APPEND_UTF8 (doc, "databaseName", mongoc_apm_command_failed_get_database_name (apm_command)); - - { - bson_error_t error; - mongoc_apm_command_failed_get_error (apm_command, &error); - BSON_APPEND_UTF8 (doc, "failure", error.message); - } - - BSON_APPEND_INT64 (doc, "requestId", mongoc_apm_command_failed_get_request_id (apm_command)); - - BSON_APPEND_INT64 (doc, "operationId", mongoc_apm_command_failed_get_operation_id (apm_command)); - - BSON_APPEND_UTF8 (doc, "connectionId", mongoc_apm_command_failed_get_host (apm_command)->host_and_port); - BSON_APPEND_INT64 ( - doc, "serverConnectionId", mongoc_apm_command_failed_get_server_connection_id_int64 (apm_command)); - - { - const bson_oid_t *const service_id = mongoc_apm_command_failed_get_service_id (apm_command); - - if (service_id) { - BSON_APPEND_OID (doc, "serviceId", service_id); - } + if (should_observe_event (entity, event)) { + // Transfer ownership of observed serialized events to the event list + LL_APPEND (entity->events, event); + } else { + // Discard serialized events we are not observing + event_destroy (event); } } - static void -store_event_serialize_succeeded (bson_t *doc, const void *apm_command_vp) +command_started (const mongoc_apm_command_started_t *started) { - const mongoc_apm_command_succeeded_t *const apm_command = apm_command_vp; - - BSON_APPEND_INT64 (doc, "duration", mongoc_apm_command_succeeded_get_duration (apm_command)); - - // Spec: The test runner MAY omit the command field for CommandStartedEvent - // and reply field for CommandSucceededEvent. - // BSON_APPEND_DOCUMENT ( - // doc, "reply", mongoc_apm_command_succeeded_get_reply (apm_command)); + entity_t *entity = (entity_t *) mongoc_apm_command_started_get_context (started); + const bson_oid_t *const service_id = mongoc_apm_command_started_get_service_id (started); + const bool is_sensitive = mongoc_apm_is_sensitive_command_message ( + mongoc_apm_command_started_get_command_name (started), mongoc_apm_command_started_get_command (started)); - BSON_APPEND_UTF8 (doc, "commandName", mongoc_apm_command_succeeded_get_command_name (apm_command)); + bson_t *serialized = bson_new (); + bsonBuildAppend ( + *serialized, + kv ("databaseName", cstr (mongoc_apm_command_started_get_database_name (started))), + kv ("commandName", cstr (mongoc_apm_command_started_get_command_name (started))), + kv ("requestId", int64 (mongoc_apm_command_started_get_request_id (started))), + kv ("operationId", int64 (mongoc_apm_command_started_get_operation_id (started))), + kv ("connectionId", cstr (mongoc_apm_command_started_get_host (started)->host_and_port)), + kv ("serverConnectionId", int64 (mongoc_apm_command_started_get_server_connection_id_int64 (started))), + if (service_id, then (kv ("serviceId", oid (service_id)))), + kv ("command", bson (*mongoc_apm_command_started_get_command (started)))); - BSON_APPEND_UTF8 (doc, "databaseName", mongoc_apm_command_succeeded_get_database_name (apm_command)); - - BSON_APPEND_INT64 (doc, "requestId", mongoc_apm_command_succeeded_get_request_id (apm_command)); - - BSON_APPEND_INT64 (doc, "operationId", mongoc_apm_command_succeeded_get_operation_id (apm_command)); - - BSON_APPEND_UTF8 (doc, "connectionId", mongoc_apm_command_succeeded_get_host (apm_command)->host_and_port); - - BSON_APPEND_INT64 ( - doc, "serverConnectionId", mongoc_apm_command_succeeded_get_server_connection_id_int64 (apm_command)); - - { - const bson_oid_t *const service_id = mongoc_apm_command_succeeded_get_service_id (apm_command); - - if (service_id) { - BSON_APPEND_OID (doc, "serviceId", service_id); - } - } + event_store_or_destroy (entity, event_new ("commandStartedEvent", serialized, is_sensitive)); } - static void -store_event_to_entities (entity_t *entity, command_callback_funcs_t funcs, const char *type, const void *apm_command) +command_failed (const mongoc_apm_command_failed_t *failed) { - BSON_ASSERT_PARAM (entity); - BSON_ASSERT_PARAM (type); - - BSON_ASSERT (entity->entity_map); - - entity_map_t *const em = entity->entity_map; - - store_event_t *const begin = (store_event_t *) entity->store_events.data; - store_event_t *const end = begin + entity->store_events.len; - - const int64_t usecs = usecs_since_epoch (); - const double secs = (double) usecs / 1000000.0; - - bson_error_t error = {0}; - - for (store_event_t *iter = begin; iter != end; ++iter) { - if (bson_strcasecmp (iter->type, type) == 0) { - mongoc_array_t *arr = entity_map_get_bson_array (em, iter->entity_id, &error); - ASSERT_OR_PRINT (arr, error); - - bson_t *doc = bson_new (); - - // Spec: the following fields MUST be stored with each event document: - BSON_APPEND_UTF8 (doc, "name", type); - BSON_APPEND_DOUBLE (doc, "observedAt", secs); - - // The event subscriber MUST serialize the events it receives into a - // document, using the documented properties of the event as field - // names, and append the document to the list stored in the specified - // entity. - funcs.serialize (doc, apm_command); - - _mongoc_array_append_val (arr, doc); // Transfer ownership. - } - } + entity_t *entity = (entity_t *) mongoc_apm_command_failed_get_context (failed); + const bson_oid_t *const service_id = mongoc_apm_command_failed_get_service_id (failed); + const bool is_sensitive = mongoc_apm_is_sensitive_command_message ( + mongoc_apm_command_failed_get_command_name (failed), mongoc_apm_command_failed_get_reply (failed)); + bson_error_t error; + mongoc_apm_command_failed_get_error (failed, &error); + + bson_t *serialized = bson_new (); + bsonBuildAppend ( + *serialized, + kv ("duration", int64 (mongoc_apm_command_failed_get_duration (failed))), + kv ("commandName", cstr (mongoc_apm_command_failed_get_command_name (failed))), + kv ("databaseName", cstr (mongoc_apm_command_failed_get_database_name (failed))), + kv ("requestId", int64 (mongoc_apm_command_failed_get_request_id (failed))), + kv ("operationId", int64 (mongoc_apm_command_failed_get_operation_id (failed))), + kv ("connectionId", cstr (mongoc_apm_command_failed_get_host (failed)->host_and_port)), + kv ("serverConnectionId", int64 (mongoc_apm_command_failed_get_server_connection_id_int64 (failed))), + if (service_id, then (kv ("serviceId", oid (service_id)))), + kv ("failure", cstr (error.message))); + + event_store_or_destroy (entity, event_new ("commandFailedEvent", serialized, is_sensitive)); } - static void -apm_command_callback (command_callback_funcs_t funcs, const char *type, const void *apm_command) +command_succeeded (const mongoc_apm_command_succeeded_t *succeeded) { - BSON_ASSERT_PARAM (type); - BSON_ASSERT_PARAM (apm_command); - - BSON_ASSERT (funcs.get_context); - entity_t *const entity = (entity_t *) funcs.get_context (apm_command); - - observe_event (entity, funcs, type, apm_command); - store_event_to_entities (entity, funcs, type, apm_command); -} - + entity_t *entity = (entity_t *) mongoc_apm_command_succeeded_get_context (succeeded); + const bson_oid_t *const service_id = mongoc_apm_command_succeeded_get_service_id (succeeded); + const bool is_sensitive = mongoc_apm_is_sensitive_command_message ( + mongoc_apm_command_succeeded_get_command_name (succeeded), mongoc_apm_command_succeeded_get_reply (succeeded)); -static void -command_started (const mongoc_apm_command_started_t *started) -{ - command_callback_funcs_t funcs = { - .get_context = _apm_func (started, get_context), - .get_command = _apm_func (started, get_command), - .get_reply = NULL, - .get_command_name = _apm_func (started, get_command_name), - .get_database_name = _apm_func (started, get_database_name), - .get_request_id = _apm_func (started, get_request_id), - .get_operation_id = _apm_func (started, get_operation_id), - .get_service_id = _apm_func (started, get_service_id), - .get_host = _apm_func (started, get_host), - .get_server_connection_id = _apm_func (started, get_server_connection_id_int64), - .serialize = store_event_serialize_started, - }; + bson_t *serialized = bson_new (); + bsonBuildAppend ( + *serialized, + kv ("duration", int64 (mongoc_apm_command_succeeded_get_duration (succeeded))), + kv ("commandName", cstr (mongoc_apm_command_succeeded_get_command_name (succeeded))), + kv ("databaseName", cstr (mongoc_apm_command_succeeded_get_database_name (succeeded))), + kv ("requestId", int64 (mongoc_apm_command_succeeded_get_request_id (succeeded))), + kv ("operationId", int64 (mongoc_apm_command_succeeded_get_operation_id (succeeded))), + kv ("connectionId", cstr (mongoc_apm_command_succeeded_get_host (succeeded)->host_and_port)), + kv ("serverConnectionId", int64 (mongoc_apm_command_succeeded_get_server_connection_id_int64 (succeeded))), + if (service_id, then (kv ("serviceId", oid (service_id)))), + kv ("reply", bson (*mongoc_apm_command_succeeded_get_reply (succeeded)))); - apm_command_callback (funcs, "commandStartedEvent", started); + event_store_or_destroy (entity, event_new ("commandSucceededEvent", serialized, is_sensitive)); } static void -command_failed (const mongoc_apm_command_failed_t *failed) +server_changed (const mongoc_apm_server_changed_t *changed) { - command_callback_funcs_t funcs = { - .get_context = _apm_func (failed, get_context), - .get_command = NULL, - .get_reply = _apm_func (failed, get_reply), - .get_command_name = _apm_func (failed, get_command_name), - .get_database_name = _apm_func (failed, get_database_name), - .get_request_id = _apm_func (failed, get_request_id), - .get_operation_id = _apm_func (failed, get_operation_id), - .get_service_id = _apm_func (failed, get_service_id), - .get_host = _apm_func (failed, get_host), - .get_server_connection_id = _apm_func (failed, get_server_connection_id_int64), - .serialize = store_event_serialize_failed, - }; - - apm_command_callback (funcs, "commandFailedEvent", failed); -} + entity_t *entity = (entity_t *) mongoc_apm_server_changed_get_context (changed); + bson_oid_t topology_id; + mongoc_apm_server_changed_get_topology_id (changed, &topology_id); -static void -command_succeeded (const mongoc_apm_command_succeeded_t *succeeded) -{ - command_callback_funcs_t funcs = { - .get_context = _apm_func (succeeded, get_context), - .get_command = NULL, - .get_reply = _apm_func (succeeded, get_reply), - .get_command_name = _apm_func (succeeded, get_command_name), - .get_database_name = _apm_func (succeeded, get_database_name), - .get_request_id = _apm_func (succeeded, get_request_id), - .get_operation_id = _apm_func (succeeded, get_operation_id), - .get_service_id = _apm_func (succeeded, get_service_id), - .get_host = _apm_func (succeeded, get_host), - .get_server_connection_id = _apm_func (succeeded, get_server_connection_id_int64), - .serialize = store_event_serialize_succeeded, - }; + bson_t *serialized = bson_new (); + bsonBuildAppend ( + *serialized, + kv ("address", cstr (mongoc_apm_server_changed_get_host (changed)->host_and_port)), + kv ("topologyId", oid (&topology_id)), + kv ("previousDescription", + bson ( + *mongoc_server_description_hello_response (mongoc_apm_server_changed_get_previous_description (changed)))), + kv ("newDescription", + bson (*mongoc_server_description_hello_response (mongoc_apm_server_changed_get_new_description (changed))))); - apm_command_callback (funcs, "commandSucceededEvent", succeeded); + event_store_or_destroy (entity, event_new ("serverDescriptionChangedEvent", serialized, false)); } static void @@ -597,11 +403,17 @@ set_command_succeeded_cb (mongoc_apm_callbacks_t *callbacks) mongoc_apm_set_command_succeeded_cb (callbacks, command_succeeded); } +static void +set_server_changed_cb (mongoc_apm_callbacks_t *callbacks) +{ + mongoc_apm_set_server_changed_cb (callbacks, server_changed); +} + // Note: multiple invocations of this function is okay, since all it does // is set the appropriate pointer in `callbacks`, and the callback function(s) // being used is always the same for a given type. static void -set_command_callback (mongoc_apm_callbacks_t *callbacks, const char *type) +set_event_callback (mongoc_apm_callbacks_t *callbacks, const char *type) { typedef void (*set_func_t) (mongoc_apm_callbacks_t *); @@ -610,10 +422,11 @@ set_command_callback (mongoc_apm_callbacks_t *callbacks, const char *type) set_func_t set; } command_to_cb_t; - const command_to_cb_t commands[] = { + static const command_to_cb_t commands[] = { {.type = "commandStartedEvent", .set = set_command_started_cb}, {.type = "commandFailedEvent", .set = set_command_failed_cb}, {.type = "commandSucceededEvent", .set = set_command_succeeded_cb}, + {.type = "serverDescriptionChangedEvent", .set = set_server_changed_cb}, {.type = NULL, .set = NULL}, }; @@ -625,6 +438,14 @@ set_command_callback (mongoc_apm_callbacks_t *callbacks, const char *type) } } +static bool +is_supported_event_type (const char *type) +{ + return (0 == bson_strcasecmp (type, "commandStartedEvent") || 0 == bson_strcasecmp (type, "commandFailedEvent") || + 0 == bson_strcasecmp (type, "commandSucceededEvent") || + 0 == bson_strcasecmp (type, "serverDescriptionChangedEvent")); +} + static void add_observe_event (entity_t *entity, const char *type) { @@ -641,7 +462,6 @@ add_store_event (entity_t *entity, const char *type, const char *entity_id) _mongoc_array_append_val (&entity->store_events, event); } - entity_t * entity_client_new (entity_map_t *em, bson_t *bson, bson_error_t *error) { @@ -683,12 +503,9 @@ entity_client_new (entity_map_t *em, bson_t *bson, bson_error_t *error) // Ensure all elements are strings: when (not(type (utf8)), error ("Every 'observeEvents' element must be a string")), // Dispatch based on the event name: - when (anyOf (iStrEqual ("commandStartedEvent"), - iStrEqual ("commandFailedEvent"), - iStrEqual ("commandSucceededEvent")), - do ({ + when (eval (is_supported_event_type (bson_iter_utf8 (&bsonVisitIter, NULL))), do ({ const char *const type = bson_iter_utf8 (&bsonVisitIter, NULL); - set_command_callback (callbacks, type); + set_event_callback (callbacks, type); add_observe_event (entity, type); })), // Unsupported (but known) event names: @@ -732,34 +549,32 @@ entity_client_new (entity_map_t *em, bson_t *bson, bson_error_t *error) *p = bsonAs (boolean); })), // Which events should be available as entities: - find (key ("storeEventsAsEntities"), - if (not(type (array)), then (error ("'storeEventsAsEntities' must be an array"))), - visitEach (parse ( - find (keyWithType ("id", utf8), storeStrRef (store_entity_id), do ({ - if (!entity_map_add_bson_array (em, store_entity_id, error)) { - test_error ("failed to create storeEventsAsEntities " - "entity '%s': %s", - store_entity_id, - error->message); - } - })), - find (keyWithType ("events", array), - visitEach (case (when (not(type (utf8)), - error ("Every 'storeEventsAsEntities.events' " - "element must be a string")), - when (anyOf (iStrEqual ("commandStartedEvent"), - iStrEqual ("commandFailedEvent"), - iStrEqual ("commandSucceededEvent")), - do ({ - const char *const type = bson_iter_utf8 (&bsonVisitIter, NULL); - set_command_callback (callbacks, type); - add_store_event (entity, type, store_entity_id); - })), - when (eval (is_unsupported_event_type (bson_iter_utf8 (&bsonVisitIter, NULL))), - do (MONGOC_DEBUG ("Skipping unsupported event type '%s'", bsonAs (cstr)))), - else (do (test_error ("Unknown event type '%s'", bsonAs (cstr))))))), - visitOthers ( - errorf (err, "Unexpected field '%s' in storeEventsAsEntities", bson_iter_key (&bsonVisitIter)))))), + find ( + key ("storeEventsAsEntities"), + if (not(type (array)), then (error ("'storeEventsAsEntities' must be an array"))), + visitEach (parse ( + find (keyWithType ("id", utf8), storeStrRef (store_entity_id), do ({ + if (!entity_map_add_bson_array (em, store_entity_id, error)) { + test_error ("failed to create storeEventsAsEntities " + "entity '%s': %s", + store_entity_id, + error->message); + } + })), + find (keyWithType ("events", array), + visitEach (case (when (not(type (utf8)), + error ("Every 'storeEventsAsEntities.events' " + "element must be a string")), + when (eval (is_supported_event_type (bson_iter_utf8 (&bsonVisitIter, NULL))), do ({ + const char *const type = bson_iter_utf8 (&bsonVisitIter, NULL); + set_event_callback (callbacks, type); + add_store_event (entity, type, store_entity_id); + })), + when (eval (is_unsupported_event_type (bson_iter_utf8 (&bsonVisitIter, NULL))), + do (MONGOC_DEBUG ("Skipping unsupported event type '%s'", bsonAs (cstr)))), + else (do (test_error ("Unknown event type '%s'", bsonAs (cstr))))))), + visitOthers ( + errorf (err, "Unexpected field '%s' in storeEventsAsEntities", bson_iter_key (&bsonVisitIter)))))), // Log messages to observe: find (key ("observeLogMessages"), if (not(type (doc)), then (error ("'observeLogMessages' must be a document"))), @@ -2218,20 +2033,11 @@ event_list_to_string (event_t *events) str = mcommon_string_new (""); LL_FOREACH (events, eiter) { - mcommon_string_append_printf (str, "- %s:", eiter->type); - if (eiter->command_name) { - mcommon_string_append_printf (str, " cmd=%s", eiter->command_name); - } - if (eiter->database_name) { - mcommon_string_append_printf (str, " db=%s", eiter->database_name); - } - if (eiter->command) { - mcommon_string_append_printf (str, " sent %s", tmp_json (eiter->command)); - } - if (eiter->reply) { - mcommon_string_append_printf (str, " received %s", tmp_json (eiter->reply)); - } - mcommon_string_append (str, "\n"); + mcommon_string_append_printf (str, + "- %s: %s (%s)\n", + eiter->type, + tmp_json (eiter->serialized), + eiter->is_sensitive_command ? "marked SENSITIVE" : "not sensitive"); } return mcommon_string_free (str, false); } diff --git a/src/libmongoc/tests/unified/entity-map.h b/src/libmongoc/tests/unified/entity-map.h index bdc2d323105..3b5aee4191b 100644 --- a/src/libmongoc/tests/unified/entity-map.h +++ b/src/libmongoc/tests/unified/entity-map.h @@ -24,14 +24,10 @@ #include "test-diagnostics.h" typedef struct _event_t { - char *type; - char *command_name; - char *database_name; - bson_t *command; - bson_t *reply; - bson_oid_t service_id; - int64_t server_connection_id; struct _event_t *next; + const char *type; // Non-owning + bson_t *serialized; + bool is_sensitive_command; } event_t; typedef struct _log_message_t { diff --git a/src/libmongoc/tests/unified/operation.c b/src/libmongoc/tests/unified/operation.c index d5c9be53870..6db151f1062 100644 --- a/src/libmongoc/tests/unified/operation.c +++ b/src/libmongoc/tests/unified/operation.c @@ -2813,20 +2813,20 @@ assert_lsid_on_last_two_commands (test_t *test, operation_t *op, result_t *resul goto done; } - if (!bson_iter_init_find (&iter, a->command, "lsid")) { - test_set_error (error, "unable to find lsid in second to last commandStartedEvent: %s", tmp_json (a->command)); + if (bson_iter_init (&iter, a->serialized) && bson_iter_find_descendant (&iter, "command.lsid", &iter)) { + bson_iter_bson (&iter, &a_lsid); + } else { + test_set_error (error, "unable to find lsid in second to last commandStartedEvent: %s", tmp_json (a->serialized)); goto done; } - bson_iter_bson (&iter, &a_lsid); - - if (!bson_iter_init_find (&iter, b->command, "lsid")) { - test_set_error (error, "unable to find lsid in second to last commandStartedEvent: %s", tmp_json (b->command)); + if (bson_iter_init (&iter, b->serialized) && bson_iter_find_descendant (&iter, "command.lsid", &iter)) { + bson_iter_bson (&iter, &b_lsid); + } else { + test_set_error (error, "unable to find lsid in last commandStartedEvent: %s", tmp_json (b->serialized)); goto done; } - bson_iter_bson (&iter, &b_lsid); - if (check_same != bson_equal (&a_lsid, &b_lsid)) { test_set_error (error, "expected $lsid's to be%s equal, but got: %s and %s", @@ -3864,7 +3864,22 @@ operation_wait_for_event (test_t *test, operation_t *op, result_t *result, bson_ if (count >= *expected_count) { break; } - ASSERT_CMPINT64 (bson_get_monotonic_time () - start_time, <, (int64_t) WAIT_FOR_EVENT_TIMEOUT_MS * 1000); + + int64_t duration = bson_get_monotonic_time () - start_time; + if (duration >= (int64_t) WAIT_FOR_EVENT_TIMEOUT_MS * 1000) { + char *event_list_string = event_list_to_string (client->events); + test_diagnostics_error_info ("all captured events for client:\n%s", event_list_string); + bson_free (event_list_string); + test_diagnostics_error_info ("checking for expected event: %s\n", tmp_json (expected_event)); + test_set_error (error, + "waitForEvent timed out with %" PRId64 " of %" PRId64 + " matches needed. waited %dms (max %dms)", + count, + *expected_count, + (int) (duration / 1000), + (int) WAIT_FOR_EVENT_TIMEOUT_MS); + goto done; + }; // If any events are coming, they will be from a different thread. // To keep the tests simple, this polls until match or timeout. diff --git a/src/libmongoc/tests/unified/runner.c b/src/libmongoc/tests/unified/runner.c index 94b425a8c1c..d86d9291c4c 100644 --- a/src/libmongoc/tests/unified/runner.c +++ b/src/libmongoc/tests/unified/runner.c @@ -986,6 +986,11 @@ test_check_event (test_t *test, bson_t *expected, event_t *actual, bson_error_t bool *expected_has_service_id = NULL; bool *expected_has_server_connection_id = NULL; + BSON_ASSERT_PARAM (test); + BSON_ASSERT_PARAM (expected); + BSON_ASSERT_PARAM (actual); + BSON_ASSERT_PARAM (error); + if (bson_count_keys (expected) != 1) { test_set_error (error, "expected 1 key in expected event, but got: %s", tmp_json (expected)); goto done; @@ -1017,71 +1022,115 @@ test_check_event (test_t *test, bson_t *expected, event_t *actual, bson_error_t } if (expected_command) { - bson_val_t *expected_val; - bson_val_t *actual_val; - - if (!actual || !actual->command) { - test_set_error (error, "Expected a value but got NULL"); + if (!bson_iter_init_find (&iter, actual->serialized, "command")) { + test_set_error (error, "event.command expected but missing"); goto done; } - - expected_val = bson_val_from_bson (expected_command); - actual_val = bson_val_from_bson (actual->command); - - if (!entity_map_match (test->entity_map, expected_val, actual_val, false, error)) { - bson_val_destroy (expected_val); - bson_val_destroy (actual_val); + if (!BSON_ITER_HOLDS_DOCUMENT (&iter)) { + test_set_error (error, "Unexpected type event.command, should be document"); goto done; } + bson_val_t *expected_val = bson_val_from_bson (expected_command); + bson_val_t *actual_val = bson_val_from_iter (&iter); + bool is_match = entity_map_match (test->entity_map, expected_val, actual_val, false, error); bson_val_destroy (expected_val); bson_val_destroy (actual_val); + if (!is_match) { + goto done; + } } - if (expected_command_name && 0 != strcmp (expected_command_name, actual->command_name)) { - test_set_error (error, "expected commandName: %s, but got: %s", expected_command_name, actual->command_name); - goto done; + if (expected_command_name) { + if (!bson_iter_init_find (&iter, actual->serialized, "commandName")) { + test_set_error (error, "event.commandName expected but missing"); + goto done; + } + if (!BSON_ITER_HOLDS_UTF8 (&iter)) { + test_set_error (error, "Unexpected type for event.commandName, should be string"); + goto done; + } + const char *actual_command_name = bson_iter_utf8 (&iter, NULL); + if (0 != strcmp (expected_command_name, actual_command_name)) { + test_set_error (error, "expected commandName: %s, but got: %s", expected_command_name, actual_command_name); + goto done; + } } - if (expected_database_name && 0 != strcmp (expected_database_name, actual->database_name)) { - test_set_error (error, "expected databaseName: %s, but got: %s", expected_database_name, actual->database_name); - goto done; + if (expected_database_name) { + if (!bson_iter_init_find (&iter, actual->serialized, "databaseName")) { + test_set_error (error, "event.databaseName expected but missing"); + goto done; + } + if (!BSON_ITER_HOLDS_UTF8 (&iter)) { + test_set_error (error, "Unexpected type for event.databaseName, should be string"); + goto done; + } + const char *actual_database_name = bson_iter_utf8 (&iter, NULL); + if (0 != strcmp (expected_database_name, actual_database_name)) { + test_set_error (error, "expected databaseName: %s, but got: %s", expected_database_name, actual_database_name); + goto done; + } } if (expected_reply) { - bson_val_t *expected_val = bson_val_from_bson (expected_reply); - bson_val_t *actual_val = bson_val_from_bson (actual->reply); - if (!entity_map_match (test->entity_map, expected_val, actual_val, false, error)) { - bson_val_destroy (expected_val); - bson_val_destroy (actual_val); + if (!bson_iter_init_find (&iter, actual->serialized, "reply")) { + test_set_error (error, "event.reply expected but missing"); + goto done; + } + if (!BSON_ITER_HOLDS_DOCUMENT (&iter)) { + test_set_error (error, "Unexpected type for event.reply, should be document"); goto done; } + bson_val_t *expected_val = bson_val_from_bson (expected_reply); + bson_val_t *actual_val = bson_val_from_iter (&iter); + bool is_match = entity_map_match (test->entity_map, expected_val, actual_val, false, error); bson_val_destroy (expected_val); bson_val_destroy (actual_val); + if (!is_match) { + goto done; + } } if (expected_has_service_id) { - bool has_service_id = !mcommon_oid_is_zero (&actual->service_id); - char oid_str[25]; - bson_oid_to_string (&actual->service_id, oid_str); - - if (*expected_has_service_id && !has_service_id) { - test_error ("expected serviceId, but got none"); + if (!bson_iter_init_find (&iter, actual->serialized, "serviceId")) { + test_set_error (error, "event.serviceId field expected but missing"); + goto done; + } + if (!BSON_ITER_HOLDS_OID (&iter)) { + test_set_error (error, "Unexpected type for event.serviceId, should be ObjectId"); + goto done; } - if (!*expected_has_service_id && has_service_id) { - test_error ("expected no serviceId, but got %s", oid_str); + const bson_oid_t *actual_oid = bson_iter_oid (&iter); + bool actual_has_service_id = !mcommon_oid_is_zero (actual_oid); + char actual_oid_str[25]; + bson_oid_to_string (actual_oid, actual_oid_str); + + if (*expected_has_service_id && !actual_has_service_id) { + test_error ("expected nonzero serviceId, but found zero"); + } + if (!*expected_has_service_id && actual_has_service_id) { + test_error ("expected zeroed serviceId, but found nonzero value: %s", actual_oid_str); } } if (expected_has_server_connection_id) { - const bool has_server_connection_id = actual->server_connection_id != MONGOC_NO_SERVER_CONNECTION_ID; + if (!bson_iter_init_find (&iter, actual->serialized, "serverConnectionId")) { + test_set_error (error, "event.serverConnectionId expected but missing"); + goto done; + } + if (!BSON_ITER_HOLDS_INT64 (&iter)) { + test_set_error (error, "Unexpected type for event.serverConnectionId, should be int64"); + goto done; + } + int64_t actual_server_connection_id = bson_iter_int64 (&iter); + const bool has_server_connection_id = actual_server_connection_id != MONGOC_NO_SERVER_CONNECTION_ID; if (*expected_has_server_connection_id && !has_server_connection_id) { - test_error ("expected server connectionId, but got none"); + test_error ("expected server connectionId, but got MONGOC_NO_SERVER_CONNECTION_ID"); } - if (!*expected_has_server_connection_id && has_server_connection_id) { - test_error ("expected no server connectionId, but got %" PRId64, actual->server_connection_id); + test_error ("expected MONGOC_NO_SERVER_CONNECTION_ID, but got %" PRId64, actual_server_connection_id); } } @@ -1188,9 +1237,7 @@ test_check_expected_events_for_client (test_t *test, bson_t *expected_events_for done: if (!ret) { if (entity && entity->events) { - char *event_list_string = NULL; - - event_list_string = event_list_to_string (entity->events); + char *event_list_string = event_list_to_string (entity->events); test_diagnostics_error_info ("all captured events:\n%s", event_list_string); bson_free (event_list_string); } From b466d3b4ed9fb3891756ef7f44de25453b263414 Mon Sep 17 00:00:00 2001 From: Micah Scott Date: Fri, 15 Nov 2024 16:04:04 -0800 Subject: [PATCH 072/139] unified test runner debug output for events and structured logs --- src/libmongoc/tests/unified/entity-map.c | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/libmongoc/tests/unified/entity-map.c b/src/libmongoc/tests/unified/entity-map.c index 13a31ef293f..1401ce283f0 100644 --- a/src/libmongoc/tests/unified/entity-map.c +++ b/src/libmongoc/tests/unified/entity-map.c @@ -145,6 +145,11 @@ event_new (const char *type, bson_t *serialized, bool is_sensitive_command) BSON_APPEND_UTF8 (serialized, "name", type); BSON_APPEND_DOUBLE (serialized, "observedAt", secs); + MONGOC_DEBUG ("new event: %s %s (%s)", + type, + tmp_json (serialized), + is_sensitive_command ? "marked SENSITIVE" : "not sensitive"); + event_t *event = bson_malloc0 (sizeof *event); event->type = type; // Borrowed event->serialized = serialized; // Takes ownership @@ -172,6 +177,12 @@ log_message_new (const mongoc_structured_log_entry_t *entry) log_message->component = mongoc_structured_log_entry_get_component (entry); log_message->level = mongoc_structured_log_entry_get_level (entry); log_message->message = mongoc_structured_log_entry_message_as_bson (entry); + + MONGOC_DEBUG ("new structured log: %s %s %s", + mongoc_structured_log_get_level_name (log_message->level), + mongoc_structured_log_get_component_name (log_message->component), + tmp_json (log_message->message)); + return log_message; } From ab418aa8708f7887c227cc1c4974af1be16b20b8 Mon Sep 17 00:00:00 2001 From: Micah Scott Date: Mon, 18 Nov 2024 07:42:53 -0800 Subject: [PATCH 073/139] Remove unnecessary cast advice in sdam-monitoring example --- src/libmongoc/examples/example-sdam-monitoring.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/libmongoc/examples/example-sdam-monitoring.c b/src/libmongoc/examples/example-sdam-monitoring.c index de617c87fbb..b5afd515c1f 100644 --- a/src/libmongoc/examples/example-sdam-monitoring.c +++ b/src/libmongoc/examples/example-sdam-monitoring.c @@ -108,14 +108,13 @@ topology_changed (const mongoc_apm_topology_changed_t *event) prefs = mongoc_read_prefs_new (MONGOC_READ_SECONDARY); - /* it is safe, and unfortunately necessary, to cast away const here */ - if (mongoc_topology_description_has_readable_server ((mongoc_topology_description_t *) new_td, prefs)) { + if (mongoc_topology_description_has_readable_server (new_td, prefs)) { printf (" secondary AVAILABLE\n"); } else { printf (" secondary UNAVAILABLE\n"); } - if (mongoc_topology_description_has_writable_server ((mongoc_topology_description_t *) new_td)) { + if (mongoc_topology_description_has_writable_server (new_td)) { printf (" primary AVAILABLE\n"); } else { printf (" primary UNAVAILABLE\n"); From aad715b2d792e8ffd2cdf7cb59ea2de600094f19 Mon Sep 17 00:00:00 2001 From: Micah Scott Date: Mon, 18 Nov 2024 09:33:36 -0800 Subject: [PATCH 074/139] explicitly initiate a single-threaded topology scan in waitForEvent blocking --- src/libmongoc/tests/unified/operation.c | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/src/libmongoc/tests/unified/operation.c b/src/libmongoc/tests/unified/operation.c index 6db151f1062..73e80bfa89b 100644 --- a/src/libmongoc/tests/unified/operation.c +++ b/src/libmongoc/tests/unified/operation.c @@ -3881,9 +3881,20 @@ operation_wait_for_event (test_t *test, operation_t *op, result_t *result, bson_ goto done; }; - // If any events are coming, they will be from a different thread. - // To keep the tests simple, this polls until match or timeout. - // (Same approach used by json-test-operations, outside the unified tests) + // @todo Re-examine this once we have support for connection pools in the unified test + // runner. Without pooling, all events we could be waiting on would be coming + // from single-threaded (blocking) topology scans. There's no reason for those + // to happen without some explicit activity, and the tests aren't written to account + // for this. We could work around it by sending a command (and hiding its excess events/logs) + // or by directly initiating a scan. + { + mongoc_client_t *mc_client = entity_map_get_client (test->entity_map, client_id, error); + if (!mc_client) { + goto done; + } + _mongoc_topology_do_blocking_scan (mc_client->topology, error); + } + _mongoc_usleep (WAIT_FOR_EVENT_TICK_MS * 1000); } From 4add6376a0e2dc9a288c7bc45b2f1178de8bd543 Mon Sep 17 00:00:00 2001 From: Micah Scott Date: Mon, 18 Nov 2024 09:38:33 -0800 Subject: [PATCH 075/139] event observe/store arrays must copy their strings These aren't long lived strings, they might be part of a temporary bson document like in createEntities. --- src/libmongoc/tests/unified/entity-map.c | 28 +++++++++++++++++++----- src/libmongoc/tests/unified/entity-map.h | 6 ++--- 2 files changed, 25 insertions(+), 9 deletions(-) diff --git a/src/libmongoc/tests/unified/entity-map.c b/src/libmongoc/tests/unified/entity-map.c index 1401ce283f0..5ea9db132a7 100644 --- a/src/libmongoc/tests/unified/entity-map.c +++ b/src/libmongoc/tests/unified/entity-map.c @@ -460,16 +460,14 @@ is_supported_event_type (const char *type) static void add_observe_event (entity_t *entity, const char *type) { - observe_event_t event = {.type = type}; - + observe_event_t event = {.type = bson_strdup (type)}; _mongoc_array_append_val (&entity->observe_events, event); } static void add_store_event (entity_t *entity, const char *type, const char *entity_id) { - store_event_t event = {.type = type, .entity_id = entity_id}; - + store_event_t event = {.type = bson_strdup (type), .entity_id = bson_strdup (entity_id)}; _mongoc_array_append_val (&entity->store_events, event); } @@ -653,6 +651,7 @@ entity_client_new (entity_map_t *em, bson_t *bson, bson_error_t *error) } if (can_reduce_heartbeat && em->reduced_heartbeat) { + // @todo This option is needed for both single-threaded and pooled clients, but only works on pooled mongoc_uri_set_option_as_int32 (uri, MONGOC_URI_HEARTBEATFREQUENCYMS, REDUCED_HEARTBEAT_FREQUENCY_MS); } @@ -663,6 +662,7 @@ entity_client_new (entity_map_t *em, bson_t *bson, bson_error_t *error) mongoc_client_set_apm_callbacks (client, callbacks, entity); if (can_reduce_heartbeat && em->reduced_heartbeat) { + // @todo Examine whether this is needed in addition to the URI param above client->topology->min_heartbeat_frequency_msec = REDUCED_MIN_HEARTBEAT_FREQUENCY_MS; } @@ -1671,8 +1671,24 @@ entity_destroy (entity_t *entity) } } - _mongoc_array_destroy (&entity->observe_events); - _mongoc_array_destroy (&entity->store_events); + { + observe_event_t *const begin = (observe_event_t *) entity->observe_events.data; + observe_event_t *const end = begin + entity->observe_events.len; + for (observe_event_t *iter = begin; iter != end; ++iter) { + bson_free (iter->type); + } + _mongoc_array_destroy (&entity->observe_events); + } + { + store_event_t *const begin = (store_event_t *) entity->store_events.data; + store_event_t *const end = begin + entity->store_events.len; + for (store_event_t *iter = begin; iter != end; ++iter) { + bson_free (iter->type); + bson_free (iter->entity_id); + } + _mongoc_array_destroy (&entity->store_events); + } + bson_destroy (entity->ignore_command_monitoring_events); bson_free (entity->type); bson_free (entity->id); diff --git a/src/libmongoc/tests/unified/entity-map.h b/src/libmongoc/tests/unified/entity-map.h index 3b5aee4191b..b293d27bb74 100644 --- a/src/libmongoc/tests/unified/entity-map.h +++ b/src/libmongoc/tests/unified/entity-map.h @@ -38,12 +38,12 @@ typedef struct _log_message_t { } log_message_t; typedef struct _observe_event_t { - const char *type; // Non-owning. Type of event to observe. + char *type; // Type of event to observe. } observe_event_t; typedef struct _store_event_t { - const char *entity_id; // Non-owning. Target entity to store event. - const char *type; // Non-owning. Type of event to store. + char *entity_id; // Target entity to store event. + char *type; // Type of event to store. } store_event_t; typedef struct _entity_t { From c7b6be76ff463a6d892f9343e9146938f601f432 Mon Sep 17 00:00:00 2001 From: Micah Scott Date: Mon, 18 Nov 2024 10:04:51 -0800 Subject: [PATCH 076/139] Portability fix for structured_log atomic early-out Use a separate atomic flag instead of trying to make an atomic func pointer, to avoid non-portable hacks. --- src/libmongoc/src/mongoc/mongoc-structured-log.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/libmongoc/src/mongoc/mongoc-structured-log.c b/src/libmongoc/src/mongoc/mongoc-structured-log.c index 450ce92eacd..d0eea5dadae 100644 --- a/src/libmongoc/src/mongoc/mongoc-structured-log.c +++ b/src/libmongoc/src/mongoc/mongoc-structured-log.c @@ -36,7 +36,8 @@ static struct { bson_mutex_t func_mutex; // Mutex prevents func reentrancy, ensures atomic updates to (func, user_data) mongoc_structured_log_func_t func; void *user_data; - FILE *stream; // Only used by the default handler + FILE *stream; // Only used by the default handler + int func_is_null_atomic; // Always (func == NULL), effectively bool. Portably atomic to read without mutex. int32_t max_document_length; int component_level_table[STRUCTURED_LOG_COMPONENT_TABLE_SIZE]; // Really mongoc_structured_log_level_t; int typed to @@ -73,8 +74,9 @@ void mongoc_structured_log_set_handler (mongoc_structured_log_func_t log_func, void *user_data) { bson_mutex_lock (&gStructuredLog.func_mutex); - mcommon_atomic_ptr_exchange ((void *) &gStructuredLog.func, (void *) log_func, mcommon_memory_order_relaxed); + gStructuredLog.func = log_func; gStructuredLog.user_data = user_data; + mcommon_atomic_int_exchange (&gStructuredLog.func_is_null_atomic, log_func == NULL, mcommon_memory_order_relaxed); bson_mutex_unlock (&gStructuredLog.func_mutex); } @@ -117,7 +119,7 @@ mongoc_structured_log_get_max_level_for_component (mongoc_structured_log_compone bool _mongoc_structured_log_should_log (const mongoc_structured_log_envelope_t *envelope) { - return mcommon_atomic_ptr_fetch ((void *) &gStructuredLog.func, mcommon_memory_order_relaxed) && + return !mcommon_atomic_int_fetch ((void *) &gStructuredLog.func_is_null_atomic, mcommon_memory_order_relaxed) && envelope->level <= mongoc_structured_log_get_max_level_for_component (envelope->component); } From 5e607a55b136ed633f1583d7d2817b6e1ef8adde Mon Sep 17 00:00:00 2001 From: Micah Scott Date: Mon, 18 Nov 2024 11:32:35 -0800 Subject: [PATCH 077/139] log level aliases, and tests for component/level names --- .../src/mongoc/mongoc-structured-log.c | 37 ++++++-- .../tests/test-mongoc-structured-log.c | 93 +++++++++++++++++++ 2 files changed, 124 insertions(+), 6 deletions(-) diff --git a/src/libmongoc/src/mongoc/mongoc-structured-log.c b/src/libmongoc/src/mongoc/mongoc-structured-log.c index d0eea5dadae..4ecdc890b67 100644 --- a/src/libmongoc/src/mongoc/mongoc-structured-log.c +++ b/src/libmongoc/src/mongoc/mongoc-structured-log.c @@ -27,10 +27,20 @@ mongoc_structured_log_default_handler (const mongoc_structured_log_entry_t *entr #define STRUCTURED_LOG_COMPONENT_TABLE_SIZE (1 + (size_t) MONGOC_STRUCTURED_LOG_COMPONENT_CONNECTION) +// Canonical names for log components +static const char *gStructuredLogComponentNames[] = {"command", "topology", "serverSelection", "connection"}; + +// Canonical names for log levels static const char *gStructuredLogLevelNames[] = { "Emergency", "Alert", "Critical", "Error", "Warning", "Notice", "Informational", "Debug", "Trace"}; -static const char *gStructuredLogComponentNames[] = {"command", "topology", "serverSelection", "connection"}; +// Additional valid names for log levels +static const struct { + const char *name; + mongoc_structured_log_level_t level; +} gStructuredLogLevelAliases[] = {{.name = "off", .level = (mongoc_structured_log_level_t) 0}, + {.name = "warn", .level = MONGOC_STRUCTURED_LOG_LEVEL_WARNING}, + {.name = "info", .level = MONGOC_STRUCTURED_LOG_LEVEL_INFO}}; static struct { bson_mutex_t func_mutex; // Mutex prevents func reentrancy, ensures atomic updates to (func, user_data) @@ -159,11 +169,26 @@ mongoc_structured_log_get_level_name (mongoc_structured_log_level_t level) bool mongoc_structured_log_get_named_level (const char *name, mongoc_structured_log_level_t *out) { - const size_t table_size = sizeof gStructuredLogLevelNames / sizeof gStructuredLogLevelNames[0]; - for (unsigned table_index = 0; table_index < table_size; table_index++) { - if (!strcasecmp (name, gStructuredLogLevelNames[table_index])) { - *out = (mongoc_structured_log_level_t) table_index; - return true; + // First check canonical names + { + const size_t table_size = sizeof gStructuredLogLevelNames / sizeof gStructuredLogLevelNames[0]; + for (unsigned table_index = 0; table_index < table_size; table_index++) { + if (!strcasecmp (name, gStructuredLogLevelNames[table_index])) { + *out = (mongoc_structured_log_level_t) table_index; + return true; + } + } + } + // Check additional acceptable names + { + const size_t table_size = sizeof gStructuredLogLevelAliases / sizeof gStructuredLogLevelAliases[0]; + for (unsigned table_index = 0; table_index < table_size; table_index++) { + const char *alias = gStructuredLogLevelAliases[table_index].name; + mongoc_structured_log_level_t level = gStructuredLogLevelAliases[table_index].level; + if (!strcasecmp (name, alias)) { + *out = level; + return true; + } } } return false; diff --git a/src/libmongoc/tests/test-mongoc-structured-log.c b/src/libmongoc/tests/test-mongoc-structured-log.c index fa0742abd32..cd4daf81103 100644 --- a/src/libmongoc/tests/test-mongoc-structured-log.c +++ b/src/libmongoc/tests/test-mongoc-structured-log.c @@ -470,6 +470,97 @@ test_structured_log_duration (void) bson_destroy (assumption.expected_bson); } +void +test_structured_log_level_names (void) +{ + mongoc_structured_log_level_t level = (mongoc_structured_log_level_t) -1; + + // Alias, off = 0 + ASSERT (mongoc_structured_log_get_named_level ("off", &level)); + ASSERT_CMPINT (0, ==, level); + ASSERT_CMPINT (MONGOC_STRUCTURED_LOG_LEVEL_EMERGENCY, ==, level); + ASSERT_CMPSTR (mongoc_structured_log_get_level_name (level), "Emergency"); + + ASSERT (mongoc_structured_log_get_named_level ("emergency", &level)); + ASSERT_CMPINT (0, ==, level); + ASSERT_CMPINT (MONGOC_STRUCTURED_LOG_LEVEL_EMERGENCY, ==, level); + ASSERT_CMPSTR (mongoc_structured_log_get_level_name (level), "Emergency"); + + ASSERT (mongoc_structured_log_get_named_level ("alert", &level)); + ASSERT_CMPINT (1, ==, level); + ASSERT_CMPINT (MONGOC_STRUCTURED_LOG_LEVEL_ALERT, ==, level); + ASSERT_CMPSTR (mongoc_structured_log_get_level_name (level), "Alert"); + + ASSERT (mongoc_structured_log_get_named_level ("critical", &level)); + ASSERT_CMPINT (2, ==, level); + ASSERT_CMPINT (MONGOC_STRUCTURED_LOG_LEVEL_CRITICAL, ==, level); + ASSERT_CMPSTR (mongoc_structured_log_get_level_name (level), "Critical"); + + ASSERT (mongoc_structured_log_get_named_level ("error", &level)); + ASSERT_CMPINT (3, ==, level); + ASSERT_CMPINT (MONGOC_STRUCTURED_LOG_LEVEL_ERROR, ==, level); + ASSERT_CMPSTR (mongoc_structured_log_get_level_name (level), "Error"); + + // Alias, warn = Warning + ASSERT (mongoc_structured_log_get_named_level ("warn", &level)); + ASSERT_CMPINT (4, ==, level); + ASSERT_CMPINT (MONGOC_STRUCTURED_LOG_LEVEL_WARNING, ==, level); + ASSERT_CMPSTR (mongoc_structured_log_get_level_name (level), "Warning"); + + ASSERT (mongoc_structured_log_get_named_level ("warning", &level)); + ASSERT_CMPINT (4, ==, level); + ASSERT_CMPINT (MONGOC_STRUCTURED_LOG_LEVEL_WARNING, ==, level); + ASSERT_CMPSTR (mongoc_structured_log_get_level_name (level), "Warning"); + + ASSERT (mongoc_structured_log_get_named_level ("notice", &level)); + ASSERT_CMPINT (5, ==, level); + ASSERT_CMPINT (MONGOC_STRUCTURED_LOG_LEVEL_NOTICE, ==, level); + ASSERT_CMPSTR (mongoc_structured_log_get_level_name (level), "Notice"); + + // Alias, info = Informational + ASSERT (mongoc_structured_log_get_named_level ("info", &level)); + ASSERT_CMPINT (6, ==, level); + ASSERT_CMPINT (MONGOC_STRUCTURED_LOG_LEVEL_INFO, ==, level); + ASSERT_CMPSTR (mongoc_structured_log_get_level_name (level), "Informational"); + + ASSERT (mongoc_structured_log_get_named_level ("informational", &level)); + ASSERT_CMPINT (6, ==, level); + ASSERT_CMPINT (MONGOC_STRUCTURED_LOG_LEVEL_INFO, ==, level); + ASSERT_CMPSTR (mongoc_structured_log_get_level_name (level), "Informational"); + + ASSERT (mongoc_structured_log_get_named_level ("debug", &level)); + ASSERT_CMPINT (7, ==, level); + ASSERT_CMPINT (MONGOC_STRUCTURED_LOG_LEVEL_DEBUG, ==, level); + ASSERT_CMPSTR (mongoc_structured_log_get_level_name (level), "Debug"); + + ASSERT (mongoc_structured_log_get_named_level ("trace", &level)); + ASSERT_CMPINT (8, ==, level); + ASSERT_CMPINT (MONGOC_STRUCTURED_LOG_LEVEL_TRACE, ==, level); + ASSERT_CMPSTR (mongoc_structured_log_get_level_name (level), "Trace"); +} + +void +test_structured_log_component_names (void) +{ + mongoc_structured_log_component_t component = (mongoc_structured_log_component_t) -1; + + ASSERT (mongoc_structured_log_get_named_component ("Command", &component)); + ASSERT_CMPINT (MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND, ==, component); + ASSERT_CMPSTR (mongoc_structured_log_get_component_name (component), "command"); + + ASSERT (mongoc_structured_log_get_named_component ("Topology", &component)); + ASSERT_CMPINT (MONGOC_STRUCTURED_LOG_COMPONENT_TOPOLOGY, ==, component); + ASSERT_CMPSTR (mongoc_structured_log_get_component_name (component), "topology"); + + ASSERT (mongoc_structured_log_get_named_component ("ServerSelection", &component)); + ASSERT_CMPINT (MONGOC_STRUCTURED_LOG_COMPONENT_SERVER_SELECTION, ==, component); + ASSERT_CMPSTR (mongoc_structured_log_get_component_name (component), "serverSelection"); + + ASSERT (mongoc_structured_log_get_named_component ("Connection", &component)); + ASSERT_CMPINT (MONGOC_STRUCTURED_LOG_COMPONENT_CONNECTION, ==, component); + ASSERT_CMPSTR (mongoc_structured_log_get_component_name (component), "connection"); +} + void test_structured_log_install (TestSuite *suite) { @@ -481,4 +572,6 @@ test_structured_log_install (TestSuite *suite) TestSuite_Add (suite, "/structured_log/server_description", test_structured_log_server_description); TestSuite_Add (suite, "/structured_log/command", test_structured_log_command); TestSuite_Add (suite, "/structured_log/duration", test_structured_log_duration); + TestSuite_Add (suite, "/structured_log/level_names", test_structured_log_level_names); + TestSuite_Add (suite, "/structured_log/component_names", test_structured_log_component_names); } From d585c01122ce036f387a2bd004f621afcff93b43 Mon Sep 17 00:00:00 2001 From: Micah Scott Date: Mon, 18 Nov 2024 11:55:21 -0800 Subject: [PATCH 078/139] support MONGODB_LOG_PATH=stdout --- .../src/mongoc/mongoc-structured-log.c | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/src/libmongoc/src/mongoc/mongoc-structured-log.c b/src/libmongoc/src/mongoc/mongoc-structured-log.c index 4ecdc890b67..aebb4ef5b08 100644 --- a/src/libmongoc/src/mongoc/mongoc-structured-log.c +++ b/src/libmongoc/src/mongoc/mongoc-structured-log.c @@ -268,14 +268,19 @@ _mongoc_structured_log_init (void) static FILE * _mongoc_structured_log_open_stream (void) { - const char *log_target = getenv ("MONGODB_LOG_PATH"); - bool log_to_stderr = !log_target || !strcmp (log_target, "stderr"); - FILE *log_stream = log_to_stderr ? stderr : fopen (log_target, "a"); - if (!log_stream) { - MONGOC_ERROR ("Cannot open log file %s for writing", log_target); + const char *path = getenv ("MONGODB_LOG_PATH"); + if (!path || !strcmp (path, "stderr")) { + return stderr; + } + if (!strcmp (path, "stdout")) { + return stdout; + } + FILE *file = fopen (path, "a"); + if (!file) { + MONGOC_ERROR ("Cannot open log file %s for writing", path); exit (EXIT_FAILURE); } - return log_stream; + return file; } static FILE * @@ -285,6 +290,8 @@ _mongoc_structured_log_get_stream (void) if (log_stream) { return log_stream; } + // Note that log_stream may be the global stderr/stdout streams, + // or an allocated FILE that is never closed. log_stream = _mongoc_structured_log_open_stream (); gStructuredLog.stream = log_stream; return log_stream; From f1574d38a0aac72aa7d25c11bba599f8b5d7a186 Mon Sep 17 00:00:00 2001 From: Micah Scott Date: Mon, 18 Nov 2024 12:11:54 -0800 Subject: [PATCH 079/139] MSVC compatibility, move eval to outer macro --- src/libmongoc/src/mongoc/mongoc-structured-log-private.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libmongoc/src/mongoc/mongoc-structured-log-private.h b/src/libmongoc/src/mongoc/mongoc-structured-log-private.h index 8cfdd7358fe..4823331fe00 100644 --- a/src/libmongoc/src/mongoc/mongoc-structured-log-private.h +++ b/src/libmongoc/src/mongoc/mongoc-structured-log-private.h @@ -52,7 +52,7 @@ BSON_BEGIN_DECLS * once the table is built. */ #define mongoc_structured_log(_level, _component, ...) \ - _mongoc_structured_log_with_end_of_list (_level, _component, __VA_ARGS__, end_of_list ()) + _bsonDSL_eval (_mongoc_structured_log_with_end_of_list (_level, _component, __VA_ARGS__, end_of_list ())) #define _mongoc_structured_log_with_end_of_list(_level, _component, _message, ...) \ do { \ @@ -67,7 +67,7 @@ BSON_BEGIN_DECLS } while (0) #define _mongoc_structured_log_items_to_stages(...) \ - _bsonDSL_eval (_bsonDSL_mapMacro (_mongoc_structured_log_item_to_stages, ~, __VA_ARGS__)) + _bsonDSL_mapMacro (_mongoc_structured_log_item_to_stages, ~, __VA_ARGS__) #define _mongoc_structured_log_flag_expr(_action, _constant, _counter) | (_constant##_##_action) From 6cd4fd13cabee68f870806bc024df03785f6444f Mon Sep 17 00:00:00 2001 From: Micah Scott Date: Mon, 18 Nov 2024 14:47:54 -0800 Subject: [PATCH 080/139] Document the public structured logging API --- src/libmongoc/doc/logging.rst | 135 ++---------------- .../doc/mongoc_structured_log_component_t.rst | 34 +++++ ...goc_structured_log_entry_get_component.rst | 26 ++++ .../mongoc_structured_log_entry_get_level.rst | 26 ++++ ...c_structured_log_entry_message_as_bson.rst | 30 ++++ .../doc/mongoc_structured_log_entry_t.rst | 30 ++++ .../doc/mongoc_structured_log_func_t.rst | 31 ++++ ...ngoc_structured_log_get_component_name.rst | 27 ++++ .../mongoc_structured_log_get_level_name.rst | 27 ++++ ...ctured_log_get_max_level_for_component.rst | 27 ++++ ...goc_structured_log_get_named_component.rst | 30 ++++ .../mongoc_structured_log_get_named_level.rst | 30 ++++ .../doc/mongoc_structured_log_level_t.rst | 37 +++++ .../doc/mongoc_structured_log_set_handler.rst | 33 +++++ ...d_log_set_max_level_for_all_components.rst | 25 ++++ ...ctured_log_set_max_level_for_component.rst | 28 ++++ src/libmongoc/doc/structured_log.rst | 114 +++++++++++++++ src/libmongoc/doc/unstructured_log.rst | 132 +++++++++++++++++ 18 files changed, 696 insertions(+), 126 deletions(-) create mode 100644 src/libmongoc/doc/mongoc_structured_log_component_t.rst create mode 100644 src/libmongoc/doc/mongoc_structured_log_entry_get_component.rst create mode 100644 src/libmongoc/doc/mongoc_structured_log_entry_get_level.rst create mode 100644 src/libmongoc/doc/mongoc_structured_log_entry_message_as_bson.rst create mode 100644 src/libmongoc/doc/mongoc_structured_log_entry_t.rst create mode 100644 src/libmongoc/doc/mongoc_structured_log_func_t.rst create mode 100644 src/libmongoc/doc/mongoc_structured_log_get_component_name.rst create mode 100644 src/libmongoc/doc/mongoc_structured_log_get_level_name.rst create mode 100644 src/libmongoc/doc/mongoc_structured_log_get_max_level_for_component.rst create mode 100644 src/libmongoc/doc/mongoc_structured_log_get_named_component.rst create mode 100644 src/libmongoc/doc/mongoc_structured_log_get_named_level.rst create mode 100644 src/libmongoc/doc/mongoc_structured_log_level_t.rst create mode 100644 src/libmongoc/doc/mongoc_structured_log_set_handler.rst create mode 100644 src/libmongoc/doc/mongoc_structured_log_set_max_level_for_all_components.rst create mode 100644 src/libmongoc/doc/mongoc_structured_log_set_max_level_for_component.rst create mode 100644 src/libmongoc/doc/structured_log.rst create mode 100644 src/libmongoc/doc/unstructured_log.rst diff --git a/src/libmongoc/doc/logging.rst b/src/libmongoc/doc/logging.rst index ea35eb3fadd..1a0b7cd1cfd 100644 --- a/src/libmongoc/doc/logging.rst +++ b/src/libmongoc/doc/logging.rst @@ -3,133 +3,16 @@ Logging ======= -MongoDB C driver Logging Abstraction +The MongoDB C driver has two different types of logging available: -Synopsis --------- +* The original ``mongoc_log`` facility supports freeform string messages that originate from the driver itself or from application code. This has been retroactively termed "unstructured logging". +* A new ``mongoc_structured_log`` facility reports messages from the driver itself using a BSON format defined across driver implementations by the `MongoDB Logging Specification `_. -.. code-block:: c +These two systems are configured and used independently. - typedef enum { - MONGOC_LOG_LEVEL_ERROR, - MONGOC_LOG_LEVEL_CRITICAL, - MONGOC_LOG_LEVEL_WARNING, - MONGOC_LOG_LEVEL_MESSAGE, - MONGOC_LOG_LEVEL_INFO, - MONGOC_LOG_LEVEL_DEBUG, - MONGOC_LOG_LEVEL_TRACE, - } mongoc_log_level_t; - - #define MONGOC_ERROR(...) - #define MONGOC_CRITICAL(...) - #define MONGOC_WARNING(...) - #define MONGOC_MESSAGE(...) - #define MONGOC_INFO(...) - #define MONGOC_DEBUG(...) - - typedef void (*mongoc_log_func_t) (mongoc_log_level_t log_level, - const char *log_domain, - const char *message, - void *user_data); - - void - mongoc_log_set_handler (mongoc_log_func_t log_func, void *user_data); - void - mongoc_log (mongoc_log_level_t log_level, - const char *log_domain, - const char *format, - ...); - const char * - mongoc_log_level_str (mongoc_log_level_t log_level); - void - mongoc_log_default_handler (mongoc_log_level_t log_level, - const char *log_domain, - const char *message, - void *user_data); - void - mongoc_log_trace_enable (void); - void - mongoc_log_trace_disable (void); - -The MongoDB C driver comes with an abstraction for logging that you can use in your application, or integrate with an existing logging system. - -Macros ------- - -To make logging a little less painful, various helper macros are provided. See the following example. - -.. code-block:: c - - #undef MONGOC_LOG_DOMAIN - #define MONGOC_LOG_DOMAIN "my-custom-domain" - - MONGOC_WARNING ("An error occurred: %s", strerror (errno)); - -.. _custom_log_handlers: - -Custom Log Handlers -------------------- - -The default log handler prints a timestamp and the log message to ``stdout``, or to ``stderr`` for warnings, critical messages, and errors. - You can override the handler with ``mongoc_log_set_handler()``. - Your handler function is called in a mutex for thread safety. - -For example, you could register a custom handler to suppress messages at INFO level and below: - -.. code-block:: c - - void - my_logger (mongoc_log_level_t log_level, - const char *log_domain, - const char *message, - void *user_data) - { - /* smaller values are more important */ - if (log_level < MONGOC_LOG_LEVEL_INFO) { - mongoc_log_default_handler (log_level, log_domain, message, user_data); - } - } - - int - main (int argc, char *argv[]) - { - mongoc_log_set_handler (my_logger, NULL); - mongoc_init (); - - /* ... your code ... */ - - mongoc_cleanup (); - return 0; - } - -Note that in the example above ``mongoc_log_set_handler()`` is called before ``mongoc_init()``. -Otherwise, some log traces could not be processed by the log handler. - -To restore the default handler: - -.. code-block:: c - - mongoc_log_set_handler (mongoc_log_default_handler, NULL); - -Disable logging ---------------- - -To disable all logging, including warnings, critical messages and errors, provide an empty log handler: - -.. code-block:: c - - mongoc_log_set_handler (NULL, NULL); - -Tracing -------- - -If compiling your own copy of the MongoDB C driver, consider configuring with ``-DENABLE_TRACING=ON`` to enable function tracing and hex dumps of network packets to ``STDERR`` and ``STDOUT`` during development and debugging. - -This is especially useful when debugging what may be going on internally in the driver. - -Trace messages can be enabled and disabled by calling ``mongoc_log_trace_enable()`` and ``mongoc_log_trace_disable()`` - -.. note:: - - Compiling the driver with ``-DENABLE_TRACING=ON`` will affect its performance. Disabling tracing with ``mongoc_log_trace_disable()`` significantly reduces the overhead, but cannot remove it completely. +.. toctree:: + :titlesonly: + :maxdepth: 1 + unstructured_log + structured_log diff --git a/src/libmongoc/doc/mongoc_structured_log_component_t.rst b/src/libmongoc/doc/mongoc_structured_log_component_t.rst new file mode 100644 index 00000000000..9538d60d12a --- /dev/null +++ b/src/libmongoc/doc/mongoc_structured_log_component_t.rst @@ -0,0 +1,34 @@ +:man_page: mongoc_structured_log_component_t + +mongoc_structured_log_component_t +================================= + +Synopsis +-------- + +.. code-block:: c + + typedef enum { + MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND = 0, + MONGOC_STRUCTURED_LOG_COMPONENT_TOPOLOGY = 1, + MONGOC_STRUCTURED_LOG_COMPONENT_SERVER_SELECTION = 2, + MONGOC_STRUCTURED_LOG_COMPONENT_CONNECTION = 3, + } mongoc_structured_log_component_t; + +``mongoc_structured_log_component_t`` enumerates the structured logging components. +Applications should never rely on having an exhaustive list of all log components. +Instead, use :symbol:`mongoc_structured_log_set_max_level_for_all_components` to set a default level if needed. + +Functions +--------- + +.. toctree:: + :titlesonly: + :maxdepth: 1 + + mongoc_structured_log_get_component_name + mongoc_structured_log_get_named_component + +.. seealso:: + + | :doc:`structured_log` diff --git a/src/libmongoc/doc/mongoc_structured_log_entry_get_component.rst b/src/libmongoc/doc/mongoc_structured_log_entry_get_component.rst new file mode 100644 index 00000000000..313d30ce7be --- /dev/null +++ b/src/libmongoc/doc/mongoc_structured_log_entry_get_component.rst @@ -0,0 +1,26 @@ +:man_page: mongoc_structured_log_entry_get_component + +mongoc_structured_log_entry_get_component +========================================= + +Synopsis +-------- + +.. code-block:: c + + mongoc_structured_log_component_t + mongoc_structured_log_entry_get_component (const mongoc_structured_log_entry_t *entry); + +Parameters +---------- + +* ``entry``: A :symbol:`mongoc_structured_log_entry_t` pointer. + +Returns +------- + +The :symbol:`mongoc_structured_log_component_t` associated with this log entry. + +.. seealso:: + + | :doc:`structured_log` diff --git a/src/libmongoc/doc/mongoc_structured_log_entry_get_level.rst b/src/libmongoc/doc/mongoc_structured_log_entry_get_level.rst new file mode 100644 index 00000000000..22f9b03a022 --- /dev/null +++ b/src/libmongoc/doc/mongoc_structured_log_entry_get_level.rst @@ -0,0 +1,26 @@ +:man_page: mongoc_structured_log_entry_get_level + +mongoc_structured_log_entry_get_level +===================================== + +Synopsis +-------- + +.. code-block:: c + + mongoc_structured_log_level_t + mongoc_structured_log_entry_get_level (const mongoc_structured_log_entry_t *entry); + +Parameters +---------- + +* ``entry``: A :symbol:`mongoc_structured_log_entry_t` pointer. + +Returns +------- + +The :symbol:`mongoc_structured_log_level_t` associated with this log entry. + +.. seealso:: + + | :doc:`structured_log` diff --git a/src/libmongoc/doc/mongoc_structured_log_entry_message_as_bson.rst b/src/libmongoc/doc/mongoc_structured_log_entry_message_as_bson.rst new file mode 100644 index 00000000000..fe002712709 --- /dev/null +++ b/src/libmongoc/doc/mongoc_structured_log_entry_message_as_bson.rst @@ -0,0 +1,30 @@ +:man_page: mongoc_structured_log_entry_message_as_bson + +mongoc_structured_log_entry_message_as_bson +=========================================== + +Synopsis +-------- + +.. code-block:: c + + bson_t * + mongoc_structured_log_entry_message_as_bson (const mongoc_structured_log_entry_t *entry); + +Make a new copy, as a bson_t, of the log entry's standardized BSON representation. +When possible, a log handler should avoid serializing log messages that will be discarded. +Each call allocates an independent copy of the message that must be freed. + +Parameters +---------- + +* ``entry``: A :symbol:`mongoc_structured_log_entry_t` pointer. + +Returns +------- + +A new allocated :symbol:`bson_t` that must be freed with a call to :symbol:`bson_destroy`. + +.. seealso:: + + | :doc:`structured_log` diff --git a/src/libmongoc/doc/mongoc_structured_log_entry_t.rst b/src/libmongoc/doc/mongoc_structured_log_entry_t.rst new file mode 100644 index 00000000000..310a099262f --- /dev/null +++ b/src/libmongoc/doc/mongoc_structured_log_entry_t.rst @@ -0,0 +1,30 @@ +:man_page: mongoc_structured_log_entry_t + +mongoc_structured_log_entry_t +============================= + +Synopsis +-------- + +.. code-block:: c + + typedef struct mongoc_structured_log_entry_t mongoc_structured_log_entry_t; + +``mongoc_structured_log_entry_t`` is an opaque structure which represents the temporary state of an in-progress log entry. +It can only be used during a :symbol:`mongoc_structured_log_func_t`, it is not valid after the log handler returns. +Use the functions below to query individual aspects of the log entry. + +Functions +--------- + +.. toctree:: + :titlesonly: + :maxdepth: 1 + + mongoc_structured_log_entry_get_component + mongoc_structured_log_entry_get_level + mongoc_structured_log_entry_message_as_bson + +.. seealso:: + + | :doc:`structured_log` diff --git a/src/libmongoc/doc/mongoc_structured_log_func_t.rst b/src/libmongoc/doc/mongoc_structured_log_func_t.rst new file mode 100644 index 00000000000..7b3e16eb872 --- /dev/null +++ b/src/libmongoc/doc/mongoc_structured_log_func_t.rst @@ -0,0 +1,31 @@ +:man_page: mongoc_structured_log_func_t + +mongoc_structured_log_func_t +============================ + +Synopsis +-------- + +.. code-block:: c + + typedef void (*mongoc_structured_log_func_t) + (const mongoc_structured_log_entry_t *entry, void *user_data); + +Callback function for :symbol:`mongoc_structured_log_set_handler`. +Not required to be re-entrant, mutual exclusion is provided by the logging facility. + +Handlers may use any operating system or ``libbson`` functions but MUST not use any ``libmongoc`` functions except: + +* :symbol:`mongoc_structured_log_entry_get_component` +* :symbol:`mongoc_structured_log_entry_get_level` +* :symbol:`mongoc_structured_log_entry_message_as_bson` + +Parameters +---------- + +* ``entry``: A :symbol:`mongoc_structured_log_entry_t` pointer, only valid during the handler invocation. +* ``user_data``: Optional user data from :symbol:`mongoc_structured_log_set_handler`. + +.. seealso:: + + | :doc:`structured_log` diff --git a/src/libmongoc/doc/mongoc_structured_log_get_component_name.rst b/src/libmongoc/doc/mongoc_structured_log_get_component_name.rst new file mode 100644 index 00000000000..66dc03d8d60 --- /dev/null +++ b/src/libmongoc/doc/mongoc_structured_log_get_component_name.rst @@ -0,0 +1,27 @@ +:man_page: mongoc_structured_log_get_component_name + +mongoc_structured_log_get_component_name +======================================== + +Synopsis +-------- + +.. code-block:: c + + const char * + mongoc_structured_log_get_component_name (mongoc_structured_log_component_t component); + +Parameters +---------- + +* ``component``: Log component as a :symbol:`mongoc_structured_log_component_t`. + +Returns +------- + +If the component is known, returns a pointer to a constant string that should not be freed. +If the component has no known name, returns NULL. + +.. seealso:: + + | :doc:`structured_log` diff --git a/src/libmongoc/doc/mongoc_structured_log_get_level_name.rst b/src/libmongoc/doc/mongoc_structured_log_get_level_name.rst new file mode 100644 index 00000000000..93fc0860de6 --- /dev/null +++ b/src/libmongoc/doc/mongoc_structured_log_get_level_name.rst @@ -0,0 +1,27 @@ +:man_page: mongoc_structured_log_get_level_name + +mongoc_structured_log_get_level_name +==================================== + +Synopsis +-------- + +.. code-block:: c + + const char * + mongoc_structured_log_get_level_name (mongoc_structured_log_level_t level); + +Parameters +---------- + +* ``level``: Log level as a :symbol:`mongoc_structured_log_level_t`. + +Returns +------- + +If the level is known, returns a pointer to a constant string that should not be freed. +If the level has no known name, returns NULL. + +.. seealso:: + + | :doc:`structured_log` diff --git a/src/libmongoc/doc/mongoc_structured_log_get_max_level_for_component.rst b/src/libmongoc/doc/mongoc_structured_log_get_max_level_for_component.rst new file mode 100644 index 00000000000..8cab64920f2 --- /dev/null +++ b/src/libmongoc/doc/mongoc_structured_log_get_max_level_for_component.rst @@ -0,0 +1,27 @@ +:man_page: mongoc_structured_log_get_max_level_for_component + +mongoc_structured_log_get_max_level_for_component +================================================= + +Synopsis +-------- + +.. code-block:: c + + mongoc_structured_log_level_t + mongoc_structured_log_get_max_level_for_component (mongoc_structured_log_component_t component); + +Parameters +---------- + +* ``component``: Log component as a :symbol:`mongoc_structured_log_component_t`. + +Returns +------- + +Returns the current maximum log level for a specific component. +This may be the last value set with :symbol:`mongoc_structured_log_set_max_level_for_component` or :symbol:`mongoc_structured_log_set_max_level_for_all_components`, or it may be the default obtained from environment variables. + +.. seealso:: + + | :doc:`structured_log` diff --git a/src/libmongoc/doc/mongoc_structured_log_get_named_component.rst b/src/libmongoc/doc/mongoc_structured_log_get_named_component.rst new file mode 100644 index 00000000000..bfac67862c5 --- /dev/null +++ b/src/libmongoc/doc/mongoc_structured_log_get_named_component.rst @@ -0,0 +1,30 @@ +:man_page: mongoc_structured_log_get_named_component + +mongoc_structured_log_get_named_component +========================================= + +Synopsis +-------- + +.. code-block:: c + + bool + mongoc_structured_log_get_named_component (const char *name, mongoc_structured_log_component_t *out); + +Look up a component by name. Case insensitive. + +Parameters +---------- + +* ``name``: A name to look up as a log component. +* ``out``: On success, the corresponding :symbol:`mongoc_structured_log_component_t` is written here. + +Returns +------- + +If the component name is known, returns ``true`` and writes the component enum to ``*out``. +If the component name is not known, returns ``false`` and does not write ``*out``. + +.. seealso:: + + | :doc:`structured_log` diff --git a/src/libmongoc/doc/mongoc_structured_log_get_named_level.rst b/src/libmongoc/doc/mongoc_structured_log_get_named_level.rst new file mode 100644 index 00000000000..63ed45278ac --- /dev/null +++ b/src/libmongoc/doc/mongoc_structured_log_get_named_level.rst @@ -0,0 +1,30 @@ +:man_page: mongoc_structured_log_get_named_level + +mongoc_structured_log_get_named_level +===================================== + +Synopsis +-------- + +.. code-block:: c + + bool + mongoc_structured_log_get_named_level (const char *name, mongoc_structured_log_level_t *out); + +Look up a log level by name. Case insensitive. + +Parameters +---------- + +* ``name``: A name to look up as a log level. +* ``out``: On success, the corresponding :symbol:`mongoc_structured_log_level_t` is written here. + +Returns +------- + +If the level name is known, returns ``true`` and writes the level enum to ``*out``. +If the level name is not known, returns ``false`` and does not write ``*out``. + +.. seealso:: + + | :doc:`structured_log` diff --git a/src/libmongoc/doc/mongoc_structured_log_level_t.rst b/src/libmongoc/doc/mongoc_structured_log_level_t.rst new file mode 100644 index 00000000000..96108f41e6b --- /dev/null +++ b/src/libmongoc/doc/mongoc_structured_log_level_t.rst @@ -0,0 +1,37 @@ +:man_page: mongoc_structured_log_level_t + +mongoc_structured_log_level_t +============================= + +Synopsis +-------- + +.. code-block:: c + + typedef enum { + MONGOC_STRUCTURED_LOG_LEVEL_EMERGENCY = 0, + MONGOC_STRUCTURED_LOG_LEVEL_ALERT = 1, + MONGOC_STRUCTURED_LOG_LEVEL_CRITICAL = 2, + MONGOC_STRUCTURED_LOG_LEVEL_ERROR = 3, + MONGOC_STRUCTURED_LOG_LEVEL_WARNING = 4, + MONGOC_STRUCTURED_LOG_LEVEL_NOTICE = 5, + MONGOC_STRUCTURED_LOG_LEVEL_INFO = 6, + MONGOC_STRUCTURED_LOG_LEVEL_DEBUG = 7, + MONGOC_STRUCTURED_LOG_LEVEL_TRACE = 8, + } mongoc_structured_log_level_t; + +``mongoc_structured_log_level_t`` enumerates the available log levels for use with structured logging. + +Functions +--------- + +.. toctree:: + :titlesonly: + :maxdepth: 1 + + mongoc_structured_log_get_level_name + mongoc_structured_log_get_named_level + +.. seealso:: + + | :doc:`structured_log` diff --git a/src/libmongoc/doc/mongoc_structured_log_set_handler.rst b/src/libmongoc/doc/mongoc_structured_log_set_handler.rst new file mode 100644 index 00000000000..7c575434d38 --- /dev/null +++ b/src/libmongoc/doc/mongoc_structured_log_set_handler.rst @@ -0,0 +1,33 @@ +:man_page: mongoc_structured_log_set_handler + +mongoc_structured_log_set_handler +================================= + +Synopsis +-------- + +.. code-block:: c + + void + mongoc_structured_log_set_handler (mongoc_structured_log_func_t log_func, void *user_data); + +Sets the function to be called to handle structured log messages, as a :symbol:`mongoc_structured_log_func_t`. + +The callback is given a :symbol:`mongoc_structured_log_entry_t` as a handle for obtaining additional information about the log message. +This entry pointer is only valid during a callback, because it's a low cost reference to temporary data. +Callbacks are not required to be re-entrant, mutual exclusion is provided by the logging facility. +Handlers must take care not to re-enter libmongoc except in proscribed ways. See :symbol:`mongoc_structured_log_func_t`. + +There is a single global handler per process. +Any call to this function will replace the default handler. +If the ``log_func`` is set to NULL, structured logging will be disabled. + +Parameters +---------- + +* ``log_func``: The handler to install, a :symbol:`mongoc_structured_log_func_t`, or NULL to disable structured logging. +* ``user_data``: Optional user data, passed on to the handler. + +.. seealso:: + + | :doc:`structured_log` diff --git a/src/libmongoc/doc/mongoc_structured_log_set_max_level_for_all_components.rst b/src/libmongoc/doc/mongoc_structured_log_set_max_level_for_all_components.rst new file mode 100644 index 00000000000..5aeb16e0de6 --- /dev/null +++ b/src/libmongoc/doc/mongoc_structured_log_set_max_level_for_all_components.rst @@ -0,0 +1,25 @@ +:man_page: mongoc_structured_log_set_max_level_for_all_components + +mongoc_structured_log_set_max_level_for_all_components +====================================================== + +Synopsis +-------- + +.. code-block:: c + + void + mongoc_structured_log_set_max_level_for_all_components (mongoc_structured_log_level_t level); + +Sets all per-component maximum log levels to the same value. +Only log messages at or below this severity level will be passed to :symbol:`mongoc_structured_log_func_t`. +Effective even for logging components not known at compile-time. + +Parameters +---------- + +* ``level``: The max log level for all components, as a :symbol:`mongoc_structured_log_level_t`. + +.. seealso:: + + | :doc:`structured_log` diff --git a/src/libmongoc/doc/mongoc_structured_log_set_max_level_for_component.rst b/src/libmongoc/doc/mongoc_structured_log_set_max_level_for_component.rst new file mode 100644 index 00000000000..5aa187e4cab --- /dev/null +++ b/src/libmongoc/doc/mongoc_structured_log_set_max_level_for_component.rst @@ -0,0 +1,28 @@ +:man_page: mongoc_structured_log_set_max_level_for_component + +mongoc_structured_log_set_max_level_for_component +================================================= + +Synopsis +-------- + +.. code-block:: c + + void + mongoc_structured_log_set_max_level_for_component (mongoc_structured_log_component_t component, + mongoc_structured_log_level_t level); + +Sets the maximum log level per-component. +Only log messages at or below this severity level will be passed to :symbol:`mongoc_structured_log_func_t`. + +By default, each component's log level comes from the environment variables ``MONGOC_LOG_`` and ``MONGOC_LOG_ALL``. + +Parameters +---------- + +* ``component``: The component to set a max log level. for, as a :symbol:`mongoc_structured_log_component_t`. +* ``level``: The new max log level for this component, as a :symbol:`mongoc_structured_log_level_t`. + +.. seealso:: + + | :doc:`structured_log` diff --git a/src/libmongoc/doc/structured_log.rst b/src/libmongoc/doc/structured_log.rst new file mode 100644 index 00000000000..70bde01a93e --- /dev/null +++ b/src/libmongoc/doc/structured_log.rst @@ -0,0 +1,114 @@ +:man_page: mongoc_structured_log + +Structured Logging +================== + +This document describes a newer "structured" logging facility which reports messages from the driver itself using a BSON format defined across driver implementations by the `MongoDB Logging Specification `_. +See :doc:`unstructured_log` for the original freeform logging facility. + +These two systems are configured and used independently. The structured logging system has independent settings for handler and log levels. + +Defaults +-------- + +By default, it follows the behavior outlined by the common `MongoDB Logging Specification `_: + +* Log levels are set from the environment variables ``MONGODB_LOG_ALL``, ``MONGODB_LOG_COMMAND``, ``MONGODB_LOG_TOPOLOGY``, ``MONGODB_LOG_SERVER_SELECTION``, expecting a value from the `severity level table `_: ``off``, ``emergency``, ``alert``, ``critical``, ``error``, ``warning``, ``warn``, ``notice``, ``informational``, ``info``, ``debug``, ``trace``. +* A default handler is installed, which logs text representations of each message to a file given by ``MONGODB_LOG_PATH``, which may be a full path or one of the special values ``stdout`` or ``stderr``. By default, logs go to stderr. + +Levels and Components +--------------------- + +Log levels and components are defined as :symbol:`mongoc_structured_log_level_t` and :symbol:`mongoc_structured_log_component_t` enumerations. Utilities are provided to convert between these values and their standard string representations. The string values are case-insensitive. + +.. code-block:: c + + typedef enum { + MONGOC_STRUCTURED_LOG_LEVEL_EMERGENCY = 0, + MONGOC_STRUCTURED_LOG_LEVEL_ALERT = 1, + MONGOC_STRUCTURED_LOG_LEVEL_CRITICAL = 2, + MONGOC_STRUCTURED_LOG_LEVEL_ERROR = 3, + MONGOC_STRUCTURED_LOG_LEVEL_WARNING = 4, + MONGOC_STRUCTURED_LOG_LEVEL_NOTICE = 5, + MONGOC_STRUCTURED_LOG_LEVEL_INFO = 6, + MONGOC_STRUCTURED_LOG_LEVEL_DEBUG = 7, + MONGOC_STRUCTURED_LOG_LEVEL_TRACE = 8, + } mongoc_structured_log_level_t; + + typedef enum { + MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND = 0, + MONGOC_STRUCTURED_LOG_COMPONENT_TOPOLOGY = 1, + MONGOC_STRUCTURED_LOG_COMPONENT_SERVER_SELECTION = 2, + MONGOC_STRUCTURED_LOG_COMPONENT_CONNECTION = 3, + } mongoc_structured_log_component_t; + +.. toctree:: + :titlesonly: + :maxdepth: 1 + + mongoc_structured_log_level_t + mongoc_structured_log_component_t + +.. seealso:: + + mongoc_structured_log_get_level_name + mongoc_structured_log_get_named_level + mongoc_structured_log_get_component_name + mongoc_structured_log_get_named_component + + +Log Filtering +------------- + +Structured log messages may be filtered in two ways: + +* A maximum log level can be set per-component. +* A log handler function can ignore messages based on any criteria. + +The max level settings are configured by the environment variables above, and they may be altered or queried by applications. +To reduce overhead for unwanted logging, messages should be ignored as early as possible. +If it's possible to filter messages based on only their component and level, this should be done by handlers before requesting BSON serialization by calling :symbol:`mongoc_structured_log_entry_message_as_bson`. + +.. toctree:: + :titlesonly: + :maxdepth: 1 + + mongoc_structured_log_set_max_level_for_all_components + mongoc_structured_log_set_max_level_for_component + mongoc_structured_log_get_max_level_for_component + +Log Handlers +------------ + +There can be one global log handler set at a time. +This handler is called with a :symbol:`mongoc_structured_log_entry_t` that can be queried for further details. +Log handlers need not be re-entrant. +The logging subsystem will ensure mutual exclusion. + +.. toctree:: + :titlesonly: + :maxdepth: 1 + + mongoc_structured_log_func_t + mongoc_structured_log_set_handler + +Log Entries +----------- + +Each log entry is represented within the handler by a short-lived :symbol:`mongoc_structured_log_entry_t` pointer. +During the handler, this pointer can be used to access the individual properties of an entry: its level, component, and message. + +The message will be assembled as a :symbol:`bson_t` only when explicitly requested by a call to :symbol:`mongoc_structured_log_entry_message_as_bson`. +This results in a standalone document that may be retained for any amount of time and must be explicitly destroyed. + +.. toctree:: + :titlesonly: + :maxdepth: 1 + + mongoc_structured_log_entry_t + +.. seealso:: + + mongoc_structured_log_entry_get_component + mongoc_structured_log_entry_get_level + mongoc_structured_log_entry_message_as_bson diff --git a/src/libmongoc/doc/unstructured_log.rst b/src/libmongoc/doc/unstructured_log.rst new file mode 100644 index 00000000000..e84cac1f0fa --- /dev/null +++ b/src/libmongoc/doc/unstructured_log.rst @@ -0,0 +1,132 @@ +:man_page: mongoc_unstructured_log + +Unstructured Logging +==================== + +This is the original logging facility that supports freeform string messages originating from the driver itself or from application code. This has been retroactively termed "unstructured logging". +See :doc:`structured_log` for the newer standardized logging facility. + +.. code-block:: c + + typedef enum { + MONGOC_LOG_LEVEL_ERROR, + MONGOC_LOG_LEVEL_CRITICAL, + MONGOC_LOG_LEVEL_WARNING, + MONGOC_LOG_LEVEL_MESSAGE, + MONGOC_LOG_LEVEL_INFO, + MONGOC_LOG_LEVEL_DEBUG, + MONGOC_LOG_LEVEL_TRACE, + } mongoc_log_level_t; + + #define MONGOC_ERROR(...) + #define MONGOC_CRITICAL(...) + #define MONGOC_WARNING(...) + #define MONGOC_MESSAGE(...) + #define MONGOC_INFO(...) + #define MONGOC_DEBUG(...) + + typedef void (*mongoc_log_func_t) (mongoc_log_level_t log_level, + const char *log_domain, + const char *message, + void *user_data); + + void + mongoc_log_set_handler (mongoc_log_func_t log_func, void *user_data); + void + mongoc_log (mongoc_log_level_t log_level, + const char *log_domain, + const char *format, + ...); + const char * + mongoc_log_level_str (mongoc_log_level_t log_level); + void + mongoc_log_default_handler (mongoc_log_level_t log_level, + const char *log_domain, + const char *message, + void *user_data); + void + mongoc_log_trace_enable (void); + void + mongoc_log_trace_disable (void); + +This abstraction can be used for logging in your application, or you can integrate the driver with an existing logging system. + +Macros +------ + +To make logging a little less painful, various helper macros are provided. See the following example. + +.. code-block:: c + + #undef MONGOC_LOG_DOMAIN + #define MONGOC_LOG_DOMAIN "my-custom-domain" + + MONGOC_WARNING ("An error occurred: %s", strerror (errno)); + +.. _custom_log_handlers: + +Custom Log Handlers +------------------- + +The default log handler prints a timestamp and the log message to ``stdout``, or to ``stderr`` for warnings, critical messages, and errors. + You can override the handler with ``mongoc_log_set_handler()``. + Your handler function is called in a mutex for thread safety. + +For example, you could register a custom handler to suppress messages at INFO level and below: + +.. code-block:: c + + void + my_logger (mongoc_log_level_t log_level, + const char *log_domain, + const char *message, + void *user_data) + { + /* smaller values are more important */ + if (log_level < MONGOC_LOG_LEVEL_INFO) { + mongoc_log_default_handler (log_level, log_domain, message, user_data); + } + } + + int + main (int argc, char *argv[]) + { + mongoc_log_set_handler (my_logger, NULL); + mongoc_init (); + + /* ... your code ... */ + + mongoc_cleanup (); + return 0; + } + +Note that in the example above ``mongoc_log_set_handler()`` is called before ``mongoc_init()``. +Otherwise, some log traces could not be processed by the log handler. + +To restore the default handler: + +.. code-block:: c + + mongoc_log_set_handler (mongoc_log_default_handler, NULL); + +Disable logging +--------------- + +To disable all logging, including warnings, critical messages and errors, provide an empty log handler: + +.. code-block:: c + + mongoc_log_set_handler (NULL, NULL); + +Tracing +------- + +If compiling your own copy of the MongoDB C driver, consider configuring with ``-DENABLE_TRACING=ON`` to enable function tracing and hex dumps of network packets to ``STDERR`` and ``STDOUT`` during development and debugging. + +This is especially useful when debugging what may be going on internally in the driver. + +Trace messages can be enabled and disabled by calling ``mongoc_log_trace_enable()`` and ``mongoc_log_trace_disable()`` + +.. note:: + + Compiling the driver with ``-DENABLE_TRACING=ON`` will affect its performance. Disabling tracing with ``mongoc_log_trace_disable()`` significantly reduces the overhead, but cannot remove it completely. From e11ad3c615446500d412b8c3a64d5583eeb4c01c Mon Sep 17 00:00:00 2001 From: Micah Scott Date: Mon, 18 Nov 2024 14:55:52 -0800 Subject: [PATCH 081/139] skip MONGOC_NO_SERVER_CONNECTION_ID in structured log serialization --- src/libmongoc/src/mongoc/mongoc-structured-log.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/libmongoc/src/mongoc/mongoc-structured-log.c b/src/libmongoc/src/mongoc/mongoc-structured-log.c index aebb4ef5b08..578af69673a 100644 --- a/src/libmongoc/src/mongoc/mongoc-structured-log.c +++ b/src/libmongoc/src/mongoc/mongoc-structured-log.c @@ -621,7 +621,10 @@ _mongoc_structured_log_append_server_description (bson_t *bson, const mongoc_str BSON_APPEND_INT32 (bson, "serverPort", sd->host.port); } if (flags & MONGOC_STRUCTURED_LOG_SERVER_DESCRIPTION_SERVER_CONNECTION_ID) { - BSON_APPEND_INT64 (bson, "serverConnectionId", sd->server_connection_id); + int64_t server_connection_id = sd->server_connection_id; + if (MONGOC_NO_SERVER_CONNECTION_ID != server_connection_id) { + BSON_APPEND_INT64 (bson, "serverConnectionId", server_connection_id); + } } if (flags & MONGOC_STRUCTURED_LOG_SERVER_DESCRIPTION_SERVICE_ID) { if (!mcommon_oid_is_zero (&sd->service_id)) { From abc810342d3353e9af4ca996f85c9cebc8cdb112 Mon Sep 17 00:00:00 2001 From: Micah Scott Date: Mon, 18 Nov 2024 15:36:17 -0800 Subject: [PATCH 082/139] Add a simple structured-logging example --- src/libmongoc/.gitignore | 1 + src/libmongoc/CMakeLists.txt | 1 + src/libmongoc/doc/structured_log.rst | 6 ++ .../examples/example-structured-log.c | 91 +++++++++++++++++++ 4 files changed, 99 insertions(+) create mode 100644 src/libmongoc/examples/example-structured-log.c diff --git a/src/libmongoc/.gitignore b/src/libmongoc/.gitignore index cd9a9c49cc5..6c54a64bf95 100644 --- a/src/libmongoc/.gitignore +++ b/src/libmongoc/.gitignore @@ -34,6 +34,7 @@ example-scram example-sdam-monitoring example-session example-start-at-optime +example-structured-log example-transaction example-update fam diff --git a/src/libmongoc/CMakeLists.txt b/src/libmongoc/CMakeLists.txt index b2c707e2355..7c570c2fe80 100644 --- a/src/libmongoc/CMakeLists.txt +++ b/src/libmongoc/CMakeLists.txt @@ -1301,6 +1301,7 @@ if (ENABLE_EXAMPLES AND ENABLE_SHARED) mongoc_add_example (example-scram ${PROJECT_SOURCE_DIR}/examples/example-scram.c) mongoc_add_example (example-sdam-monitoring ${PROJECT_SOURCE_DIR}/examples/example-sdam-monitoring.c) mongoc_add_example (example-session ${PROJECT_SOURCE_DIR}/examples/example-session.c) + mongoc_add_example (example-structured-log ${PROJECT_SOURCE_DIR}/examples/example-structured-log.c) mongoc_add_example (example-transaction ${PROJECT_SOURCE_DIR}/examples/example-transaction.c) mongoc_add_example (example-update ${PROJECT_SOURCE_DIR}/examples/example-update.c) mongoc_add_example (find-and-modify ${PROJECT_SOURCE_DIR}/examples/find-and-modify.c) diff --git a/src/libmongoc/doc/structured_log.rst b/src/libmongoc/doc/structured_log.rst index 70bde01a93e..91a33f8b2e5 100644 --- a/src/libmongoc/doc/structured_log.rst +++ b/src/libmongoc/doc/structured_log.rst @@ -107,6 +107,12 @@ This results in a standalone document that may be retained for any amount of tim mongoc_structured_log_entry_t +Example +------- +.. literalinclude:: ../examples/example-structured-log.c + :language: c + :caption: example-structured-log.c + .. seealso:: mongoc_structured_log_entry_get_component diff --git a/src/libmongoc/examples/example-structured-log.c b/src/libmongoc/examples/example-structured-log.c new file mode 100644 index 00000000000..caf895e4c52 --- /dev/null +++ b/src/libmongoc/examples/example-structured-log.c @@ -0,0 +1,91 @@ +/* gcc example-structured-log.c -o example-structured-log \ + * $(pkg-config --cflags --libs libmongoc-1.0) */ + +#include +#include +#include + +static void +example_handler (const mongoc_structured_log_entry_t *entry, void *user_data) +{ + mongoc_structured_log_component_t component = mongoc_structured_log_entry_get_component (entry); + mongoc_structured_log_level_t level = mongoc_structured_log_entry_get_level (entry); + + printf ("Log component=%s level=%s\n", + mongoc_structured_log_get_component_name (component), + mongoc_structured_log_get_level_name (level)); + + /* + * At this point, the handler might make additional filtering decisions + * before asking for a bson_t. As an example, let's log the component and + * level for all messages but only show contents for command logs. + */ + if (component == MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND) { + bson_t *message = mongoc_structured_log_entry_message_as_bson (entry); + char *json = bson_as_relaxed_extended_json (message, NULL); + printf ("Log body: %s\n", json); + bson_destroy (message); + bson_free (json); + } +} + +int +main (void) +{ + const char *uri_string = "mongodb://localhost:27017"; + int result = EXIT_FAILURE; + bson_error_t error; + mongoc_uri_t *uri = NULL; + mongoc_client_t *client = NULL; + + /* + * Initialize libmongoc before configuring logs + */ + mongoc_init (); + + /* + * For demonstration purposes, set up a handler that receives all possible + * log messages. In a real app, you would set this from some configurable + * source or simply use the default behavior which sets them from environment + * variables. + */ + mongoc_structured_log_set_max_level_for_all_components (MONGOC_STRUCTURED_LOG_LEVEL_TRACE); + mongoc_structured_log_set_handler (example_handler, NULL); + + /* + * Create a MongoDB URI object. This example assumes a local server. + */ + uri = mongoc_uri_new_with_error (uri_string, &error); + if (!uri) { + fprintf (stderr, "URI parse error: %s\n", error.message); + goto done; + } + + /* + * Create a new client instance. + */ + client = mongoc_client_new_from_uri (uri); + if (!client) { + goto done; + } + + /* + * Do some work that we'll see logs from. This example just sends a 'ping' command. + */ + bson_t *command = BCON_NEW ("ping", BCON_INT32 (1)); + bson_t reply; + bool command_ret = mongoc_client_command_simple (client, "admin", command, NULL, &reply, &error); + bson_destroy (command); + bson_destroy (&reply); + if (!command_ret) { + fprintf (stderr, "Command error: %s\n", error.message); + goto done; + } + + result = EXIT_SUCCESS; +done: + mongoc_uri_destroy (uri); + mongoc_client_destroy (client); + mongoc_cleanup (); + return result; +} From 5dc2469e06793a0f5bc7672f773e94cd5dd2e522 Mon Sep 17 00:00:00 2001 From: Micah Scott Date: Tue, 19 Nov 2024 08:29:28 -0800 Subject: [PATCH 083/139] Make structured logging thread-safe Structured log handlers are now required to be thread-safe too. This implementation uses a shared lock (rwlock) to synchronize handler changes, and another mutex for the default log handler which covers only stream output. --- src/libmongoc/CMakeLists.txt | 3 +- .../doc/mongoc_structured_log_func_t.rst | 3 +- .../doc/mongoc_structured_log_set_handler.rst | 12 +++++-- src/libmongoc/doc/structured_log.rst | 7 ++-- .../examples/example-structured-log.c | 14 ++++++++ .../src/mongoc/mongoc-structured-log.c | 35 +++++++++++-------- .../tests/test-mongoc-structured-log.c | 32 ++++++++++------- src/libmongoc/tests/unified/entity-map.c | 5 +++ src/libmongoc/tests/unified/entity-map.h | 2 ++ src/libmongoc/tests/unified/runner.c | 7 ++++ 10 files changed, 88 insertions(+), 32 deletions(-) diff --git a/src/libmongoc/CMakeLists.txt b/src/libmongoc/CMakeLists.txt index 7c570c2fe80..aa1bce19a8d 100644 --- a/src/libmongoc/CMakeLists.txt +++ b/src/libmongoc/CMakeLists.txt @@ -1296,12 +1296,13 @@ if (ENABLE_EXAMPLES AND ENABLE_SHARED) mongoc_add_example (example-gridfs ${PROJECT_SOURCE_DIR}/examples/example-gridfs.c) mongoc_add_example (example-gridfs-bucket ${PROJECT_SOURCE_DIR}/examples/example-gridfs-bucket.c) if (NOT WIN32 AND ENABLE_EXAMPLES) + # Examples that use pthreads mongoc_add_example (example-pool ${PROJECT_SOURCE_DIR}/examples/example-pool.c) + mongoc_add_example (example-structured-log ${PROJECT_SOURCE_DIR}/examples/example-structured-log.c) endif () mongoc_add_example (example-scram ${PROJECT_SOURCE_DIR}/examples/example-scram.c) mongoc_add_example (example-sdam-monitoring ${PROJECT_SOURCE_DIR}/examples/example-sdam-monitoring.c) mongoc_add_example (example-session ${PROJECT_SOURCE_DIR}/examples/example-session.c) - mongoc_add_example (example-structured-log ${PROJECT_SOURCE_DIR}/examples/example-structured-log.c) mongoc_add_example (example-transaction ${PROJECT_SOURCE_DIR}/examples/example-transaction.c) mongoc_add_example (example-update ${PROJECT_SOURCE_DIR}/examples/example-update.c) mongoc_add_example (find-and-modify ${PROJECT_SOURCE_DIR}/examples/find-and-modify.c) diff --git a/src/libmongoc/doc/mongoc_structured_log_func_t.rst b/src/libmongoc/doc/mongoc_structured_log_func_t.rst index 7b3e16eb872..6247a9569d6 100644 --- a/src/libmongoc/doc/mongoc_structured_log_func_t.rst +++ b/src/libmongoc/doc/mongoc_structured_log_func_t.rst @@ -12,7 +12,8 @@ Synopsis (const mongoc_structured_log_entry_t *entry, void *user_data); Callback function for :symbol:`mongoc_structured_log_set_handler`. -Not required to be re-entrant, mutual exclusion is provided by the logging facility. +Structured log handlers must be thread-safe. +Logging may occur simultaneously on multiple application threads and/or background threads. Handlers may use any operating system or ``libbson`` functions but MUST not use any ``libmongoc`` functions except: diff --git a/src/libmongoc/doc/mongoc_structured_log_set_handler.rst b/src/libmongoc/doc/mongoc_structured_log_set_handler.rst index 7c575434d38..090cebfc5c5 100644 --- a/src/libmongoc/doc/mongoc_structured_log_set_handler.rst +++ b/src/libmongoc/doc/mongoc_structured_log_set_handler.rst @@ -15,8 +15,16 @@ Sets the function to be called to handle structured log messages, as a :symbol:` The callback is given a :symbol:`mongoc_structured_log_entry_t` as a handle for obtaining additional information about the log message. This entry pointer is only valid during a callback, because it's a low cost reference to temporary data. -Callbacks are not required to be re-entrant, mutual exclusion is provided by the logging facility. -Handlers must take care not to re-enter libmongoc except in proscribed ways. See :symbol:`mongoc_structured_log_func_t`. + +This function is thread-safe. +It provides atomic updates to the handler function and data. +If handlers are ongoing, blocks until they complete before applying the change. + +The callback handler itself must be thread-safe. +Logging can occur simultaneously on multiple application or background threads. + +Handlers must not re-enter libmongoc except as allowed to access the log entry parameter. +See :symbol:`mongoc_structured_log_func_t`. There is a single global handler per process. Any call to this function will replace the default handler. diff --git a/src/libmongoc/doc/structured_log.rst b/src/libmongoc/doc/structured_log.rst index 91a33f8b2e5..0ea7b4f224e 100644 --- a/src/libmongoc/doc/structured_log.rst +++ b/src/libmongoc/doc/structured_log.rst @@ -82,8 +82,11 @@ Log Handlers There can be one global log handler set at a time. This handler is called with a :symbol:`mongoc_structured_log_entry_t` that can be queried for further details. -Log handlers need not be re-entrant. -The logging subsystem will ensure mutual exclusion. + +.. note:: + + Structured log handlers must be thread-safe. + This differs from unstructured logging, which provides a global mutex. .. toctree:: :titlesonly: diff --git a/src/libmongoc/examples/example-structured-log.c b/src/libmongoc/examples/example-structured-log.c index caf895e4c52..1f1dd5565ec 100644 --- a/src/libmongoc/examples/example-structured-log.c +++ b/src/libmongoc/examples/example-structured-log.c @@ -2,15 +2,26 @@ * $(pkg-config --cflags --libs libmongoc-1.0) */ #include +#include #include #include +static pthread_mutex_t handler_mutex; + static void example_handler (const mongoc_structured_log_entry_t *entry, void *user_data) { mongoc_structured_log_component_t component = mongoc_structured_log_entry_get_component (entry); mongoc_structured_log_level_t level = mongoc_structured_log_entry_get_level (entry); + /* + * Structured log handlers need to be thread-safe. + * Many apps will be happy to use a global mutex in their logging handler, + * but high performance multithreaded apps may prefer dispatching log + * messages asynchronously with thread-safe data structures. + */ + pthread_mutex_lock (&handler_mutex); + printf ("Log component=%s level=%s\n", mongoc_structured_log_get_component_name (component), mongoc_structured_log_get_level_name (level)); @@ -27,6 +38,8 @@ example_handler (const mongoc_structured_log_entry_t *entry, void *user_data) bson_destroy (message); bson_free (json); } + + pthread_mutex_unlock (&handler_mutex); } int @@ -49,6 +62,7 @@ main (void) * source or simply use the default behavior which sets them from environment * variables. */ + pthread_mutex_init (&handler_mutex, NULL); mongoc_structured_log_set_max_level_for_all_components (MONGOC_STRUCTURED_LOG_LEVEL_TRACE); mongoc_structured_log_set_handler (example_handler, NULL); diff --git a/src/libmongoc/src/mongoc/mongoc-structured-log.c b/src/libmongoc/src/mongoc/mongoc-structured-log.c index 578af69673a..9e54e8daaf5 100644 --- a/src/libmongoc/src/mongoc/mongoc-structured-log.c +++ b/src/libmongoc/src/mongoc/mongoc-structured-log.c @@ -20,6 +20,7 @@ #include "mongoc-util-private.h" #include "mongoc-apm-private.h" #include "common-atomic-private.h" +#include "common-thread-private.h" #include "common-oid-private.h" static void @@ -43,7 +44,8 @@ static const struct { {.name = "info", .level = MONGOC_STRUCTURED_LOG_LEVEL_INFO}}; static struct { - bson_mutex_t func_mutex; // Mutex prevents func reentrancy, ensures atomic updates to (func, user_data) + bson_shared_mutex_t func_mutex; // Handlers run concurrently, but updates to (func, user_data) synchronized + bson_mutex_t stream_mutex; // Only used by the default handler mongoc_structured_log_func_t func; void *user_data; FILE *stream; // Only used by the default handler @@ -83,20 +85,20 @@ mongoc_structured_log_entry_get_component (const mongoc_structured_log_entry_t * void mongoc_structured_log_set_handler (mongoc_structured_log_func_t log_func, void *user_data) { - bson_mutex_lock (&gStructuredLog.func_mutex); + bson_shared_mutex_lock (&gStructuredLog.func_mutex); // Waits for handler invocations to end gStructuredLog.func = log_func; gStructuredLog.user_data = user_data; mcommon_atomic_int_exchange (&gStructuredLog.func_is_null_atomic, log_func == NULL, mcommon_memory_order_relaxed); - bson_mutex_unlock (&gStructuredLog.func_mutex); + bson_shared_mutex_unlock (&gStructuredLog.func_mutex); } void mongoc_structured_log_get_handler (mongoc_structured_log_func_t *log_func, void **user_data) { - bson_mutex_lock (&gStructuredLog.func_mutex); + bson_shared_mutex_lock_shared (&gStructuredLog.func_mutex); *log_func = gStructuredLog.func; *user_data = gStructuredLog.user_data; - bson_mutex_unlock (&gStructuredLog.func_mutex); + bson_shared_mutex_unlock_shared (&gStructuredLog.func_mutex); } void @@ -136,12 +138,12 @@ _mongoc_structured_log_should_log (const mongoc_structured_log_envelope_t *envel void _mongoc_structured_log_with_entry (const mongoc_structured_log_entry_t *entry) { - bson_mutex_lock (&gStructuredLog.func_mutex); + bson_shared_mutex_lock_shared (&gStructuredLog.func_mutex); mongoc_structured_log_func_t func = gStructuredLog.func; if (func) { func (entry, gStructuredLog.user_data); } - bson_mutex_unlock (&gStructuredLog.func_mutex); + bson_shared_mutex_unlock_shared (&gStructuredLog.func_mutex); } static bool @@ -242,7 +244,8 @@ _mongoc_structured_log_get_max_document_length_from_env (void) void _mongoc_structured_log_init (void) { - bson_mutex_init (&gStructuredLog.func_mutex); + bson_shared_mutex_init (&gStructuredLog.func_mutex); + bson_mutex_init (&gStructuredLog.stream_mutex); gStructuredLog.max_document_length = _mongoc_structured_log_get_max_document_length_from_env (); mongoc_structured_log_level_t level; @@ -286,6 +289,7 @@ _mongoc_structured_log_open_stream (void) static FILE * _mongoc_structured_log_get_stream (void) { + // Not re-entrant; protected by the stream_lock. FILE *log_stream = gStructuredLog.stream; if (log_stream) { return log_stream; @@ -300,17 +304,20 @@ _mongoc_structured_log_get_stream (void) static void mongoc_structured_log_default_handler (const mongoc_structured_log_entry_t *entry, void *user_data) { + // We can serialize the message before taking the stream lock bson_t *bson_message = mongoc_structured_log_entry_message_as_bson (entry); char *json_message = bson_as_relaxed_extended_json (bson_message, NULL); + bson_destroy (bson_message); + + const char *level_name = mongoc_structured_log_get_level_name (mongoc_structured_log_entry_get_level (entry)); + const char *component_name = + mongoc_structured_log_get_component_name (mongoc_structured_log_entry_get_component (entry)); - fprintf (_mongoc_structured_log_get_stream (), - "MONGODB_LOG %s %s %s\n", - mongoc_structured_log_get_level_name (mongoc_structured_log_entry_get_level (entry)), - mongoc_structured_log_get_component_name (mongoc_structured_log_entry_get_component (entry)), - json_message); + bson_mutex_lock (&gStructuredLog.stream_mutex); + fprintf (_mongoc_structured_log_get_stream (), "MONGODB_LOG %s %s %s\n", level_name, component_name, json_message); + bson_mutex_unlock (&gStructuredLog.stream_mutex); bson_free (json_message); - bson_destroy (bson_message); } char * diff --git a/src/libmongoc/tests/test-mongoc-structured-log.c b/src/libmongoc/tests/test-mongoc-structured-log.c index cd4daf81103..e58d702bb9b 100644 --- a/src/libmongoc/tests/test-mongoc-structured-log.c +++ b/src/libmongoc/tests/test-mongoc-structured-log.c @@ -15,6 +15,7 @@ */ #include +#include #include "mongoc/mongoc-structured-log-private.h" #include "TestSuite.h" @@ -23,7 +24,7 @@ typedef struct log_assumption { mongoc_structured_log_envelope_t expected_envelope; bson_t *expected_bson; int expected_calls; - int calls; + int calls_atomic; } log_assumption; typedef struct structured_log_state { @@ -50,9 +51,8 @@ structured_log_func (const mongoc_structured_log_entry_t *entry, void *user_data { struct log_assumption *assumption = (struct log_assumption *) user_data; - assumption->calls++; - - ASSERT_CMPINT (assumption->calls, <=, assumption->expected_calls); + int calls = 1 + mcommon_atomic_int_fetch_add (&assumption->calls_atomic, 1, mcommon_memory_order_seq_cst); + ASSERT_CMPINT (calls, <=, assumption->expected_calls); ASSERT_CMPINT (entry->envelope.level, ==, assumption->expected_envelope.level); ASSERT_CMPINT (entry->envelope.component, ==, assumption->expected_envelope.component); @@ -96,7 +96,8 @@ test_structured_log_plain (void) mongoc_structured_log ( MONGOC_STRUCTURED_LOG_LEVEL_WARNING, MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND, "Plain log entry"); - ASSERT_CMPINT (assumption.calls, ==, 1); + int calls = mcommon_atomic_int_fetch (&assumption.calls_atomic, mcommon_memory_order_seq_cst); + ASSERT_CMPINT (calls, ==, 1); restore_state (old_state); bson_destroy (assumption.expected_bson); } @@ -120,7 +121,8 @@ test_structured_log_plain_with_extra_data (void) "Plain log entry with extra data", int32 ("extra", 1)); - ASSERT_CMPINT (assumption.calls, ==, 1); + int calls = mcommon_atomic_int_fetch (&assumption.calls_atomic, mcommon_memory_order_seq_cst); + ASSERT_CMPINT (calls, ==, 1); restore_state (old_state); bson_destroy (assumption.expected_bson); } @@ -184,7 +186,8 @@ test_structured_log_basic_data_types (void) boolean ("kFalse", false), boolean (NULL, true)); - ASSERT_CMPINT (assumption.calls, ==, 1); + int calls = mcommon_atomic_int_fetch (&assumption.calls_atomic, mcommon_memory_order_seq_cst); + ASSERT_CMPINT (calls, ==, 1); restore_state (old_state); bson_destroy (assumption.expected_bson); bson_destroy (bson_str_n); @@ -218,7 +221,8 @@ test_structured_log_json (void) bson_as_json ("kNull", NULL), bson_as_json (NULL, NULL)); - ASSERT_CMPINT (assumption.calls, ==, 1); + int calls = mcommon_atomic_int_fetch (&assumption.calls_atomic, mcommon_memory_order_seq_cst); + ASSERT_CMPINT (calls, ==, 1); restore_state (old_state); bson_destroy (assumption.expected_bson); bson_destroy (json_doc); @@ -253,7 +257,8 @@ test_structured_log_oid (void) oid_as_hex ("kNull", NULL), oid_as_hex (NULL, NULL)); - ASSERT_CMPINT (assumption.calls, ==, 1); + int calls = mcommon_atomic_int_fetch (&assumption.calls_atomic, mcommon_memory_order_seq_cst); + ASSERT_CMPINT (calls, ==, 1); restore_state (old_state); bson_destroy (assumption.expected_bson); } @@ -326,7 +331,8 @@ test_structured_log_server_description (void) server_description (&server_description_1, SERVER_HOST, SERVER_PORT, SERVER_CONNECTION_ID, SERVICE_ID), server_description (&server_description_2, SERVER_HOST, SERVER_PORT, SERVER_CONNECTION_ID, SERVICE_ID)); - ASSERT_CMPINT (assumption.calls, ==, 1); + int calls = mcommon_atomic_int_fetch (&assumption.calls_atomic, mcommon_memory_order_seq_cst); + ASSERT_CMPINT (calls, ==, 1); restore_state (old_state); bson_destroy (assumption.expected_bson); } @@ -424,7 +430,8 @@ test_structured_log_command (void) cmd_name_failure ("authenticate", reply_doc, &server_error), cmd_name_failure ("authenticate", reply_doc, &client_error)); - ASSERT_CMPINT (assumption.calls, ==, 1); + int calls = mcommon_atomic_int_fetch (&assumption.calls_atomic, mcommon_memory_order_seq_cst); + ASSERT_CMPINT (calls, ==, 1); restore_state (old_state); bson_destroy (assumption.expected_bson); bson_destroy (cmd_doc); @@ -465,7 +472,8 @@ test_structured_log_duration (void) monotonic_time_duration (10), monotonic_time_duration (10000000999)); - ASSERT_CMPINT (assumption.calls, ==, 1); + int calls = mcommon_atomic_int_fetch (&assumption.calls_atomic, mcommon_memory_order_seq_cst); + ASSERT_CMPINT (calls, ==, 1); restore_state (old_state); bson_destroy (assumption.expected_bson); } diff --git a/src/libmongoc/tests/unified/entity-map.c b/src/libmongoc/tests/unified/entity-map.c index 5ea9db132a7..4bf72ffb036 100644 --- a/src/libmongoc/tests/unified/entity-map.c +++ b/src/libmongoc/tests/unified/entity-map.c @@ -206,6 +206,7 @@ entity_new (entity_map_t *em, const char *type) entity->entity_map = em; _mongoc_array_init (&entity->observe_events, sizeof (observe_event_t)); _mongoc_array_init (&entity->store_events, sizeof (store_event_t)); + bson_mutex_init (&entity->log_messages_mutex); return entity; } @@ -217,7 +218,9 @@ structured_log_cb (const mongoc_structured_log_entry_t *entry, void *user_data) if (!test_is_suppressing_structured_logs ()) { entity_t *entity = (entity_t *) user_data; log_message_t *log_message = log_message_new (entry); + bson_mutex_lock (&entity->log_messages_mutex); LL_APPEND (entity->log_messages, log_message); + bson_mutex_unlock (&entity->log_messages_mutex); } } @@ -1664,6 +1667,7 @@ entity_destroy (entity_t *entity) } } { + // No reason to take the log_messages_mutex here; log handlers are stopped above when we delete clients. log_message_t *log_message, *tmp; LL_FOREACH_SAFE (entity->log_messages, log_message, tmp) { @@ -1689,6 +1693,7 @@ entity_destroy (entity_t *entity) _mongoc_array_destroy (&entity->store_events); } + bson_mutex_destroy (&entity->log_messages_mutex); bson_destroy (entity->ignore_command_monitoring_events); bson_free (entity->type); bson_free (entity->id); diff --git a/src/libmongoc/tests/unified/entity-map.h b/src/libmongoc/tests/unified/entity-map.h index b293d27bb74..ac4ffb99e5a 100644 --- a/src/libmongoc/tests/unified/entity-map.h +++ b/src/libmongoc/tests/unified/entity-map.h @@ -20,6 +20,7 @@ #include "bson/bson.h" #include "mongoc/mongoc.h" #include "mongoc-array-private.h" +#include "common-thread-private.h" #include "bsonutil/bson-match.h" #include "test-diagnostics.h" @@ -54,6 +55,7 @@ typedef struct _entity_t { bool *observe_sensitive_commands; struct _entity_t *next; event_t *events; + bson_mutex_t log_messages_mutex; log_message_t *log_messages; struct _entity_map_t *entity_map; // Parent entity map. mongoc_array_t observe_events; // observe_event_t [N]. diff --git a/src/libmongoc/tests/unified/runner.c b/src/libmongoc/tests/unified/runner.c index d86d9291c4c..83bbc7fc43e 100644 --- a/src/libmongoc/tests/unified/runner.c +++ b/src/libmongoc/tests/unified/runner.c @@ -1421,6 +1421,7 @@ test_check_expected_log_messages_for_client (test_t *test, bson_error_t *error) { bool ret = false; + bson_mutex_t *locked = NULL; bson_parser_t *bp = bson_parser_new (); char *client_id; @@ -1441,6 +1442,9 @@ test_check_expected_log_messages_for_client (test_t *test, goto done; } + locked = &entity->log_messages_mutex; + bson_mutex_lock (locked); + log_message_t *actual_message_iter = entity->log_messages; bson_iter_t expected_message_iter; bool expected_message_iter_ok = @@ -1490,6 +1494,9 @@ test_check_expected_log_messages_for_client (test_t *test, ret = true; done: + if (locked) { + bson_mutex_unlock (locked); + } bson_parser_destroy_with_parsed_fields (bp); return ret; } From b264a020dd550f11ab0326e739614cf50f0e18cf Mon Sep 17 00:00:00 2001 From: Micah Scott Date: Tue, 19 Nov 2024 10:06:42 -0800 Subject: [PATCH 084/139] typo --- src/libmongoc/src/mongoc/mongoc-structured-log-private.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libmongoc/src/mongoc/mongoc-structured-log-private.h b/src/libmongoc/src/mongoc/mongoc-structured-log-private.h index 4823331fe00..6c995dd5279 100644 --- a/src/libmongoc/src/mongoc/mongoc-structured-log-private.h +++ b/src/libmongoc/src/mongoc/mongoc-structured-log-private.h @@ -31,7 +31,7 @@ BSON_BEGIN_DECLS #define MONGOC_STRUCTURED_LOG_DEFAULT_MAX_DOCUMENT_LENGTH 1000 /** - * @def mognoc_structured_log(level, component, message, ...) + * @def mongoc_structured_log(level, component, message, ...) * @brief Write to the libmongoc structured log. * * @param level Log level, as a mongoc_structured_log_level_t expression From 925f5bfdb8b192fe4069fcf3cb98336ac5d5ceb2 Mon Sep 17 00:00:00 2001 From: Micah Scott Date: Tue, 19 Nov 2024 10:17:35 -0800 Subject: [PATCH 085/139] utf8 was supposed to be char not void --- src/libmongoc/src/mongoc/mongoc-structured-log-private.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libmongoc/src/mongoc/mongoc-structured-log-private.h b/src/libmongoc/src/mongoc/mongoc-structured-log-private.h index 6c995dd5279..41f1043c6f2 100644 --- a/src/libmongoc/src/mongoc/mongoc-structured-log-private.h +++ b/src/libmongoc/src/mongoc/mongoc-structured-log-private.h @@ -288,13 +288,13 @@ struct mongoc_structured_log_builder_stage_t { const bson_error_t *error; const mongoc_cmd_t *cmd; const mongoc_server_description_t *server_description; - const void *utf8; + const char *utf8; } arg1; union { bool boolean; bson_oid_t *oid; const bson_t *bson; - const void *utf8; + const char *utf8; int32_t int32; int64_t int64; mongoc_structured_log_cmd_flags_t cmd_flags; From 0364ea06225d382ba650e89857d0813a97efd4c1 Mon Sep 17 00:00:00 2001 From: Micah Scott Date: Tue, 19 Nov 2024 11:25:02 -0800 Subject: [PATCH 086/139] CDRIVER-4535 Updates to logging spec Revises error behavior for environment variables, and adds a documented way to allow the environment to override programmatic settings. --- ...ctured_log_set_max_level_for_component.rst | 3 +- ...structured_log_set_max_levels_from_env.rst | 28 +++++++++ src/libmongoc/doc/structured_log.rst | 10 +++- .../examples/example-structured-log.c | 14 +++-- .../src/mongoc/mongoc-structured-log.c | 58 +++++++++++++------ .../src/mongoc/mongoc-structured-log.h | 23 ++++++++ 6 files changed, 111 insertions(+), 25 deletions(-) create mode 100644 src/libmongoc/doc/mongoc_structured_log_set_max_levels_from_env.rst diff --git a/src/libmongoc/doc/mongoc_structured_log_set_max_level_for_component.rst b/src/libmongoc/doc/mongoc_structured_log_set_max_level_for_component.rst index 5aa187e4cab..d20eeb1b15c 100644 --- a/src/libmongoc/doc/mongoc_structured_log_set_max_level_for_component.rst +++ b/src/libmongoc/doc/mongoc_structured_log_set_max_level_for_component.rst @@ -15,7 +15,8 @@ Synopsis Sets the maximum log level per-component. Only log messages at or below this severity level will be passed to :symbol:`mongoc_structured_log_func_t`. -By default, each component's log level comes from the environment variables ``MONGOC_LOG_`` and ``MONGOC_LOG_ALL``. +By default, each component's log level may come from environment variables. +See :symbol:`mongoc_structured_log_set_max_levels_from_env`. Parameters ---------- diff --git a/src/libmongoc/doc/mongoc_structured_log_set_max_levels_from_env.rst b/src/libmongoc/doc/mongoc_structured_log_set_max_levels_from_env.rst new file mode 100644 index 00000000000..49c32ff2678 --- /dev/null +++ b/src/libmongoc/doc/mongoc_structured_log_set_max_levels_from_env.rst @@ -0,0 +1,28 @@ +:man_page: mongoc_structured_log_set_max_levels_from_env + +mongoc_structured_log_set_max_levels_from_env +============================================= + +Synopsis +-------- + +.. code-block:: c + + void + mongoc_structured_log_set_max_levels_from_env (void) + +Sets any maximum log levels requested by environment variables: ``MONGODB_LOG_ALL`` for all components, followed by per-component log levels ``MONGODB_LOG_COMMAND``, ``MONGODB_LOG_CONNECTION``, ``MONGODB_LOG_TOPOLOGY``, and ``MONGODB_LOG_SERVER_SELECTION``. + +Expects the value to be recognizable by :symbol:`mongoc_structured_log_get_named_level`. +Parse errors may cause a warning message. + +Component levels with no valid environment variable setting will be left unmodified. + +Normally this happens automatically during :symbol:`mongoc_init`, and it provides defaults that can be overridden programmatically by calls to :symbol:`mongoc_structured_log_set_max_level_for_component` and :symbol:`mongoc_structured_log_set_max_level_for_all_components`. + +For applications that desire the opposite behavior, where environment variables may override programmatic settings, they may call ``mongoc_structured_log_set_max_levels_from_env()`` after calling :symbol:`mongoc_structured_log_set_max_level_for_component` and :symbol:`mongoc_structured_log_set_max_level_for_all_components`. +This will process the environment a second time, allowing it to override customized defaults. + +.. seealso:: + + | :doc:`structured_log` diff --git a/src/libmongoc/doc/structured_log.rst b/src/libmongoc/doc/structured_log.rst index 0ea7b4f224e..282133f1827 100644 --- a/src/libmongoc/doc/structured_log.rst +++ b/src/libmongoc/doc/structured_log.rst @@ -11,10 +11,13 @@ These two systems are configured and used independently. The structured logging Defaults -------- -By default, it follows the behavior outlined by the common `MongoDB Logging Specification `_: +During :symbol:`mongoc_init`: -* Log levels are set from the environment variables ``MONGODB_LOG_ALL``, ``MONGODB_LOG_COMMAND``, ``MONGODB_LOG_TOPOLOGY``, ``MONGODB_LOG_SERVER_SELECTION``, expecting a value from the `severity level table `_: ``off``, ``emergency``, ``alert``, ``critical``, ``error``, ``warning``, ``warn``, ``notice``, ``informational``, ``info``, ``debug``, ``trace``. -* A default handler is installed, which logs text representations of each message to a file given by ``MONGODB_LOG_PATH``, which may be a full path or one of the special values ``stdout`` or ``stderr``. By default, logs go to stderr. +* Default log levels are set from the environment variables ``MONGODB_LOG_ALL``, ``MONGODB_LOG_COMMAND``, ``MONGODB_LOG_TOPOLOGY``, ``MONGODB_LOG_SERVER_SELECTION``, expecting a value from the `severity level table `_: ``off``, ``emergency``, ``alert``, ``critical``, ``error``, ``warning``, ``warn``, ``notice``, ``informational``, ``info``, ``debug``, ``trace``. +* A default handler is installed, which logs text representations of each message to a file given by ``MONGODB_LOG_PATH``, which may be a full path or one of the special values ``stdout`` or ``stderr``. If no valid path is given, logs are written to stderr. + +Normally environment variables provide defaults that can be overridden programmatically. +To request the opposite behavior, where your programmatic defaults can be overridden by the environment, see :symbol:`mongoc_structured_log_set_max_levels_from_env`. Levels and Components --------------------- @@ -75,6 +78,7 @@ If it's possible to filter messages based on only their component and level, thi mongoc_structured_log_set_max_level_for_all_components mongoc_structured_log_set_max_level_for_component + mongoc_structured_log_set_max_levels_from_env mongoc_structured_log_get_max_level_for_component Log Handlers diff --git a/src/libmongoc/examples/example-structured-log.c b/src/libmongoc/examples/example-structured-log.c index 1f1dd5565ec..418e1c2ea60 100644 --- a/src/libmongoc/examples/example-structured-log.c +++ b/src/libmongoc/examples/example-structured-log.c @@ -57,15 +57,21 @@ main (void) mongoc_init (); /* - * For demonstration purposes, set up a handler that receives all possible - * log messages. In a real app, you would set this from some configurable - * source or simply use the default behavior which sets them from environment - * variables. + * For demonstration purposes, set up a handler that receives all possible log messages. */ pthread_mutex_init (&handler_mutex, NULL); mongoc_structured_log_set_max_level_for_all_components (MONGOC_STRUCTURED_LOG_LEVEL_TRACE); mongoc_structured_log_set_handler (example_handler, NULL); + /* + * By default libmongoc proceses log options from the environment first, + * and then allows you to apply programmatic overrides. To request the + * opposite behavior, allowing the environment to override programmatic + * defaults, you can ask for the environment to be re-read after setting + * your own defaults. + */ + mongoc_structured_log_set_max_levels_from_env (); + /* * Create a MongoDB URI object. This example assumes a local server. */ diff --git a/src/libmongoc/src/mongoc/mongoc-structured-log.c b/src/libmongoc/src/mongoc/mongoc-structured-log.c index 9e54e8daaf5..bd2ed239bc4 100644 --- a/src/libmongoc/src/mongoc/mongoc-structured-log.c +++ b/src/libmongoc/src/mongoc/mongoc-structured-log.c @@ -147,7 +147,9 @@ _mongoc_structured_log_with_entry (const mongoc_structured_log_entry_t *entry) } static bool -_mongoc_structured_log_get_log_level_from_env (const char *variable, mongoc_structured_log_level_t *out) +_mongoc_structured_log_get_log_level_from_env (const char *variable, + mongoc_structured_log_level_t *out, + int volatile *err_count_atomic) { const char *level = getenv (variable); if (!level) { @@ -156,8 +158,11 @@ _mongoc_structured_log_get_log_level_from_env (const char *variable, mongoc_stru if (mongoc_structured_log_get_named_level (level, out)) { return true; } - MONGOC_ERROR ("Invalid log level '%s' read from environment variable %s", level, variable); - exit (EXIT_FAILURE); + // Only report the first instance of each error + if (0 == mcommon_atomic_int_fetch_add (err_count_atomic, 1, mcommon_memory_order_seq_cst)) { + MONGOC_ERROR ("Invalid log level '%s' read from environment variable %s", level, variable); + } + return false; } const char * @@ -238,7 +243,7 @@ _mongoc_structured_log_get_max_document_length_from_env (void) } MONGOC_ERROR ("Invalid length '%s' read from environment variable %s", max_length_str, variable); - exit (EXIT_FAILURE); + return MONGOC_STRUCTURED_LOG_DEFAULT_MAX_DOCUMENT_LENGTH; } void @@ -247,24 +252,43 @@ _mongoc_structured_log_init (void) bson_shared_mutex_init (&gStructuredLog.func_mutex); bson_mutex_init (&gStructuredLog.stream_mutex); gStructuredLog.max_document_length = _mongoc_structured_log_get_max_document_length_from_env (); + mongoc_structured_log_set_max_level_for_all_components (MONGOC_STRUCTURED_LOG_DEFAULT_LEVEL); + mongoc_structured_log_set_max_levels_from_env (); +} +void +mongoc_structured_log_set_max_levels_from_env (void) +{ mongoc_structured_log_level_t level; - if (!_mongoc_structured_log_get_log_level_from_env ("MONGODB_LOG_ALL", &level)) { - level = MONGOC_STRUCTURED_LOG_DEFAULT_LEVEL; + { + static int err_count_atomic = 0; + if (_mongoc_structured_log_get_log_level_from_env ("MONGODB_LOG_ALL", &level, &err_count_atomic)) { + mongoc_structured_log_set_max_level_for_all_components (level); + } } - mongoc_structured_log_set_max_level_for_all_components (level); - - if (_mongoc_structured_log_get_log_level_from_env ("MONGODB_LOG_COMMAND", &level)) { - mongoc_structured_log_set_max_level_for_component (MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND, level); + { + static int err_count_atomic = 0; + if (_mongoc_structured_log_get_log_level_from_env ("MONGODB_LOG_COMMAND", &level, &err_count_atomic)) { + mongoc_structured_log_set_max_level_for_component (MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND, level); + } } - if (_mongoc_structured_log_get_log_level_from_env ("MONGODB_LOG_CONNECTION", &level)) { - mongoc_structured_log_set_max_level_for_component (MONGOC_STRUCTURED_LOG_COMPONENT_CONNECTION, level); + { + static int err_count_atomic = 0; + if (_mongoc_structured_log_get_log_level_from_env ("MONGODB_LOG_CONNECTION", &level, &err_count_atomic)) { + mongoc_structured_log_set_max_level_for_component (MONGOC_STRUCTURED_LOG_COMPONENT_CONNECTION, level); + } } - if (_mongoc_structured_log_get_log_level_from_env ("MONGODB_LOG_TOPOLOGY", &level)) { - mongoc_structured_log_set_max_level_for_component (MONGOC_STRUCTURED_LOG_COMPONENT_TOPOLOGY, level); + { + static int err_count_atomic = 0; + if (_mongoc_structured_log_get_log_level_from_env ("MONGODB_LOG_TOPOLOGY", &level, &err_count_atomic)) { + mongoc_structured_log_set_max_level_for_component (MONGOC_STRUCTURED_LOG_COMPONENT_TOPOLOGY, level); + } } - if (_mongoc_structured_log_get_log_level_from_env ("MONGODB_LOG_SERVER_SELECTION", &level)) { - mongoc_structured_log_set_max_level_for_component (MONGOC_STRUCTURED_LOG_COMPONENT_SERVER_SELECTION, level); + { + static int err_count_atomic = 0; + if (_mongoc_structured_log_get_log_level_from_env ("MONGODB_LOG_SERVER_SELECTION", &level, &err_count_atomic)) { + mongoc_structured_log_set_max_level_for_component (MONGOC_STRUCTURED_LOG_COMPONENT_SERVER_SELECTION, level); + } } } @@ -281,7 +305,7 @@ _mongoc_structured_log_open_stream (void) FILE *file = fopen (path, "a"); if (!file) { MONGOC_ERROR ("Cannot open log file %s for writing", path); - exit (EXIT_FAILURE); + return stderr; } return file; } diff --git a/src/libmongoc/src/mongoc/mongoc-structured-log.h b/src/libmongoc/src/mongoc/mongoc-structured-log.h index a3c7ae5072e..c29ec5cf924 100644 --- a/src/libmongoc/src/mongoc/mongoc-structured-log.h +++ b/src/libmongoc/src/mongoc/mongoc-structured-log.h @@ -98,6 +98,29 @@ mongoc_structured_log_set_max_level_for_component (mongoc_structured_log_compone MONGOC_EXPORT (void) mongoc_structured_log_set_max_level_for_all_components (mongoc_structured_log_level_t level); +/** + * mongoc_structured_log_set_max_levels_from_env: + * + * Sets any maximum log levels requested by environment variables: "MONGODB_LOG_ALL" + * for all components, followed by per-component log levels "MONGODB_LOG_COMMAND", + * "MONGODB_LOG_CONNECTION", "MONGODB_LOG_TOPOLOGY", and "MONGODB_LOG_SERVER_SELECTION". + * + * Component levels with no valid environment variable setting will be left unmodified. + * + * Normally this happens automatically during mongoc_init(), and it provides defaults + * that can be overridden programmatically by calls to mongoc_structured_log_set_max_level_for_component() + * and mongoc_structured_log_set_max_level_for_all_components(). + * + * For applications that desire the opposite behavior, where environment variables may + * override programmatic settings, they may call mongoc_structured_log_set_max_levels_from_env() + * after calling mongoc_structured_log_set_max_level_for_component() and + * mongoc_structured_log_set_max_level_for_all_components(). This will process the environment + * a second time, allowing it to override customized defaults. + */ + +MONGOC_EXPORT (void) +mongoc_structured_log_set_max_levels_from_env (void); + /** * mongoc_structured_log_get_max_level_for_component: * @component: A logging component to check the log level limit for. From 7ee09f1298c028e30b6e9daf40c53d8c45068de0 Mon Sep 17 00:00:00 2001 From: Micah Scott Date: Tue, 19 Nov 2024 14:20:12 -0800 Subject: [PATCH 087/139] Environment variable problems should be warnings, not errors --- src/libmongoc/src/mongoc/mongoc-structured-log.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/libmongoc/src/mongoc/mongoc-structured-log.c b/src/libmongoc/src/mongoc/mongoc-structured-log.c index bd2ed239bc4..5df923cf02e 100644 --- a/src/libmongoc/src/mongoc/mongoc-structured-log.c +++ b/src/libmongoc/src/mongoc/mongoc-structured-log.c @@ -160,7 +160,7 @@ _mongoc_structured_log_get_log_level_from_env (const char *variable, } // Only report the first instance of each error if (0 == mcommon_atomic_int_fetch_add (err_count_atomic, 1, mcommon_memory_order_seq_cst)) { - MONGOC_ERROR ("Invalid log level '%s' read from environment variable %s", level, variable); + MONGOC_WARNING ("Invalid log level '%s' read from environment variable %s. Ignoring it.", level, variable); } return false; } @@ -242,7 +242,7 @@ _mongoc_structured_log_get_max_document_length_from_env (void) return (int32_t) int_value; } - MONGOC_ERROR ("Invalid length '%s' read from environment variable %s", max_length_str, variable); + MONGOC_WARNING ("Invalid length '%s' read from environment variable %s. Ignoring it.", max_length_str, variable); return MONGOC_STRUCTURED_LOG_DEFAULT_MAX_DOCUMENT_LENGTH; } @@ -304,7 +304,7 @@ _mongoc_structured_log_open_stream (void) } FILE *file = fopen (path, "a"); if (!file) { - MONGOC_ERROR ("Cannot open log file %s for writing", path); + MONGOC_WARNING ("Cannot open log file '%s' for writing. Logging to stderr instead.", path); return stderr; } return file; From d161667e460dc469d66b1016c8d6cbc3dee873cb Mon Sep 17 00:00:00 2001 From: Micah Scott Date: Tue, 19 Nov 2024 14:10:46 -0800 Subject: [PATCH 088/139] allow using and configuring structured logging prior to mongoc_init Includes an optimization to should_log that combines the init check, the func==NULL early out, and the level check into a single operation. Might be useful for extending structured log support to early init, if we decide to go that route. --- ...structured_log_set_max_levels_from_env.rst | 3 +- src/libmongoc/doc/structured_log.rst | 4 +- .../examples/example-structured-log.c | 11 +- src/libmongoc/src/mongoc/mongoc-init.c | 2 - .../mongoc/mongoc-structured-log-private.h | 3 - .../src/mongoc/mongoc-structured-log.c | 246 +++++++++++++----- .../src/mongoc/mongoc-structured-log.h | 11 +- 7 files changed, 198 insertions(+), 82 deletions(-) diff --git a/src/libmongoc/doc/mongoc_structured_log_set_max_levels_from_env.rst b/src/libmongoc/doc/mongoc_structured_log_set_max_levels_from_env.rst index 49c32ff2678..411198b46f6 100644 --- a/src/libmongoc/doc/mongoc_structured_log_set_max_levels_from_env.rst +++ b/src/libmongoc/doc/mongoc_structured_log_set_max_levels_from_env.rst @@ -18,7 +18,8 @@ Parse errors may cause a warning message. Component levels with no valid environment variable setting will be left unmodified. -Normally this happens automatically during :symbol:`mongoc_init`, and it provides defaults that can be overridden programmatically by calls to :symbol:`mongoc_structured_log_set_max_level_for_component` and :symbol:`mongoc_structured_log_set_max_level_for_all_components`. +Normally this happens automatically when log levels are automatically initialized on first use. +The resulting default values can be overridden programmatically by calls to :symbol:`mongoc_structured_log_set_max_level_for_component` and :symbol:`mongoc_structured_log_set_max_level_for_all_components`. For applications that desire the opposite behavior, where environment variables may override programmatic settings, they may call ``mongoc_structured_log_set_max_levels_from_env()`` after calling :symbol:`mongoc_structured_log_set_max_level_for_component` and :symbol:`mongoc_structured_log_set_max_level_for_all_components`. This will process the environment a second time, allowing it to override customized defaults. diff --git a/src/libmongoc/doc/structured_log.rst b/src/libmongoc/doc/structured_log.rst index 282133f1827..b03e2d9c83b 100644 --- a/src/libmongoc/doc/structured_log.rst +++ b/src/libmongoc/doc/structured_log.rst @@ -6,12 +6,12 @@ Structured Logging This document describes a newer "structured" logging facility which reports messages from the driver itself using a BSON format defined across driver implementations by the `MongoDB Logging Specification `_. See :doc:`unstructured_log` for the original freeform logging facility. -These two systems are configured and used independently. The structured logging system has independent settings for handler and log levels. +These two systems are configured and used independently. The structured logging system has independent settings for handler and log levels. Defaults -------- -During :symbol:`mongoc_init`: +When the structured logging subsystem is first used, possibly before or during :symbol:`mongoc_init`: * Default log levels are set from the environment variables ``MONGODB_LOG_ALL``, ``MONGODB_LOG_COMMAND``, ``MONGODB_LOG_TOPOLOGY``, ``MONGODB_LOG_SERVER_SELECTION``, expecting a value from the `severity level table `_: ``off``, ``emergency``, ``alert``, ``critical``, ``error``, ``warning``, ``warn``, ``notice``, ``informational``, ``info``, ``debug``, ``trace``. * A default handler is installed, which logs text representations of each message to a file given by ``MONGODB_LOG_PATH``, which may be a full path or one of the special values ``stdout`` or ``stderr``. If no valid path is given, logs are written to stderr. diff --git a/src/libmongoc/examples/example-structured-log.c b/src/libmongoc/examples/example-structured-log.c index 418e1c2ea60..576c040ab62 100644 --- a/src/libmongoc/examples/example-structured-log.c +++ b/src/libmongoc/examples/example-structured-log.c @@ -51,11 +51,6 @@ main (void) mongoc_uri_t *uri = NULL; mongoc_client_t *client = NULL; - /* - * Initialize libmongoc before configuring logs - */ - mongoc_init (); - /* * For demonstration purposes, set up a handler that receives all possible log messages. */ @@ -72,6 +67,12 @@ main (void) */ mongoc_structured_log_set_max_levels_from_env (); + /* + * This is the main libmongoc initialization, but structured logging + * can be used earlier. It's automatically initialized on first use. + */ + mongoc_init (); + /* * Create a MongoDB URI object. This example assumes a local server. */ diff --git a/src/libmongoc/src/mongoc/mongoc-init.c b/src/libmongoc/src/mongoc/mongoc-init.c index ca1f08ae2dc..279b67d94a7 100644 --- a/src/libmongoc/src/mongoc/mongoc-init.c +++ b/src/libmongoc/src/mongoc/mongoc-init.c @@ -97,8 +97,6 @@ mongoc_cyrus_mutex_free (void *mutex) static BSON_ONCE_FUN (_mongoc_do_init) { - _mongoc_structured_log_init (); - #ifdef MONGOC_ENABLE_SASL_CYRUS int status; #endif diff --git a/src/libmongoc/src/mongoc/mongoc-structured-log-private.h b/src/libmongoc/src/mongoc/mongoc-structured-log-private.h index 41f1043c6f2..549d5a149bc 100644 --- a/src/libmongoc/src/mongoc/mongoc-structured-log-private.h +++ b/src/libmongoc/src/mongoc/mongoc-structured-log-private.h @@ -314,9 +314,6 @@ struct mongoc_structured_log_entry_t { const mongoc_structured_log_builder_stage_t *builder; // Required }; -void -_mongoc_structured_log_init (void); - char * mongoc_structured_log_document_to_json (const bson_t *document, size_t *length); diff --git a/src/libmongoc/src/mongoc/mongoc-structured-log.c b/src/libmongoc/src/mongoc/mongoc-structured-log.c index 5df923cf02e..3ce10d54d8e 100644 --- a/src/libmongoc/src/mongoc/mongoc-structured-log.c +++ b/src/libmongoc/src/mongoc/mongoc-structured-log.c @@ -23,11 +23,10 @@ #include "common-thread-private.h" #include "common-oid-private.h" -static void -mongoc_structured_log_default_handler (const mongoc_structured_log_entry_t *entry, void *user_data); - #define STRUCTURED_LOG_COMPONENT_TABLE_SIZE (1 + (size_t) MONGOC_STRUCTURED_LOG_COMPONENT_CONNECTION) +static BSON_ONCE_FUN (_mongoc_structured_log_init_once); + // Canonical names for log components static const char *gStructuredLogComponentNames[] = {"command", "topology", "serverSelection", "connection"}; @@ -43,20 +42,56 @@ static const struct { {.name = "warn", .level = MONGOC_STRUCTURED_LOG_LEVEL_WARNING}, {.name = "info", .level = MONGOC_STRUCTURED_LOG_LEVEL_INFO}}; -static struct { - bson_shared_mutex_t func_mutex; // Handlers run concurrently, but updates to (func, user_data) synchronized - bson_mutex_t stream_mutex; // Only used by the default handler - mongoc_structured_log_func_t func; - void *user_data; - FILE *stream; // Only used by the default handler - int func_is_null_atomic; // Always (func == NULL), effectively bool. Portably atomic to read without mutex. +// Values for gStructuredLog.state_atomic +typedef enum { + MONGOC_STRUCTURED_LOG_STATE_UNINITIALIZED = 0, // Must _mongoc_structured_log_ensure_init + MONGOC_STRUCTURED_LOG_STATE_INACTIVE = 1, // Early out + MONGOC_STRUCTURED_LOG_STATE_ACTIVE = 2, // Maybe active, log level added to this base +} mongoc_structured_log_state_t; +static struct { + /* Pre-computed table combining state_atomic and component_level_table. + * Individual values are atomic, read with relaxed memory order. + * Table rebuilds are protected by a mutex. */ + struct { + int component_level_plus_state[STRUCTURED_LOG_COMPONENT_TABLE_SIZE]; + bson_mutex_t build_mutex; + } combined_table; + + // Handler state, updated under exclusive lock, callable/gettable with shared lock + struct { + bson_shared_mutex_t mutex; + mongoc_structured_log_func_t func; + void *user_data; + } handler; + + // State only used by the default handler + struct { + bson_mutex_t stream_mutex; + FILE *stream; + } default_handler; + + // Configured max document length, only modified during initialization int32_t max_document_length; - int component_level_table[STRUCTURED_LOG_COMPONENT_TABLE_SIZE]; // Really mongoc_structured_log_level_t; int typed to - // support atomic fetch -} gStructuredLog = { - .func = mongoc_structured_log_default_handler, -}; + + /* Main storage for public per-component max log levels. + * Atomic mongoc_structured_log_level_t, seq_cst memory order. + * Used only for getting/setting individual component levels. + * Not used directly by should_log, which relies on 'combined_table'. */ + int component_level_table[STRUCTURED_LOG_COMPONENT_TABLE_SIZE]; + + /* Tracks whether we might be uninitialized or have no handler. + * Set when updating 'handler', read when recomputing 'combined_table'. + * Atomic mongoc_structured_log_state_t, seq_cst memory order. */ + int state_atomic; +} gStructuredLog; + +static BSON_INLINE void +_mongoc_structured_log_ensure_init (void) +{ + static bson_once_t init_once = BSON_ONCE_INIT; + bson_once (&init_once, &_mongoc_structured_log_init_once); +} bson_t * mongoc_structured_log_entry_message_as_bson (const mongoc_structured_log_entry_t *entry) @@ -82,68 +117,135 @@ mongoc_structured_log_entry_get_component (const mongoc_structured_log_entry_t * return entry->envelope.component; } +static BSON_INLINE mongoc_structured_log_level_t +_mongoc_structured_log_get_max_level_for_component (mongoc_structured_log_component_t component) +{ + unsigned table_index = (unsigned) component; + BSON_ASSERT (table_index < STRUCTURED_LOG_COMPONENT_TABLE_SIZE); + return mcommon_atomic_int_fetch (&gStructuredLog.component_level_table[table_index], mcommon_memory_order_seq_cst); +} + +mongoc_structured_log_level_t +mongoc_structured_log_get_max_level_for_component (mongoc_structured_log_component_t component) +{ + _mongoc_structured_log_ensure_init (); + return _mongoc_structured_log_get_max_level_for_component (component); +} + +static void +_mongoc_structured_log_update_level_plus_state (void) +{ + // Pre-calculate a table of values that combine per-component level and global state. + // Needs to be updated when the handler is set/unset or when any level setting changes. + bson_mutex_lock (&gStructuredLog.combined_table.build_mutex); + int state = mcommon_atomic_int_fetch (&gStructuredLog.state_atomic, mcommon_memory_order_seq_cst); + for (unsigned table_index = 0; table_index < STRUCTURED_LOG_COMPONENT_TABLE_SIZE; table_index++) { + mongoc_structured_log_component_t component = (mongoc_structured_log_component_t) table_index; + mcommon_atomic_int_exchange (&gStructuredLog.combined_table.component_level_plus_state[table_index], + state == MONGOC_STRUCTURED_LOG_STATE_ACTIVE + ? (int) state + + (int) _mongoc_structured_log_get_max_level_for_component (component) + : (int) state, + mcommon_memory_order_relaxed); + } + bson_mutex_unlock (&gStructuredLog.combined_table.build_mutex); +} + +static void +_mongoc_structured_log_set_handler (mongoc_structured_log_func_t log_func, void *user_data) +{ + bson_shared_mutex_lock (&gStructuredLog.handler.mutex); // Waits for handler invocations to end + gStructuredLog.handler.func = log_func; + gStructuredLog.handler.user_data = user_data; + mcommon_atomic_int_exchange (&gStructuredLog.state_atomic, + log_func == NULL ? MONGOC_STRUCTURED_LOG_STATE_INACTIVE + : MONGOC_STRUCTURED_LOG_STATE_ACTIVE, + mcommon_memory_order_seq_cst); + bson_shared_mutex_unlock (&gStructuredLog.handler.mutex); +} + void mongoc_structured_log_set_handler (mongoc_structured_log_func_t log_func, void *user_data) { - bson_shared_mutex_lock (&gStructuredLog.func_mutex); // Waits for handler invocations to end - gStructuredLog.func = log_func; - gStructuredLog.user_data = user_data; - mcommon_atomic_int_exchange (&gStructuredLog.func_is_null_atomic, log_func == NULL, mcommon_memory_order_relaxed); - bson_shared_mutex_unlock (&gStructuredLog.func_mutex); + _mongoc_structured_log_ensure_init (); + _mongoc_structured_log_set_handler (log_func, user_data); + _mongoc_structured_log_update_level_plus_state (); } void mongoc_structured_log_get_handler (mongoc_structured_log_func_t *log_func, void **user_data) { - bson_shared_mutex_lock_shared (&gStructuredLog.func_mutex); - *log_func = gStructuredLog.func; - *user_data = gStructuredLog.user_data; - bson_shared_mutex_unlock_shared (&gStructuredLog.func_mutex); + _mongoc_structured_log_ensure_init (); + bson_shared_mutex_lock_shared (&gStructuredLog.handler.mutex); + *log_func = gStructuredLog.handler.func; + *user_data = gStructuredLog.handler.user_data; + bson_shared_mutex_unlock_shared (&gStructuredLog.handler.mutex); } -void -mongoc_structured_log_set_max_level_for_component (mongoc_structured_log_component_t component, - mongoc_structured_log_level_t level) +static void +_mongoc_structured_log_set_max_level_for_component (mongoc_structured_log_component_t component, + mongoc_structured_log_level_t level) { BSON_ASSERT (level >= MONGOC_STRUCTURED_LOG_LEVEL_EMERGENCY && level <= MONGOC_STRUCTURED_LOG_LEVEL_TRACE); unsigned table_index = (unsigned) component; BSON_ASSERT (table_index < STRUCTURED_LOG_COMPONENT_TABLE_SIZE); mcommon_atomic_int_exchange ( - &gStructuredLog.component_level_table[table_index], level, mcommon_memory_order_relaxed); + &gStructuredLog.component_level_table[table_index], level, mcommon_memory_order_seq_cst); } void -mongoc_structured_log_set_max_level_for_all_components (mongoc_structured_log_level_t level) +mongoc_structured_log_set_max_level_for_component (mongoc_structured_log_component_t component, + mongoc_structured_log_level_t level) +{ + _mongoc_structured_log_ensure_init (); + _mongoc_structured_log_set_max_level_for_component (component, level); + _mongoc_structured_log_update_level_plus_state (); +} + +static void +_mongoc_structured_log_set_max_level_for_all_components (mongoc_structured_log_level_t level) { for (int component = 0; component < STRUCTURED_LOG_COMPONENT_TABLE_SIZE; component++) { - mongoc_structured_log_set_max_level_for_component ((mongoc_structured_log_component_t) component, level); + _mongoc_structured_log_set_max_level_for_component ((mongoc_structured_log_component_t) component, level); } } -mongoc_structured_log_level_t -mongoc_structured_log_get_max_level_for_component (mongoc_structured_log_component_t component) +void +mongoc_structured_log_set_max_level_for_all_components (mongoc_structured_log_level_t level) { - unsigned table_index = (unsigned) component; - BSON_ASSERT (table_index < STRUCTURED_LOG_COMPONENT_TABLE_SIZE); - return mcommon_atomic_int_fetch (&gStructuredLog.component_level_table[table_index], mcommon_memory_order_relaxed); + _mongoc_structured_log_ensure_init (); + _mongoc_structured_log_set_max_level_for_all_components (level); + _mongoc_structured_log_update_level_plus_state (); } bool _mongoc_structured_log_should_log (const mongoc_structured_log_envelope_t *envelope) { - return !mcommon_atomic_int_fetch ((void *) &gStructuredLog.func_is_null_atomic, mcommon_memory_order_relaxed) && - envelope->level <= mongoc_structured_log_get_max_level_for_component (envelope->component); + unsigned table_index = (unsigned) envelope->component; + BSON_ASSERT (table_index < STRUCTURED_LOG_COMPONENT_TABLE_SIZE); + const int volatile *level_plus_state_ptr = &gStructuredLog.combined_table.component_level_plus_state[table_index]; + + // One atomic fetch gives us the per-component level, the global + // enable/disable state, and lets us detect whether initialization is needed. + int level_plus_state = mcommon_atomic_int_fetch (level_plus_state_ptr, mcommon_memory_order_relaxed); + if (BSON_UNLIKELY (level_plus_state == (int) MONGOC_STRUCTURED_LOG_STATE_UNINITIALIZED)) { + _mongoc_structured_log_ensure_init (); + level_plus_state = mcommon_atomic_int_fetch (level_plus_state_ptr, mcommon_memory_order_relaxed); + } + return envelope->level + MONGOC_STRUCTURED_LOG_STATE_ACTIVE <= level_plus_state; } void _mongoc_structured_log_with_entry (const mongoc_structured_log_entry_t *entry) { - bson_shared_mutex_lock_shared (&gStructuredLog.func_mutex); - mongoc_structured_log_func_t func = gStructuredLog.func; - if (func) { - func (entry, gStructuredLog.user_data); + // should_log has guaranteed that we've run init by now. + // No guarantee that there's actually a handler set but it's likely. + bson_shared_mutex_lock_shared (&gStructuredLog.handler.mutex); + mongoc_structured_log_func_t func = gStructuredLog.handler.func; + if (BSON_LIKELY (func)) { + func (entry, gStructuredLog.handler.user_data); } - bson_shared_mutex_unlock_shared (&gStructuredLog.func_mutex); + bson_shared_mutex_unlock_shared (&gStructuredLog.handler.mutex); } static bool @@ -246,52 +348,50 @@ _mongoc_structured_log_get_max_document_length_from_env (void) return MONGOC_STRUCTURED_LOG_DEFAULT_MAX_DOCUMENT_LENGTH; } -void -_mongoc_structured_log_init (void) -{ - bson_shared_mutex_init (&gStructuredLog.func_mutex); - bson_mutex_init (&gStructuredLog.stream_mutex); - gStructuredLog.max_document_length = _mongoc_structured_log_get_max_document_length_from_env (); - mongoc_structured_log_set_max_level_for_all_components (MONGOC_STRUCTURED_LOG_DEFAULT_LEVEL); - mongoc_structured_log_set_max_levels_from_env (); -} - -void -mongoc_structured_log_set_max_levels_from_env (void) +static void +_mongoc_structured_log_set_max_levels_from_env (void) { mongoc_structured_log_level_t level; { static int err_count_atomic = 0; if (_mongoc_structured_log_get_log_level_from_env ("MONGODB_LOG_ALL", &level, &err_count_atomic)) { - mongoc_structured_log_set_max_level_for_all_components (level); + _mongoc_structured_log_set_max_level_for_all_components (level); } } { static int err_count_atomic = 0; if (_mongoc_structured_log_get_log_level_from_env ("MONGODB_LOG_COMMAND", &level, &err_count_atomic)) { - mongoc_structured_log_set_max_level_for_component (MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND, level); + _mongoc_structured_log_set_max_level_for_component (MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND, level); } } { static int err_count_atomic = 0; if (_mongoc_structured_log_get_log_level_from_env ("MONGODB_LOG_CONNECTION", &level, &err_count_atomic)) { - mongoc_structured_log_set_max_level_for_component (MONGOC_STRUCTURED_LOG_COMPONENT_CONNECTION, level); + _mongoc_structured_log_set_max_level_for_component (MONGOC_STRUCTURED_LOG_COMPONENT_CONNECTION, level); } } { static int err_count_atomic = 0; if (_mongoc_structured_log_get_log_level_from_env ("MONGODB_LOG_TOPOLOGY", &level, &err_count_atomic)) { - mongoc_structured_log_set_max_level_for_component (MONGOC_STRUCTURED_LOG_COMPONENT_TOPOLOGY, level); + _mongoc_structured_log_set_max_level_for_component (MONGOC_STRUCTURED_LOG_COMPONENT_TOPOLOGY, level); } } { static int err_count_atomic = 0; if (_mongoc_structured_log_get_log_level_from_env ("MONGODB_LOG_SERVER_SELECTION", &level, &err_count_atomic)) { - mongoc_structured_log_set_max_level_for_component (MONGOC_STRUCTURED_LOG_COMPONENT_SERVER_SELECTION, level); + _mongoc_structured_log_set_max_level_for_component (MONGOC_STRUCTURED_LOG_COMPONENT_SERVER_SELECTION, level); } } } +void +mongoc_structured_log_set_max_levels_from_env (void) +{ + _mongoc_structured_log_ensure_init (); + _mongoc_structured_log_set_max_levels_from_env (); + _mongoc_structured_log_update_level_plus_state (); +} + static FILE * _mongoc_structured_log_open_stream (void) { @@ -313,15 +413,15 @@ _mongoc_structured_log_open_stream (void) static FILE * _mongoc_structured_log_get_stream (void) { - // Not re-entrant; protected by the stream_lock. - FILE *log_stream = gStructuredLog.stream; + // Not re-entrant; protected by the default_handler.stream_mutex. + FILE *log_stream = gStructuredLog.default_handler.stream; if (log_stream) { return log_stream; } // Note that log_stream may be the global stderr/stdout streams, // or an allocated FILE that is never closed. log_stream = _mongoc_structured_log_open_stream (); - gStructuredLog.stream = log_stream; + gStructuredLog.default_handler.stream = log_stream; return log_stream; } @@ -337,13 +437,31 @@ mongoc_structured_log_default_handler (const mongoc_structured_log_entry_t *entr const char *component_name = mongoc_structured_log_get_component_name (mongoc_structured_log_entry_get_component (entry)); - bson_mutex_lock (&gStructuredLog.stream_mutex); + bson_mutex_lock (&gStructuredLog.default_handler.stream_mutex); fprintf (_mongoc_structured_log_get_stream (), "MONGODB_LOG %s %s %s\n", level_name, component_name, json_message); - bson_mutex_unlock (&gStructuredLog.stream_mutex); + bson_mutex_unlock (&gStructuredLog.default_handler.stream_mutex); bson_free (json_message); } +static BSON_ONCE_FUN (_mongoc_structured_log_init_once) +{ + bson_shared_mutex_init (&gStructuredLog.handler.mutex); + bson_mutex_init (&gStructuredLog.default_handler.stream_mutex); + bson_mutex_init (&gStructuredLog.combined_table.build_mutex); + + gStructuredLog.max_document_length = _mongoc_structured_log_get_max_document_length_from_env (); + + _mongoc_structured_log_set_max_level_for_all_components (MONGOC_STRUCTURED_LOG_DEFAULT_LEVEL); + _mongoc_structured_log_set_max_levels_from_env (); + + // The default handler replaces MONGOC_STRUCTURED_LOG_STATE_MAYBE_UNINITIALIZED, this must be last. + _mongoc_structured_log_set_handler (mongoc_structured_log_default_handler, NULL); + _mongoc_structured_log_update_level_plus_state (); + + BSON_ONCE_RETURN; +} + char * mongoc_structured_log_document_to_json (const bson_t *document, size_t *length) { diff --git a/src/libmongoc/src/mongoc/mongoc-structured-log.h b/src/libmongoc/src/mongoc/mongoc-structured-log.h index c29ec5cf924..afaf8178a7d 100644 --- a/src/libmongoc/src/mongoc/mongoc-structured-log.h +++ b/src/libmongoc/src/mongoc/mongoc-structured-log.h @@ -79,8 +79,8 @@ mongoc_structured_log_set_handler (mongoc_structured_log_func_t log_func, void * * Sets the maximum log level per-component. Only log messages at or below * this severity level will be passed to mongoc_structured_log_func_t. * - * By default, each component's log level comes from the environment - * variables `MONGOC_LOG_` captured during mongoc_init(). + * By default, each component's log level may be set by environment variables. + * See mongoc_structured_log_set_max_levels_from_env(). */ MONGOC_EXPORT (void) mongoc_structured_log_set_max_level_for_component (mongoc_structured_log_component_t component, @@ -107,9 +107,10 @@ mongoc_structured_log_set_max_level_for_all_components (mongoc_structured_log_le * * Component levels with no valid environment variable setting will be left unmodified. * - * Normally this happens automatically during mongoc_init(), and it provides defaults - * that can be overridden programmatically by calls to mongoc_structured_log_set_max_level_for_component() - * and mongoc_structured_log_set_max_level_for_all_components(). + * Normally this happens automatically when log levels are automatically initialized on + * first use. The resulting default values can be overridden programmatically by calls + * to mongoc_structured_log_set_max_level_for_component() and + * mongoc_structured_log_set_max_level_for_all_components(). * * For applications that desire the opposite behavior, where environment variables may * override programmatic settings, they may call mongoc_structured_log_set_max_levels_from_env() From d346f78592875624eb2dddde5cb6dbae8b2d81bf Mon Sep 17 00:00:00 2001 From: Micah Scott Date: Tue, 19 Nov 2024 16:29:03 -0800 Subject: [PATCH 089/139] typo --- .../doc/mongoc_structured_log_set_max_levels_from_env.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libmongoc/doc/mongoc_structured_log_set_max_levels_from_env.rst b/src/libmongoc/doc/mongoc_structured_log_set_max_levels_from_env.rst index 411198b46f6..e221ccdcf6d 100644 --- a/src/libmongoc/doc/mongoc_structured_log_set_max_levels_from_env.rst +++ b/src/libmongoc/doc/mongoc_structured_log_set_max_levels_from_env.rst @@ -18,7 +18,7 @@ Parse errors may cause a warning message. Component levels with no valid environment variable setting will be left unmodified. -Normally this happens automatically when log levels are automatically initialized on first use. +Normally this happens automatically when log levels are initialized on first use. The resulting default values can be overridden programmatically by calls to :symbol:`mongoc_structured_log_set_max_level_for_component` and :symbol:`mongoc_structured_log_set_max_level_for_all_components`. For applications that desire the opposite behavior, where environment variables may override programmatic settings, they may call ``mongoc_structured_log_set_max_levels_from_env()`` after calling :symbol:`mongoc_structured_log_set_max_level_for_component` and :symbol:`mongoc_structured_log_set_max_level_for_all_components`. From 1f2a0355a0f0372b75bbb04c117045bacdde820e Mon Sep 17 00:00:00 2001 From: Micah Scott Date: Tue, 19 Nov 2024 16:47:05 -0800 Subject: [PATCH 090/139] structured log header no longer needed in mongoc-init --- src/libmongoc/src/mongoc/mongoc-init.c | 1 - 1 file changed, 1 deletion(-) diff --git a/src/libmongoc/src/mongoc/mongoc-init.c b/src/libmongoc/src/mongoc/mongoc-init.c index 279b67d94a7..4e72f0ec436 100644 --- a/src/libmongoc/src/mongoc/mongoc-init.c +++ b/src/libmongoc/src/mongoc/mongoc-init.c @@ -19,7 +19,6 @@ #include "mongoc-config.h" #include "mongoc-counters-private.h" -#include "mongoc-structured-log-private.h" #include "mongoc-init.h" #include "mongoc-handshake-private.h" From c69de085fbe0fde4833c3c70744bdb8c7818e783 Mon Sep 17 00:00:00 2001 From: Micah Scott Date: Tue, 19 Nov 2024 16:59:28 -0800 Subject: [PATCH 091/139] doc a few more safe functions for use inside mongoc_structured_log_func_t --- src/libmongoc/doc/mongoc_structured_log_func_t.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/libmongoc/doc/mongoc_structured_log_func_t.rst b/src/libmongoc/doc/mongoc_structured_log_func_t.rst index 6247a9569d6..910d7824932 100644 --- a/src/libmongoc/doc/mongoc_structured_log_func_t.rst +++ b/src/libmongoc/doc/mongoc_structured_log_func_t.rst @@ -20,6 +20,11 @@ Handlers may use any operating system or ``libbson`` functions but MUST not use * :symbol:`mongoc_structured_log_entry_get_component` * :symbol:`mongoc_structured_log_entry_get_level` * :symbol:`mongoc_structured_log_entry_message_as_bson` +* :symbol:`mongoc_structured_log_get_level_name` +* :symbol:`mongoc_structured_log_get_named_level` +* :symbol:`mongoc_structured_log_get_component_name` +* :symbol:`mongoc_structured_log_get_named_component` +* :symbol:`mongoc_structured_log_get_max_level_for_component` Parameters ---------- From ef42f4fec34057c987b41f19dd797030dd50dc67 Mon Sep 17 00:00:00 2001 From: Micah Scott Date: Tue, 19 Nov 2024 17:31:44 -0800 Subject: [PATCH 092/139] table build mutex should cover the entire instigating setter Setter operations (handler, level) now acquire a table build mutex first, which serializes the setters with each other and ensures the table build state is consistent. These locks don't impact the fast paths: handler invocation can take a single shared lock still, public log level getters don't require the mutex, and should_log doesn't need a mutex. --- .../src/mongoc/mongoc-structured-log.c | 64 ++++++++++++------- 1 file changed, 42 insertions(+), 22 deletions(-) diff --git a/src/libmongoc/src/mongoc/mongoc-structured-log.c b/src/libmongoc/src/mongoc/mongoc-structured-log.c index 3ce10d54d8e..447ecc4f500 100644 --- a/src/libmongoc/src/mongoc/mongoc-structured-log.c +++ b/src/libmongoc/src/mongoc/mongoc-structured-log.c @@ -42,17 +42,25 @@ static const struct { {.name = "warn", .level = MONGOC_STRUCTURED_LOG_LEVEL_WARNING}, {.name = "info", .level = MONGOC_STRUCTURED_LOG_LEVEL_INFO}}; -// Values for gStructuredLog.state_atomic +// Values for gStructuredLog.handler.state typedef enum { - MONGOC_STRUCTURED_LOG_STATE_UNINITIALIZED = 0, // Must _mongoc_structured_log_ensure_init - MONGOC_STRUCTURED_LOG_STATE_INACTIVE = 1, // Early out - MONGOC_STRUCTURED_LOG_STATE_ACTIVE = 2, // Maybe active, log level added to this base + MONGOC_STRUCTURED_LOG_STATE_UNINITIALIZED = 0, // Must ensure_init. This state is never re-entered once exited. + MONGOC_STRUCTURED_LOG_STATE_INACTIVE = 1, // Handler disabled. Always < MONGOC_STRUCTURED_LOG_STATE_ACTIVE. + MONGOC_STRUCTURED_LOG_STATE_ACTIVE = 2, // Maybe active. Value is also a base to which log levels are added. } mongoc_structured_log_state_t; static struct { - /* Pre-computed table combining state_atomic and component_level_table. + /* Pre-computed table combining handler.state and component_level_table. * Individual values are atomic, read with relaxed memory order. - * Table rebuilds are protected by a mutex. */ + * + * Table rebuilds are protected by a mutex. The build_mutex should + * be held while updating the table and while performing the instigating + * level or handler change. + * + * When acquiring both the build_mutex and an exclusive lock on handler.mutex + * to update the handler, the build mutex must always be acquired first. + * Log handlers must not be allowed to call functions that acquire the build_mutex. + */ struct { int component_level_plus_state[STRUCTURED_LOG_COMPONENT_TABLE_SIZE]; bson_mutex_t build_mutex; @@ -63,6 +71,7 @@ static struct { bson_shared_mutex_t mutex; mongoc_structured_log_func_t func; void *user_data; + mongoc_structured_log_state_t state; } handler; // State only used by the default handler @@ -79,11 +88,6 @@ static struct { * Used only for getting/setting individual component levels. * Not used directly by should_log, which relies on 'combined_table'. */ int component_level_table[STRUCTURED_LOG_COMPONENT_TABLE_SIZE]; - - /* Tracks whether we might be uninitialized or have no handler. - * Set when updating 'handler', read when recomputing 'combined_table'. - * Atomic mongoc_structured_log_state_t, seq_cst memory order. */ - int state_atomic; } gStructuredLog; static BSON_INLINE void @@ -135,20 +139,19 @@ mongoc_structured_log_get_max_level_for_component (mongoc_structured_log_compone static void _mongoc_structured_log_update_level_plus_state (void) { + // The caller MUST be holding the build_mutex throughout the operation which caused this table update. // Pre-calculate a table of values that combine per-component level and global state. // Needs to be updated when the handler is set/unset or when any level setting changes. - bson_mutex_lock (&gStructuredLog.combined_table.build_mutex); - int state = mcommon_atomic_int_fetch (&gStructuredLog.state_atomic, mcommon_memory_order_seq_cst); + mongoc_structured_log_state_t handler_state = gStructuredLog.handler.state; for (unsigned table_index = 0; table_index < STRUCTURED_LOG_COMPONENT_TABLE_SIZE; table_index++) { mongoc_structured_log_component_t component = (mongoc_structured_log_component_t) table_index; mcommon_atomic_int_exchange (&gStructuredLog.combined_table.component_level_plus_state[table_index], - state == MONGOC_STRUCTURED_LOG_STATE_ACTIVE - ? (int) state + + handler_state == MONGOC_STRUCTURED_LOG_STATE_ACTIVE + ? (int) MONGOC_STRUCTURED_LOG_STATE_ACTIVE + (int) _mongoc_structured_log_get_max_level_for_component (component) - : (int) state, + : (int) handler_state, mcommon_memory_order_relaxed); } - bson_mutex_unlock (&gStructuredLog.combined_table.build_mutex); } static void @@ -157,10 +160,8 @@ _mongoc_structured_log_set_handler (mongoc_structured_log_func_t log_func, void bson_shared_mutex_lock (&gStructuredLog.handler.mutex); // Waits for handler invocations to end gStructuredLog.handler.func = log_func; gStructuredLog.handler.user_data = user_data; - mcommon_atomic_int_exchange (&gStructuredLog.state_atomic, - log_func == NULL ? MONGOC_STRUCTURED_LOG_STATE_INACTIVE - : MONGOC_STRUCTURED_LOG_STATE_ACTIVE, - mcommon_memory_order_seq_cst); + gStructuredLog.handler.state = + log_func == NULL ? MONGOC_STRUCTURED_LOG_STATE_INACTIVE : MONGOC_STRUCTURED_LOG_STATE_ACTIVE; bson_shared_mutex_unlock (&gStructuredLog.handler.mutex); } @@ -168,14 +169,18 @@ void mongoc_structured_log_set_handler (mongoc_structured_log_func_t log_func, void *user_data) { _mongoc_structured_log_ensure_init (); + + bson_mutex_lock (&gStructuredLog.combined_table.build_mutex); _mongoc_structured_log_set_handler (log_func, user_data); _mongoc_structured_log_update_level_plus_state (); + bson_mutex_unlock (&gStructuredLog.combined_table.build_mutex); } void mongoc_structured_log_get_handler (mongoc_structured_log_func_t *log_func, void **user_data) { _mongoc_structured_log_ensure_init (); + bson_shared_mutex_lock_shared (&gStructuredLog.handler.mutex); *log_func = gStructuredLog.handler.func; *user_data = gStructuredLog.handler.user_data; @@ -198,8 +203,11 @@ mongoc_structured_log_set_max_level_for_component (mongoc_structured_log_compone mongoc_structured_log_level_t level) { _mongoc_structured_log_ensure_init (); + + bson_mutex_lock (&gStructuredLog.combined_table.build_mutex); _mongoc_structured_log_set_max_level_for_component (component, level); _mongoc_structured_log_update_level_plus_state (); + bson_mutex_unlock (&gStructuredLog.combined_table.build_mutex); } static void @@ -214,8 +222,11 @@ void mongoc_structured_log_set_max_level_for_all_components (mongoc_structured_log_level_t level) { _mongoc_structured_log_ensure_init (); + + bson_mutex_lock (&gStructuredLog.combined_table.build_mutex); _mongoc_structured_log_set_max_level_for_all_components (level); _mongoc_structured_log_update_level_plus_state (); + bson_mutex_unlock (&gStructuredLog.combined_table.build_mutex); } bool @@ -388,8 +399,11 @@ void mongoc_structured_log_set_max_levels_from_env (void) { _mongoc_structured_log_ensure_init (); + + bson_mutex_lock (&gStructuredLog.combined_table.build_mutex); _mongoc_structured_log_set_max_levels_from_env (); _mongoc_structured_log_update_level_plus_state (); + bson_mutex_unlock (&gStructuredLog.combined_table.build_mutex); } static FILE * @@ -452,13 +466,19 @@ static BSON_ONCE_FUN (_mongoc_structured_log_init_once) gStructuredLog.max_document_length = _mongoc_structured_log_get_max_document_length_from_env (); + bson_mutex_lock (&gStructuredLog.combined_table.build_mutex); + _mongoc_structured_log_set_max_level_for_all_components (MONGOC_STRUCTURED_LOG_DEFAULT_LEVEL); _mongoc_structured_log_set_max_levels_from_env (); - // The default handler replaces MONGOC_STRUCTURED_LOG_STATE_MAYBE_UNINITIALIZED, this must be last. + // The default handler replaces MONGOC_STRUCTURED_LOG_STATE_MAYBE_UNINITIALIZED. + // Other threads may immediately submit log entries, so this must happen after + // log level setup and just before we unlock the build_mutex and return. _mongoc_structured_log_set_handler (mongoc_structured_log_default_handler, NULL); _mongoc_structured_log_update_level_plus_state (); + bson_mutex_unlock (&gStructuredLog.combined_table.build_mutex); + BSON_ONCE_RETURN; } From ffff0ca5ab6b74e61f85468360574164059c4c68 Mon Sep 17 00:00:00 2001 From: Micah Scott Date: Wed, 20 Nov 2024 08:11:26 -0800 Subject: [PATCH 093/139] comment only, additional text about handler api restrictions --- src/libmongoc/doc/mongoc_structured_log_func_t.rst | 4 ++++ src/libmongoc/src/mongoc/mongoc-structured-log.h | 9 +++++++++ 2 files changed, 13 insertions(+) diff --git a/src/libmongoc/doc/mongoc_structured_log_func_t.rst b/src/libmongoc/doc/mongoc_structured_log_func_t.rst index 910d7824932..0eeaa93fd52 100644 --- a/src/libmongoc/doc/mongoc_structured_log_func_t.rst +++ b/src/libmongoc/doc/mongoc_structured_log_func_t.rst @@ -26,6 +26,10 @@ Handlers may use any operating system or ``libbson`` functions but MUST not use * :symbol:`mongoc_structured_log_get_named_component` * :symbol:`mongoc_structured_log_get_max_level_for_component` +Use of other ``libmongoc`` APIs will have undefined consequences, including deadlocks and unbounded recursion. + +Applications that wish to use MongoDB itself as a logging destination will need to store the serialized messages temporarily and insert them asynchronously from outside the log handler. + Parameters ---------- diff --git a/src/libmongoc/src/mongoc/mongoc-structured-log.h b/src/libmongoc/src/mongoc/mongoc-structured-log.h index afaf8178a7d..095a19ed2e3 100644 --- a/src/libmongoc/src/mongoc/mongoc-structured-log.h +++ b/src/libmongoc/src/mongoc/mongoc-structured-log.h @@ -67,6 +67,15 @@ typedef void (*mongoc_structured_log_func_t) (const mongoc_structured_log_entry_ * mongoc_structured_log_entry_get_component to filter log messages, and * selectively call mongoc_structured_log_entry_message_as_bson to create * a bson_t for log messages only when needed. + * + * Any operating system or libbson APIs may be used by the callback, but + * libmongoc must be used only in limited ways to prevent deadlocks or + * unbounded recursion. See libmongoc/doc/mongoc_structured_log_func_t.rst + * for a complete list of allowed APIs for handlers to use. + * + * Applications that wish to use MongoDB itself as a logging destination will + * need to store the serialized messages temporarily and insert them later + * from outside the log handler. */ MONGOC_EXPORT (void) mongoc_structured_log_set_handler (mongoc_structured_log_func_t log_func, void *user_data); From bb71ad2c231403a4867ad6dba46c6ce346755393 Mon Sep 17 00:00:00 2001 From: Micah Scott Date: Wed, 20 Nov 2024 13:52:46 -0800 Subject: [PATCH 094/139] unified tests: avoid duplicating the table of supported events --- src/libmongoc/tests/unified/entity-map.c | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/src/libmongoc/tests/unified/entity-map.c b/src/libmongoc/tests/unified/entity-map.c index 4bf72ffb036..2591795b99f 100644 --- a/src/libmongoc/tests/unified/entity-map.c +++ b/src/libmongoc/tests/unified/entity-map.c @@ -423,10 +423,12 @@ set_server_changed_cb (mongoc_apm_callbacks_t *callbacks) mongoc_apm_set_server_changed_cb (callbacks, server_changed); } -// Note: multiple invocations of this function is okay, since all it does -// is set the appropriate pointer in `callbacks`, and the callback function(s) -// being used is always the same for a given type. -static void +/* Set a callback for the indicated event type in a mongoc_apm_callbacks_t. + * Safe to call multiple times for the same event: callbacks for a specific + * event type are always the same. Returns 'true' if the event is known and + * 'false' if unknown. If 'callbacks' is NULL, validates the 'type' without + * taking any other action. */ +static bool set_event_callback (mongoc_apm_callbacks_t *callbacks, const char *type) { typedef void (*set_func_t) (mongoc_apm_callbacks_t *); @@ -446,18 +448,19 @@ set_event_callback (mongoc_apm_callbacks_t *callbacks, const char *type) for (const command_to_cb_t *iter = commands; iter->type; ++iter) { if (bson_strcasecmp (type, iter->type) == 0) { - iter->set (callbacks); - return; + if (callbacks) { + iter->set (callbacks); + } + return true; } } + return false; } static bool is_supported_event_type (const char *type) { - return (0 == bson_strcasecmp (type, "commandStartedEvent") || 0 == bson_strcasecmp (type, "commandFailedEvent") || - 0 == bson_strcasecmp (type, "commandSucceededEvent") || - 0 == bson_strcasecmp (type, "serverDescriptionChangedEvent")); + return set_event_callback (NULL, type); } static void From ef38227f53f7019945ace9a1bde19c1e09348a33 Mon Sep 17 00:00:00 2001 From: Micah Scott Date: Mon, 25 Nov 2024 13:51:16 -0800 Subject: [PATCH 095/139] In waitForEvent, provoke lazy opening of topology descriptions --- src/libmongoc/tests/unified/operation.c | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/src/libmongoc/tests/unified/operation.c b/src/libmongoc/tests/unified/operation.c index 73e80bfa89b..bc48ae394fa 100644 --- a/src/libmongoc/tests/unified/operation.c +++ b/src/libmongoc/tests/unified/operation.c @@ -3881,21 +3881,19 @@ operation_wait_for_event (test_t *test, operation_t *op, result_t *result, bson_ goto done; }; + _mongoc_usleep (WAIT_FOR_EVENT_TICK_MS * 1000); + // @todo Re-examine this once we have support for connection pools in the unified test // runner. Without pooling, all events we could be waiting on would be coming - // from single-threaded (blocking) topology scans. There's no reason for those - // to happen without some explicit activity, and the tests aren't written to account - // for this. We could work around it by sending a command (and hiding its excess events/logs) - // or by directly initiating a scan. + // from single-threaded (blocking) topology scans, or from lazily opening the topology + // description when it's first used. Request server selection after blocking, to + // handle either of these cases. { mongoc_client_t *mc_client = entity_map_get_client (test->entity_map, client_id, error); - if (!mc_client) { - goto done; + if (mc_client) { + mongoc_topology_select_server_id (mc_client->topology, MONGOC_SS_READ, NULL, NULL, NULL, error); } - _mongoc_topology_do_blocking_scan (mc_client->topology, error); } - - _mongoc_usleep (WAIT_FOR_EVENT_TICK_MS * 1000); } result_from_ok (result); From 7110257adc745d677acd1c448c6b7a92f9f8f9cb Mon Sep 17 00:00:00 2001 From: Micah Scott Date: Mon, 25 Nov 2024 17:15:23 -0800 Subject: [PATCH 096/139] factor out bson_error_t serialization into a structured log "error()" item --- .../mongoc/mongoc-structured-log-private.h | 18 +++++- .../src/mongoc/mongoc-structured-log.c | 30 +++++++++- .../tests/test-mongoc-structured-log.c | 57 +++++++++++++++++++ 3 files changed, 101 insertions(+), 4 deletions(-) diff --git a/src/libmongoc/src/mongoc/mongoc-structured-log-private.h b/src/libmongoc/src/mongoc/mongoc-structured-log-private.h index 549d5a149bc..76f00942eb0 100644 --- a/src/libmongoc/src/mongoc/mongoc-structured-log-private.h +++ b/src/libmongoc/src/mongoc/mongoc-structured-log-private.h @@ -138,6 +138,18 @@ BSON_BEGIN_DECLS #define _mongoc_structured_log_item_boolean(_key_or_null, _value_boolean) \ {.func = _mongoc_structured_log_append_boolean, .arg1.utf8 = (_key_or_null), .arg2.boolean = (_value_boolean)}, +/** + * @def error(key, value) + * @brief Structured log item, bson_error_t document + * + * Serializes the fields of a bson_error_t as a subdocument with the indicated key. + * + * @param key Key as a NULL-terminated const char * expression, or NULL to skip this item. + * @param value Error as a const bson_error_t * expression, or NULL for a null value. + */ +#define _mongoc_structured_log_item_error(_key_or_null, _error_or_null) \ + {.func = _mongoc_structured_log_append_error, .arg1.utf8 = (_key_or_null), .arg2.error = (_error_or_null)}, + /** * @def oid_as_hex(key, value) * @brief Structured log item, bson_oid_t converted to a hex string @@ -286,13 +298,14 @@ struct mongoc_structured_log_builder_stage_t { mongoc_structured_log_builder_func_t func; // NULL sentinel here union { const bson_error_t *error; + const char *utf8; const mongoc_cmd_t *cmd; const mongoc_server_description_t *server_description; - const char *utf8; } arg1; union { bool boolean; bson_oid_t *oid; + const bson_error_t *error; const bson_t *bson; const char *utf8; int32_t int32; @@ -344,6 +357,9 @@ _mongoc_structured_log_append_int64 (bson_t *bson, const mongoc_structured_log_b const mongoc_structured_log_builder_stage_t * _mongoc_structured_log_append_boolean (bson_t *bson, const mongoc_structured_log_builder_stage_t *stage); +const mongoc_structured_log_builder_stage_t * +_mongoc_structured_log_append_error (bson_t *bson, const mongoc_structured_log_builder_stage_t *stage); + const mongoc_structured_log_builder_stage_t * _mongoc_structured_log_append_oid_as_hex (bson_t *bson, const mongoc_structured_log_builder_stage_t *stage); diff --git a/src/libmongoc/src/mongoc/mongoc-structured-log.c b/src/libmongoc/src/mongoc/mongoc-structured-log.c index 447ecc4f500..c9641e04fff 100644 --- a/src/libmongoc/src/mongoc/mongoc-structured-log.c +++ b/src/libmongoc/src/mongoc/mongoc-structured-log.c @@ -684,6 +684,32 @@ _mongoc_structured_log_append_cmd_name_reply (bson_t *bson, const mongoc_structu return stage + 1; } +static void +_mongoc_structured_log_append_error_contents (bson_t *bson, const bson_error_t *error) +{ + BSON_APPEND_INT32 (bson, "code", error->code); + BSON_APPEND_INT32 (bson, "domain", error->domain); + BSON_APPEND_UTF8 (bson, "message", error->message); +} + +const mongoc_structured_log_builder_stage_t * +_mongoc_structured_log_append_error (bson_t *bson, const mongoc_structured_log_builder_stage_t *stage) +{ + const char *key_or_null = stage->arg1.utf8; + const bson_error_t *error_or_null = stage->arg2.error; + if (key_or_null) { + if (error_or_null) { + bson_t child; + BSON_ASSERT (BSON_APPEND_DOCUMENT_BEGIN (bson, key_or_null, &child)); + _mongoc_structured_log_append_error_contents (&child, error_or_null); + BSON_ASSERT (bson_append_document_end (bson, &child)); + } else { + bson_append_null (bson, key_or_null, -1); + } + } + return stage + 1; +} + static void _mongoc_structured_log_append_redacted_cmd_failure (bson_t *bson, bool is_sensitive, @@ -713,9 +739,7 @@ _mongoc_structured_log_append_redacted_cmd_failure (bson_t *bson, // Client-side errors converted directly from bson_error_t, never redacted bson_t failure; BSON_ASSERT (BSON_APPEND_DOCUMENT_BEGIN (bson, "failure", &failure)); - BSON_APPEND_INT32 (&failure, "code", error->code); - BSON_APPEND_INT32 (&failure, "domain", error->domain); - BSON_APPEND_UTF8 (&failure, "message", error->message); + _mongoc_structured_log_append_error_contents (&failure, error); BSON_ASSERT (bson_append_document_end (bson, &failure)); } } diff --git a/src/libmongoc/tests/test-mongoc-structured-log.c b/src/libmongoc/tests/test-mongoc-structured-log.c index e58d702bb9b..67b7872fd82 100644 --- a/src/libmongoc/tests/test-mongoc-structured-log.c +++ b/src/libmongoc/tests/test-mongoc-structured-log.c @@ -44,6 +44,8 @@ static BSON_INLINE void restore_state (structured_log_state state) { mongoc_structured_log_set_handler (state.handler, state.data); + mongoc_structured_log_set_max_level_for_all_components (MONGOC_STRUCTURED_LOG_LEVEL_DEBUG); + mongoc_structured_log_set_max_levels_from_env (); } static void @@ -92,6 +94,7 @@ test_structured_log_plain (void) structured_log_state old_state = save_state (); mongoc_structured_log_set_handler (structured_log_func, &assumption); + mongoc_structured_log_set_max_level_for_all_components (MONGOC_STRUCTURED_LOG_LEVEL_DEBUG); mongoc_structured_log ( MONGOC_STRUCTURED_LOG_LEVEL_WARNING, MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND, "Plain log entry"); @@ -115,6 +118,7 @@ test_structured_log_plain_with_extra_data (void) structured_log_state old_state = save_state (); mongoc_structured_log_set_handler (structured_log_func, &assumption); + mongoc_structured_log_set_max_level_for_all_components (MONGOC_STRUCTURED_LOG_LEVEL_DEBUG); mongoc_structured_log (MONGOC_STRUCTURED_LOG_LEVEL_WARNING, MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND, @@ -165,6 +169,7 @@ test_structured_log_basic_data_types (void) structured_log_state old_state = save_state (); mongoc_structured_log_set_handler (structured_log_func, &assumption); + mongoc_structured_log_set_max_level_for_all_components (MONGOC_STRUCTURED_LOG_LEVEL_DEBUG); mongoc_structured_log (MONGOC_STRUCTURED_LOG_LEVEL_WARNING, MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND, @@ -213,6 +218,7 @@ test_structured_log_json (void) structured_log_state old_state = save_state (); mongoc_structured_log_set_handler (structured_log_func, &assumption); + mongoc_structured_log_set_max_level_for_all_components (MONGOC_STRUCTURED_LOG_LEVEL_DEBUG); mongoc_structured_log (MONGOC_STRUCTURED_LOG_LEVEL_WARNING, MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND, @@ -249,6 +255,7 @@ test_structured_log_oid (void) structured_log_state old_state = save_state (); mongoc_structured_log_set_handler (structured_log_func, &assumption); + mongoc_structured_log_set_max_level_for_all_components (MONGOC_STRUCTURED_LOG_LEVEL_DEBUG); mongoc_structured_log (MONGOC_STRUCTURED_LOG_LEVEL_WARNING, MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND, @@ -263,6 +270,52 @@ test_structured_log_oid (void) bson_destroy (assumption.expected_bson); } +void +test_structured_log_error (void) +{ + struct log_assumption assumption = { + .expected_envelope.level = MONGOC_STRUCTURED_LOG_LEVEL_INFO, + .expected_envelope.component = MONGOC_STRUCTURED_LOG_COMPONENT_SERVER_SELECTION, + .expected_envelope.message = "Log entry with bson_error_t values", + .expected_bson = BCON_NEW ("message", + BCON_UTF8 ("Log entry with bson_error_t values"), + "failure", + "{", + "code", + BCON_INT32 (0xabab5555), + "domain", + BCON_INT32 (0x87654321), + "message", + BCON_UTF8 ("Some Text"), + "}", + "null", + BCON_NULL), + .expected_calls = 1, + }; + + const bson_error_t err = { + .domain = 0x87654321, + .code = 0xabab5555, + .message = "Some Text", + }; + + structured_log_state old_state = save_state (); + mongoc_structured_log_set_handler (structured_log_func, &assumption); + mongoc_structured_log_set_max_level_for_all_components (MONGOC_STRUCTURED_LOG_LEVEL_INFO); + + mongoc_structured_log (MONGOC_STRUCTURED_LOG_LEVEL_INFO, + MONGOC_STRUCTURED_LOG_COMPONENT_SERVER_SELECTION, + "Log entry with bson_error_t values", + error ("failure", &err), + error (NULL, NULL), + error ("null", NULL)); + + int calls = mcommon_atomic_int_fetch (&assumption.calls_atomic, mcommon_memory_order_seq_cst); + ASSERT_CMPINT (calls, ==, 1); + restore_state (old_state); + bson_destroy (assumption.expected_bson); +} + void test_structured_log_server_description (void) { @@ -317,6 +370,7 @@ test_structured_log_server_description (void) structured_log_state old_state = save_state (); mongoc_structured_log_set_handler (structured_log_func, &assumption); + mongoc_structured_log_set_max_level_for_all_components (MONGOC_STRUCTURED_LOG_LEVEL_DEBUG); mongoc_structured_log ( MONGOC_STRUCTURED_LOG_LEVEL_WARNING, @@ -416,6 +470,7 @@ test_structured_log_command (void) structured_log_state old_state = save_state (); mongoc_structured_log_set_handler (structured_log_func, &assumption); + mongoc_structured_log_set_max_level_for_all_components (MONGOC_STRUCTURED_LOG_LEVEL_DEBUG); mongoc_structured_log (MONGOC_STRUCTURED_LOG_LEVEL_WARNING, MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND, @@ -464,6 +519,7 @@ test_structured_log_duration (void) structured_log_state old_state = save_state (); mongoc_structured_log_set_handler (structured_log_func, &assumption); + mongoc_structured_log_set_max_level_for_all_components (MONGOC_STRUCTURED_LOG_LEVEL_DEBUG); mongoc_structured_log (MONGOC_STRUCTURED_LOG_LEVEL_WARNING, MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND, @@ -577,6 +633,7 @@ test_structured_log_install (TestSuite *suite) TestSuite_Add (suite, "/structured_log/basic_data_types", test_structured_log_basic_data_types); TestSuite_Add (suite, "/structured_log/json", test_structured_log_json); TestSuite_Add (suite, "/structured_log/oid", test_structured_log_oid); + TestSuite_Add (suite, "/structured_log/error", test_structured_log_error); TestSuite_Add (suite, "/structured_log/server_description", test_structured_log_server_description); TestSuite_Add (suite, "/structured_log/command", test_structured_log_command); TestSuite_Add (suite, "/structured_log/duration", test_structured_log_duration); From 98471c5c4140b4149e15b2aed584e35a60c67351 Mon Sep 17 00:00:00 2001 From: Micah Scott Date: Wed, 27 Nov 2024 08:12:38 -0800 Subject: [PATCH 097/139] typos, null/nul One of these was correct but most of them had a copy-paste typo i missed earlier. null is a pointer, nul is an ascii character. --- .../src/mongoc/mongoc-structured-log-private.h | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/libmongoc/src/mongoc/mongoc-structured-log-private.h b/src/libmongoc/src/mongoc/mongoc-structured-log-private.h index 76f00942eb0..679bc1477a1 100644 --- a/src/libmongoc/src/mongoc/mongoc-structured-log-private.h +++ b/src/libmongoc/src/mongoc/mongoc-structured-log-private.h @@ -89,7 +89,7 @@ BSON_BEGIN_DECLS * @def utf8_n(key, value, value_len) * @brief Structured log item, referencing a utf8 string value with explicit length. * - * @param key Document key as a NULL-terminated const char * expression, or NULL to skip this item. + * @param key Document key as a NUL-terminated const char * expression, or NULL to skip this item. * @param value UTF8 value as a const char * expression, or NULL for a null value. May have embedded NUL bytes. * @param value_len UTF8 value length in bytes, as an int32_t expression. */ @@ -100,7 +100,7 @@ BSON_BEGIN_DECLS * @def utf8_nn(key, key_len, value, value_len) * @brief Structured log item, referencing a utf8 string with explicit key and value lengths. * - * @param key Key as a NULL-terminated const char * expression, or NULL to skip this item. + * @param key Key as a NUL-terminated const char * expression, or NULL to skip this item. * @param value UTF8 value as a const char * expression, or NULL for a null value. May have embedded NUL bytes. * @param value_len UTF8 value length in bytes, as an int32_t expression. */ @@ -112,7 +112,7 @@ BSON_BEGIN_DECLS * @def int32(key, value) * @brief Structured log item, 32-bit integer * - * @param key Key as a NULL-terminated const char * expression, or NULL to skip this item. + * @param key Key as a NUL-terminated const char * expression, or NULL to skip this item. * @param value Value as an int32_t expression. */ #define _mongoc_structured_log_item_int32(_key_or_null, _value_int32) \ @@ -122,7 +122,7 @@ BSON_BEGIN_DECLS * @def int64(key, value) * @brief Structured log item, 64-bit integer * - * @param key Key as a NULL-terminated const char * expression, or NULL to skip this item. + * @param key Key as a NUL-terminated const char * expression, or NULL to skip this item. * @param value Value as an int64_t expression. */ #define _mongoc_structured_log_item_int64(_key_or_null, _value_int64) \ @@ -132,7 +132,7 @@ BSON_BEGIN_DECLS * @def boolean(key, value) * @brief Structured log item, boolean * - * @param key Key as a NULL-terminated const char * expression, or NULL to skip this item. + * @param key Key as a NUL-terminated const char * expression, or NULL to skip this item. * @param value Value as a bool expression. */ #define _mongoc_structured_log_item_boolean(_key_or_null, _value_boolean) \ @@ -144,7 +144,7 @@ BSON_BEGIN_DECLS * * Serializes the fields of a bson_error_t as a subdocument with the indicated key. * - * @param key Key as a NULL-terminated const char * expression, or NULL to skip this item. + * @param key Key as a NUL-terminated const char * expression, or NULL to skip this item. * @param value Error as a const bson_error_t * expression, or NULL for a null value. */ #define _mongoc_structured_log_item_error(_key_or_null, _error_or_null) \ @@ -154,7 +154,7 @@ BSON_BEGIN_DECLS * @def oid_as_hex(key, value) * @brief Structured log item, bson_oid_t converted to a hex string * - * @param key Key as a NULL-terminated const char * expression, or NULL to skip this item. + * @param key Key as a NUL-terminated const char * expression, or NULL to skip this item. * @param value OID to convert as a const bson_oid_t * expression, or NULL for a null value. */ #define _mongoc_structured_log_item_oid_as_hex(_key_or_null, _value_oid) \ @@ -167,7 +167,7 @@ BSON_BEGIN_DECLS * Always uses relaxed extended JSON format, and the current applicable * maximum document length for structured logging. * - * @param key Key as a NULL-terminated const char * expression, or NULL to skip this item. + * @param key Key as a NUL-terminated const char * expression, or NULL to skip this item. * @param value BSON to convert as a const bson_t * expression, or NULL for a null value. */ #define _mongoc_structured_log_item_bson_as_json(_key_or_null, _value_bson) \ From 1957ebbd759254fc055a1e555a9f5afab411c49a Mon Sep 17 00:00:00 2001 From: Micah Scott Date: Wed, 27 Nov 2024 09:01:35 -0800 Subject: [PATCH 098/139] Let's start a trend of making bson serialization a shared internal API Serialization will be needed for other data types too, and this seems better than having each consumer do their own bson serialization. This design appends fields to an existing bson document. Chose the name to be distinct from 'serialize' (which is vague, and often used in a concurrency context in libmongoc) and 'as_bson' which is not clear about whether the document is being allocated or not. --- .../src/mongoc/mongoc-error-private.h | 3 +++ src/libmongoc/src/mongoc/mongoc-error.c | 9 +++++++ .../src/mongoc/mongoc-structured-log.c | 24 +++++++------------ 3 files changed, 20 insertions(+), 16 deletions(-) diff --git a/src/libmongoc/src/mongoc/mongoc-error-private.h b/src/libmongoc/src/mongoc/mongoc-error-private.h index f3bd4f0578b..d7d37bc84bc 100644 --- a/src/libmongoc/src/mongoc/mongoc-error-private.h +++ b/src/libmongoc/src/mongoc/mongoc-error-private.h @@ -93,4 +93,7 @@ _mongoc_error_is_auth (const bson_error_t *error); void _mongoc_error_append (bson_error_t *error, const char *s); +bool +_mongoc_error_append_contents_to_bson (const bson_error_t *error, bson_t *doc_in_out); + BSON_END_DECLS diff --git a/src/libmongoc/src/mongoc/mongoc-error.c b/src/libmongoc/src/mongoc/mongoc-error.c index 8b172d430f3..3a17afc60a3 100644 --- a/src/libmongoc/src/mongoc/mongoc-error.c +++ b/src/libmongoc/src/mongoc/mongoc-error.c @@ -320,3 +320,12 @@ _mongoc_error_append (bson_error_t *error, const char *s) const size_t remaining = sizeof (error->message) - error_len; bson_strncpy (error->message + error_len, s, remaining); } + +bool +_mongoc_error_append_contents_to_bson (const bson_error_t *error, bson_t *bson) +{ + BSON_ASSERT_PARAM (error); + BSON_ASSERT_PARAM (bson); + return (BSON_APPEND_INT32 (bson, "code", error->code) && BSON_APPEND_INT32 (bson, "domain", error->domain) && + BSON_APPEND_UTF8 (bson, "message", error->message)); +} diff --git a/src/libmongoc/src/mongoc/mongoc-structured-log.c b/src/libmongoc/src/mongoc/mongoc-structured-log.c index c9641e04fff..ff0e6574d0e 100644 --- a/src/libmongoc/src/mongoc/mongoc-structured-log.c +++ b/src/libmongoc/src/mongoc/mongoc-structured-log.c @@ -14,14 +14,14 @@ * limitations under the License. */ -#include "mongoc-structured-log.h" -#include "mongoc-structured-log-private.h" -#include "mongoc-thread-private.h" -#include "mongoc-util-private.h" -#include "mongoc-apm-private.h" #include "common-atomic-private.h" -#include "common-thread-private.h" #include "common-oid-private.h" +#include "common-thread-private.h" +#include "mongoc-apm-private.h" +#include "mongoc-error-private.h" +#include "mongoc-structured-log-private.h" +#include "mongoc-structured-log.h" +#include "mongoc-util-private.h" #define STRUCTURED_LOG_COMPONENT_TABLE_SIZE (1 + (size_t) MONGOC_STRUCTURED_LOG_COMPONENT_CONNECTION) @@ -684,14 +684,6 @@ _mongoc_structured_log_append_cmd_name_reply (bson_t *bson, const mongoc_structu return stage + 1; } -static void -_mongoc_structured_log_append_error_contents (bson_t *bson, const bson_error_t *error) -{ - BSON_APPEND_INT32 (bson, "code", error->code); - BSON_APPEND_INT32 (bson, "domain", error->domain); - BSON_APPEND_UTF8 (bson, "message", error->message); -} - const mongoc_structured_log_builder_stage_t * _mongoc_structured_log_append_error (bson_t *bson, const mongoc_structured_log_builder_stage_t *stage) { @@ -701,7 +693,7 @@ _mongoc_structured_log_append_error (bson_t *bson, const mongoc_structured_log_b if (error_or_null) { bson_t child; BSON_ASSERT (BSON_APPEND_DOCUMENT_BEGIN (bson, key_or_null, &child)); - _mongoc_structured_log_append_error_contents (&child, error_or_null); + BSON_ASSERT (_mongoc_error_append_contents_to_bson (error_or_null, &child)); BSON_ASSERT (bson_append_document_end (bson, &child)); } else { bson_append_null (bson, key_or_null, -1); @@ -739,7 +731,7 @@ _mongoc_structured_log_append_redacted_cmd_failure (bson_t *bson, // Client-side errors converted directly from bson_error_t, never redacted bson_t failure; BSON_ASSERT (BSON_APPEND_DOCUMENT_BEGIN (bson, "failure", &failure)); - _mongoc_structured_log_append_error_contents (&failure, error); + BSON_ASSERT (_mongoc_error_append_contents_to_bson (error, &failure)); BSON_ASSERT (bson_append_document_end (bson, &failure)); } } From 686fb1ced61c54ab544afe1bc87bc2e71d52636f Mon Sep 17 00:00:00 2001 From: Micah Scott Date: Wed, 27 Nov 2024 09:13:03 -0800 Subject: [PATCH 099/139] Don't ASSERT in the event a log message is truncated to the max BSON document size --- .../src/mongoc/mongoc-structured-log.c | 29 ++++++++++--------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/src/libmongoc/src/mongoc/mongoc-structured-log.c b/src/libmongoc/src/mongoc/mongoc-structured-log.c index ff0e6574d0e..d670741a589 100644 --- a/src/libmongoc/src/mongoc/mongoc-structured-log.c +++ b/src/libmongoc/src/mongoc/mongoc-structured-log.c @@ -692,9 +692,10 @@ _mongoc_structured_log_append_error (bson_t *bson, const mongoc_structured_log_b if (key_or_null) { if (error_or_null) { bson_t child; - BSON_ASSERT (BSON_APPEND_DOCUMENT_BEGIN (bson, key_or_null, &child)); - BSON_ASSERT (_mongoc_error_append_contents_to_bson (error_or_null, &child)); - BSON_ASSERT (bson_append_document_end (bson, &child)); + if (BSON_APPEND_DOCUMENT_BEGIN (bson, key_or_null, &child)) { + _mongoc_error_append_contents_to_bson (error_or_null, &child); + bson_append_document_end (bson, &child); + } } else { bson_append_null (bson, key_or_null, -1); } @@ -714,15 +715,16 @@ _mongoc_structured_log_append_redacted_cmd_failure (bson_t *bson, // Redacted server-side message, must be a document with at most 'code', 'codeName', 'errorLabels' bson_t failure; bson_iter_t iter; - BSON_ASSERT (BSON_APPEND_DOCUMENT_BEGIN (bson, "failure", &failure)); - bson_iter_init (&iter, reply); - while (bson_iter_next (&iter)) { - const char *key = bson_iter_key (&iter); - if (!strcmp (key, "code") || !strcmp (key, "codeName") || !strcmp (key, "errorLabels")) { - bson_append_iter (&failure, key, bson_iter_key_len (&iter), &iter); + if (BSON_APPEND_DOCUMENT_BEGIN (bson, "failure", &failure)) { + bson_iter_init (&iter, reply); + while (bson_iter_next (&iter)) { + const char *key = bson_iter_key (&iter); + if (!strcmp (key, "code") || !strcmp (key, "codeName") || !strcmp (key, "errorLabels")) { + bson_append_iter (&failure, key, bson_iter_key_len (&iter), &iter); + } } + bson_append_document_end (bson, &failure); } - BSON_ASSERT (bson_append_document_end (bson, &failure)); } else { // Non-redacted server side message, pass through BSON_APPEND_DOCUMENT (bson, "failure", reply); @@ -730,9 +732,10 @@ _mongoc_structured_log_append_redacted_cmd_failure (bson_t *bson, } else { // Client-side errors converted directly from bson_error_t, never redacted bson_t failure; - BSON_ASSERT (BSON_APPEND_DOCUMENT_BEGIN (bson, "failure", &failure)); - BSON_ASSERT (_mongoc_error_append_contents_to_bson (error, &failure)); - BSON_ASSERT (bson_append_document_end (bson, &failure)); + if (BSON_APPEND_DOCUMENT_BEGIN (bson, "failure", &failure)) { + _mongoc_error_append_contents_to_bson (error, &failure); + bson_append_document_end (bson, &failure); + } } } From d647e486cef73ccd3959546a14ddf8ff32ea0ed9 Mon Sep 17 00:00:00 2001 From: Micah Scott Date: Wed, 27 Nov 2024 11:46:34 -0800 Subject: [PATCH 100/139] refactor shared private serialization This changes responsibilities somewhat, and makes the serialization used by logging shareable with other systems that need serialization, like the unified test runner. Now every type of content is selected explicitly using a bit flag, the same flags that carry over from inside structured logging items. Serialization moves out of mongoc-structured-log.c into individual implementation files. --- src/libmongoc/src/mongoc/mongoc-cmd-private.h | 12 +++ src/libmongoc/src/mongoc/mongoc-cmd.c | 56 ++++++++++++++ .../src/mongoc/mongoc-error-private.h | 13 +++- src/libmongoc/src/mongoc/mongoc-error.c | 15 +++- .../mongoc-server-description-private.h | 12 +++ .../src/mongoc/mongoc-server-description.c | 36 +++++++++ .../mongoc/mongoc-structured-log-private.h | 28 ++----- .../src/mongoc/mongoc-structured-log.c | 77 +++---------------- 8 files changed, 160 insertions(+), 89 deletions(-) diff --git a/src/libmongoc/src/mongoc/mongoc-cmd-private.h b/src/libmongoc/src/mongoc/mongoc-cmd-private.h index 6ceedad6b88..d150c677656 100644 --- a/src/libmongoc/src/mongoc/mongoc-cmd-private.h +++ b/src/libmongoc/src/mongoc/mongoc-cmd-private.h @@ -143,6 +143,18 @@ _mongoc_cmd_append_payload_as_array (const mongoc_cmd_t *cmd, bson_t *out); void _mongoc_cmd_append_server_api (bson_t *command_body, const mongoc_server_api_t *api); +typedef enum { + MONGOC_LOGGED_CMD_CONTENT_FLAG_COMMAND = (1 << 0), + MONGOC_LOGGED_CMD_CONTENT_FLAG_DATABASE_NAME = (1 << 1), + MONGOC_LOGGED_CMD_CONTENT_FLAG_COMMAND_NAME = (1 << 2), + MONGOC_LOGGED_CMD_CONTENT_FLAG_OPERATION_ID = (1 << 3), +} mongoc_logged_cmd_content_flags_t; + +bool +mongoc_cmd_append_logged_contents_to_bson (const mongoc_cmd_t *cmd, + bson_t *bson, + mongoc_logged_cmd_content_flags_t flags); + BSON_END_DECLS diff --git a/src/libmongoc/src/mongoc/mongoc-cmd.c b/src/libmongoc/src/mongoc/mongoc-cmd.c index 9817a8d493a..8d9f229997f 100644 --- a/src/libmongoc/src/mongoc/mongoc-cmd.c +++ b/src/libmongoc/src/mongoc/mongoc-cmd.c @@ -15,12 +15,14 @@ */ +#include "mongoc-apm-private.h" #include "mongoc-cmd-private.h" #include "mongoc-read-prefs-private.h" #include "mongoc-trace-private.h" #include "mongoc-client-private.h" #include "mongoc-read-concern-private.h" #include "mongoc-server-api-private.h" +#include "mongoc-structured-log-private.h" #include "mongoc-write-concern-private.h" /* For strcasecmp on Windows */ #include "mongoc-util-private.h" @@ -1035,3 +1037,57 @@ _mongoc_cmd_append_server_api (bson_t *command_body, const mongoc_server_api_t * bson_append_bool (command_body, "apiDeprecationErrors", -1, api->deprecation_errors.value); } } + +bool +mongoc_cmd_append_logged_contents_to_bson (const mongoc_cmd_t *cmd, + bson_t *bson, + mongoc_logged_cmd_content_flags_t flags) +{ + BSON_ASSERT_PARAM (cmd); + + if ((flags & MONGOC_LOGGED_CMD_CONTENT_FLAG_DATABASE_NAME) && + !BSON_APPEND_UTF8 (bson, "databaseName", cmd->db_name)) { + return false; + } + if ((flags & MONGOC_LOGGED_CMD_CONTENT_FLAG_COMMAND_NAME) && + !BSON_APPEND_UTF8 (bson, "commandName", cmd->command_name)) { + return false; + } + if ((flags & MONGOC_LOGGED_CMD_CONTENT_FLAG_OPERATION_ID) && + !BSON_APPEND_INT64 (bson, "operationId", cmd->operation_id)) { + return false; + } + if (flags & MONGOC_LOGGED_CMD_CONTENT_FLAG_COMMAND) { + if (mongoc_apm_is_sensitive_command_message (cmd->command_name, cmd->command)) { + if (!BSON_APPEND_UTF8 (bson, "command", "{}")) { + return false; + } + } else { + bson_t *command_copy = NULL; + + if (cmd->payloads_count > 0) { + // @todo This is a performance bottleneck, we shouldn't be copying + // a potentially large command to serialize a potentially very + // small part of it. We should be appending JSON to a single buffer + // for all nesting levels, constrained by length limit, while visiting + // borrowed references to each command attribute and each payload. CDRIVER-4814 + command_copy = bson_copy (cmd->command); + _mongoc_cmd_append_payload_as_array (cmd, command_copy); + } + + size_t json_length; + char *json = mongoc_structured_log_document_to_json (command_copy ? command_copy : cmd->command, &json_length); + bool ok = false; + if (json) { + const char *key = "command"; + ok = bson_append_utf8 (bson, key, strlen (key), json, json_length); + bson_free (json); + } + bson_destroy (command_copy); + if (!ok) { + return false; + } + } + } + return true; +} diff --git a/src/libmongoc/src/mongoc/mongoc-error-private.h b/src/libmongoc/src/mongoc/mongoc-error-private.h index d7d37bc84bc..eb9310f5c72 100644 --- a/src/libmongoc/src/mongoc/mongoc-error-private.h +++ b/src/libmongoc/src/mongoc/mongoc-error-private.h @@ -16,6 +16,9 @@ #include "mongoc-prelude.h" +#ifndef MONGOC_ERROR_PRIVATE_H +#define MONGOC_ERROR_PRIVATE_H + #include #include @@ -93,7 +96,15 @@ _mongoc_error_is_auth (const bson_error_t *error); void _mongoc_error_append (bson_error_t *error, const char *s); +typedef enum { + MONGOC_ERROR_CONTENT_FLAG_CODE = (1 << 0), + MONGOC_ERROR_CONTENT_FLAG_DOMAIN = (1 << 1), + MONGOC_ERROR_CONTENT_FLAG_MESSAGE = (1 << 2), +} mongoc_error_content_flags_t; + bool -_mongoc_error_append_contents_to_bson (const bson_error_t *error, bson_t *doc_in_out); +mongoc_error_append_contents_to_bson (const bson_error_t *error, bson_t *bson, mongoc_error_content_flags_t flags); BSON_END_DECLS + +#endif /* MONGOC_ERROR_PRIVATE_H */ diff --git a/src/libmongoc/src/mongoc/mongoc-error.c b/src/libmongoc/src/mongoc/mongoc-error.c index 3a17afc60a3..5fd6998f57c 100644 --- a/src/libmongoc/src/mongoc/mongoc-error.c +++ b/src/libmongoc/src/mongoc/mongoc-error.c @@ -322,10 +322,19 @@ _mongoc_error_append (bson_error_t *error, const char *s) } bool -_mongoc_error_append_contents_to_bson (const bson_error_t *error, bson_t *bson) +mongoc_error_append_contents_to_bson (const bson_error_t *error, bson_t *bson, mongoc_error_content_flags_t flags) { BSON_ASSERT_PARAM (error); BSON_ASSERT_PARAM (bson); - return (BSON_APPEND_INT32 (bson, "code", error->code) && BSON_APPEND_INT32 (bson, "domain", error->domain) && - BSON_APPEND_UTF8 (bson, "message", error->message)); + + if ((flags & MONGOC_ERROR_CONTENT_FLAG_CODE) && !BSON_APPEND_INT32 (bson, "code", error->code)) { + return false; + } + if ((flags & MONGOC_ERROR_CONTENT_FLAG_DOMAIN) && !BSON_APPEND_INT32 (bson, "domain", error->domain)) { + return false; + } + if ((flags & MONGOC_ERROR_CONTENT_FLAG_MESSAGE) && !BSON_APPEND_UTF8 (bson, "message", error->message)) { + return false; + } + return true; } diff --git a/src/libmongoc/src/mongoc/mongoc-server-description-private.h b/src/libmongoc/src/mongoc/mongoc-server-description-private.h index 17391404181..125718b68c8 100644 --- a/src/libmongoc/src/mongoc/mongoc-server-description-private.h +++ b/src/libmongoc/src/mongoc/mongoc-server-description-private.h @@ -221,4 +221,16 @@ mongoc_server_description_set_topology_version (mongoc_server_description_t *sd, bool mongoc_server_description_has_service_id (const mongoc_server_description_t *description); +typedef enum { + MONGOC_SERVER_DESCRIPTION_CONTENT_FLAG_SERVER_HOST = (1 << 0), + MONGOC_SERVER_DESCRIPTION_CONTENT_FLAG_SERVER_PORT = (1 << 1), + MONGOC_SERVER_DESCRIPTION_CONTENT_FLAG_SERVER_CONNECTION_ID = (1 << 2), + MONGOC_SERVER_DESCRIPTION_CONTENT_FLAG_SERVICE_ID = (1 << 3), +} mongoc_server_description_content_flags_t; + +bool +mongoc_server_description_append_contents_to_bson (const mongoc_server_description_t *sd, + bson_t *bson, + mongoc_server_description_content_flags_t flags); + #endif diff --git a/src/libmongoc/src/mongoc/mongoc-server-description.c b/src/libmongoc/src/mongoc/mongoc-server-description.c index 5fd94e36992..17d962031ae 100644 --- a/src/libmongoc/src/mongoc/mongoc-server-description.c +++ b/src/libmongoc/src/mongoc/mongoc-server-description.c @@ -1221,3 +1221,39 @@ mongoc_server_description_has_service_id (const mongoc_server_description_t *des { return !mcommon_oid_is_zero (&description->service_id); } + +bool +mongoc_server_description_append_contents_to_bson (const mongoc_server_description_t *sd, + bson_t *bson, + mongoc_server_description_content_flags_t flags) +{ + BSON_ASSERT_PARAM (sd); + BSON_ASSERT_PARAM (bson); + + if ((flags & MONGOC_SERVER_DESCRIPTION_CONTENT_FLAG_SERVER_HOST) && + !BSON_APPEND_UTF8 (bson, "serverHost", sd->host.host)) { + return false; + } + if ((flags & MONGOC_SERVER_DESCRIPTION_CONTENT_FLAG_SERVER_PORT) && + !BSON_APPEND_INT32 (bson, "serverPort", sd->host.port)) { + return false; + } + if (flags & MONGOC_SERVER_DESCRIPTION_CONTENT_FLAG_SERVER_CONNECTION_ID) { + int64_t server_connection_id = sd->server_connection_id; + if (MONGOC_NO_SERVER_CONNECTION_ID != server_connection_id) { + if (!BSON_APPEND_INT64 (bson, "serverConnectionId", server_connection_id)) { + return false; + } + } + } + if (flags & MONGOC_SERVER_DESCRIPTION_CONTENT_FLAG_SERVICE_ID) { + if (mongoc_server_description_has_service_id (sd)) { + char str[25]; + bson_oid_to_string (&sd->service_id, str); + if (!BSON_APPEND_UTF8 (bson, "serviceId", str)) { + return false; + } + } + } + return true; +} diff --git a/src/libmongoc/src/mongoc/mongoc-structured-log-private.h b/src/libmongoc/src/mongoc/mongoc-structured-log-private.h index 679bc1477a1..6ea1ea99903 100644 --- a/src/libmongoc/src/mongoc/mongoc-structured-log-private.h +++ b/src/libmongoc/src/mongoc/mongoc-structured-log-private.h @@ -21,9 +21,10 @@ #include #include -#include "mongoc-structured-log.h" #include "mongoc-cmd-private.h" +#include "mongoc-error-private.h" #include "mongoc-server-description-private.h" +#include "mongoc-structured-log.h" BSON_BEGIN_DECLS @@ -184,7 +185,7 @@ BSON_BEGIN_DECLS {.func = _mongoc_structured_log_append_cmd, \ .arg1.cmd = (_cmd), \ .arg2.cmd_flags = \ - (0 _bsonDSL_mapMacro (_mongoc_structured_log_flag_expr, MONGOC_STRUCTURED_LOG_CMD, __VA_ARGS__))}, + (0 _bsonDSL_mapMacro (_mongoc_structured_log_flag_expr, MONGOC_LOGGED_CMD_CONTENT_FLAG, __VA_ARGS__))}, /** * @def cmd_reply(cmd, reply) @@ -255,8 +256,8 @@ BSON_BEGIN_DECLS #define _mongoc_structured_log_item_server_description(_server_description, ...) \ {.func = _mongoc_structured_log_append_server_description, \ .arg1.server_description = (_server_description), \ - .arg2.server_description_flags = (0 _bsonDSL_mapMacro ( \ - _mongoc_structured_log_flag_expr, MONGOC_STRUCTURED_LOG_SERVER_DESCRIPTION, __VA_ARGS__))}, + .arg2.server_description_flags = \ + (0 _bsonDSL_mapMacro (_mongoc_structured_log_flag_expr, MONGOC_SERVER_DESCRIPTION_CONTENT_FLAG, __VA_ARGS__))}, /** * @def monotonic_time_duration(duration) @@ -274,20 +275,6 @@ typedef struct mongoc_structured_log_builder_stage_t mongoc_structured_log_build typedef const mongoc_structured_log_builder_stage_t *(*mongoc_structured_log_builder_func_t) ( bson_t *bson, const mongoc_structured_log_builder_stage_t *stage); -typedef enum { - MONGOC_STRUCTURED_LOG_CMD_COMMAND = (1 << 0), - MONGOC_STRUCTURED_LOG_CMD_DATABASE_NAME = (1 << 1), - MONGOC_STRUCTURED_LOG_CMD_COMMAND_NAME = (1 << 2), - MONGOC_STRUCTURED_LOG_CMD_OPERATION_ID = (1 << 3), -} mongoc_structured_log_cmd_flags_t; - -typedef enum { - MONGOC_STRUCTURED_LOG_SERVER_DESCRIPTION_SERVER_HOST = (1 << 0), - MONGOC_STRUCTURED_LOG_SERVER_DESCRIPTION_SERVER_PORT = (1 << 1), - MONGOC_STRUCTURED_LOG_SERVER_DESCRIPTION_SERVER_CONNECTION_ID = (1 << 2), - MONGOC_STRUCTURED_LOG_SERVER_DESCRIPTION_SERVICE_ID = (1 << 3), -} mongoc_structured_log_server_description_flags_t; - struct mongoc_structured_log_builder_stage_t { // Why "stages" instead of a variable size argument list per item? // This approach keeps function pointers and other types of data @@ -310,8 +297,9 @@ struct mongoc_structured_log_builder_stage_t { const char *utf8; int32_t int32; int64_t int64; - mongoc_structured_log_cmd_flags_t cmd_flags; - mongoc_structured_log_server_description_flags_t server_description_flags; + mongoc_error_content_flags_t error_flags; + mongoc_logged_cmd_content_flags_t cmd_flags; + mongoc_server_description_content_flags_t server_description_flags; } arg2; // Avoid adding an arg3, prefer to use additional stages }; diff --git a/src/libmongoc/src/mongoc/mongoc-structured-log.c b/src/libmongoc/src/mongoc/mongoc-structured-log.c index d670741a589..7e33ca40aba 100644 --- a/src/libmongoc/src/mongoc/mongoc-structured-log.c +++ b/src/libmongoc/src/mongoc/mongoc-structured-log.c @@ -596,46 +596,8 @@ const mongoc_structured_log_builder_stage_t * _mongoc_structured_log_append_cmd (bson_t *bson, const mongoc_structured_log_builder_stage_t *stage) { const mongoc_cmd_t *cmd = stage->arg1.cmd; - const mongoc_structured_log_cmd_flags_t flags = stage->arg2.cmd_flags; - - BSON_ASSERT (cmd); - - if (flags & MONGOC_STRUCTURED_LOG_CMD_DATABASE_NAME) { - BSON_APPEND_UTF8 (bson, "databaseName", cmd->db_name); - } - if (flags & MONGOC_STRUCTURED_LOG_CMD_COMMAND_NAME) { - BSON_APPEND_UTF8 (bson, "commandName", cmd->command_name); - } - if (flags & MONGOC_STRUCTURED_LOG_CMD_OPERATION_ID) { - BSON_APPEND_INT64 (bson, "operationId", cmd->operation_id); - } - if (flags & MONGOC_STRUCTURED_LOG_CMD_COMMAND) { - if (mongoc_apm_is_sensitive_command_message (cmd->command_name, cmd->command)) { - BSON_APPEND_UTF8 (bson, "command", "{}"); - } else { - bson_t *command_copy = NULL; - - if (cmd->payloads_count > 0) { - // @todo This is a performance bottleneck, we shouldn't be copying - // a potentially large command to serialize a potentially very - // small part of it. We should be appending JSON to a single buffer - // for all nesting levels, constrained by length limit, while visiting - // borrowed references to each command attribute and each payload. CDRIVER-4814 - command_copy = bson_copy (cmd->command); - _mongoc_cmd_append_payload_as_array (cmd, command_copy); - } - - size_t json_length; - char *json = mongoc_structured_log_document_to_json (command_copy ? command_copy : cmd->command, &json_length); - if (json) { - const char *key = "command"; - bson_append_utf8 (bson, key, strlen (key), json, json_length); - bson_free (json); - } - - bson_destroy (command_copy); - } - } + const mongoc_logged_cmd_content_flags_t flags = stage->arg2.cmd_flags; + mongoc_cmd_append_logged_contents_to_bson (cmd, bson, flags); return stage + 1; } @@ -693,7 +655,10 @@ _mongoc_structured_log_append_error (bson_t *bson, const mongoc_structured_log_b if (error_or_null) { bson_t child; if (BSON_APPEND_DOCUMENT_BEGIN (bson, key_or_null, &child)) { - _mongoc_error_append_contents_to_bson (error_or_null, &child); + mongoc_error_append_contents_to_bson (error_or_null, + &child, + MONGOC_ERROR_CONTENT_FLAG_MESSAGE | MONGOC_ERROR_CONTENT_FLAG_CODE | + MONGOC_ERROR_CONTENT_FLAG_DOMAIN); bson_append_document_end (bson, &child); } } else { @@ -733,7 +698,10 @@ _mongoc_structured_log_append_redacted_cmd_failure (bson_t *bson, // Client-side errors converted directly from bson_error_t, never redacted bson_t failure; if (BSON_APPEND_DOCUMENT_BEGIN (bson, "failure", &failure)) { - _mongoc_error_append_contents_to_bson (error, &failure); + mongoc_error_append_contents_to_bson (error, + &failure, + MONGOC_ERROR_CONTENT_FLAG_MESSAGE | MONGOC_ERROR_CONTENT_FLAG_CODE | + MONGOC_ERROR_CONTENT_FLAG_DOMAIN); bson_append_document_end (bson, &failure); } } @@ -798,28 +766,7 @@ const mongoc_structured_log_builder_stage_t * _mongoc_structured_log_append_server_description (bson_t *bson, const mongoc_structured_log_builder_stage_t *stage) { const mongoc_server_description_t *sd = stage->arg1.server_description; - const mongoc_structured_log_server_description_flags_t flags = stage->arg2.server_description_flags; - - BSON_ASSERT (sd); - - if (flags & MONGOC_STRUCTURED_LOG_SERVER_DESCRIPTION_SERVER_HOST) { - BSON_APPEND_UTF8 (bson, "serverHost", sd->host.host); - } - if (flags & MONGOC_STRUCTURED_LOG_SERVER_DESCRIPTION_SERVER_PORT) { - BSON_APPEND_INT32 (bson, "serverPort", sd->host.port); - } - if (flags & MONGOC_STRUCTURED_LOG_SERVER_DESCRIPTION_SERVER_CONNECTION_ID) { - int64_t server_connection_id = sd->server_connection_id; - if (MONGOC_NO_SERVER_CONNECTION_ID != server_connection_id) { - BSON_APPEND_INT64 (bson, "serverConnectionId", server_connection_id); - } - } - if (flags & MONGOC_STRUCTURED_LOG_SERVER_DESCRIPTION_SERVICE_ID) { - if (!mcommon_oid_is_zero (&sd->service_id)) { - char str[25]; - bson_oid_to_string (&sd->service_id, str); - BSON_APPEND_UTF8 (bson, "serviceId", str); - } - } + const mongoc_server_description_content_flags_t flags = stage->arg2.server_description_flags; + mongoc_server_description_append_contents_to_bson (sd, bson, flags); return stage + 1; } From 585cdf1277588739bf19dfc7e81d657a880f650a Mon Sep 17 00:00:00 2001 From: Micah Scott Date: Tue, 3 Dec 2024 17:33:39 -0800 Subject: [PATCH 101/139] redesign structured log options and concurrency to support per-client and per-pool instances --- .../mongoc_client_pool_set_apm_callbacks.rst | 2 +- ...oc_client_pool_set_structured_log_opts.rst | 37 ++ src/libmongoc/doc/mongoc_client_pool_t.rst | 1 + .../doc/mongoc_client_set_apm_callbacks.rst | 2 +- .../mongoc_client_set_structured_log_opts.rst | 36 + src/libmongoc/doc/mongoc_client_t.rst | 1 + .../doc/mongoc_structured_log_component_t.rst | 2 +- ...goc_structured_log_entry_get_component.rst | 4 +- .../mongoc_structured_log_entry_get_level.rst | 4 +- ...c_structured_log_entry_message_as_bson.rst | 4 +- .../doc/mongoc_structured_log_func_t.rst | 23 +- ...ngoc_structured_log_get_component_name.rst | 4 +- .../mongoc_structured_log_get_level_name.rst | 4 +- ...ctured_log_get_max_level_for_component.rst | 27 - ...goc_structured_log_get_named_component.rst | 4 +- .../mongoc_structured_log_get_named_level.rst | 4 +- .../mongoc_structured_log_opts_destroy.rst | 23 + ...d_log_opts_get_max_level_for_component.rst | 30 + .../doc/mongoc_structured_log_opts_new.rst | 65 ++ ...mongoc_structured_log_opts_set_handler.rst | 36 + ..._opts_set_max_level_for_all_components.rst | 32 + ...d_log_opts_set_max_level_for_component.rst | 36 + ...tured_log_opts_set_max_levels_from_env.rst | 36 + .../doc/mongoc_structured_log_opts_t.rst | 35 + .../doc/mongoc_structured_log_set_handler.rst | 41 -- ...d_log_set_max_level_for_all_components.rst | 25 - ...ctured_log_set_max_level_for_component.rst | 29 - ...structured_log_set_max_levels_from_env.rst | 29 - src/libmongoc/doc/structured_log.rst | 66 +- .../examples/example-structured-log.c | 72 +- src/libmongoc/src/mongoc/mongoc-client-pool.c | 28 +- src/libmongoc/src/mongoc/mongoc-client-pool.h | 3 + .../src/mongoc/mongoc-client-private.h | 1 - src/libmongoc/src/mongoc/mongoc-client.c | 27 +- src/libmongoc/src/mongoc/mongoc-client.h | 3 + src/libmongoc/src/mongoc/mongoc-cluster.c | 3 + src/libmongoc/src/mongoc/mongoc-cmd-private.h | 12 - src/libmongoc/src/mongoc/mongoc-cmd.c | 54 -- .../src/mongoc/mongoc-cursor-legacy.c | 1 + src/libmongoc/src/mongoc/mongoc-cursor.c | 7 +- .../mongoc/mongoc-structured-log-private.h | 146 ++-- .../src/mongoc/mongoc-structured-log.c | 626 ++++++++++-------- .../src/mongoc/mongoc-structured-log.h | 150 +---- .../src/mongoc/mongoc-topology-private.h | 7 + src/libmongoc/src/mongoc/mongoc-topology.c | 35 + .../tests/test-mongoc-structured-log.c | 246 ++++--- src/libmongoc/tests/unified/entity-map.c | 11 +- 47 files changed, 1228 insertions(+), 846 deletions(-) create mode 100644 src/libmongoc/doc/mongoc_client_pool_set_structured_log_opts.rst create mode 100644 src/libmongoc/doc/mongoc_client_set_structured_log_opts.rst delete mode 100644 src/libmongoc/doc/mongoc_structured_log_get_max_level_for_component.rst create mode 100644 src/libmongoc/doc/mongoc_structured_log_opts_destroy.rst create mode 100644 src/libmongoc/doc/mongoc_structured_log_opts_get_max_level_for_component.rst create mode 100644 src/libmongoc/doc/mongoc_structured_log_opts_new.rst create mode 100644 src/libmongoc/doc/mongoc_structured_log_opts_set_handler.rst create mode 100644 src/libmongoc/doc/mongoc_structured_log_opts_set_max_level_for_all_components.rst create mode 100644 src/libmongoc/doc/mongoc_structured_log_opts_set_max_level_for_component.rst create mode 100644 src/libmongoc/doc/mongoc_structured_log_opts_set_max_levels_from_env.rst create mode 100644 src/libmongoc/doc/mongoc_structured_log_opts_t.rst delete mode 100644 src/libmongoc/doc/mongoc_structured_log_set_handler.rst delete mode 100644 src/libmongoc/doc/mongoc_structured_log_set_max_level_for_all_components.rst delete mode 100644 src/libmongoc/doc/mongoc_structured_log_set_max_level_for_component.rst delete mode 100644 src/libmongoc/doc/mongoc_structured_log_set_max_levels_from_env.rst diff --git a/src/libmongoc/doc/mongoc_client_pool_set_apm_callbacks.rst b/src/libmongoc/doc/mongoc_client_pool_set_apm_callbacks.rst index 50e794b7207..1f33f929db2 100644 --- a/src/libmongoc/doc/mongoc_client_pool_set_apm_callbacks.rst +++ b/src/libmongoc/doc/mongoc_client_pool_set_apm_callbacks.rst @@ -15,7 +15,7 @@ Synopsis Register a set of callbacks to receive Application Performance Monitoring events. -The ``callbacks`` are copied by the pool and may be destroyed at any time after. If a ``context`` is passed, it is the application's responsibility to ensure ``context`` remains valid for the lifetime of the pool. +The ``callbacks`` are copied by the pool and may be safely destroyed by the caller after this API call completes. If a ``context`` is passed, it is the application's responsibility to ensure ``context`` remains valid for the lifetime of the pool. Parameters ---------- diff --git a/src/libmongoc/doc/mongoc_client_pool_set_structured_log_opts.rst b/src/libmongoc/doc/mongoc_client_pool_set_structured_log_opts.rst new file mode 100644 index 00000000000..47e84fcb8c6 --- /dev/null +++ b/src/libmongoc/doc/mongoc_client_pool_set_structured_log_opts.rst @@ -0,0 +1,37 @@ +:man_page: mongoc_client_pool_set_structured_log_opts + +mongoc_client_pool_set_structured_log_opts() +============================================ + +Synopsis +-------- + +.. code-block:: c + + void + mongoc_client_pool_set_structured_log_opts (mongoc_client_pool_t *pool, + const mongoc_structured_log_opts_t *opts); + +Reconfigures this client pool's structured logging subsystem. See :doc:`structured_log`. + +The :symbol:`mongoc_structured_log_opts_t` is copied by the pool and may be safely destroyed by the caller after this API call completes. +The application is responsible for ensuring any ``user_data`` referenced by ``opts`` remains valid for the lifetime of the pool. + +By default, the :symbol:`mongoc_client_pool_t` will have log options captured from the environment during :symbol:`mongoc_client_pool_new`. +See :symbol:`mongoc_structured_log_opts_new` for a list of the supported options. + +The structured logging subsystem may be disabled by passing NULL as ``opts`` or equivalently by passing NULL as the :symbol:`mongoc_structured_log_func_t` in :symbol:`mongoc_structured_log_opts_set_handler`. + +Parameters +---------- + +* ``pool``: A :symbol:`mongoc_client_pool_t`. +* ``opts``: A :symbol:`mongoc_structured_log_opts_t` allocated with :symbol:`mongoc_structured_log_opts_new`, or NULL to disable structured logging. + +.. include:: includes/mongoc_client_pool_call_once.txt + +Thread safety within the handler is the application's responsibility. Handlers may be invoked concurrently by multiple pool users. + +.. seealso:: + + | :doc:`structured_log` diff --git a/src/libmongoc/doc/mongoc_client_pool_t.rst b/src/libmongoc/doc/mongoc_client_pool_t.rst index 783540fb9c5..8dbe1d8b44a 100644 --- a/src/libmongoc/doc/mongoc_client_pool_t.rst +++ b/src/libmongoc/doc/mongoc_client_pool_t.rst @@ -43,5 +43,6 @@ Example mongoc_client_pool_set_error_api mongoc_client_pool_set_server_api mongoc_client_pool_set_ssl_opts + mongoc_client_pool_set_structured_log_opts mongoc_client_pool_try_pop diff --git a/src/libmongoc/doc/mongoc_client_set_apm_callbacks.rst b/src/libmongoc/doc/mongoc_client_set_apm_callbacks.rst index 7201a8d3906..2615974ec80 100644 --- a/src/libmongoc/doc/mongoc_client_set_apm_callbacks.rst +++ b/src/libmongoc/doc/mongoc_client_set_apm_callbacks.rst @@ -15,7 +15,7 @@ Synopsis Register a set of callbacks to receive Application Performance Monitoring events. -The ``callbacks`` are copied by the client and may be destroyed at any time after. If a ``context`` is passed, it is the application's responsibility to ensure ``context`` remains valid for the lifetime of the client. +The ``callbacks`` are copied by the client and may be safely destroyed by the caller after this API call completes. If a ``context`` is passed, it is the application's responsibility to ensure ``context`` remains valid for the lifetime of the client. Parameters ---------- diff --git a/src/libmongoc/doc/mongoc_client_set_structured_log_opts.rst b/src/libmongoc/doc/mongoc_client_set_structured_log_opts.rst new file mode 100644 index 00000000000..760cf49868a --- /dev/null +++ b/src/libmongoc/doc/mongoc_client_set_structured_log_opts.rst @@ -0,0 +1,36 @@ +:man_page: mongoc_client_set_structured_log_opts + +mongoc_client_set_structured_log_opts() +======================================= + +Synopsis +-------- + +.. code-block:: c + + void + mongoc_client_set_structured_log_opts (mongoc_client_t *client, + const mongoc_structured_log_opts_t *opts); + +Reconfigures this client's structured logging subsystem. See :doc:`structured_log`. + +This function must not be called on clients that are part of a :symbol:`mongoc_client_pool_t`. +Structured logging for pools must be configured with :symbol:`mongoc_client_pool_set_structured_log_opts` before the first call to :symbol:`mongoc_client_pool_pop`. + +The :symbol:`mongoc_structured_log_opts_t` is copied by the client and may be safely destroyed by the caller after this API call completes. +The application is responsible for ensuring any ``user_data`` referenced by ``opts`` remains valid for the lifetime of the client. + +By default, the :symbol:`mongoc_client_t` will have log options captured from the environment during :symbol:`mongoc_client_new`. +See :symbol:`mongoc_structured_log_opts_new` for a list of the supported options. + +The structured logging subsystem may be disabled by passing NULL as ``opts`` or equivalently by passing NULL as the :symbol:`mongoc_structured_log_func_t` in :symbol:`mongoc_structured_log_opts_set_handler`. + +Parameters +---------- + +* ``client``: A :symbol:`mongoc_client_t`. +* ``opts``: A :symbol:`mongoc_structured_log_opts_t` allocated with :symbol:`mongoc_structured_log_opts_new`, or NULL to disable structured logging. + +.. seealso:: + + | :doc:`structured_log` diff --git a/src/libmongoc/doc/mongoc_client_t.rst b/src/libmongoc/doc/mongoc_client_t.rst index 06747c93b6d..e3dabf9e74f 100644 --- a/src/libmongoc/doc/mongoc_client_t.rst +++ b/src/libmongoc/doc/mongoc_client_t.rst @@ -91,6 +91,7 @@ Example mongoc_client_set_sockettimeoutms mongoc_client_set_ssl_opts mongoc_client_set_stream_initiator + mongoc_client_set_structured_log_opts mongoc_client_set_write_concern mongoc_client_start_session mongoc_client_watch diff --git a/src/libmongoc/doc/mongoc_structured_log_component_t.rst b/src/libmongoc/doc/mongoc_structured_log_component_t.rst index 9538d60d12a..58a3d9bb6e1 100644 --- a/src/libmongoc/doc/mongoc_structured_log_component_t.rst +++ b/src/libmongoc/doc/mongoc_structured_log_component_t.rst @@ -17,7 +17,7 @@ Synopsis ``mongoc_structured_log_component_t`` enumerates the structured logging components. Applications should never rely on having an exhaustive list of all log components. -Instead, use :symbol:`mongoc_structured_log_set_max_level_for_all_components` to set a default level if needed. +Instead, use :symbol:`mongoc_structured_log_opts_set_max_level_for_all_components` to set a default level if needed. Functions --------- diff --git a/src/libmongoc/doc/mongoc_structured_log_entry_get_component.rst b/src/libmongoc/doc/mongoc_structured_log_entry_get_component.rst index 313d30ce7be..df89472db1a 100644 --- a/src/libmongoc/doc/mongoc_structured_log_entry_get_component.rst +++ b/src/libmongoc/doc/mongoc_structured_log_entry_get_component.rst @@ -1,7 +1,7 @@ :man_page: mongoc_structured_log_entry_get_component -mongoc_structured_log_entry_get_component -========================================= +mongoc_structured_log_entry_get_component() +=========================================== Synopsis -------- diff --git a/src/libmongoc/doc/mongoc_structured_log_entry_get_level.rst b/src/libmongoc/doc/mongoc_structured_log_entry_get_level.rst index 22f9b03a022..14c6eb6a985 100644 --- a/src/libmongoc/doc/mongoc_structured_log_entry_get_level.rst +++ b/src/libmongoc/doc/mongoc_structured_log_entry_get_level.rst @@ -1,7 +1,7 @@ :man_page: mongoc_structured_log_entry_get_level -mongoc_structured_log_entry_get_level -===================================== +mongoc_structured_log_entry_get_level() +======================================= Synopsis -------- diff --git a/src/libmongoc/doc/mongoc_structured_log_entry_message_as_bson.rst b/src/libmongoc/doc/mongoc_structured_log_entry_message_as_bson.rst index fe002712709..9a94e9dd8f1 100644 --- a/src/libmongoc/doc/mongoc_structured_log_entry_message_as_bson.rst +++ b/src/libmongoc/doc/mongoc_structured_log_entry_message_as_bson.rst @@ -1,7 +1,7 @@ :man_page: mongoc_structured_log_entry_message_as_bson -mongoc_structured_log_entry_message_as_bson -=========================================== +mongoc_structured_log_entry_message_as_bson() +============================================= Synopsis -------- diff --git a/src/libmongoc/doc/mongoc_structured_log_func_t.rst b/src/libmongoc/doc/mongoc_structured_log_func_t.rst index 0eeaa93fd52..d2f3f6a0ca6 100644 --- a/src/libmongoc/doc/mongoc_structured_log_func_t.rst +++ b/src/libmongoc/doc/mongoc_structured_log_func_t.rst @@ -11,30 +11,15 @@ Synopsis typedef void (*mongoc_structured_log_func_t) (const mongoc_structured_log_entry_t *entry, void *user_data); -Callback function for :symbol:`mongoc_structured_log_set_handler`. -Structured log handlers must be thread-safe. -Logging may occur simultaneously on multiple application threads and/or background threads. - -Handlers may use any operating system or ``libbson`` functions but MUST not use any ``libmongoc`` functions except: - -* :symbol:`mongoc_structured_log_entry_get_component` -* :symbol:`mongoc_structured_log_entry_get_level` -* :symbol:`mongoc_structured_log_entry_message_as_bson` -* :symbol:`mongoc_structured_log_get_level_name` -* :symbol:`mongoc_structured_log_get_named_level` -* :symbol:`mongoc_structured_log_get_component_name` -* :symbol:`mongoc_structured_log_get_named_component` -* :symbol:`mongoc_structured_log_get_max_level_for_component` - -Use of other ``libmongoc`` APIs will have undefined consequences, including deadlocks and unbounded recursion. - -Applications that wish to use MongoDB itself as a logging destination will need to store the serialized messages temporarily and insert them asynchronously from outside the log handler. +Callback function for :symbol:`mongoc_structured_log_opts_set_handler`. +Structured log handlers must be thread-safe if they will be used with :symbol:`mongoc_client_pool_t`. +Handlers must avoid unbounded recursion, preferably by avoiding the use of any ``libmongoc`` client or pool which uses the same handler. Parameters ---------- * ``entry``: A :symbol:`mongoc_structured_log_entry_t` pointer, only valid during the handler invocation. -* ``user_data``: Optional user data from :symbol:`mongoc_structured_log_set_handler`. +* ``user_data``: Optional user data from :symbol:`mongoc_structured_log_opts_set_handler`. .. seealso:: diff --git a/src/libmongoc/doc/mongoc_structured_log_get_component_name.rst b/src/libmongoc/doc/mongoc_structured_log_get_component_name.rst index 66dc03d8d60..bb3a6593be4 100644 --- a/src/libmongoc/doc/mongoc_structured_log_get_component_name.rst +++ b/src/libmongoc/doc/mongoc_structured_log_get_component_name.rst @@ -1,7 +1,7 @@ :man_page: mongoc_structured_log_get_component_name -mongoc_structured_log_get_component_name -======================================== +mongoc_structured_log_get_component_name() +========================================== Synopsis -------- diff --git a/src/libmongoc/doc/mongoc_structured_log_get_level_name.rst b/src/libmongoc/doc/mongoc_structured_log_get_level_name.rst index 93fc0860de6..29ae9f9a205 100644 --- a/src/libmongoc/doc/mongoc_structured_log_get_level_name.rst +++ b/src/libmongoc/doc/mongoc_structured_log_get_level_name.rst @@ -1,7 +1,7 @@ :man_page: mongoc_structured_log_get_level_name -mongoc_structured_log_get_level_name -==================================== +mongoc_structured_log_get_level_name() +====================================== Synopsis -------- diff --git a/src/libmongoc/doc/mongoc_structured_log_get_max_level_for_component.rst b/src/libmongoc/doc/mongoc_structured_log_get_max_level_for_component.rst deleted file mode 100644 index 8cab64920f2..00000000000 --- a/src/libmongoc/doc/mongoc_structured_log_get_max_level_for_component.rst +++ /dev/null @@ -1,27 +0,0 @@ -:man_page: mongoc_structured_log_get_max_level_for_component - -mongoc_structured_log_get_max_level_for_component -================================================= - -Synopsis --------- - -.. code-block:: c - - mongoc_structured_log_level_t - mongoc_structured_log_get_max_level_for_component (mongoc_structured_log_component_t component); - -Parameters ----------- - -* ``component``: Log component as a :symbol:`mongoc_structured_log_component_t`. - -Returns -------- - -Returns the current maximum log level for a specific component. -This may be the last value set with :symbol:`mongoc_structured_log_set_max_level_for_component` or :symbol:`mongoc_structured_log_set_max_level_for_all_components`, or it may be the default obtained from environment variables. - -.. seealso:: - - | :doc:`structured_log` diff --git a/src/libmongoc/doc/mongoc_structured_log_get_named_component.rst b/src/libmongoc/doc/mongoc_structured_log_get_named_component.rst index bfac67862c5..a941fce3044 100644 --- a/src/libmongoc/doc/mongoc_structured_log_get_named_component.rst +++ b/src/libmongoc/doc/mongoc_structured_log_get_named_component.rst @@ -1,7 +1,7 @@ :man_page: mongoc_structured_log_get_named_component -mongoc_structured_log_get_named_component -========================================= +mongoc_structured_log_get_named_component() +=========================================== Synopsis -------- diff --git a/src/libmongoc/doc/mongoc_structured_log_get_named_level.rst b/src/libmongoc/doc/mongoc_structured_log_get_named_level.rst index 63ed45278ac..74849684356 100644 --- a/src/libmongoc/doc/mongoc_structured_log_get_named_level.rst +++ b/src/libmongoc/doc/mongoc_structured_log_get_named_level.rst @@ -1,7 +1,7 @@ :man_page: mongoc_structured_log_get_named_level -mongoc_structured_log_get_named_level -===================================== +mongoc_structured_log_get_named_level() +======================================= Synopsis -------- diff --git a/src/libmongoc/doc/mongoc_structured_log_opts_destroy.rst b/src/libmongoc/doc/mongoc_structured_log_opts_destroy.rst new file mode 100644 index 00000000000..2f1f15ee61e --- /dev/null +++ b/src/libmongoc/doc/mongoc_structured_log_opts_destroy.rst @@ -0,0 +1,23 @@ +:man_page: mongoc_structured_log_opts_destroy + +mongoc_structured_log_opts_destroy() +==================================== + +Synopsis +-------- + +.. code-block:: c + + void + mongoc_structured_log_opts_destroy (mongoc_structured_log_opts_t *opts); + +Parameters +---------- + +* ``opts``: Pointer to a :symbol:`mongoc_structured_log_opts_t` allocated with :symbol:`mongoc_structured_log_opts_new`, or NULL. + +Description +----------- + +This function releases all resources associated with a :symbol:`mongoc_structured_log_opts_t`. +Does nothing if ``opts`` is NULL. diff --git a/src/libmongoc/doc/mongoc_structured_log_opts_get_max_level_for_component.rst b/src/libmongoc/doc/mongoc_structured_log_opts_get_max_level_for_component.rst new file mode 100644 index 00000000000..fb8315f1063 --- /dev/null +++ b/src/libmongoc/doc/mongoc_structured_log_opts_get_max_level_for_component.rst @@ -0,0 +1,30 @@ +:man_page: mongoc_structured_log_opts_get_max_level_for_component + +mongoc_structured_log_opts_get_max_level_for_component() +======================================================== + +Synopsis +-------- + +.. code-block:: c + + mongoc_structured_log_level_t + mongoc_structured_log_opts_get_max_level_for_component (const mongoc_structured_log_opts_t *opts, + mongoc_structured_log_component_t component); + +Parameters +---------- + +* ``opts``: Structured log options, allocated with :symbol:`mongoc_structured_log_opts_new`. +* ``component``: Log component as a :symbol:`mongoc_structured_log_component_t`. + +Returns +------- + +Returns the configured maximum log level for a specific component. +This may be the last value set with :symbol:`mongoc_structured_log_opts_set_max_level_for_component` or :symbol:`mongoc_structured_log_opts_set_max_level_for_all_components`, or it may be the default obtained from environment variables. +If an invalid or unknown component enum is given, returns the lowest log level. + +.. seealso:: + + | :doc:`structured_log` diff --git a/src/libmongoc/doc/mongoc_structured_log_opts_new.rst b/src/libmongoc/doc/mongoc_structured_log_opts_new.rst new file mode 100644 index 00000000000..b4f8d0f790f --- /dev/null +++ b/src/libmongoc/doc/mongoc_structured_log_opts_new.rst @@ -0,0 +1,65 @@ +:man_page: mongoc_structured_log_opts_new + +mongoc_structured_log_opts_new() +================================ + +Synopsis +-------- + +.. code-block:: c + + mongoc_structured_log_opts_t * + mongoc_structured_log_opts_new (void); + +Creates a new :symbol:`mongoc_structured_log_opts_t`, filled with defaults captured from the current environment. + +Sets a default log handler which would write a text representation of each log message to ``stderr``, ``stdout``, or another file configurable using ``MONGODB_LOG_PATH``. +This setting has no effect if the default handler is replaced using :symbol:`mongoc_structured_log_opts_set_handler`. + +Environment variable errors are non-fatal, and result in one-time warnings delivered as an unstructured log. + +Per-component maximum levels are initialized equivalently to: + +.. code-block:: c + + mongoc_structured_log_opts_set_max_level_for_all_components(opts, MONGOC_STRUCTURED_LOG_LEVEL_WARNING); + mongoc_structured_log_opts_set_max_levels_from_env(opts); + +Environment Variables +--------------------- + +This is a full list of the captured environment variables. + +* ``MONGODB_LOG_MAX_DOCUMENT_LENGTH``: Maximum length for JSON-serialized documents that appear within a log message. + It may be a number, in bytes, or ``unlimited``. + By default, the limit is 1000 bytes. + This limit affects interior documents like commands and replies, not the total length of a structured log message. + +* ``MONGODB_LOG_PATH``: A file path or one of the special strings ``stderr`` or ``stdout``, specifying the destination for structured logs seen by the default handler. + By default, it writes to ``stderr``. + This path will be captured during ``mongoc_structured_log_opts_new()``, but it will not immediately be opened. + If the file can't be opened, a warning is then written to the unstructured log and the handler writes structured logs to ``stderr`` instead. + +* ``MONGODB_LOG_COMMAND``: A log level name to set as the maximum for ``MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND``. +* ``MONGODB_LOG_TOPOLOGY``: A log level name to set as the maximum for ``MONGOC_STRUCTURED_LOG_COMPONENT_TOPOLOGY``. +* ``MONGODB_LOG_SERVER_SELECTION``: A log level name to set as the maximum for ``MONGOC_STRUCTURED_LOG_COMPONENT_SERVER_SELECTION``. +* ``MONGODB_LOG_CONNECTION``: A log level name to set as the maximum for ``MONGOC_STRUCTURED_LOG_COMPONENT_CONNECTION``. +* ``MONGODB_LOG_ALL``: A log level name applied to all components not otherwise specified. + +Note that log level names are always case insensitive. +This is a full list of recognized names, including allowed aliases: + +* ``emergency``, ``off`` +* ``alert`` +* ``critical`` +* ``error`` +* ``warning``, ``warn`` +* ``notice`` +* ``informational``, ``info`` +* ``debug`` +* ``trace`` + +Returns +------- + +A newly allocated :symbol:`mongoc_structured_log_opts_t`. diff --git a/src/libmongoc/doc/mongoc_structured_log_opts_set_handler.rst b/src/libmongoc/doc/mongoc_structured_log_opts_set_handler.rst new file mode 100644 index 00000000000..b817c20fa13 --- /dev/null +++ b/src/libmongoc/doc/mongoc_structured_log_opts_set_handler.rst @@ -0,0 +1,36 @@ +:man_page: mongoc_structured_log_opts_set_handler + +mongoc_structured_log_opts_set_handler() +======================================== + +Synopsis +-------- + +.. code-block:: c + + void + mongoc_structured_log_opts_set_handler (mongoc_structured_log_opts_t *opts, + mongoc_structured_log_func_t log_func, + void *user_data); + +Sets the function to be called to handle structured log messages, as a :symbol:`mongoc_structured_log_func_t`. + +The callback is given a :symbol:`mongoc_structured_log_entry_t` as a handle for obtaining additional information about the log message. +This entry pointer is only valid during a callback, because it's a low cost reference to temporary data. + +Structured log handlers must be thread-safe if they will be used with :symbol:`mongoc_client_pool_t`. +Handlers must avoid unbounded recursion, preferably by avoiding the use of any ``libmongoc`` client or pool which uses the same handler. + +This function always replaces the default log handler from :symbol:`mongoc_structured_log_opts_new`, if it was still set. +If the ``log_func`` is set to NULL, structured logging will be disabled. + +Parameters +---------- + +* ``opts``: Structured log options, allocated with :symbol:`mongoc_structured_log_opts_new`. +* ``log_func``: The handler to install, a :symbol:`mongoc_structured_log_func_t`, or NULL to disable structured logging. +* ``user_data``: Optional user data, passed on to the handler. + +.. seealso:: + + | :doc:`structured_log` diff --git a/src/libmongoc/doc/mongoc_structured_log_opts_set_max_level_for_all_components.rst b/src/libmongoc/doc/mongoc_structured_log_opts_set_max_level_for_all_components.rst new file mode 100644 index 00000000000..32ca1feb21a --- /dev/null +++ b/src/libmongoc/doc/mongoc_structured_log_opts_set_max_level_for_all_components.rst @@ -0,0 +1,32 @@ +:man_page: mongoc_structured_log_opts_set_max_level_for_all_components + +mongoc_structured_log_opts_set_max_level_for_all_components() +============================================================= + +Synopsis +-------- + +.. code-block:: c + + bool + mongoc_structured_log_opts_set_max_level_for_all_components (mongoc_structured_log_opts_t *opts, + mongoc_structured_log_level_t level); + +Sets all per-component maximum log levels to the same value. +Only log messages at or below this severity level will be passed to :symbol:`mongoc_structured_log_func_t`. +Effective even for logging components not known at compile-time. + +Parameters +---------- + +* ``opts``: Structured log options, allocated with :symbol:`mongoc_structured_log_opts_new`. +* ``level``: The max log level for all components, as a :symbol:`mongoc_structured_log_level_t`. + +Returns +------- + +Returns ``true`` on success, or ``false`` if the supplied parameters were incorrect. + +.. seealso:: + + | :doc:`structured_log` diff --git a/src/libmongoc/doc/mongoc_structured_log_opts_set_max_level_for_component.rst b/src/libmongoc/doc/mongoc_structured_log_opts_set_max_level_for_component.rst new file mode 100644 index 00000000000..a72df802c60 --- /dev/null +++ b/src/libmongoc/doc/mongoc_structured_log_opts_set_max_level_for_component.rst @@ -0,0 +1,36 @@ +:man_page: mongoc_structured_log_opts_set_max_level_for_component + +mongoc_structured_log_opts_set_max_level_for_component() +======================================================== + +Synopsis +-------- + +.. code-block:: c + + bool + mongoc_structured_log_opts_set_max_level_for_component (mongoc_structured_log_opts_t *opts, + mongoc_structured_log_component_t component, + mongoc_structured_log_level_t level); + +Sets the maximum log level per-component. +Only log messages at or below this severity level will be passed to :symbol:`mongoc_structured_log_func_t`. + +By default, each component's log level may come from environment variables. +See :symbol:`mongoc_structured_log_opts_set_max_levels_from_env`. + +Parameters +---------- + +* ``opts``: Structured log options, allocated with :symbol:`mongoc_structured_log_opts_new`. +* ``component``: The component to set a max log level. for, as a :symbol:`mongoc_structured_log_component_t`. +* ``level``: The new max log level for this component, as a :symbol:`mongoc_structured_log_level_t`. + +Returns +------- + +Returns ``true`` on success, or ``false`` if the supplied parameters were incorrect. + +.. seealso:: + + | :doc:`structured_log` diff --git a/src/libmongoc/doc/mongoc_structured_log_opts_set_max_levels_from_env.rst b/src/libmongoc/doc/mongoc_structured_log_opts_set_max_levels_from_env.rst new file mode 100644 index 00000000000..f48692c6dcb --- /dev/null +++ b/src/libmongoc/doc/mongoc_structured_log_opts_set_max_levels_from_env.rst @@ -0,0 +1,36 @@ +:man_page: mongoc_structured_log_opts_set_max_levels_from_env + +mongoc_structured_log_opts_set_max_levels_from_env() +==================================================== + +Synopsis +-------- + +.. code-block:: c + + bool + mongoc_structured_log_opts_set_max_levels_from_env (mongoc_structured_log_opts_t *opts); + +Sets any maximum log levels requested by environment variables: ``MONGODB_LOG_ALL`` for all components, followed by per-component log levels ``MONGODB_LOG_COMMAND``, ``MONGODB_LOG_CONNECTION``, ``MONGODB_LOG_TOPOLOGY``, and ``MONGODB_LOG_SERVER_SELECTION``. + +Expects the value to be recognizable by :symbol:`mongoc_structured_log_get_named_level`. +Parse errors may cause a warning message, delivered via unstructured logging. + +Component levels with no valid environment variable setting will be left unmodified. + +This happens automatically when :symbol:`mongoc_structured_log_opts_new` establishes defaults. +Any subsequent programmatic modifications to the :symbol:`mongoc_structured_log_opts_t` will override the environment variable settings. +For applications that desire the opposite behavior, where environment variables may override programmatic settings, they may call ``mongoc_structured_log_opts_set_max_levels_from_env()`` after calling :symbol:`mongoc_structured_log_opts_set_max_level_for_component` and :symbol:`mongoc_structured_log_opts_set_max_level_for_all_components`. +This will process the environment a second time, allowing it to override customized defaults. + +Returns +------- + +Returns ``true`` on success. +If warnings are encountered in the environment, returns ``false`` and may log additional information to the unstructured logging facility. +Note that, by design, these errors are by default non-fatal. +When :symbol:`mongoc_structured_log_opts_new` internally calls this function, it ignores the return value. + +.. seealso:: + + | :doc:`structured_log` diff --git a/src/libmongoc/doc/mongoc_structured_log_opts_t.rst b/src/libmongoc/doc/mongoc_structured_log_opts_t.rst new file mode 100644 index 00000000000..414c24d804a --- /dev/null +++ b/src/libmongoc/doc/mongoc_structured_log_opts_t.rst @@ -0,0 +1,35 @@ +:man_page: mongoc_structured_log_opts_t + +mongoc_structured_log_opts_t +============================ + +Synopsis +-------- + +.. code-block:: c + + typedef struct mongoc_structured_log_opts_t mongoc_structured_log_opts_t; + +``mongoc_structured_log_opts_t`` is an opaque type that contains options for the structured logging subsystem: per-component log levels, a maximum logged document length, and a handler function. + +Create a ``mongoc_structured_log_opts_t`` with :symbol:`mongoc_structured_log_opts_new`, set options and a callback on it, then pass it to :symbol:`mongoc_client_set_structured_log_opts` or :symbol:`mongoc_client_pool_set_structured_log_opts`. +Must be destroyed by calling :symbol:`mongoc_structured_log_opts_destroy`. + +Functions +--------- + +.. toctree:: + :titlesonly: + :maxdepth: 1 + + mongoc_structured_log_opts_new + mongoc_structured_log_opts_destroy + mongoc_structured_log_opts_set_handler + mongoc_structured_log_opts_set_max_level_for_component + mongoc_structured_log_opts_set_max_level_for_all_components + mongoc_structured_log_opts_set_max_levels_from_env + mongoc_structured_log_opts_get_max_level_for_component + +.. seealso:: + + | :doc:`structured_log` diff --git a/src/libmongoc/doc/mongoc_structured_log_set_handler.rst b/src/libmongoc/doc/mongoc_structured_log_set_handler.rst deleted file mode 100644 index 090cebfc5c5..00000000000 --- a/src/libmongoc/doc/mongoc_structured_log_set_handler.rst +++ /dev/null @@ -1,41 +0,0 @@ -:man_page: mongoc_structured_log_set_handler - -mongoc_structured_log_set_handler -================================= - -Synopsis --------- - -.. code-block:: c - - void - mongoc_structured_log_set_handler (mongoc_structured_log_func_t log_func, void *user_data); - -Sets the function to be called to handle structured log messages, as a :symbol:`mongoc_structured_log_func_t`. - -The callback is given a :symbol:`mongoc_structured_log_entry_t` as a handle for obtaining additional information about the log message. -This entry pointer is only valid during a callback, because it's a low cost reference to temporary data. - -This function is thread-safe. -It provides atomic updates to the handler function and data. -If handlers are ongoing, blocks until they complete before applying the change. - -The callback handler itself must be thread-safe. -Logging can occur simultaneously on multiple application or background threads. - -Handlers must not re-enter libmongoc except as allowed to access the log entry parameter. -See :symbol:`mongoc_structured_log_func_t`. - -There is a single global handler per process. -Any call to this function will replace the default handler. -If the ``log_func`` is set to NULL, structured logging will be disabled. - -Parameters ----------- - -* ``log_func``: The handler to install, a :symbol:`mongoc_structured_log_func_t`, or NULL to disable structured logging. -* ``user_data``: Optional user data, passed on to the handler. - -.. seealso:: - - | :doc:`structured_log` diff --git a/src/libmongoc/doc/mongoc_structured_log_set_max_level_for_all_components.rst b/src/libmongoc/doc/mongoc_structured_log_set_max_level_for_all_components.rst deleted file mode 100644 index 5aeb16e0de6..00000000000 --- a/src/libmongoc/doc/mongoc_structured_log_set_max_level_for_all_components.rst +++ /dev/null @@ -1,25 +0,0 @@ -:man_page: mongoc_structured_log_set_max_level_for_all_components - -mongoc_structured_log_set_max_level_for_all_components -====================================================== - -Synopsis --------- - -.. code-block:: c - - void - mongoc_structured_log_set_max_level_for_all_components (mongoc_structured_log_level_t level); - -Sets all per-component maximum log levels to the same value. -Only log messages at or below this severity level will be passed to :symbol:`mongoc_structured_log_func_t`. -Effective even for logging components not known at compile-time. - -Parameters ----------- - -* ``level``: The max log level for all components, as a :symbol:`mongoc_structured_log_level_t`. - -.. seealso:: - - | :doc:`structured_log` diff --git a/src/libmongoc/doc/mongoc_structured_log_set_max_level_for_component.rst b/src/libmongoc/doc/mongoc_structured_log_set_max_level_for_component.rst deleted file mode 100644 index d20eeb1b15c..00000000000 --- a/src/libmongoc/doc/mongoc_structured_log_set_max_level_for_component.rst +++ /dev/null @@ -1,29 +0,0 @@ -:man_page: mongoc_structured_log_set_max_level_for_component - -mongoc_structured_log_set_max_level_for_component -================================================= - -Synopsis --------- - -.. code-block:: c - - void - mongoc_structured_log_set_max_level_for_component (mongoc_structured_log_component_t component, - mongoc_structured_log_level_t level); - -Sets the maximum log level per-component. -Only log messages at or below this severity level will be passed to :symbol:`mongoc_structured_log_func_t`. - -By default, each component's log level may come from environment variables. -See :symbol:`mongoc_structured_log_set_max_levels_from_env`. - -Parameters ----------- - -* ``component``: The component to set a max log level. for, as a :symbol:`mongoc_structured_log_component_t`. -* ``level``: The new max log level for this component, as a :symbol:`mongoc_structured_log_level_t`. - -.. seealso:: - - | :doc:`structured_log` diff --git a/src/libmongoc/doc/mongoc_structured_log_set_max_levels_from_env.rst b/src/libmongoc/doc/mongoc_structured_log_set_max_levels_from_env.rst deleted file mode 100644 index e221ccdcf6d..00000000000 --- a/src/libmongoc/doc/mongoc_structured_log_set_max_levels_from_env.rst +++ /dev/null @@ -1,29 +0,0 @@ -:man_page: mongoc_structured_log_set_max_levels_from_env - -mongoc_structured_log_set_max_levels_from_env -============================================= - -Synopsis --------- - -.. code-block:: c - - void - mongoc_structured_log_set_max_levels_from_env (void) - -Sets any maximum log levels requested by environment variables: ``MONGODB_LOG_ALL`` for all components, followed by per-component log levels ``MONGODB_LOG_COMMAND``, ``MONGODB_LOG_CONNECTION``, ``MONGODB_LOG_TOPOLOGY``, and ``MONGODB_LOG_SERVER_SELECTION``. - -Expects the value to be recognizable by :symbol:`mongoc_structured_log_get_named_level`. -Parse errors may cause a warning message. - -Component levels with no valid environment variable setting will be left unmodified. - -Normally this happens automatically when log levels are initialized on first use. -The resulting default values can be overridden programmatically by calls to :symbol:`mongoc_structured_log_set_max_level_for_component` and :symbol:`mongoc_structured_log_set_max_level_for_all_components`. - -For applications that desire the opposite behavior, where environment variables may override programmatic settings, they may call ``mongoc_structured_log_set_max_levels_from_env()`` after calling :symbol:`mongoc_structured_log_set_max_level_for_component` and :symbol:`mongoc_structured_log_set_max_level_for_all_components`. -This will process the environment a second time, allowing it to override customized defaults. - -.. seealso:: - - | :doc:`structured_log` diff --git a/src/libmongoc/doc/structured_log.rst b/src/libmongoc/doc/structured_log.rst index b03e2d9c83b..4ea91b0626a 100644 --- a/src/libmongoc/doc/structured_log.rst +++ b/src/libmongoc/doc/structured_log.rst @@ -6,18 +6,37 @@ Structured Logging This document describes a newer "structured" logging facility which reports messages from the driver itself using a BSON format defined across driver implementations by the `MongoDB Logging Specification `_. See :doc:`unstructured_log` for the original freeform logging facility. -These two systems are configured and used independently. The structured logging system has independent settings for handler and log levels. +These two systems are configured and used independently. -Defaults --------- +Unstructured logging is global to the entire process, but structured logging is configured separately for each :symbol:`mongoc_client_t` or :symbol:`mongoc_client_pool_t`. +See :symbol:`mongoc_client_set_structured_log_opts` and :symbol:`mongoc_client_pool_set_structured_log_opts`. -When the structured logging subsystem is first used, possibly before or during :symbol:`mongoc_init`: +Options +------- + +Structured log settings are tracked explicitly by a :symbol:`mongoc_structured_log_opts_t` instance. -* Default log levels are set from the environment variables ``MONGODB_LOG_ALL``, ``MONGODB_LOG_COMMAND``, ``MONGODB_LOG_TOPOLOGY``, ``MONGODB_LOG_SERVER_SELECTION``, expecting a value from the `severity level table `_: ``off``, ``emergency``, ``alert``, ``critical``, ``error``, ``warning``, ``warn``, ``notice``, ``informational``, ``info``, ``debug``, ``trace``. -* A default handler is installed, which logs text representations of each message to a file given by ``MONGODB_LOG_PATH``, which may be a full path or one of the special values ``stdout`` or ``stderr``. If no valid path is given, logs are written to stderr. +Like other drivers supporting structured logging, we take default settings from environment variables and offer additional optional programmatic configuration. +Environment variables are captured during :symbol:`mongoc_structured_log_opts_new`, refer there for a full list of the supported variables. Normally environment variables provide defaults that can be overridden programmatically. -To request the opposite behavior, where your programmatic defaults can be overridden by the environment, see :symbol:`mongoc_structured_log_set_max_levels_from_env`. +To request the opposite behavior, where your programmatic defaults can be overridden by the environment, see :symbol:`mongoc_structured_log_opts_set_max_levels_from_env`. + +Structured log messages may be filtered in arbitrary ways by the handler, but as both a performance optimization and a convenience, a built-in filter limits the maximum log level of reported messages with a per-component setting. + +.. toctree:: + :titlesonly: + :maxdepth: 1 + + mongoc_structured_log_opts_t + mongoc_structured_log_opts_new + mongoc_structured_log_opts_destroy + mongoc_structured_log_opts_set_handler + mongoc_structured_log_opts_get_max_level_for_component + mongoc_structured_log_opts_set_max_level_for_component + mongoc_structured_log_opts_set_max_level_for_all_components + mongoc_structured_log_opts_set_max_levels_from_env + Levels and Components --------------------- @@ -60,44 +79,23 @@ Log levels and components are defined as :symbol:`mongoc_structured_log_level_t` mongoc_structured_log_get_named_component -Log Filtering -------------- - -Structured log messages may be filtered in two ways: - -* A maximum log level can be set per-component. -* A log handler function can ignore messages based on any criteria. - -The max level settings are configured by the environment variables above, and they may be altered or queried by applications. -To reduce overhead for unwanted logging, messages should be ignored as early as possible. -If it's possible to filter messages based on only their component and level, this should be done by handlers before requesting BSON serialization by calling :symbol:`mongoc_structured_log_entry_message_as_bson`. - -.. toctree:: - :titlesonly: - :maxdepth: 1 - - mongoc_structured_log_set_max_level_for_all_components - mongoc_structured_log_set_max_level_for_component - mongoc_structured_log_set_max_levels_from_env - mongoc_structured_log_get_max_level_for_component - Log Handlers ------------ -There can be one global log handler set at a time. -This handler is called with a :symbol:`mongoc_structured_log_entry_t` that can be queried for further details. +Each :symbol:`mongoc_client_t` or :symbol:`mongoc_client_pool_t` has its own instance of the structured logging subsystem, with its own settings and handler. + +When using :symbol:`mongoc_client_pool_t`, the pooled clients all share a common logging instance. Handlers must be thread-safe. -.. note:: +The handler is called for each log entry with a level no greater than its component's maximum. +A :symbol:`mongoc_structured_log_entry_t` pointer provides access to further details, during the handler only. - Structured log handlers must be thread-safe. - This differs from unstructured logging, which provides a global mutex. +Handlers must take care not to re-enter ``libmongoc`` with the same :symbol:`mongoc_client_t` or :symbol:`mongoc_client_pool_t` that the handler has been called by. .. toctree:: :titlesonly: :maxdepth: 1 mongoc_structured_log_func_t - mongoc_structured_log_set_handler Log Entries ----------- diff --git a/src/libmongoc/examples/example-structured-log.c b/src/libmongoc/examples/example-structured-log.c index 576c040ab62..78bfd85cbee 100644 --- a/src/libmongoc/examples/example-structured-log.c +++ b/src/libmongoc/examples/example-structured-log.c @@ -15,10 +15,18 @@ example_handler (const mongoc_structured_log_entry_t *entry, void *user_data) mongoc_structured_log_level_t level = mongoc_structured_log_entry_get_level (entry); /* - * Structured log handlers need to be thread-safe. - * Many apps will be happy to use a global mutex in their logging handler, - * but high performance multithreaded apps may prefer dispatching log - * messages asynchronously with thread-safe data structures. + * With a single-threaded mongoc_client_t, handlers will always be called + * by the thread that owns the client. On a mongoc_client_pool_t, handlers + * are shared by multiple threads and must be reentrant. + * + * Note that unstructured logging includes a global mutex in the API, + * but structured logging allows applications to avoid lock contention + * even when multiple threads are issuing commands simultaneously. + * + * Simple apps like this example can achieve thread safety by adding their + * own global mutex. For other apps, this would be a performance bottleneck + * and it would be more appropriate for handlers to process their log + * messages concurrently. */ pthread_mutex_lock (&handler_mutex); @@ -49,14 +57,33 @@ main (void) int result = EXIT_FAILURE; bson_error_t error; mongoc_uri_t *uri = NULL; + mongoc_structured_log_opts_t *log_opts = NULL; mongoc_client_t *client = NULL; + mongoc_client_pool_t *pool = NULL; + + /* + * Note that structured logging only applies per-client or per-pool, + * and it won't be used during or before mongoc_init. + */ + mongoc_init (); + + /* + * Logging options are represented by a mongoc_structured_log_opts_t, + * which can be copied into a mongoc_client_t or mongoc_client_pool_t + * using mongoc_client_set_structured_log_opts() or + * mongoc_client_pool_set_structured_log_opts(), respectively. + * + * Default settings are captured from the environment into + * this structure when it's constructed. + */ + log_opts = mongoc_structured_log_opts_new (); /* * For demonstration purposes, set up a handler that receives all possible log messages. */ pthread_mutex_init (&handler_mutex, NULL); - mongoc_structured_log_set_max_level_for_all_components (MONGOC_STRUCTURED_LOG_LEVEL_TRACE); - mongoc_structured_log_set_handler (example_handler, NULL); + mongoc_structured_log_opts_set_max_level_for_all_components (log_opts, MONGOC_STRUCTURED_LOG_LEVEL_TRACE); + mongoc_structured_log_opts_set_handler (log_opts, example_handler, NULL); /* * By default libmongoc proceses log options from the environment first, @@ -65,13 +92,7 @@ main (void) * defaults, you can ask for the environment to be re-read after setting * your own defaults. */ - mongoc_structured_log_set_max_levels_from_env (); - - /* - * This is the main libmongoc initialization, but structured logging - * can be used earlier. It's automatically initialized on first use. - */ - mongoc_init (); + mongoc_structured_log_opts_set_max_levels_from_env (log_opts); /* * Create a MongoDB URI object. This example assumes a local server. @@ -83,21 +104,35 @@ main (void) } /* - * Create a new client instance. + * Create a new client pool. */ - client = mongoc_client_new_from_uri (uri); - if (!client) { + pool = mongoc_client_pool_new (uri); + if (!pool) { goto done; } /* - * Do some work that we'll see logs from. This example just sends a 'ping' command. + * Set the client pool's log options. + * This must happen only once, and only before the first mongoc_client_pool_pop. + * There's no need to keep log_opts after this point. + */ + mongoc_client_pool_set_structured_log_opts (pool, log_opts); + + /* + * Check out a client, and do some work that we'll see logs from. + * This example just sends a 'ping' command. */ + client = mongoc_client_pool_pop (pool); + if (!client) { + goto done; + } + bson_t *command = BCON_NEW ("ping", BCON_INT32 (1)); bson_t reply; bool command_ret = mongoc_client_command_simple (client, "admin", command, NULL, &reply, &error); bson_destroy (command); bson_destroy (&reply); + mongoc_client_pool_push (pool, client); if (!command_ret) { fprintf (stderr, "Command error: %s\n", error.message); goto done; @@ -106,7 +141,8 @@ main (void) result = EXIT_SUCCESS; done: mongoc_uri_destroy (uri); - mongoc_client_destroy (client); + mongoc_structured_log_opts_destroy (log_opts); + mongoc_client_pool_destroy (pool); mongoc_cleanup (); return result; } diff --git a/src/libmongoc/src/mongoc/mongoc-client-pool.c b/src/libmongoc/src/mongoc/mongoc-client-pool.c index 8dd7c8604e0..a8ab4deb47e 100644 --- a/src/libmongoc/src/mongoc/mongoc-client-pool.c +++ b/src/libmongoc/src/mongoc/mongoc-client-pool.c @@ -47,16 +47,17 @@ struct _mongoc_client_pool_t { uint32_t max_pool_size; uint32_t size; #ifdef MONGOC_ENABLE_SSL - bool ssl_opts_set; mongoc_ssl_opt_t ssl_opts; + bool ssl_opts_set; #endif bool apm_callbacks_set; + bool error_api_set; + bool structured_log_opts_set; + bool client_initialized; mongoc_apm_callbacks_t apm_callbacks; void *apm_context; int32_t error_api_version; - bool error_api_set; mongoc_server_api_t *api; - bool client_initialized; // `last_known_serverids` is a sorted array of uint32_t. mongoc_array_t last_known_serverids; }; @@ -281,7 +282,6 @@ _initialize_new_client (mongoc_client_pool_t *pool, mongoc_client_t *client) client, pool->topology->scanner->initiator, pool->topology->scanner->initiator_context); pool->client_initialized = true; - client->is_pooled = true; client->error_api_version = pool->error_api_version; _mongoc_client_set_apm_callbacks_private (client, &pool->apm_callbacks, pool->apm_context); @@ -608,6 +608,26 @@ mongoc_client_pool_set_apm_callbacks (mongoc_client_pool_t *pool, mongoc_apm_cal return true; } +void +mongoc_client_pool_set_structured_log_opts (mongoc_client_pool_t *pool, const mongoc_structured_log_opts_t *opts) +{ + BSON_ASSERT_PARAM (pool); + // opts is optional + + /* The documented restriction for most pool options: They can be set at most once, + * and only before the first client is initialized. Structured logging is generally + * expected to warn but not quit when encountering initialization errors. */ + if (pool->structured_log_opts_set) { + MONGOC_WARNING ("mongoc_client_pool_set_structured_log_opts can only be called once per pool"); + } else if (pool->client_initialized) { + MONGOC_WARNING ("mongoc_client_pool_set_structured_log_opts can only be called before mongoc_client_pool_pop"); + } else { + // Now we can be sure no other threads are relying on concurrent access to the instance yet. + mongoc_topology_set_structured_log_opts (pool->topology, opts); + pool->structured_log_opts_set = true; + } +} + bool mongoc_client_pool_set_error_api (mongoc_client_pool_t *pool, int32_t version) { diff --git a/src/libmongoc/src/mongoc/mongoc-client-pool.h b/src/libmongoc/src/mongoc/mongoc-client-pool.h index 779aec7d5aa..ae785841b3d 100644 --- a/src/libmongoc/src/mongoc/mongoc-client-pool.h +++ b/src/libmongoc/src/mongoc/mongoc-client-pool.h @@ -28,6 +28,7 @@ #ifdef MONGOC_ENABLE_SSL #include "mongoc-ssl.h" #endif +#include "mongoc-structured-log.h" #include "mongoc-uri.h" @@ -69,6 +70,8 @@ mongoc_client_pool_enable_auto_encryption (mongoc_client_pool_t *pool, bson_error_t *error); MONGOC_EXPORT (bool) mongoc_client_pool_set_server_api (mongoc_client_pool_t *pool, const mongoc_server_api_t *api, bson_error_t *error); +MONGOC_EXPORT (void) +mongoc_client_pool_set_structured_log_opts (mongoc_client_pool_t *pool, const mongoc_structured_log_opts_t *opts); BSON_END_DECLS diff --git a/src/libmongoc/src/mongoc/mongoc-client-private.h b/src/libmongoc/src/mongoc/mongoc-client-private.h index 39d540e07a8..ef88ea87c25 100644 --- a/src/libmongoc/src/mongoc/mongoc-client-private.h +++ b/src/libmongoc/src/mongoc/mongoc-client-private.h @@ -92,7 +92,6 @@ struct _mongoc_client_t { mongoc_uri_t *uri; mongoc_cluster_t cluster; bool in_exhaust; - bool is_pooled; mongoc_stream_initiator_t initiator; void *initiator_data; diff --git a/src/libmongoc/src/mongoc/mongoc-client.c b/src/libmongoc/src/mongoc/mongoc-client.c index 8d0bf509958..fd1454ccb02 100644 --- a/src/libmongoc/src/mongoc/mongoc-client.c +++ b/src/libmongoc/src/mongoc/mongoc-client.c @@ -1123,8 +1123,10 @@ _mongoc_client_new_from_topology (mongoc_topology_t *topology) } #endif - mongoc_structured_log ( - MONGOC_STRUCTURED_LOG_LEVEL_DEBUG, MONGOC_STRUCTURED_LOG_COMPONENT_CONNECTION, "Client created"); + mongoc_structured_log (topology->structured_log, + MONGOC_STRUCTURED_LOG_LEVEL_DEBUG, + MONGOC_STRUCTURED_LOG_COMPONENT_CONNECTION, + "Client created"); mongoc_counter_clients_active_inc (); @@ -2167,6 +2169,7 @@ _mongoc_client_monitor_op_killcursors (mongoc_cluster_t *cluster, _mongoc_client_prepare_killcursors_command (cursor_id, collection, &doc); mongoc_structured_log ( + cluster->client->topology->structured_log, MONGOC_STRUCTURED_LOG_LEVEL_DEBUG, MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND, "Command started", @@ -2228,6 +2231,7 @@ _mongoc_client_monitor_op_killcursors_succeeded (mongoc_cluster_t *cluster, bson_append_array_builder_end (&doc, cursors_unknown); mongoc_structured_log ( + client->topology->structured_log, MONGOC_STRUCTURED_LOG_LEVEL_DEBUG, MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND, "Command succeeded", @@ -2286,6 +2290,7 @@ _mongoc_client_monitor_op_killcursors_failed (mongoc_cluster_t *cluster, bson_append_int32 (&doc, "ok", 2, 0); mongoc_structured_log ( + client->topology->structured_log, MONGOC_STRUCTURED_LOG_LEVEL_DEBUG, MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND, "Command failed", @@ -2647,6 +2652,22 @@ mongoc_client_set_apm_callbacks (mongoc_client_t *client, mongoc_apm_callbacks_t return _mongoc_client_set_apm_callbacks_private (client, callbacks, context); } + +void +mongoc_client_set_structured_log_opts (mongoc_client_t *client, const mongoc_structured_log_opts_t *opts) +{ + BSON_ASSERT_PARAM (client); + // opts is optional + + if (client->topology->single_threaded) { + mongoc_topology_set_structured_log_opts (client->topology, opts); + } else { + MONGOC_WARNING ("Cannot set structured log options on a pooled client, use " + "mongoc_client_pool_set_structured_log_opts before the first mongoc_client_pool_pop"); + } +} + + mongoc_server_description_t * mongoc_client_get_server_description (mongoc_client_t *client, uint32_t server_id) { @@ -2963,7 +2984,7 @@ mongoc_client_set_server_api (mongoc_client_t *client, const mongoc_server_api_t BSON_ASSERT_PARAM (client); BSON_ASSERT_PARAM (api); - if (client->is_pooled) { + if (!client->topology->single_threaded) { bson_set_error (error, MONGOC_ERROR_CLIENT, MONGOC_ERROR_CLIENT_API_FROM_POOL, diff --git a/src/libmongoc/src/mongoc/mongoc-client.h b/src/libmongoc/src/mongoc/mongoc-client.h index 1c1c4b5cbcd..7ce456aa6dd 100644 --- a/src/libmongoc/src/mongoc/mongoc-client.h +++ b/src/libmongoc/src/mongoc/mongoc-client.h @@ -36,6 +36,7 @@ #include "mongoc-ssl.h" #endif #include "mongoc-stream.h" +#include "mongoc-structured-log.h" #include "mongoc-uri.h" #include "mongoc-write-concern.h" #include "mongoc-read-concern.h" @@ -228,6 +229,8 @@ mongoc_client_set_ssl_opts (mongoc_client_t *client, const mongoc_ssl_opt_t *opt #endif MONGOC_EXPORT (bool) mongoc_client_set_apm_callbacks (mongoc_client_t *client, mongoc_apm_callbacks_t *callbacks, void *context); +MONGOC_EXPORT (void) +mongoc_client_set_structured_log_opts (mongoc_client_t *client, const mongoc_structured_log_opts_t *opts); MONGOC_EXPORT (mongoc_server_description_t *) mongoc_client_get_server_description (mongoc_client_t *client, uint32_t server_id) BSON_GNUC_WARN_UNUSED_RESULT; MONGOC_EXPORT (mongoc_server_description_t **) diff --git a/src/libmongoc/src/mongoc/mongoc-cluster.c b/src/libmongoc/src/mongoc/mongoc-cluster.c index 3a27f09de47..eb5b408d923 100644 --- a/src/libmongoc/src/mongoc/mongoc-cluster.c +++ b/src/libmongoc/src/mongoc/mongoc-cluster.c @@ -535,6 +535,7 @@ mongoc_cluster_run_command_monitored (mongoc_cluster_t *cluster, mongoc_cmd_t *c } mongoc_structured_log ( + cluster->client->topology->structured_log, MONGOC_STRUCTURED_LOG_LEVEL_DEBUG, MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND, "Command started", @@ -566,6 +567,7 @@ mongoc_cluster_run_command_monitored (mongoc_cluster_t *cluster, mongoc_cmd_t *c } mongoc_structured_log ( + cluster->client->topology->structured_log, MONGOC_STRUCTURED_LOG_LEVEL_DEBUG, MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND, "Command succeeded", @@ -599,6 +601,7 @@ mongoc_cluster_run_command_monitored (mongoc_cluster_t *cluster, mongoc_cmd_t *c int64_t duration = bson_get_monotonic_time () - started; mongoc_structured_log ( + cluster->client->topology->structured_log, MONGOC_STRUCTURED_LOG_LEVEL_DEBUG, MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND, "Command failed", diff --git a/src/libmongoc/src/mongoc/mongoc-cmd-private.h b/src/libmongoc/src/mongoc/mongoc-cmd-private.h index d150c677656..6ceedad6b88 100644 --- a/src/libmongoc/src/mongoc/mongoc-cmd-private.h +++ b/src/libmongoc/src/mongoc/mongoc-cmd-private.h @@ -143,18 +143,6 @@ _mongoc_cmd_append_payload_as_array (const mongoc_cmd_t *cmd, bson_t *out); void _mongoc_cmd_append_server_api (bson_t *command_body, const mongoc_server_api_t *api); -typedef enum { - MONGOC_LOGGED_CMD_CONTENT_FLAG_COMMAND = (1 << 0), - MONGOC_LOGGED_CMD_CONTENT_FLAG_DATABASE_NAME = (1 << 1), - MONGOC_LOGGED_CMD_CONTENT_FLAG_COMMAND_NAME = (1 << 2), - MONGOC_LOGGED_CMD_CONTENT_FLAG_OPERATION_ID = (1 << 3), -} mongoc_logged_cmd_content_flags_t; - -bool -mongoc_cmd_append_logged_contents_to_bson (const mongoc_cmd_t *cmd, - bson_t *bson, - mongoc_logged_cmd_content_flags_t flags); - BSON_END_DECLS diff --git a/src/libmongoc/src/mongoc/mongoc-cmd.c b/src/libmongoc/src/mongoc/mongoc-cmd.c index 8d9f229997f..a1d7fb449bb 100644 --- a/src/libmongoc/src/mongoc/mongoc-cmd.c +++ b/src/libmongoc/src/mongoc/mongoc-cmd.c @@ -1037,57 +1037,3 @@ _mongoc_cmd_append_server_api (bson_t *command_body, const mongoc_server_api_t * bson_append_bool (command_body, "apiDeprecationErrors", -1, api->deprecation_errors.value); } } - -bool -mongoc_cmd_append_logged_contents_to_bson (const mongoc_cmd_t *cmd, - bson_t *bson, - mongoc_logged_cmd_content_flags_t flags) -{ - BSON_ASSERT_PARAM (cmd); - - if ((flags & MONGOC_LOGGED_CMD_CONTENT_FLAG_DATABASE_NAME) && - !BSON_APPEND_UTF8 (bson, "databaseName", cmd->db_name)) { - return false; - } - if ((flags & MONGOC_LOGGED_CMD_CONTENT_FLAG_COMMAND_NAME) && - !BSON_APPEND_UTF8 (bson, "commandName", cmd->command_name)) { - return false; - } - if ((flags & MONGOC_LOGGED_CMD_CONTENT_FLAG_OPERATION_ID) && - !BSON_APPEND_INT64 (bson, "operationId", cmd->operation_id)) { - return false; - } - if (flags & MONGOC_LOGGED_CMD_CONTENT_FLAG_COMMAND) { - if (mongoc_apm_is_sensitive_command_message (cmd->command_name, cmd->command)) { - if (!BSON_APPEND_UTF8 (bson, "command", "{}")) { - return false; - } - } else { - bson_t *command_copy = NULL; - - if (cmd->payloads_count > 0) { - // @todo This is a performance bottleneck, we shouldn't be copying - // a potentially large command to serialize a potentially very - // small part of it. We should be appending JSON to a single buffer - // for all nesting levels, constrained by length limit, while visiting - // borrowed references to each command attribute and each payload. CDRIVER-4814 - command_copy = bson_copy (cmd->command); - _mongoc_cmd_append_payload_as_array (cmd, command_copy); - } - - size_t json_length; - char *json = mongoc_structured_log_document_to_json (command_copy ? command_copy : cmd->command, &json_length); - bool ok = false; - if (json) { - const char *key = "command"; - ok = bson_append_utf8 (bson, key, strlen (key), json, json_length); - bson_free (json); - } - bson_destroy (command_copy); - if (!ok) { - return false; - } - } - } - return true; -} diff --git a/src/libmongoc/src/mongoc/mongoc-cursor-legacy.c b/src/libmongoc/src/mongoc/mongoc-cursor-legacy.c index 658f1348965..0b42c576654 100644 --- a/src/libmongoc/src/mongoc/mongoc-cursor-legacy.c +++ b/src/libmongoc/src/mongoc/mongoc-cursor-legacy.c @@ -50,6 +50,7 @@ _mongoc_cursor_monitor_legacy_get_more (mongoc_cursor_t *cursor, mongoc_server_s _mongoc_cursor_prepare_getmore_command (cursor, &doc); mongoc_structured_log ( + client->topology->structured_log, MONGOC_STRUCTURED_LOG_LEVEL_DEBUG, MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND, "Command started", diff --git a/src/libmongoc/src/mongoc/mongoc-cursor.c b/src/libmongoc/src/mongoc/mongoc-cursor.c index 734b178ed95..f891e854fb3 100644 --- a/src/libmongoc/src/mongoc/mongoc-cursor.c +++ b/src/libmongoc/src/mongoc/mongoc-cursor.c @@ -647,6 +647,7 @@ _mongoc_cursor_monitor_command (mongoc_cursor_t *cursor, client = cursor->client; mongoc_structured_log ( + client->topology->structured_log, MONGOC_STRUCTURED_LOG_LEVEL_DEBUG, MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND, "Command started", @@ -739,7 +740,8 @@ _mongoc_cursor_monitor_succeeded (mongoc_cursor_t *cursor, bson_destroy (&docs_array); - mongoc_structured_log (MONGOC_STRUCTURED_LOG_LEVEL_DEBUG, + mongoc_structured_log (client->topology->structured_log, + MONGOC_STRUCTURED_LOG_LEVEL_DEBUG, MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND, "Command succeeded", int32 ("requestId", client->cluster.request_id), @@ -796,7 +798,8 @@ _mongoc_cursor_monitor_failed (mongoc_cursor_t *cursor, bsonBuildDecl (reply, kv ("ok", int32 (0))); char *db = bson_strndup (cursor->ns, cursor->dblen); - mongoc_structured_log (MONGOC_STRUCTURED_LOG_LEVEL_DEBUG, + mongoc_structured_log (client->topology->structured_log, + MONGOC_STRUCTURED_LOG_LEVEL_DEBUG, MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND, "Command failed", int32 ("requestId", client->cluster.request_id), diff --git a/src/libmongoc/src/mongoc/mongoc-structured-log-private.h b/src/libmongoc/src/mongoc/mongoc-structured-log-private.h index 6ea1ea99903..895e5a2e4df 100644 --- a/src/libmongoc/src/mongoc/mongoc-structured-log-private.h +++ b/src/libmongoc/src/mongoc/mongoc-structured-log-private.h @@ -28,19 +28,45 @@ BSON_BEGIN_DECLS +typedef struct mongoc_structured_log_instance_t mongoc_structured_log_instance_t; + #define MONGOC_STRUCTURED_LOG_DEFAULT_LEVEL MONGOC_STRUCTURED_LOG_LEVEL_WARNING #define MONGOC_STRUCTURED_LOG_DEFAULT_MAX_DOCUMENT_LENGTH 1000 /** - * @def mongoc_structured_log(level, component, message, ...) + * @brief Allocate a new instance of the structured logging system + * @param opts Options, copied into the new instance. + * + * Must be paired with mongoc_structured_log_instance_destroy(). + * + * Once created, an instance has immutable settings. To change the handler + * or the max level per component filters, the instance will be replaced. + * One instance can be used by mongoc_structured_log() calls on multiple + * threads concurrently. + */ +mongoc_structured_log_instance_t * +mongoc_structured_log_instance_new (const mongoc_structured_log_opts_t *opts); + +/** + * @brief Destroy an instance of the structured logging system. + * + * All threads must be finished using the mongoc_structured_log_instance_t + * before it is destroyed. + */ +void +mongoc_structured_log_instance_destroy (mongoc_structured_log_instance_t *instance); + +/** + * @def mongoc_structured_log(instance, level, component, message, ...) * @brief Write to the libmongoc structured log. * + * @param instance Structured log instance, as a mongoc_structured_log_instance_t* expression * @param level Log level, as a mongoc_structured_log_level_t expression * @param component Log component, as a mongoc_structured_log_component_t expression * @param message Log message, as a const char* expression * @param ... Optional list of log 'items' that specify additional information to include. * - * The level, component, and message expressions are always evaluated. + * The instance, level, component, and message expressions are always evaluated. * Any expressions in the optional items list are only evaluated if the log * hasn't been disabled by a component's maximum log level setting or by * unsetting the global structured log handler. @@ -52,19 +78,22 @@ BSON_BEGIN_DECLS * building the table of item information and _mongoc_structured_log_with_entry() * once the table is built. */ -#define mongoc_structured_log(_level, _component, ...) \ - _bsonDSL_eval (_mongoc_structured_log_with_end_of_list (_level, _component, __VA_ARGS__, end_of_list ())) - -#define _mongoc_structured_log_with_end_of_list(_level, _component, _message, ...) \ - do { \ - mongoc_structured_log_entry_t _entry = { \ - .envelope.level = (_level), .envelope.component = (_component), .envelope.message = (_message)}; \ - if (_mongoc_structured_log_should_log (&_entry.envelope)) { \ - const mongoc_structured_log_builder_stage_t _builder[] = { \ - _mongoc_structured_log_items_to_stages (__VA_ARGS__)}; \ - _entry.builder = _builder; \ - _mongoc_structured_log_with_entry (&_entry); \ - } \ +#define mongoc_structured_log(_structured_log_instance, _level, _component, ...) \ + _bsonDSL_eval (_mongoc_structured_log_with_end_of_list ( \ + _structured_log_instance, _level, _component, __VA_ARGS__, end_of_list ())) + +#define _mongoc_structured_log_with_end_of_list(_structured_log_instance, _level, _component, _message, ...) \ + do { \ + mongoc_structured_log_entry_t _entry = {.envelope.instance = (_structured_log_instance), \ + .envelope.level = (_level), \ + .envelope.component = (_component), \ + .envelope.message = (_message)}; \ + if (_mongoc_structured_log_should_log (&_entry.envelope)) { \ + const mongoc_structured_log_builder_stage_t _builder[] = { \ + _mongoc_structured_log_items_to_stages (__VA_ARGS__)}; \ + _entry.builder = _builder; \ + _mongoc_structured_log_with_entry (&_entry); \ + } \ } while (0) #define _mongoc_structured_log_items_to_stages(...) \ @@ -174,6 +203,13 @@ BSON_BEGIN_DECLS #define _mongoc_structured_log_item_bson_as_json(_key_or_null, _value_bson) \ {.func = _mongoc_structured_log_append_bson_as_json, .arg1.utf8 = (_key_or_null), .arg2.bson = (_value_bson)}, +typedef enum { + MONGOC_STRUCTURED_LOG_CMD_CONTENT_FLAG_COMMAND = (1 << 0), + MONGOC_STRUCTURED_LOG_CMD_CONTENT_FLAG_DATABASE_NAME = (1 << 1), + MONGOC_STRUCTURED_LOG_CMD_CONTENT_FLAG_COMMAND_NAME = (1 << 2), + MONGOC_STRUCTURED_LOG_CMD_CONTENT_FLAG_OPERATION_ID = (1 << 3), +} mongoc_structured_log_cmd_content_flags_t; + /** * @def cmd(cmd, ...) * @brief Structured log item, mongoc_cmd_t fields with automatic redaction @@ -185,7 +221,7 @@ BSON_BEGIN_DECLS {.func = _mongoc_structured_log_append_cmd, \ .arg1.cmd = (_cmd), \ .arg2.cmd_flags = \ - (0 _bsonDSL_mapMacro (_mongoc_structured_log_flag_expr, MONGOC_LOGGED_CMD_CONTENT_FLAG, __VA_ARGS__))}, + (0 _bsonDSL_mapMacro (_mongoc_structured_log_flag_expr, MONGOC_STRUCTURED_LOG_CMD_CONTENT_FLAG, __VA_ARGS__))}, /** * @def cmd_reply(cmd, reply) @@ -273,7 +309,7 @@ BSON_BEGIN_DECLS typedef struct mongoc_structured_log_builder_stage_t mongoc_structured_log_builder_stage_t; typedef const mongoc_structured_log_builder_stage_t *(*mongoc_structured_log_builder_func_t) ( - bson_t *bson, const mongoc_structured_log_builder_stage_t *stage); + bson_t *bson, const mongoc_structured_log_builder_stage_t *stage, const mongoc_structured_log_opts_t *opts); struct mongoc_structured_log_builder_stage_t { // Why "stages" instead of a variable size argument list per item? @@ -298,13 +334,14 @@ struct mongoc_structured_log_builder_stage_t { int32_t int32; int64_t int64; mongoc_error_content_flags_t error_flags; - mongoc_logged_cmd_content_flags_t cmd_flags; + mongoc_structured_log_cmd_content_flags_t cmd_flags; mongoc_server_description_content_flags_t server_description_flags; } arg2; // Avoid adding an arg3, prefer to use additional stages }; typedef struct mongoc_structured_log_envelope_t { + mongoc_structured_log_instance_t *instance; mongoc_structured_log_level_t level; mongoc_structured_log_component_t component; const char *message; @@ -315,11 +352,10 @@ struct mongoc_structured_log_entry_t { const mongoc_structured_log_builder_stage_t *builder; // Required }; -char * -mongoc_structured_log_document_to_json (const bson_t *document, size_t *length); - void -mongoc_structured_log_get_handler (mongoc_structured_log_func_t *log_func, void **user_data); +mongoc_structured_log_get_handler (const mongoc_structured_log_opts_t *opts, + mongoc_structured_log_func_t *log_func, + void **user_data); bool _mongoc_structured_log_should_log (const mongoc_structured_log_envelope_t *envelope); @@ -328,57 +364,89 @@ void _mongoc_structured_log_with_entry (const mongoc_structured_log_entry_t *entry); const mongoc_structured_log_builder_stage_t * -_mongoc_structured_log_append_utf8 (bson_t *bson, const mongoc_structured_log_builder_stage_t *stage); +_mongoc_structured_log_append_utf8 (bson_t *bson, + const mongoc_structured_log_builder_stage_t *stage, + const mongoc_structured_log_opts_t *opts); const mongoc_structured_log_builder_stage_t * -_mongoc_structured_log_append_utf8_n_stage0 (bson_t *bson, const mongoc_structured_log_builder_stage_t *stage); +_mongoc_structured_log_append_utf8_n_stage0 (bson_t *bson, + const mongoc_structured_log_builder_stage_t *stage, + const mongoc_structured_log_opts_t *opts); const mongoc_structured_log_builder_stage_t * -_mongoc_structured_log_append_utf8_n_stage1 (bson_t *bson, const mongoc_structured_log_builder_stage_t *stage); +_mongoc_structured_log_append_utf8_n_stage1 (bson_t *bson, + const mongoc_structured_log_builder_stage_t *stage, + const mongoc_structured_log_opts_t *opts); const mongoc_structured_log_builder_stage_t * -_mongoc_structured_log_append_int32 (bson_t *bson, const mongoc_structured_log_builder_stage_t *stage); +_mongoc_structured_log_append_int32 (bson_t *bson, + const mongoc_structured_log_builder_stage_t *stage, + const mongoc_structured_log_opts_t *opts); const mongoc_structured_log_builder_stage_t * -_mongoc_structured_log_append_int64 (bson_t *bson, const mongoc_structured_log_builder_stage_t *stage); +_mongoc_structured_log_append_int64 (bson_t *bson, + const mongoc_structured_log_builder_stage_t *stage, + const mongoc_structured_log_opts_t *opts); const mongoc_structured_log_builder_stage_t * -_mongoc_structured_log_append_boolean (bson_t *bson, const mongoc_structured_log_builder_stage_t *stage); +_mongoc_structured_log_append_boolean (bson_t *bson, + const mongoc_structured_log_builder_stage_t *stage, + const mongoc_structured_log_opts_t *opts); const mongoc_structured_log_builder_stage_t * -_mongoc_structured_log_append_error (bson_t *bson, const mongoc_structured_log_builder_stage_t *stage); +_mongoc_structured_log_append_error (bson_t *bson, + const mongoc_structured_log_builder_stage_t *stage, + const mongoc_structured_log_opts_t *opts); const mongoc_structured_log_builder_stage_t * -_mongoc_structured_log_append_oid_as_hex (bson_t *bson, const mongoc_structured_log_builder_stage_t *stage); +_mongoc_structured_log_append_oid_as_hex (bson_t *bson, + const mongoc_structured_log_builder_stage_t *stage, + const mongoc_structured_log_opts_t *opts); const mongoc_structured_log_builder_stage_t * -_mongoc_structured_log_append_bson_as_json (bson_t *bson, const mongoc_structured_log_builder_stage_t *stage); +_mongoc_structured_log_append_bson_as_json (bson_t *bson, + const mongoc_structured_log_builder_stage_t *stage, + const mongoc_structured_log_opts_t *opts); const mongoc_structured_log_builder_stage_t * -_mongoc_structured_log_append_cmd (bson_t *bson, const mongoc_structured_log_builder_stage_t *stage); +_mongoc_structured_log_append_cmd (bson_t *bson, + const mongoc_structured_log_builder_stage_t *stage, + const mongoc_structured_log_opts_t *opts); const mongoc_structured_log_builder_stage_t * -_mongoc_structured_log_append_cmd_reply (bson_t *bson, const mongoc_structured_log_builder_stage_t *stage); +_mongoc_structured_log_append_cmd_reply (bson_t *bson, + const mongoc_structured_log_builder_stage_t *stage, + const mongoc_structured_log_opts_t *opts); const mongoc_structured_log_builder_stage_t * -_mongoc_structured_log_append_cmd_name_reply (bson_t *bson, const mongoc_structured_log_builder_stage_t *stage); +_mongoc_structured_log_append_cmd_name_reply (bson_t *bson, + const mongoc_structured_log_builder_stage_t *stage, + const mongoc_structured_log_opts_t *opts); const mongoc_structured_log_builder_stage_t * -_mongoc_structured_log_append_cmd_failure_stage0 (bson_t *bson, const mongoc_structured_log_builder_stage_t *stage); +_mongoc_structured_log_append_cmd_failure_stage0 (bson_t *bson, + const mongoc_structured_log_builder_stage_t *stage, + const mongoc_structured_log_opts_t *opts); const mongoc_structured_log_builder_stage_t * -_mongoc_structured_log_append_cmd_failure_stage1 (bson_t *bson, const mongoc_structured_log_builder_stage_t *stage); +_mongoc_structured_log_append_cmd_failure_stage1 (bson_t *bson, + const mongoc_structured_log_builder_stage_t *stage, + const mongoc_structured_log_opts_t *opts); const mongoc_structured_log_builder_stage_t * _mongoc_structured_log_append_cmd_name_failure_stage0 (bson_t *bson, - const mongoc_structured_log_builder_stage_t *stage); + const mongoc_structured_log_builder_stage_t *stage, + const mongoc_structured_log_opts_t *opts); const mongoc_structured_log_builder_stage_t * _mongoc_structured_log_append_cmd_name_failure_stage1 (bson_t *bson, - const mongoc_structured_log_builder_stage_t *stage); + const mongoc_structured_log_builder_stage_t *stage, + const mongoc_structured_log_opts_t *opts); const mongoc_structured_log_builder_stage_t * -_mongoc_structured_log_append_server_description (bson_t *bson, const mongoc_structured_log_builder_stage_t *stage); +_mongoc_structured_log_append_server_description (bson_t *bson, + const mongoc_structured_log_builder_stage_t *stage, + const mongoc_structured_log_opts_t *opts); BSON_END_DECLS diff --git a/src/libmongoc/src/mongoc/mongoc-structured-log.c b/src/libmongoc/src/mongoc/mongoc-structured-log.c index 7e33ca40aba..348d89b9de0 100644 --- a/src/libmongoc/src/mongoc/mongoc-structured-log.c +++ b/src/libmongoc/src/mongoc/mongoc-structured-log.c @@ -25,7 +25,9 @@ #define STRUCTURED_LOG_COMPONENT_TABLE_SIZE (1 + (size_t) MONGOC_STRUCTURED_LOG_COMPONENT_CONNECTION) -static BSON_ONCE_FUN (_mongoc_structured_log_init_once); +// Environment variables with default level for each log component +static const char *gStructuredLogComponentEnvVars[] = { + "MONGODB_LOG_COMMAND", "MONGODB_LOG_TOPOLOGY", "MONGODB_LOG_SERVER_SELECTION", "MONGODB_LOG_CONNECTION"}; // Canonical names for log components static const char *gStructuredLogComponentNames[] = {"command", "topology", "serverSelection", "connection"}; @@ -42,69 +44,36 @@ static const struct { {.name = "warn", .level = MONGOC_STRUCTURED_LOG_LEVEL_WARNING}, {.name = "info", .level = MONGOC_STRUCTURED_LOG_LEVEL_INFO}}; -// Values for gStructuredLog.handler.state -typedef enum { - MONGOC_STRUCTURED_LOG_STATE_UNINITIALIZED = 0, // Must ensure_init. This state is never re-entered once exited. - MONGOC_STRUCTURED_LOG_STATE_INACTIVE = 1, // Handler disabled. Always < MONGOC_STRUCTURED_LOG_STATE_ACTIVE. - MONGOC_STRUCTURED_LOG_STATE_ACTIVE = 2, // Maybe active. Value is also a base to which log levels are added. -} mongoc_structured_log_state_t; - -static struct { - /* Pre-computed table combining handler.state and component_level_table. - * Individual values are atomic, read with relaxed memory order. - * - * Table rebuilds are protected by a mutex. The build_mutex should - * be held while updating the table and while performing the instigating - * level or handler change. - * - * When acquiring both the build_mutex and an exclusive lock on handler.mutex - * to update the handler, the build mutex must always be acquired first. - * Log handlers must not be allowed to call functions that acquire the build_mutex. - */ - struct { - int component_level_plus_state[STRUCTURED_LOG_COMPONENT_TABLE_SIZE]; - bson_mutex_t build_mutex; - } combined_table; - - // Handler state, updated under exclusive lock, callable/gettable with shared lock - struct { - bson_shared_mutex_t mutex; - mongoc_structured_log_func_t func; - void *user_data; - mongoc_structured_log_state_t state; - } handler; - - // State only used by the default handler - struct { - bson_mutex_t stream_mutex; - FILE *stream; - } default_handler; - - // Configured max document length, only modified during initialization +// Shared mutable data for the default handler +typedef struct mongoc_structured_log_default_handler_shared_t { + bson_mutex_t mutex; + FILE *stream; + bool stream_fclose_on_destroy; +} mongoc_structured_log_default_handler_shared_t; + +struct mongoc_structured_log_opts_t { + mongoc_structured_log_func_t handler_func; + void *handler_user_data; + mongoc_structured_log_level_t max_level_per_component[STRUCTURED_LOG_COMPONENT_TABLE_SIZE]; int32_t max_document_length; + char *default_handler_path; +}; - /* Main storage for public per-component max log levels. - * Atomic mongoc_structured_log_level_t, seq_cst memory order. - * Used only for getting/setting individual component levels. - * Not used directly by should_log, which relies on 'combined_table'. */ - int component_level_table[STRUCTURED_LOG_COMPONENT_TABLE_SIZE]; -} gStructuredLog; - -static BSON_INLINE void -_mongoc_structured_log_ensure_init (void) -{ - static bson_once_t init_once = BSON_ONCE_INIT; - bson_once (&init_once, &_mongoc_structured_log_init_once); -} +struct mongoc_structured_log_instance_t { + struct mongoc_structured_log_opts_t opts; // Immutable capture of log_opts, func != NULL + mongoc_structured_log_default_handler_shared_t default_handler_shared; // Inner mutability +}; bson_t * mongoc_structured_log_entry_message_as_bson (const mongoc_structured_log_entry_t *entry) { + BSON_ASSERT_PARAM (entry); bson_t *bson = bson_new (); BSON_APPEND_UTF8 (bson, "message", entry->envelope.message); const mongoc_structured_log_builder_stage_t *stage = entry->builder; + const mongoc_structured_log_opts_t *opts = &entry->envelope.instance->opts; while (stage->func) { - stage = stage->func (bson, stage); + stage = stage->func (bson, stage, opts); } return bson; } @@ -112,151 +81,100 @@ mongoc_structured_log_entry_message_as_bson (const mongoc_structured_log_entry_t mongoc_structured_log_level_t mongoc_structured_log_entry_get_level (const mongoc_structured_log_entry_t *entry) { + BSON_ASSERT_PARAM (entry); return entry->envelope.level; } mongoc_structured_log_component_t mongoc_structured_log_entry_get_component (const mongoc_structured_log_entry_t *entry) { + BSON_ASSERT_PARAM (entry); return entry->envelope.component; } -static BSON_INLINE mongoc_structured_log_level_t -_mongoc_structured_log_get_max_level_for_component (mongoc_structured_log_component_t component) -{ - unsigned table_index = (unsigned) component; - BSON_ASSERT (table_index < STRUCTURED_LOG_COMPONENT_TABLE_SIZE); - return mcommon_atomic_int_fetch (&gStructuredLog.component_level_table[table_index], mcommon_memory_order_seq_cst); -} - mongoc_structured_log_level_t -mongoc_structured_log_get_max_level_for_component (mongoc_structured_log_component_t component) -{ - _mongoc_structured_log_ensure_init (); - return _mongoc_structured_log_get_max_level_for_component (component); -} - -static void -_mongoc_structured_log_update_level_plus_state (void) +mongoc_structured_log_opts_get_max_level_for_component (const mongoc_structured_log_opts_t *opts, + mongoc_structured_log_component_t component) { - // The caller MUST be holding the build_mutex throughout the operation which caused this table update. - // Pre-calculate a table of values that combine per-component level and global state. - // Needs to be updated when the handler is set/unset or when any level setting changes. - mongoc_structured_log_state_t handler_state = gStructuredLog.handler.state; - for (unsigned table_index = 0; table_index < STRUCTURED_LOG_COMPONENT_TABLE_SIZE; table_index++) { - mongoc_structured_log_component_t component = (mongoc_structured_log_component_t) table_index; - mcommon_atomic_int_exchange (&gStructuredLog.combined_table.component_level_plus_state[table_index], - handler_state == MONGOC_STRUCTURED_LOG_STATE_ACTIVE - ? (int) MONGOC_STRUCTURED_LOG_STATE_ACTIVE + - (int) _mongoc_structured_log_get_max_level_for_component (component) - : (int) handler_state, - mcommon_memory_order_relaxed); + BSON_ASSERT_PARAM (opts); + unsigned table_index = (unsigned) component; + if (table_index < STRUCTURED_LOG_COMPONENT_TABLE_SIZE) { + return opts->max_level_per_component[table_index]; + } else { + // As documented, unknown component enums return the lowest possible log level. + return MONGOC_STRUCTURED_LOG_LEVEL_EMERGENCY; } } -static void -_mongoc_structured_log_set_handler (mongoc_structured_log_func_t log_func, void *user_data) -{ - bson_shared_mutex_lock (&gStructuredLog.handler.mutex); // Waits for handler invocations to end - gStructuredLog.handler.func = log_func; - gStructuredLog.handler.user_data = user_data; - gStructuredLog.handler.state = - log_func == NULL ? MONGOC_STRUCTURED_LOG_STATE_INACTIVE : MONGOC_STRUCTURED_LOG_STATE_ACTIVE; - bson_shared_mutex_unlock (&gStructuredLog.handler.mutex); -} - void -mongoc_structured_log_set_handler (mongoc_structured_log_func_t log_func, void *user_data) +mongoc_structured_log_opts_set_handler (mongoc_structured_log_opts_t *opts, + mongoc_structured_log_func_t log_func, + void *user_data) { - _mongoc_structured_log_ensure_init (); - - bson_mutex_lock (&gStructuredLog.combined_table.build_mutex); - _mongoc_structured_log_set_handler (log_func, user_data); - _mongoc_structured_log_update_level_plus_state (); - bson_mutex_unlock (&gStructuredLog.combined_table.build_mutex); + BSON_ASSERT_PARAM (opts); + opts->handler_func = log_func; + opts->handler_user_data = user_data; } void -mongoc_structured_log_get_handler (mongoc_structured_log_func_t *log_func, void **user_data) +mongoc_structured_log_get_handler (const mongoc_structured_log_opts_t *opts, + mongoc_structured_log_func_t *log_func, + void **user_data) { - _mongoc_structured_log_ensure_init (); - - bson_shared_mutex_lock_shared (&gStructuredLog.handler.mutex); - *log_func = gStructuredLog.handler.func; - *user_data = gStructuredLog.handler.user_data; - bson_shared_mutex_unlock_shared (&gStructuredLog.handler.mutex); + BSON_ASSERT_PARAM (opts); + *log_func = opts->handler_func; + *user_data = opts->handler_user_data; } -static void -_mongoc_structured_log_set_max_level_for_component (mongoc_structured_log_component_t component, - mongoc_structured_log_level_t level) -{ - BSON_ASSERT (level >= MONGOC_STRUCTURED_LOG_LEVEL_EMERGENCY && level <= MONGOC_STRUCTURED_LOG_LEVEL_TRACE); - unsigned table_index = (unsigned) component; - BSON_ASSERT (table_index < STRUCTURED_LOG_COMPONENT_TABLE_SIZE); - mcommon_atomic_int_exchange ( - &gStructuredLog.component_level_table[table_index], level, mcommon_memory_order_seq_cst); -} - -void -mongoc_structured_log_set_max_level_for_component (mongoc_structured_log_component_t component, - mongoc_structured_log_level_t level) -{ - _mongoc_structured_log_ensure_init (); - - bson_mutex_lock (&gStructuredLog.combined_table.build_mutex); - _mongoc_structured_log_set_max_level_for_component (component, level); - _mongoc_structured_log_update_level_plus_state (); - bson_mutex_unlock (&gStructuredLog.combined_table.build_mutex); +bool +mongoc_structured_log_opts_set_max_level_for_component (mongoc_structured_log_opts_t *opts, + mongoc_structured_log_component_t component, + mongoc_structured_log_level_t level) +{ + BSON_ASSERT_PARAM (opts); + if (level >= MONGOC_STRUCTURED_LOG_LEVEL_EMERGENCY && level <= MONGOC_STRUCTURED_LOG_LEVEL_TRACE) { + unsigned table_index = (unsigned) component; + if (table_index < STRUCTURED_LOG_COMPONENT_TABLE_SIZE) { + opts->max_level_per_component[table_index] = level; + return true; + } + } + return false; } -static void -_mongoc_structured_log_set_max_level_for_all_components (mongoc_structured_log_level_t level) +bool +mongoc_structured_log_opts_set_max_level_for_all_components (mongoc_structured_log_opts_t *opts, + mongoc_structured_log_level_t level) { + BSON_ASSERT_PARAM (opts); for (int component = 0; component < STRUCTURED_LOG_COMPONENT_TABLE_SIZE; component++) { - _mongoc_structured_log_set_max_level_for_component ((mongoc_structured_log_component_t) component, level); + if (!mongoc_structured_log_opts_set_max_level_for_component ( + opts, (mongoc_structured_log_component_t) component, level)) { + // Fine to stop on the first error, always means 'level' is wrong and none of these will succeed. + return false; + } } -} - -void -mongoc_structured_log_set_max_level_for_all_components (mongoc_structured_log_level_t level) -{ - _mongoc_structured_log_ensure_init (); - - bson_mutex_lock (&gStructuredLog.combined_table.build_mutex); - _mongoc_structured_log_set_max_level_for_all_components (level); - _mongoc_structured_log_update_level_plus_state (); - bson_mutex_unlock (&gStructuredLog.combined_table.build_mutex); + return true; } bool _mongoc_structured_log_should_log (const mongoc_structured_log_envelope_t *envelope) { + // Note that the instance's max_level_per_component table will also be + // set to zeroes if logging is disabled. See mongoc_structured_log_instance_new. unsigned table_index = (unsigned) envelope->component; BSON_ASSERT (table_index < STRUCTURED_LOG_COMPONENT_TABLE_SIZE); - const int volatile *level_plus_state_ptr = &gStructuredLog.combined_table.component_level_plus_state[table_index]; - - // One atomic fetch gives us the per-component level, the global - // enable/disable state, and lets us detect whether initialization is needed. - int level_plus_state = mcommon_atomic_int_fetch (level_plus_state_ptr, mcommon_memory_order_relaxed); - if (BSON_UNLIKELY (level_plus_state == (int) MONGOC_STRUCTURED_LOG_STATE_UNINITIALIZED)) { - _mongoc_structured_log_ensure_init (); - level_plus_state = mcommon_atomic_int_fetch (level_plus_state_ptr, mcommon_memory_order_relaxed); - } - return envelope->level + MONGOC_STRUCTURED_LOG_STATE_ACTIVE <= level_plus_state; + return envelope->level <= envelope->instance->opts.max_level_per_component[table_index]; } void _mongoc_structured_log_with_entry (const mongoc_structured_log_entry_t *entry) { - // should_log has guaranteed that we've run init by now. - // No guarantee that there's actually a handler set but it's likely. - bson_shared_mutex_lock_shared (&gStructuredLog.handler.mutex); - mongoc_structured_log_func_t func = gStructuredLog.handler.func; - if (BSON_LIKELY (func)) { - func (entry, gStructuredLog.handler.user_data); - } - bson_shared_mutex_unlock_shared (&gStructuredLog.handler.mutex); + // By now, func is not allowed to be NULL. See mongoc_structured_log_instance_new. + mongoc_structured_log_instance_t *instance = entry->envelope.instance; + mongoc_structured_log_func_t func = instance->opts.handler_func; + BSON_ASSERT (func); + func (entry, instance->opts.handler_user_data); } static bool @@ -271,7 +189,7 @@ _mongoc_structured_log_get_log_level_from_env (const char *variable, if (mongoc_structured_log_get_named_level (level, out)) { return true; } - // Only report the first instance of each error + // Only log the first instance of each error per process if (0 == mcommon_atomic_int_fetch_add (err_count_atomic, 1, mcommon_memory_order_seq_cst)) { MONGOC_WARNING ("Invalid log level '%s' read from environment variable %s. Ignoring it.", level, variable); } @@ -289,6 +207,9 @@ mongoc_structured_log_get_level_name (mongoc_structured_log_level_t level) bool mongoc_structured_log_get_named_level (const char *name, mongoc_structured_log_level_t *out) { + BSON_ASSERT_PARAM (name); + BSON_ASSERT_PARAM (out); + // First check canonical names { const size_t table_size = sizeof gStructuredLogLevelNames / sizeof gStructuredLogLevelNames[0]; @@ -325,6 +246,9 @@ mongoc_structured_log_get_component_name (mongoc_structured_log_component_t comp bool mongoc_structured_log_get_named_component (const char *name, mongoc_structured_log_component_t *out) { + BSON_ASSERT_PARAM (name); + BSON_ASSERT_PARAM (out); + const size_t table_size = sizeof gStructuredLogComponentNames / sizeof gStructuredLogComponentNames[0]; for (unsigned table_index = 0; table_index < table_size; table_index++) { if (!strcasecmp (name, gStructuredLogComponentNames[table_index])) { @@ -355,94 +279,98 @@ _mongoc_structured_log_get_max_document_length_from_env (void) return (int32_t) int_value; } - MONGOC_WARNING ("Invalid length '%s' read from environment variable %s. Ignoring it.", max_length_str, variable); + // Only log the first instance of each error per process + static int err_count_atomic; + if (0 == mcommon_atomic_int_fetch_add (&err_count_atomic, 1, mcommon_memory_order_seq_cst)) { + MONGOC_WARNING ("Invalid length '%s' read from environment variable %s. Ignoring it.", max_length_str, variable); + } return MONGOC_STRUCTURED_LOG_DEFAULT_MAX_DOCUMENT_LENGTH; } -static void -_mongoc_structured_log_set_max_levels_from_env (void) +bool +mongoc_structured_log_opts_set_max_levels_from_env (mongoc_structured_log_opts_t *opts) { + BSON_ASSERT_PARAM (opts); + + bool all_ok = true; mongoc_structured_log_level_t level; - { - static int err_count_atomic = 0; - if (_mongoc_structured_log_get_log_level_from_env ("MONGODB_LOG_ALL", &level, &err_count_atomic)) { - _mongoc_structured_log_set_max_level_for_all_components (level); - } - } - { - static int err_count_atomic = 0; - if (_mongoc_structured_log_get_log_level_from_env ("MONGODB_LOG_COMMAND", &level, &err_count_atomic)) { - _mongoc_structured_log_set_max_level_for_component (MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND, level); - } - } - { - static int err_count_atomic = 0; - if (_mongoc_structured_log_get_log_level_from_env ("MONGODB_LOG_CONNECTION", &level, &err_count_atomic)) { - _mongoc_structured_log_set_max_level_for_component (MONGOC_STRUCTURED_LOG_COMPONENT_CONNECTION, level); - } - } - { - static int err_count_atomic = 0; - if (_mongoc_structured_log_get_log_level_from_env ("MONGODB_LOG_TOPOLOGY", &level, &err_count_atomic)) { - _mongoc_structured_log_set_max_level_for_component (MONGOC_STRUCTURED_LOG_COMPONENT_TOPOLOGY, level); - } + + // Errors are not fatal by detault; always reported by return value, and reported the first time only via a log + // warning. + static int err_count_all_atomic; + static int err_count_per_component_atomic[STRUCTURED_LOG_COMPONENT_TABLE_SIZE]; + + if (_mongoc_structured_log_get_log_level_from_env ("MONGODB_LOG_ALL", &level, &err_count_all_atomic)) { + mongoc_structured_log_opts_set_max_level_for_all_components (opts, level); + } else { + all_ok = false; } - { - static int err_count_atomic = 0; - if (_mongoc_structured_log_get_log_level_from_env ("MONGODB_LOG_SERVER_SELECTION", &level, &err_count_atomic)) { - _mongoc_structured_log_set_max_level_for_component (MONGOC_STRUCTURED_LOG_COMPONENT_SERVER_SELECTION, level); + + for (int component = 0; component < STRUCTURED_LOG_COMPONENT_TABLE_SIZE; component++) { + if (_mongoc_structured_log_get_log_level_from_env ( + gStructuredLogComponentEnvVars[component], &level, &err_count_per_component_atomic[component])) { + mongoc_structured_log_opts_set_max_level_for_component ( + opts, (mongoc_structured_log_component_t) component, level); + } else { + all_ok = false; } } -} - -void -mongoc_structured_log_set_max_levels_from_env (void) -{ - _mongoc_structured_log_ensure_init (); - bson_mutex_lock (&gStructuredLog.combined_table.build_mutex); - _mongoc_structured_log_set_max_levels_from_env (); - _mongoc_structured_log_update_level_plus_state (); - bson_mutex_unlock (&gStructuredLog.combined_table.build_mutex); + return all_ok; } -static FILE * -_mongoc_structured_log_open_stream (void) +static void +_mongoc_structured_log_default_handler_open_stream (mongoc_structured_log_default_handler_shared_t *shared, + const char *path) { - const char *path = getenv ("MONGODB_LOG_PATH"); + // shared->mutex must already be locked + if (!path || !strcmp (path, "stderr")) { - return stderr; - } - if (!strcmp (path, "stdout")) { - return stdout; - } - FILE *file = fopen (path, "a"); - if (!file) { - MONGOC_WARNING ("Cannot open log file '%s' for writing. Logging to stderr instead.", path); - return stderr; + // Default or explicit stderr + shared->stream = stderr; + shared->stream_fclose_on_destroy = false; + } else if (!strcmp (path, "stdout")) { + shared->stream = stdout; + shared->stream_fclose_on_destroy = false; + } else { + FILE *file = fopen (path, "a"); + if (file) { + shared->stream = file; + shared->stream_fclose_on_destroy = true; + } else { + MONGOC_WARNING ("Cannot open log file '%s' for writing. Logging to stderr instead.", path); + shared->stream = stderr; + shared->stream_fclose_on_destroy = false; + } } - return file; } static FILE * -_mongoc_structured_log_get_stream (void) +_mongoc_structured_log_default_handler_get_stream (mongoc_structured_log_instance_t *instance) { - // Not re-entrant; protected by the default_handler.stream_mutex. - FILE *log_stream = gStructuredLog.default_handler.stream; - if (log_stream) { + // instance->default_handler_shared->mutex must already be locked + { + FILE *log_stream = instance->default_handler_shared.stream; + if (log_stream) { + return log_stream; + } + } + _mongoc_structured_log_default_handler_open_stream (&instance->default_handler_shared, + instance->opts.default_handler_path); + { + FILE *log_stream = instance->default_handler_shared.stream; + BSON_ASSERT (log_stream); return log_stream; } - // Note that log_stream may be the global stderr/stdout streams, - // or an allocated FILE that is never closed. - log_stream = _mongoc_structured_log_open_stream (); - gStructuredLog.default_handler.stream = log_stream; - return log_stream; } static void -mongoc_structured_log_default_handler (const mongoc_structured_log_entry_t *entry, void *user_data) +_mongoc_structured_log_default_handler (const mongoc_structured_log_entry_t *entry, void *user_data) { - // We can serialize the message before taking the stream lock + BSON_UNUSED (user_data); + mongoc_structured_log_instance_t *instance = entry->envelope.instance; + + // We can serialize the message before taking the default_handler_shared mutex bson_t *bson_message = mongoc_structured_log_entry_message_as_bson (entry); char *json_message = bson_as_relaxed_extended_json (bson_message, NULL); bson_destroy (bson_message); @@ -451,49 +379,119 @@ mongoc_structured_log_default_handler (const mongoc_structured_log_entry_t *entr const char *component_name = mongoc_structured_log_get_component_name (mongoc_structured_log_entry_get_component (entry)); - bson_mutex_lock (&gStructuredLog.default_handler.stream_mutex); - fprintf (_mongoc_structured_log_get_stream (), "MONGODB_LOG %s %s %s\n", level_name, component_name, json_message); - bson_mutex_unlock (&gStructuredLog.default_handler.stream_mutex); + bson_mutex_lock (&instance->default_handler_shared.mutex); + fprintf (_mongoc_structured_log_default_handler_get_stream (instance), + "MONGODB_LOG %s %s %s\n", + level_name, + component_name, + json_message); + bson_mutex_unlock (&instance->default_handler_shared.mutex); bson_free (json_message); } -static BSON_ONCE_FUN (_mongoc_structured_log_init_once) +static void +_mongoc_structured_log_no_handler (const mongoc_structured_log_entry_t *entry, void *user_data) +{ + // Stub, for when logging is disabled. Only possible to call at MONGOC_STRUCTURED_LOG_LEVEL_EMERGENCY. + BSON_UNUSED (entry); + BSON_UNUSED (user_data); +} + +mongoc_structured_log_opts_t * +mongoc_structured_log_opts_new (void) { - bson_shared_mutex_init (&gStructuredLog.handler.mutex); - bson_mutex_init (&gStructuredLog.default_handler.stream_mutex); - bson_mutex_init (&gStructuredLog.combined_table.build_mutex); + mongoc_structured_log_opts_t *opts = (mongoc_structured_log_opts_t *) bson_malloc0 (sizeof *opts); - gStructuredLog.max_document_length = _mongoc_structured_log_get_max_document_length_from_env (); + // Capture default state from the environment now + opts->default_handler_path = bson_strdup (getenv ("MONGODB_LOG_PATH")); + opts->max_document_length = _mongoc_structured_log_get_max_document_length_from_env (); + mongoc_structured_log_opts_set_max_level_for_all_components (opts, MONGOC_STRUCTURED_LOG_DEFAULT_LEVEL); + mongoc_structured_log_opts_set_max_levels_from_env (opts); - bson_mutex_lock (&gStructuredLog.combined_table.build_mutex); + // Set default handler. Its shared state is allocated later, as part of instance_t. + mongoc_structured_log_opts_set_handler (opts, _mongoc_structured_log_default_handler, NULL); - _mongoc_structured_log_set_max_level_for_all_components (MONGOC_STRUCTURED_LOG_DEFAULT_LEVEL); - _mongoc_structured_log_set_max_levels_from_env (); + return opts; +} + +void +mongoc_structured_log_opts_destroy (mongoc_structured_log_opts_t *opts) +{ + if (opts) { + bson_free (opts->default_handler_path); + bson_free (opts); + } +} - // The default handler replaces MONGOC_STRUCTURED_LOG_STATE_MAYBE_UNINITIALIZED. - // Other threads may immediately submit log entries, so this must happen after - // log level setup and just before we unlock the build_mutex and return. - _mongoc_structured_log_set_handler (mongoc_structured_log_default_handler, NULL); - _mongoc_structured_log_update_level_plus_state (); +mongoc_structured_log_instance_t * +mongoc_structured_log_instance_new (const mongoc_structured_log_opts_t *opts) +{ + /* Creating the instance captures an immutable copy of the options. + * We also make a transformation that simplifies the critical path in + * _mongoc_structured_log_should_log so that it only needs to check the + * per-component table: In the instance, NULL handlers are no longer + * allowed. If structured logging is disabled, the per-component table + * will be set to the lowest possible levels and a stub handler function + * is set in case of 'emergency' logs. + * + * 'opts' is optional; if NULL, structured logging is disabled. + * (To request default options, you still need to use + * mongoc_structured_log_opts_new) */ + + mongoc_structured_log_instance_t *instance = (mongoc_structured_log_instance_t *) bson_malloc0 (sizeof *instance); + bson_mutex_init (&instance->default_handler_shared.mutex); + + if (opts) { + instance->opts.default_handler_path = bson_strdup (opts->default_handler_path); + instance->opts.max_document_length = opts->max_document_length; + instance->opts.handler_func = opts->handler_func; + instance->opts.handler_user_data = opts->handler_user_data; + } + if (instance->opts.handler_func) { + if (opts) { + memcpy (instance->opts.max_level_per_component, + opts->max_level_per_component, + sizeof instance->opts.max_level_per_component); + } + } else { + // No handler; leave the max_level_per_component table zero'ed, and add a stub handler for emergency level only. + instance->opts.handler_func = _mongoc_structured_log_no_handler; + } - bson_mutex_unlock (&gStructuredLog.combined_table.build_mutex); + return instance; +} - BSON_ONCE_RETURN; +void +mongoc_structured_log_instance_destroy (mongoc_structured_log_instance_t *instance) +{ + if (instance) { + bson_mutex_destroy (&instance->default_handler_shared.mutex); + bson_free (instance->opts.default_handler_path); + if (instance->default_handler_shared.stream_fclose_on_destroy) { + fclose (instance->default_handler_shared.stream); + } + bson_free (instance); + } } -char * -mongoc_structured_log_document_to_json (const bson_t *document, size_t *length) +static char * +_mongoc_structured_log_inner_document_to_json (const bson_t *document, + size_t *length, + const mongoc_structured_log_opts_t *opts) { - bson_json_opts_t *opts = bson_json_opts_new (BSON_JSON_MODE_RELAXED, gStructuredLog.max_document_length); - char *json = bson_as_json_with_opts (document, length, opts); - bson_json_opts_destroy (opts); + bson_json_opts_t *json_opts = bson_json_opts_new (BSON_JSON_MODE_RELAXED, opts->max_document_length); + char *json = bson_as_json_with_opts (document, length, json_opts); + bson_json_opts_destroy (json_opts); return json; } const mongoc_structured_log_builder_stage_t * -_mongoc_structured_log_append_utf8 (bson_t *bson, const mongoc_structured_log_builder_stage_t *stage) +_mongoc_structured_log_append_utf8 (bson_t *bson, + const mongoc_structured_log_builder_stage_t *stage, + const mongoc_structured_log_opts_t *opts) { + BSON_UNUSED (opts); const char *key_or_null = stage->arg1.utf8; if (key_or_null) { bson_append_utf8 (bson, key_or_null, -1, stage->arg2.utf8, -1); @@ -502,8 +500,11 @@ _mongoc_structured_log_append_utf8 (bson_t *bson, const mongoc_structured_log_bu } const mongoc_structured_log_builder_stage_t * -_mongoc_structured_log_append_utf8_n_stage0 (bson_t *bson, const mongoc_structured_log_builder_stage_t *stage) +_mongoc_structured_log_append_utf8_n_stage0 (bson_t *bson, + const mongoc_structured_log_builder_stage_t *stage, + const mongoc_structured_log_opts_t *opts) { + BSON_UNUSED (opts); BSON_ASSERT (stage[1].func == _mongoc_structured_log_append_utf8_n_stage1); const char *key_or_null = stage[0].arg1.utf8; int32_t key_len = stage[0].arg2.int32; @@ -516,18 +517,24 @@ _mongoc_structured_log_append_utf8_n_stage0 (bson_t *bson, const mongoc_structur } const mongoc_structured_log_builder_stage_t * -_mongoc_structured_log_append_utf8_n_stage1 (bson_t *bson, const mongoc_structured_log_builder_stage_t *stage) +_mongoc_structured_log_append_utf8_n_stage1 (bson_t *bson, + const mongoc_structured_log_builder_stage_t *stage, + const mongoc_structured_log_opts_t *opts) { // Never called, marks the second stage in a two-stage utf8_n BSON_UNUSED (bson); BSON_UNUSED (stage); + BSON_UNUSED (opts); BSON_ASSERT (false); return NULL; } const mongoc_structured_log_builder_stage_t * -_mongoc_structured_log_append_int32 (bson_t *bson, const mongoc_structured_log_builder_stage_t *stage) +_mongoc_structured_log_append_int32 (bson_t *bson, + const mongoc_structured_log_builder_stage_t *stage, + const mongoc_structured_log_opts_t *opts) { + BSON_UNUSED (opts); const char *key_or_null = stage->arg1.utf8; if (key_or_null) { bson_append_int32 (bson, key_or_null, -1, stage->arg2.int32); @@ -536,8 +543,11 @@ _mongoc_structured_log_append_int32 (bson_t *bson, const mongoc_structured_log_b } const mongoc_structured_log_builder_stage_t * -_mongoc_structured_log_append_int64 (bson_t *bson, const mongoc_structured_log_builder_stage_t *stage) +_mongoc_structured_log_append_int64 (bson_t *bson, + const mongoc_structured_log_builder_stage_t *stage, + const mongoc_structured_log_opts_t *opts) { + BSON_UNUSED (opts); const char *key_or_null = stage->arg1.utf8; if (key_or_null) { bson_append_int64 (bson, key_or_null, -1, stage->arg2.int64); @@ -546,8 +556,11 @@ _mongoc_structured_log_append_int64 (bson_t *bson, const mongoc_structured_log_b } const mongoc_structured_log_builder_stage_t * -_mongoc_structured_log_append_boolean (bson_t *bson, const mongoc_structured_log_builder_stage_t *stage) +_mongoc_structured_log_append_boolean (bson_t *bson, + const mongoc_structured_log_builder_stage_t *stage, + const mongoc_structured_log_opts_t *opts) { + BSON_UNUSED (opts); const char *key_or_null = stage->arg1.utf8; if (key_or_null) { bson_append_bool (bson, key_or_null, -1, stage->arg2.boolean); @@ -556,8 +569,11 @@ _mongoc_structured_log_append_boolean (bson_t *bson, const mongoc_structured_log } const mongoc_structured_log_builder_stage_t * -_mongoc_structured_log_append_oid_as_hex (bson_t *bson, const mongoc_structured_log_builder_stage_t *stage) +_mongoc_structured_log_append_oid_as_hex (bson_t *bson, + const mongoc_structured_log_builder_stage_t *stage, + const mongoc_structured_log_opts_t *opts) { + BSON_UNUSED (opts); const char *key_or_null = stage->arg1.utf8; const bson_oid_t *oid_or_null = stage->arg2.oid; if (key_or_null) { @@ -573,14 +589,16 @@ _mongoc_structured_log_append_oid_as_hex (bson_t *bson, const mongoc_structured_ } const mongoc_structured_log_builder_stage_t * -_mongoc_structured_log_append_bson_as_json (bson_t *bson, const mongoc_structured_log_builder_stage_t *stage) +_mongoc_structured_log_append_bson_as_json (bson_t *bson, + const mongoc_structured_log_builder_stage_t *stage, + const mongoc_structured_log_opts_t *opts) { const char *key_or_null = stage->arg1.utf8; const bson_t *bson_or_null = stage->arg2.bson; if (key_or_null) { if (bson_or_null) { size_t json_length; - char *json = mongoc_structured_log_document_to_json (bson_or_null, &json_length); + char *json = _mongoc_structured_log_inner_document_to_json (bson_or_null, &json_length, opts); if (json) { bson_append_utf8 (bson, key_or_null, -1, json, json_length); bson_free (json); @@ -593,22 +611,65 @@ _mongoc_structured_log_append_bson_as_json (bson_t *bson, const mongoc_structure } const mongoc_structured_log_builder_stage_t * -_mongoc_structured_log_append_cmd (bson_t *bson, const mongoc_structured_log_builder_stage_t *stage) +_mongoc_structured_log_append_cmd (bson_t *bson, + const mongoc_structured_log_builder_stage_t *stage, + const mongoc_structured_log_opts_t *opts) { const mongoc_cmd_t *cmd = stage->arg1.cmd; - const mongoc_logged_cmd_content_flags_t flags = stage->arg2.cmd_flags; - mongoc_cmd_append_logged_contents_to_bson (cmd, bson, flags); + const mongoc_structured_log_cmd_content_flags_t flags = stage->arg2.cmd_flags; + BSON_ASSERT (cmd); + + if (flags & MONGOC_STRUCTURED_LOG_CMD_CONTENT_FLAG_DATABASE_NAME) { + BSON_APPEND_UTF8 (bson, "databaseName", cmd->db_name); + } + if (flags & MONGOC_STRUCTURED_LOG_CMD_CONTENT_FLAG_COMMAND_NAME) { + BSON_APPEND_UTF8 (bson, "commandName", cmd->command_name); + } + if (flags & MONGOC_STRUCTURED_LOG_CMD_CONTENT_FLAG_OPERATION_ID) { + BSON_APPEND_INT64 (bson, "operationId", cmd->operation_id); + } + if (flags & MONGOC_STRUCTURED_LOG_CMD_CONTENT_FLAG_COMMAND) { + if (mongoc_apm_is_sensitive_command_message (cmd->command_name, cmd->command)) { + BSON_APPEND_UTF8 (bson, "command", "{}"); + } else { + bson_t *command_copy = NULL; + + if (cmd->payloads_count > 0) { + // @todo This is a performance bottleneck, we shouldn't be copying + // a potentially large command to serialize a potentially very + // small part of it. We should be appending JSON to a single buffer + // for all nesting levels, constrained by length limit, while visiting + // borrowed references to each command attribute and each payload. CDRIVER-4814 + command_copy = bson_copy (cmd->command); + _mongoc_cmd_append_payload_as_array (cmd, command_copy); + } + + size_t json_length; + char *json = _mongoc_structured_log_inner_document_to_json ( + command_copy ? command_copy : cmd->command, &json_length, opts); + if (json) { + const char *key = "command"; + bson_append_utf8 (bson, key, strlen (key), json, json_length); + bson_free (json); + } + bson_destroy (command_copy); + } + } + return stage + 1; } static void -_mongoc_structured_log_append_redacted_cmd_reply (bson_t *bson, bool is_sensitive, const bson_t *reply) +_mongoc_structured_log_append_redacted_cmd_reply (bson_t *bson, + bool is_sensitive, + const bson_t *reply, + const mongoc_structured_log_opts_t *opts) { if (is_sensitive) { BSON_APPEND_UTF8 (bson, "reply", "{}"); } else { size_t json_length; - char *json = mongoc_structured_log_document_to_json (reply, &json_length); + char *json = _mongoc_structured_log_inner_document_to_json (reply, &json_length, opts); if (json) { const char *key = "reply"; bson_append_utf8 (bson, key, strlen (key), json, json_length); @@ -618,7 +679,9 @@ _mongoc_structured_log_append_redacted_cmd_reply (bson_t *bson, bool is_sensitiv } const mongoc_structured_log_builder_stage_t * -_mongoc_structured_log_append_cmd_reply (bson_t *bson, const mongoc_structured_log_builder_stage_t *stage) +_mongoc_structured_log_append_cmd_reply (bson_t *bson, + const mongoc_structured_log_builder_stage_t *stage, + const mongoc_structured_log_opts_t *opts) { const mongoc_cmd_t *cmd = stage->arg1.cmd; const bson_t *reply = stage->arg2.bson; @@ -628,12 +691,14 @@ _mongoc_structured_log_append_cmd_reply (bson_t *bson, const mongoc_structured_l bool is_sensitive = mongoc_apm_is_sensitive_command_message (cmd->command_name, cmd->command) || mongoc_apm_is_sensitive_command_message (cmd->command_name, reply); - _mongoc_structured_log_append_redacted_cmd_reply (bson, is_sensitive, reply); + _mongoc_structured_log_append_redacted_cmd_reply (bson, is_sensitive, reply, opts); return stage + 1; } const mongoc_structured_log_builder_stage_t * -_mongoc_structured_log_append_cmd_name_reply (bson_t *bson, const mongoc_structured_log_builder_stage_t *stage) +_mongoc_structured_log_append_cmd_name_reply (bson_t *bson, + const mongoc_structured_log_builder_stage_t *stage, + const mongoc_structured_log_opts_t *opts) { const char *cmd_name = stage->arg1.utf8; const bson_t *reply = stage->arg2.bson; @@ -642,12 +707,14 @@ _mongoc_structured_log_append_cmd_name_reply (bson_t *bson, const mongoc_structu BSON_ASSERT (reply); bool is_sensitive = mongoc_apm_is_sensitive_command_message (cmd_name, reply); - _mongoc_structured_log_append_redacted_cmd_reply (bson, is_sensitive, reply); + _mongoc_structured_log_append_redacted_cmd_reply (bson, is_sensitive, reply, opts); return stage + 1; } const mongoc_structured_log_builder_stage_t * -_mongoc_structured_log_append_error (bson_t *bson, const mongoc_structured_log_builder_stage_t *stage) +_mongoc_structured_log_append_error (bson_t *bson, + const mongoc_structured_log_builder_stage_t *stage, + const mongoc_structured_log_opts_t *opts) { const char *key_or_null = stage->arg1.utf8; const bson_error_t *error_or_null = stage->arg2.error; @@ -708,8 +775,11 @@ _mongoc_structured_log_append_redacted_cmd_failure (bson_t *bson, } const mongoc_structured_log_builder_stage_t * -_mongoc_structured_log_append_cmd_failure_stage0 (bson_t *bson, const mongoc_structured_log_builder_stage_t *stage) +_mongoc_structured_log_append_cmd_failure_stage0 (bson_t *bson, + const mongoc_structured_log_builder_stage_t *stage, + const mongoc_structured_log_opts_t *opts) { + BSON_UNUSED (opts); BSON_ASSERT (stage[1].func == _mongoc_structured_log_append_cmd_failure_stage1); const mongoc_cmd_t *cmd = stage[0].arg1.cmd; const bson_t *reply = stage[0].arg2.bson; @@ -726,18 +796,24 @@ _mongoc_structured_log_append_cmd_failure_stage0 (bson_t *bson, const mongoc_str } const mongoc_structured_log_builder_stage_t * -_mongoc_structured_log_append_cmd_failure_stage1 (bson_t *bson, const mongoc_structured_log_builder_stage_t *stage) +_mongoc_structured_log_append_cmd_failure_stage1 (bson_t *bson, + const mongoc_structured_log_builder_stage_t *stage, + const mongoc_structured_log_opts_t *opts) { // Never called, marks the second stage in a two-stage cmd_failure BSON_UNUSED (bson); BSON_UNUSED (stage); + BSON_UNUSED (opts); BSON_ASSERT (false); return NULL; } const mongoc_structured_log_builder_stage_t * -_mongoc_structured_log_append_cmd_name_failure_stage0 (bson_t *bson, const mongoc_structured_log_builder_stage_t *stage) +_mongoc_structured_log_append_cmd_name_failure_stage0 (bson_t *bson, + const mongoc_structured_log_builder_stage_t *stage, + const mongoc_structured_log_opts_t *opts) { + BSON_UNUSED (opts); BSON_ASSERT (stage[1].func == _mongoc_structured_log_append_cmd_name_failure_stage1); const char *cmd_name = stage[0].arg1.utf8; const bson_t *reply = stage[0].arg2.bson; @@ -753,18 +829,24 @@ _mongoc_structured_log_append_cmd_name_failure_stage0 (bson_t *bson, const mongo } const mongoc_structured_log_builder_stage_t * -_mongoc_structured_log_append_cmd_name_failure_stage1 (bson_t *bson, const mongoc_structured_log_builder_stage_t *stage) +_mongoc_structured_log_append_cmd_name_failure_stage1 (bson_t *bson, + const mongoc_structured_log_builder_stage_t *stage, + const mongoc_structured_log_opts_t *opts) { // Never called, marks the second stage in a two-stage cmd_name_failure BSON_UNUSED (bson); BSON_UNUSED (stage); + BSON_UNUSED (opts); BSON_ASSERT (false); return NULL; } const mongoc_structured_log_builder_stage_t * -_mongoc_structured_log_append_server_description (bson_t *bson, const mongoc_structured_log_builder_stage_t *stage) +_mongoc_structured_log_append_server_description (bson_t *bson, + const mongoc_structured_log_builder_stage_t *stage, + const mongoc_structured_log_opts_t *opts) { + BSON_UNUSED (opts); const mongoc_server_description_t *sd = stage->arg1.server_description; const mongoc_server_description_content_flags_t flags = stage->arg2.server_description_flags; mongoc_server_description_append_contents_to_bson (sd, bson, flags); diff --git a/src/libmongoc/src/mongoc/mongoc-structured-log.h b/src/libmongoc/src/mongoc/mongoc-structured-log.h index 095a19ed2e3..8ffbb71867a 100644 --- a/src/libmongoc/src/mongoc/mongoc-structured-log.h +++ b/src/libmongoc/src/mongoc/mongoc-structured-log.h @@ -46,165 +46,55 @@ typedef enum { typedef struct mongoc_structured_log_entry_t mongoc_structured_log_entry_t; +typedef struct mongoc_structured_log_opts_t mongoc_structured_log_opts_t; + typedef void (*mongoc_structured_log_func_t) (const mongoc_structured_log_entry_t *entry, void *user_data); -/** - * mongoc_structured_log_set_handler: - * @log_func: A function to handle structured messages. - * @user_data: User data for @log_func. - * - * Sets the function to be called to handle structured log messages. - * - * The callback is given a mongoc_structured_log_entry_t* as a handle for - * obtaining additional information about the log message. - * - * The entry pointer is only valid during a callback, because it's a low - * cost reference to temporary data. When the log message is extracted - * as a bson_t a new owned copy of this data must be made, and deferred - * transformations like bson_as_json take place. - * - * Callbacks may use mongoc_structured_log_entry_get_level and - * mongoc_structured_log_entry_get_component to filter log messages, and - * selectively call mongoc_structured_log_entry_message_as_bson to create - * a bson_t for log messages only when needed. - * - * Any operating system or libbson APIs may be used by the callback, but - * libmongoc must be used only in limited ways to prevent deadlocks or - * unbounded recursion. See libmongoc/doc/mongoc_structured_log_func_t.rst - * for a complete list of allowed APIs for handlers to use. - * - * Applications that wish to use MongoDB itself as a logging destination will - * need to store the serialized messages temporarily and insert them later - * from outside the log handler. - */ -MONGOC_EXPORT (void) -mongoc_structured_log_set_handler (mongoc_structured_log_func_t log_func, void *user_data); +MONGOC_EXPORT (mongoc_structured_log_opts_t *) +mongoc_structured_log_opts_new (void); -/** - * mongoc_structured_log_set_max_level_for_component: - * @level: Maximum log level for this component. - * @component: A logging component to set the level limit for. - * - * Sets the maximum log level per-component. Only log messages at or below - * this severity level will be passed to mongoc_structured_log_func_t. - * - * By default, each component's log level may be set by environment variables. - * See mongoc_structured_log_set_max_levels_from_env(). - */ MONGOC_EXPORT (void) -mongoc_structured_log_set_max_level_for_component (mongoc_structured_log_component_t component, - mongoc_structured_log_level_t level); +mongoc_structured_log_opts_destroy (mongoc_structured_log_opts_t *opts); -/** - * mongoc_structured_log_set_max_level_for_all_components: - * @level: Maximum log level for all components. - * - * Sets all per-component maximum log levels to the same value. - * Only log messages at or below this severity level will be passed to - * mongoc_structured_log_func_t. Effective even for logging components not - * known at compile-time. - */ MONGOC_EXPORT (void) -mongoc_structured_log_set_max_level_for_all_components (mongoc_structured_log_level_t level); +mongoc_structured_log_opts_set_handler (mongoc_structured_log_opts_t *opts, + mongoc_structured_log_func_t log_func, + void *user_data); -/** - * mongoc_structured_log_set_max_levels_from_env: - * - * Sets any maximum log levels requested by environment variables: "MONGODB_LOG_ALL" - * for all components, followed by per-component log levels "MONGODB_LOG_COMMAND", - * "MONGODB_LOG_CONNECTION", "MONGODB_LOG_TOPOLOGY", and "MONGODB_LOG_SERVER_SELECTION". - * - * Component levels with no valid environment variable setting will be left unmodified. - * - * Normally this happens automatically when log levels are automatically initialized on - * first use. The resulting default values can be overridden programmatically by calls - * to mongoc_structured_log_set_max_level_for_component() and - * mongoc_structured_log_set_max_level_for_all_components(). - * - * For applications that desire the opposite behavior, where environment variables may - * override programmatic settings, they may call mongoc_structured_log_set_max_levels_from_env() - * after calling mongoc_structured_log_set_max_level_for_component() and - * mongoc_structured_log_set_max_level_for_all_components(). This will process the environment - * a second time, allowing it to override customized defaults. - */ +MONGOC_EXPORT (bool) +mongoc_structured_log_opts_set_max_level_for_component (mongoc_structured_log_opts_t *opts, + mongoc_structured_log_component_t component, + mongoc_structured_log_level_t level); -MONGOC_EXPORT (void) -mongoc_structured_log_set_max_levels_from_env (void); +MONGOC_EXPORT (bool) +mongoc_structured_log_opts_set_max_level_for_all_components (mongoc_structured_log_opts_t *opts, + mongoc_structured_log_level_t level); + +MONGOC_EXPORT (bool) +mongoc_structured_log_opts_set_max_levels_from_env (mongoc_structured_log_opts_t *opts); -/** - * mongoc_structured_log_get_max_level_for_component: - * @component: A logging component to check the log level limit for. - * - * Returns the current maximum log level for one component. - */ MONGOC_EXPORT (mongoc_structured_log_level_t) -mongoc_structured_log_get_max_level_for_component (mongoc_structured_log_component_t component); +mongoc_structured_log_opts_get_max_level_for_component (const mongoc_structured_log_opts_t *opts, + mongoc_structured_log_component_t component); -/** - * mongoc_structured_log_entry_message_as_bson: - * @entry: A log entry to extract the message from. - * - * Returns the structured message as a new allocated bson_t - * that must be freed by bson_destroy(). - */ MONGOC_EXPORT (bson_t *) mongoc_structured_log_entry_message_as_bson (const mongoc_structured_log_entry_t *entry); -/** - * mongoc_structured_log_entry_get_level: - * @entry: A log entry to read the level from. - * - * Returns the severity level of the structured log entry. - */ MONGOC_EXPORT (mongoc_structured_log_level_t) mongoc_structured_log_entry_get_level (const mongoc_structured_log_entry_t *entry); -/** - * mongoc_structured_log_entry_get_component: - * @entry: A log entry to read the component from. - * - * Returns the component of the structured log entry. - */ MONGOC_EXPORT (mongoc_structured_log_component_t) mongoc_structured_log_entry_get_component (const mongoc_structured_log_entry_t *entry); -/** - * mongoc_structured_log_get_level_name: - * @level: A log level code. - * - * Returns the canonical name for a log level as a constant string that does - * not need to be deallocated, or NULL if the level has no known name. - */ MONGOC_EXPORT (const char *) mongoc_structured_log_get_level_name (mongoc_structured_log_level_t level); -/** - * mongoc_structured_log_get_named_level: - * @name: A log level name. - * @out: On success, writes a level here and returns true. Returns false if name is not recognized. - * - * Try to parse the string as a log level name. Case insensitive. - */ MONGOC_EXPORT (bool) mongoc_structured_log_get_named_level (const char *name, mongoc_structured_log_level_t *out); -/** - * mongoc_structured_log_get_component_name: - * @level: A log level code. - * - * Returns the canonical name for a component as a constant string that does - * not need to be deallocated, or NULL if the level has no known name. - */ MONGOC_EXPORT (const char *) mongoc_structured_log_get_component_name (mongoc_structured_log_component_t component); -/** - * mongoc_structured_log_get_named_component: - * @name: A log component name. - * @out: On success, writes a component here and returns true. Returns false if name is not recognized. - * - * Try to parse the string as a log component name. Case insensitive. - */ MONGOC_EXPORT (bool) mongoc_structured_log_get_named_component (const char *name, mongoc_structured_log_component_t *out); diff --git a/src/libmongoc/src/mongoc/mongoc-topology-private.h b/src/libmongoc/src/mongoc/mongoc-topology-private.h index e38956ed815..0ec05e1afb8 100644 --- a/src/libmongoc/src/mongoc/mongoc-topology-private.h +++ b/src/libmongoc/src/mongoc/mongoc-topology-private.h @@ -30,6 +30,7 @@ #include "mongoc-ts-pool-private.h" #include "mongoc-shared-private.h" #include "mongoc-sleep.h" +#include "mongoc-structured-log-private.h" #include #define MONGOC_TOPOLOGY_MIN_HEARTBEAT_FREQUENCY_MS 500 @@ -209,6 +210,8 @@ typedef struct _mongoc_topology_t { mongoc_set_t *rtt_monitors; bson_mutex_t apm_mutex; + mongoc_structured_log_instance_t *structured_log; + /* This is overridable for SRV polling tests to mock DNS records. */ _mongoc_rr_resolver_fn rr_resolver; @@ -237,6 +240,10 @@ mongoc_topology_set_apm_callbacks (mongoc_topology_t *topology, mongoc_apm_callbacks_t const *callbacks, void *context); +void +mongoc_topology_set_structured_log_opts (mongoc_topology_t *topology, const mongoc_structured_log_opts_t *opts); + + void mongoc_topology_destroy (mongoc_topology_t *topology); diff --git a/src/libmongoc/src/mongoc/mongoc-topology.c b/src/libmongoc/src/mongoc/mongoc-topology.c index b4125bcbb03..c5f9ece53ae 100644 --- a/src/libmongoc/src/mongoc/mongoc-topology.c +++ b/src/libmongoc/src/mongoc/mongoc-topology.c @@ -401,6 +401,11 @@ mongoc_topology_new (const mongoc_uri_t *uri, bool single_threaded) topology->session_pool = mongoc_server_session_pool_new_with_params ( _server_session_init, _server_session_destroy, _server_session_should_prune, topology); + // Capture default structured log options from the environment + mongoc_structured_log_opts_t *structured_log_opts = mongoc_structured_log_opts_new (); + topology->structured_log = mongoc_structured_log_instance_new (structured_log_opts); + mongoc_structured_log_opts_destroy (structured_log_opts); + topology->valid = false; const int32_t heartbeat_default = single_threaded ? MONGOC_TOPOLOGY_HEARTBEAT_FREQUENCY_MS_SINGLE_THREADED @@ -661,6 +666,35 @@ mongoc_topology_set_apm_callbacks (mongoc_topology_t *topology, topology->scanner->apm_context = context; } +/* + *------------------------------------------------------------------------- + * + * mongoc_topology_set_structured_log_opts -- + * + * Replace the topology's structured logging options. Options are copied. + * The structured log instance used by a client or client pool in the public + * API is internally owned by a mongoc_topology_t. + * + * This is only safe to call when no other threads may be accessing the + * structured log. On a single-threaded topology it can be used on the + * thread that owns that topology. On a multi-threaded topology it can + * only be used during initialization, before clients have been created. + * This limitation is enforced by mongoc_client_pool_set_structured_log_opts. + * + *------------------------------------------------------------------------- + */ + +void +mongoc_topology_set_structured_log_opts (mongoc_topology_t *topology, const mongoc_structured_log_opts_t *opts) +{ + BSON_ASSERT_PARAM (topology); + // opts is optional + + mongoc_structured_log_instance_destroy (topology->structured_log); + topology->structured_log = mongoc_structured_log_instance_new (opts); +} + + /* *------------------------------------------------------------------------- * @@ -714,6 +748,7 @@ mongoc_topology_destroy (mongoc_topology_t *topology) mongoc_topology_scanner_destroy (topology->scanner); mongoc_server_session_pool_free (topology->session_pool); bson_free (topology->clientSideEncryption.autoOptions.extraOptions.cryptSharedLibPath); + mongoc_structured_log_instance_destroy (topology->structured_log); mongoc_cond_destroy (&topology->cond_client); bson_mutex_destroy (&topology->tpld_modification_mtx); diff --git a/src/libmongoc/tests/test-mongoc-structured-log.c b/src/libmongoc/tests/test-mongoc-structured-log.c index 67b7872fd82..2a987f95464 100644 --- a/src/libmongoc/tests/test-mongoc-structured-log.c +++ b/src/libmongoc/tests/test-mongoc-structured-log.c @@ -15,7 +15,6 @@ */ #include -#include #include "mongoc/mongoc-structured-log-private.h" #include "TestSuite.h" @@ -24,36 +23,15 @@ typedef struct log_assumption { mongoc_structured_log_envelope_t expected_envelope; bson_t *expected_bson; int expected_calls; - int calls_atomic; + int calls; } log_assumption; -typedef struct structured_log_state { - mongoc_structured_log_func_t handler; - void *data; -} structured_log_state; - -static BSON_INLINE structured_log_state -save_state (void) -{ - structured_log_state state; - mongoc_structured_log_get_handler (&state.handler, &state.data); - return state; -} - -static BSON_INLINE void -restore_state (structured_log_state state) -{ - mongoc_structured_log_set_handler (state.handler, state.data); - mongoc_structured_log_set_max_level_for_all_components (MONGOC_STRUCTURED_LOG_LEVEL_DEBUG); - mongoc_structured_log_set_max_levels_from_env (); -} - static void structured_log_func (const mongoc_structured_log_entry_t *entry, void *user_data) { struct log_assumption *assumption = (struct log_assumption *) user_data; - int calls = 1 + mcommon_atomic_int_fetch_add (&assumption->calls_atomic, 1, mcommon_memory_order_seq_cst); + int calls = ++assumption->calls; ASSERT_CMPINT (calls, <=, assumption->expected_calls); ASSERT_CMPINT (entry->envelope.level, ==, assumption->expected_envelope.level); @@ -80,6 +58,75 @@ structured_log_func (const mongoc_structured_log_entry_t *entry, void *user_data bson_free (json_actual); bson_free (json_expected); } +void + +test_structured_log_opts (void) +{ + mongoc_structured_log_opts_t *opts = mongoc_structured_log_opts_new (); + + ASSERT_CMPINT ( + MONGOC_STRUCTURED_LOG_LEVEL_WARNING, + ==, + mongoc_structured_log_opts_get_max_level_for_component (opts, MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND)); + ASSERT_CMPINT ( + MONGOC_STRUCTURED_LOG_LEVEL_WARNING, + ==, + mongoc_structured_log_opts_get_max_level_for_component (opts, MONGOC_STRUCTURED_LOG_COMPONENT_CONNECTION)); + ASSERT_CMPINT ( + MONGOC_STRUCTURED_LOG_LEVEL_WARNING, + ==, + mongoc_structured_log_opts_get_max_level_for_component (opts, MONGOC_STRUCTURED_LOG_COMPONENT_SERVER_SELECTION)); + ASSERT_CMPINT ( + MONGOC_STRUCTURED_LOG_LEVEL_WARNING, + ==, + mongoc_structured_log_opts_get_max_level_for_component (opts, MONGOC_STRUCTURED_LOG_COMPONENT_TOPOLOGY)); + ASSERT_CMPINT ( + MONGOC_STRUCTURED_LOG_LEVEL_EMERGENCY, + ==, + mongoc_structured_log_opts_get_max_level_for_component (opts, (mongoc_structured_log_component_t) 12345)); + + ASSERT (!mongoc_structured_log_opts_set_max_level_for_all_components (opts, (mongoc_structured_log_level_t) -1)); + ASSERT (mongoc_structured_log_opts_set_max_level_for_all_components (opts, MONGOC_STRUCTURED_LOG_LEVEL_INFO)); + + ASSERT_CMPINT ( + MONGOC_STRUCTURED_LOG_LEVEL_INFO, + ==, + mongoc_structured_log_opts_get_max_level_for_component (opts, MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND)); + ASSERT_CMPINT ( + MONGOC_STRUCTURED_LOG_LEVEL_INFO, + ==, + mongoc_structured_log_opts_get_max_level_for_component (opts, MONGOC_STRUCTURED_LOG_COMPONENT_CONNECTION)); + ASSERT_CMPINT ( + MONGOC_STRUCTURED_LOG_LEVEL_INFO, + ==, + mongoc_structured_log_opts_get_max_level_for_component (opts, MONGOC_STRUCTURED_LOG_COMPONENT_SERVER_SELECTION)); + ASSERT_CMPINT ( + MONGOC_STRUCTURED_LOG_LEVEL_INFO, + ==, + mongoc_structured_log_opts_get_max_level_for_component (opts, MONGOC_STRUCTURED_LOG_COMPONENT_TOPOLOGY)); + ASSERT_CMPINT ( + MONGOC_STRUCTURED_LOG_LEVEL_EMERGENCY, + ==, + mongoc_structured_log_opts_get_max_level_for_component (opts, (mongoc_structured_log_component_t) 12345)); + + ASSERT (!mongoc_structured_log_opts_set_max_level_for_component ( + opts, (mongoc_structured_log_component_t) -1, MONGOC_STRUCTURED_LOG_LEVEL_WARNING)); + ASSERT (!mongoc_structured_log_opts_set_max_level_for_component ( + opts, MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND, (mongoc_structured_log_level_t) -1)); + ASSERT (mongoc_structured_log_opts_set_max_level_for_component ( + opts, MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND, MONGOC_STRUCTURED_LOG_LEVEL_WARNING)); + + ASSERT_CMPINT ( + MONGOC_STRUCTURED_LOG_LEVEL_WARNING, + ==, + mongoc_structured_log_opts_get_max_level_for_component (opts, MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND)); + ASSERT_CMPINT ( + MONGOC_STRUCTURED_LOG_LEVEL_INFO, + ==, + mongoc_structured_log_opts_get_max_level_for_component (opts, MONGOC_STRUCTURED_LOG_COMPONENT_CONNECTION)); + + mongoc_structured_log_opts_destroy (opts); +} void test_structured_log_plain (void) @@ -92,16 +139,18 @@ test_structured_log_plain (void) .expected_calls = 1, }; - structured_log_state old_state = save_state (); - mongoc_structured_log_set_handler (structured_log_func, &assumption); - mongoc_structured_log_set_max_level_for_all_components (MONGOC_STRUCTURED_LOG_LEVEL_DEBUG); + mongoc_structured_log_opts_t *opts = mongoc_structured_log_opts_new (); + mongoc_structured_log_opts_set_handler (opts, structured_log_func, &assumption); + mongoc_structured_log_opts_set_max_level_for_all_components (opts, MONGOC_STRUCTURED_LOG_LEVEL_DEBUG); + + mongoc_structured_log_instance_t *instance = mongoc_structured_log_instance_new (opts); + mongoc_structured_log_opts_destroy (opts); mongoc_structured_log ( - MONGOC_STRUCTURED_LOG_LEVEL_WARNING, MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND, "Plain log entry"); + instance, MONGOC_STRUCTURED_LOG_LEVEL_WARNING, MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND, "Plain log entry"); - int calls = mcommon_atomic_int_fetch (&assumption.calls_atomic, mcommon_memory_order_seq_cst); - ASSERT_CMPINT (calls, ==, 1); - restore_state (old_state); + mongoc_structured_log_instance_destroy (instance); + ASSERT_CMPINT (assumption.calls, ==, 1); bson_destroy (assumption.expected_bson); } @@ -116,18 +165,21 @@ test_structured_log_plain_with_extra_data (void) .expected_calls = 1, }; - structured_log_state old_state = save_state (); - mongoc_structured_log_set_handler (structured_log_func, &assumption); - mongoc_structured_log_set_max_level_for_all_components (MONGOC_STRUCTURED_LOG_LEVEL_DEBUG); + mongoc_structured_log_opts_t *opts = mongoc_structured_log_opts_new (); + mongoc_structured_log_opts_set_handler (opts, structured_log_func, &assumption); + mongoc_structured_log_opts_set_max_level_for_all_components (opts, MONGOC_STRUCTURED_LOG_LEVEL_DEBUG); + + mongoc_structured_log_instance_t *instance = mongoc_structured_log_instance_new (opts); + mongoc_structured_log_opts_destroy (opts); - mongoc_structured_log (MONGOC_STRUCTURED_LOG_LEVEL_WARNING, + mongoc_structured_log (instance, + MONGOC_STRUCTURED_LOG_LEVEL_WARNING, MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND, "Plain log entry with extra data", int32 ("extra", 1)); - int calls = mcommon_atomic_int_fetch (&assumption.calls_atomic, mcommon_memory_order_seq_cst); - ASSERT_CMPINT (calls, ==, 1); - restore_state (old_state); + mongoc_structured_log_instance_destroy (instance); + ASSERT_CMPINT (assumption.calls, ==, 1); bson_destroy (assumption.expected_bson); } @@ -167,11 +219,15 @@ test_structured_log_basic_data_types (void) .expected_calls = 1, }; - structured_log_state old_state = save_state (); - mongoc_structured_log_set_handler (structured_log_func, &assumption); - mongoc_structured_log_set_max_level_for_all_components (MONGOC_STRUCTURED_LOG_LEVEL_DEBUG); + mongoc_structured_log_opts_t *opts = mongoc_structured_log_opts_new (); + mongoc_structured_log_opts_set_handler (opts, structured_log_func, &assumption); + mongoc_structured_log_opts_set_max_level_for_all_components (opts, MONGOC_STRUCTURED_LOG_LEVEL_DEBUG); + + mongoc_structured_log_instance_t *instance = mongoc_structured_log_instance_new (opts); + mongoc_structured_log_opts_destroy (opts); - mongoc_structured_log (MONGOC_STRUCTURED_LOG_LEVEL_WARNING, + mongoc_structured_log (instance, + MONGOC_STRUCTURED_LOG_LEVEL_WARNING, MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND, "Log entry with all basic data types", utf8 ("kStr", "string value"), @@ -191,9 +247,8 @@ test_structured_log_basic_data_types (void) boolean ("kFalse", false), boolean (NULL, true)); - int calls = mcommon_atomic_int_fetch (&assumption.calls_atomic, mcommon_memory_order_seq_cst); - ASSERT_CMPINT (calls, ==, 1); - restore_state (old_state); + mongoc_structured_log_instance_destroy (instance); + ASSERT_CMPINT (assumption.calls, ==, 1); bson_destroy (assumption.expected_bson); bson_destroy (bson_str_n); } @@ -216,20 +271,23 @@ test_structured_log_json (void) bson_t *json_doc = BCON_NEW ("k", BCON_UTF8 ("v")); - structured_log_state old_state = save_state (); - mongoc_structured_log_set_handler (structured_log_func, &assumption); - mongoc_structured_log_set_max_level_for_all_components (MONGOC_STRUCTURED_LOG_LEVEL_DEBUG); + mongoc_structured_log_opts_t *opts = mongoc_structured_log_opts_new (); + mongoc_structured_log_opts_set_handler (opts, structured_log_func, &assumption); + mongoc_structured_log_opts_set_max_level_for_all_components (opts, MONGOC_STRUCTURED_LOG_LEVEL_DEBUG); - mongoc_structured_log (MONGOC_STRUCTURED_LOG_LEVEL_WARNING, + mongoc_structured_log_instance_t *instance = mongoc_structured_log_instance_new (opts); + mongoc_structured_log_opts_destroy (opts); + + mongoc_structured_log (instance, + MONGOC_STRUCTURED_LOG_LEVEL_WARNING, MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND, "Log entry with deferred BSON-to-JSON", bson_as_json ("kJSON", json_doc), bson_as_json ("kNull", NULL), bson_as_json (NULL, NULL)); - int calls = mcommon_atomic_int_fetch (&assumption.calls_atomic, mcommon_memory_order_seq_cst); - ASSERT_CMPINT (calls, ==, 1); - restore_state (old_state); + mongoc_structured_log_instance_destroy (instance); + ASSERT_CMPINT (assumption.calls, ==, 1); bson_destroy (assumption.expected_bson); bson_destroy (json_doc); } @@ -253,20 +311,23 @@ test_structured_log_oid (void) bson_oid_t oid; bson_oid_init_from_string (&oid, "112233445566778899aabbcc"); - structured_log_state old_state = save_state (); - mongoc_structured_log_set_handler (structured_log_func, &assumption); - mongoc_structured_log_set_max_level_for_all_components (MONGOC_STRUCTURED_LOG_LEVEL_DEBUG); + mongoc_structured_log_opts_t *opts = mongoc_structured_log_opts_new (); + mongoc_structured_log_opts_set_handler (opts, structured_log_func, &assumption); + mongoc_structured_log_opts_set_max_level_for_all_components (opts, MONGOC_STRUCTURED_LOG_LEVEL_DEBUG); + + mongoc_structured_log_instance_t *instance = mongoc_structured_log_instance_new (opts); + mongoc_structured_log_opts_destroy (opts); - mongoc_structured_log (MONGOC_STRUCTURED_LOG_LEVEL_WARNING, + mongoc_structured_log (instance, + MONGOC_STRUCTURED_LOG_LEVEL_WARNING, MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND, "Log entry with deferred OID-to-hex conversion", oid_as_hex ("kOID", &oid), oid_as_hex ("kNull", NULL), oid_as_hex (NULL, NULL)); - int calls = mcommon_atomic_int_fetch (&assumption.calls_atomic, mcommon_memory_order_seq_cst); - ASSERT_CMPINT (calls, ==, 1); - restore_state (old_state); + mongoc_structured_log_instance_destroy (instance); + ASSERT_CMPINT (assumption.calls, ==, 1); bson_destroy (assumption.expected_bson); } @@ -299,20 +360,23 @@ test_structured_log_error (void) .message = "Some Text", }; - structured_log_state old_state = save_state (); - mongoc_structured_log_set_handler (structured_log_func, &assumption); - mongoc_structured_log_set_max_level_for_all_components (MONGOC_STRUCTURED_LOG_LEVEL_INFO); + mongoc_structured_log_opts_t *opts = mongoc_structured_log_opts_new (); + mongoc_structured_log_opts_set_handler (opts, structured_log_func, &assumption); + mongoc_structured_log_opts_set_max_level_for_all_components (opts, MONGOC_STRUCTURED_LOG_LEVEL_INFO); + + mongoc_structured_log_instance_t *instance = mongoc_structured_log_instance_new (opts); + mongoc_structured_log_opts_destroy (opts); - mongoc_structured_log (MONGOC_STRUCTURED_LOG_LEVEL_INFO, + mongoc_structured_log (instance, + MONGOC_STRUCTURED_LOG_LEVEL_INFO, MONGOC_STRUCTURED_LOG_COMPONENT_SERVER_SELECTION, "Log entry with bson_error_t values", error ("failure", &err), error (NULL, NULL), error ("null", NULL)); - int calls = mcommon_atomic_int_fetch (&assumption.calls_atomic, mcommon_memory_order_seq_cst); - ASSERT_CMPINT (calls, ==, 1); - restore_state (old_state); + mongoc_structured_log_instance_destroy (instance); + ASSERT_CMPINT (assumption.calls, ==, 1); bson_destroy (assumption.expected_bson); } @@ -368,11 +432,15 @@ test_structured_log_server_description (void) .service_id = {{0}}, }; - structured_log_state old_state = save_state (); - mongoc_structured_log_set_handler (structured_log_func, &assumption); - mongoc_structured_log_set_max_level_for_all_components (MONGOC_STRUCTURED_LOG_LEVEL_DEBUG); + mongoc_structured_log_opts_t *opts = mongoc_structured_log_opts_new (); + mongoc_structured_log_opts_set_handler (opts, structured_log_func, &assumption); + mongoc_structured_log_opts_set_max_level_for_all_components (opts, MONGOC_STRUCTURED_LOG_LEVEL_DEBUG); + + mongoc_structured_log_instance_t *instance = mongoc_structured_log_instance_new (opts); + mongoc_structured_log_opts_destroy (opts); mongoc_structured_log ( + instance, MONGOC_STRUCTURED_LOG_LEVEL_WARNING, MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND, "Log entry with server description", @@ -385,9 +453,8 @@ test_structured_log_server_description (void) server_description (&server_description_1, SERVER_HOST, SERVER_PORT, SERVER_CONNECTION_ID, SERVICE_ID), server_description (&server_description_2, SERVER_HOST, SERVER_PORT, SERVER_CONNECTION_ID, SERVICE_ID)); - int calls = mcommon_atomic_int_fetch (&assumption.calls_atomic, mcommon_memory_order_seq_cst); - ASSERT_CMPINT (calls, ==, 1); - restore_state (old_state); + mongoc_structured_log_instance_destroy (instance); + ASSERT_CMPINT (assumption.calls, ==, 1); bson_destroy (assumption.expected_bson); } @@ -468,11 +535,15 @@ test_structured_log_command (void) .command = cmd_doc, }; - structured_log_state old_state = save_state (); - mongoc_structured_log_set_handler (structured_log_func, &assumption); - mongoc_structured_log_set_max_level_for_all_components (MONGOC_STRUCTURED_LOG_LEVEL_DEBUG); + mongoc_structured_log_opts_t *opts = mongoc_structured_log_opts_new (); + mongoc_structured_log_opts_set_handler (opts, structured_log_func, &assumption); + mongoc_structured_log_opts_set_max_level_for_all_components (opts, MONGOC_STRUCTURED_LOG_LEVEL_DEBUG); + + mongoc_structured_log_instance_t *instance = mongoc_structured_log_instance_new (opts); + mongoc_structured_log_opts_destroy (opts); - mongoc_structured_log (MONGOC_STRUCTURED_LOG_LEVEL_WARNING, + mongoc_structured_log (instance, + MONGOC_STRUCTURED_LOG_LEVEL_WARNING, MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND, "Log entry with command and reply fields", cmd (&cmd, COMMAND_NAME), @@ -485,9 +556,8 @@ test_structured_log_command (void) cmd_name_failure ("authenticate", reply_doc, &server_error), cmd_name_failure ("authenticate", reply_doc, &client_error)); - int calls = mcommon_atomic_int_fetch (&assumption.calls_atomic, mcommon_memory_order_seq_cst); - ASSERT_CMPINT (calls, ==, 1); - restore_state (old_state); + mongoc_structured_log_instance_destroy (instance); + ASSERT_CMPINT (assumption.calls, ==, 1); bson_destroy (assumption.expected_bson); bson_destroy (cmd_doc); bson_destroy (reply_doc); @@ -517,20 +587,23 @@ test_structured_log_duration (void) .expected_calls = 1, }; - structured_log_state old_state = save_state (); - mongoc_structured_log_set_handler (structured_log_func, &assumption); - mongoc_structured_log_set_max_level_for_all_components (MONGOC_STRUCTURED_LOG_LEVEL_DEBUG); + mongoc_structured_log_opts_t *opts = mongoc_structured_log_opts_new (); + mongoc_structured_log_opts_set_handler (opts, structured_log_func, &assumption); + mongoc_structured_log_opts_set_max_level_for_all_components (opts, MONGOC_STRUCTURED_LOG_LEVEL_DEBUG); + + mongoc_structured_log_instance_t *instance = mongoc_structured_log_instance_new (opts); + mongoc_structured_log_opts_destroy (opts); - mongoc_structured_log (MONGOC_STRUCTURED_LOG_LEVEL_WARNING, + mongoc_structured_log (instance, + MONGOC_STRUCTURED_LOG_LEVEL_WARNING, MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND, "Log entry with duration", monotonic_time_duration (1999), monotonic_time_duration (10), monotonic_time_duration (10000000999)); - int calls = mcommon_atomic_int_fetch (&assumption.calls_atomic, mcommon_memory_order_seq_cst); - ASSERT_CMPINT (calls, ==, 1); - restore_state (old_state); + mongoc_structured_log_instance_destroy (instance); + ASSERT_CMPINT (assumption.calls, ==, 1); bson_destroy (assumption.expected_bson); } @@ -628,6 +701,7 @@ test_structured_log_component_names (void) void test_structured_log_install (TestSuite *suite) { + TestSuite_Add (suite, "/structured_log/opts", test_structured_log_opts); TestSuite_Add (suite, "/structured_log/plain", test_structured_log_plain); TestSuite_Add (suite, "/structured_log/plain_with_extra_data", test_structured_log_plain_with_extra_data); TestSuite_Add (suite, "/structured_log/basic_data_types", test_structured_log_basic_data_types); diff --git a/src/libmongoc/tests/unified/entity-map.c b/src/libmongoc/tests/unified/entity-map.c index 2591795b99f..0aaa559c1f7 100644 --- a/src/libmongoc/tests/unified/entity-map.c +++ b/src/libmongoc/tests/unified/entity-map.c @@ -486,6 +486,7 @@ entity_client_new (entity_map_t *em, bson_t *bson, bson_error_t *error) bool ret = false; mongoc_apm_callbacks_t *callbacks = NULL; bson_t *uri_options = NULL; + mongoc_structured_log_opts_t *log_opts = mongoc_structured_log_opts_new (); bool use_multiple_mongoses = false; bool use_multiple_mongoses_set = false; bool can_reduce_heartbeat = false; @@ -595,8 +596,9 @@ entity_client_new (entity_map_t *em, bson_t *bson, bson_error_t *error) if (not(type (doc)), then (error ("'observeLogMessages' must be a document"))), do ({ // Initialize all components to the lowest available level, and install a handler. - mongoc_structured_log_set_max_level_for_all_components (MONGOC_STRUCTURED_LOG_LEVEL_EMERGENCY); - mongoc_structured_log_set_handler (structured_log_cb, entity); + mongoc_structured_log_opts_set_max_level_for_all_components (log_opts, + MONGOC_STRUCTURED_LOG_LEVEL_EMERGENCY); + mongoc_structured_log_opts_set_handler (log_opts, structured_log_cb, entity); }), visitEach ( if (not(type (utf8)), then (error ("Every value in 'observeLogMessages' must be a log level string"))), @@ -611,7 +613,7 @@ entity_client_new (entity_map_t *em, bson_t *bson, bson_error_t *error) if (!mongoc_structured_log_get_named_level (level_name, &level)) { test_error ("Unknown log level '%s' given in 'observeLogMessages'", component_name); } - mongoc_structured_log_set_max_level_for_component (component, level); + mongoc_structured_log_opts_set_max_level_for_component (log_opts, component, level); }))), visitOthers ( dupPath (errpath), @@ -666,6 +668,7 @@ entity_client_new (entity_map_t *em, bson_t *bson, bson_error_t *error) mongoc_client_set_error_api (client, MONGOC_ERROR_API_VERSION_2); entity->value = client; mongoc_client_set_apm_callbacks (client, callbacks, entity); + mongoc_client_set_structured_log_opts (client, log_opts); if (can_reduce_heartbeat && em->reduced_heartbeat) { // @todo Examine whether this is needed in addition to the URI param above @@ -677,6 +680,7 @@ entity_client_new (entity_map_t *em, bson_t *bson, bson_error_t *error) mongoc_uri_destroy (uri); mongoc_apm_callbacks_destroy (callbacks); mongoc_server_api_destroy (api); + mongoc_structured_log_opts_destroy (log_opts); bson_destroy (uri_options); if (!ret) { entity_destroy (entity); @@ -1603,7 +1607,6 @@ entity_destroy (entity_t *entity) mongoc_client_t *client = NULL; client = (mongoc_client_t *) entity->value; - mongoc_structured_log_set_handler (NULL, NULL); mongoc_client_destroy (client); } else if (0 == strcmp ("clientEncryption", entity->type)) { mongoc_client_encryption_t *ce = NULL; From 872cce80a20135326c0e624845f41630d9f1c4cd Mon Sep 17 00:00:00 2001 From: Micah Scott Date: Wed, 4 Dec 2024 12:57:48 -0800 Subject: [PATCH 102/139] structured_log overview: document string values for level, component --- src/libmongoc/doc/structured_log.rst | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/src/libmongoc/doc/structured_log.rst b/src/libmongoc/doc/structured_log.rst index 4ea91b0626a..411c1e9d16d 100644 --- a/src/libmongoc/doc/structured_log.rst +++ b/src/libmongoc/doc/structured_log.rst @@ -46,22 +46,22 @@ Log levels and components are defined as :symbol:`mongoc_structured_log_level_t` .. code-block:: c typedef enum { - MONGOC_STRUCTURED_LOG_LEVEL_EMERGENCY = 0, - MONGOC_STRUCTURED_LOG_LEVEL_ALERT = 1, - MONGOC_STRUCTURED_LOG_LEVEL_CRITICAL = 2, - MONGOC_STRUCTURED_LOG_LEVEL_ERROR = 3, - MONGOC_STRUCTURED_LOG_LEVEL_WARNING = 4, - MONGOC_STRUCTURED_LOG_LEVEL_NOTICE = 5, - MONGOC_STRUCTURED_LOG_LEVEL_INFO = 6, - MONGOC_STRUCTURED_LOG_LEVEL_DEBUG = 7, - MONGOC_STRUCTURED_LOG_LEVEL_TRACE = 8, + MONGOC_STRUCTURED_LOG_LEVEL_EMERGENCY = 0, // "Emergency" ("off" also accepted) + MONGOC_STRUCTURED_LOG_LEVEL_ALERT = 1, // "Alert" + MONGOC_STRUCTURED_LOG_LEVEL_CRITICAL = 2, // "Critical" + MONGOC_STRUCTURED_LOG_LEVEL_ERROR = 3, // "Error" + MONGOC_STRUCTURED_LOG_LEVEL_WARNING = 4, // "Warning" ("warn" also accepted) + MONGOC_STRUCTURED_LOG_LEVEL_NOTICE = 5, // "Notice" + MONGOC_STRUCTURED_LOG_LEVEL_INFO = 6, // "Informational" ("info" also accepted) + MONGOC_STRUCTURED_LOG_LEVEL_DEBUG = 7, // "Debug" + MONGOC_STRUCTURED_LOG_LEVEL_TRACE = 8, // "Trace" } mongoc_structured_log_level_t; typedef enum { - MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND = 0, - MONGOC_STRUCTURED_LOG_COMPONENT_TOPOLOGY = 1, - MONGOC_STRUCTURED_LOG_COMPONENT_SERVER_SELECTION = 2, - MONGOC_STRUCTURED_LOG_COMPONENT_CONNECTION = 3, + MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND = 0, // "command" + MONGOC_STRUCTURED_LOG_COMPONENT_TOPOLOGY = 1, // "topology" + MONGOC_STRUCTURED_LOG_COMPONENT_SERVER_SELECTION = 2, // "serverSelection" + MONGOC_STRUCTURED_LOG_COMPONENT_CONNECTION = 3, // "connection" } mongoc_structured_log_component_t; .. toctree:: From 7ff7ce3cbc09ecf55fcd863f840e9965f7541a10 Mon Sep 17 00:00:00 2001 From: micah Date: Wed, 4 Dec 2024 13:05:36 -0800 Subject: [PATCH 103/139] Update src/libmongoc/tests/bsonutil/bson-match.c Co-authored-by: Kevin Albertson --- src/libmongoc/tests/bsonutil/bson-match.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libmongoc/tests/bsonutil/bson-match.c b/src/libmongoc/tests/bsonutil/bson-match.c index cac7b95e26e..ecd154e424c 100644 --- a/src/libmongoc/tests/bsonutil/bson-match.c +++ b/src/libmongoc/tests/bsonutil/bson-match.c @@ -295,7 +295,7 @@ special_match_as_document (const bson_matcher_context_t *context, bson_error_t *error) { bool ret = false; - bson_t actual_as_bson; + bson_t actual_as_bson = BSON_INITIALIZER; BSON_UNUSED (user_data); bson_iter_t iter; From 310ad357cd71d1a6fd69b4d594302ae65e17bcac Mon Sep 17 00:00:00 2001 From: micah Date: Wed, 4 Dec 2024 13:08:44 -0800 Subject: [PATCH 104/139] Update src/libmongoc/tests/bsonutil/bson-match.c Co-authored-by: Kevin Albertson --- src/libmongoc/tests/bsonutil/bson-match.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/libmongoc/tests/bsonutil/bson-match.c b/src/libmongoc/tests/bsonutil/bson-match.c index ecd154e424c..29d466e2fc2 100644 --- a/src/libmongoc/tests/bsonutil/bson-match.c +++ b/src/libmongoc/tests/bsonutil/bson-match.c @@ -326,9 +326,10 @@ special_match_as_document (const bson_matcher_context_t *context, ret = bson_matcher_match (context, expected_val, actual_val, error); bson_val_destroy (actual_val); bson_val_destroy (expected_val); - -done: bson_destroy (&actual_as_bson); + +done: + return ret; } From ef4b7eac8e09ceb917e67c4dc5ad960e052ed2d1 Mon Sep 17 00:00:00 2001 From: Micah Scott Date: Wed, 4 Dec 2024 13:18:28 -0800 Subject: [PATCH 105/139] clang-format --- src/libmongoc/tests/bsonutil/bson-match.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libmongoc/tests/bsonutil/bson-match.c b/src/libmongoc/tests/bsonutil/bson-match.c index 29d466e2fc2..adcfef5b180 100644 --- a/src/libmongoc/tests/bsonutil/bson-match.c +++ b/src/libmongoc/tests/bsonutil/bson-match.c @@ -327,7 +327,7 @@ special_match_as_document (const bson_matcher_context_t *context, bson_val_destroy (actual_val); bson_val_destroy (expected_val); bson_destroy (&actual_as_bson); - + done: return ret; From 5a65c5fbfdfdc590160af3aec5cd4d37ebc73f82 Mon Sep 17 00:00:00 2001 From: Micah Scott Date: Wed, 4 Dec 2024 14:19:32 -0800 Subject: [PATCH 106/139] clarify, non-pooled clients --- src/libmongoc/doc/structured_log.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libmongoc/doc/structured_log.rst b/src/libmongoc/doc/structured_log.rst index 411c1e9d16d..151ebff83e1 100644 --- a/src/libmongoc/doc/structured_log.rst +++ b/src/libmongoc/doc/structured_log.rst @@ -82,7 +82,7 @@ Log levels and components are defined as :symbol:`mongoc_structured_log_level_t` Log Handlers ------------ -Each :symbol:`mongoc_client_t` or :symbol:`mongoc_client_pool_t` has its own instance of the structured logging subsystem, with its own settings and handler. +Each :symbol:`mongoc_client_pool_t` or standalone :symbol:`mongoc_client_t` has its own instance of the structured logging subsystem, with its own settings and handler. When using :symbol:`mongoc_client_pool_t`, the pooled clients all share a common logging instance. Handlers must be thread-safe. From b2ad83a57b31ee6eceebd9d7e142fa5478056eca Mon Sep 17 00:00:00 2001 From: Micah Scott Date: Wed, 4 Dec 2024 14:31:28 -0800 Subject: [PATCH 107/139] revert structured logging from _mongoc_client_monitor_op_killcursors See CDRIVER-5823, this code path is no longer needed and should be removed. --- src/libmongoc/src/mongoc/mongoc-client.c | 62 ++++-------------------- 1 file changed, 10 insertions(+), 52 deletions(-) diff --git a/src/libmongoc/src/mongoc/mongoc-client.c b/src/libmongoc/src/mongoc/mongoc-client.c index fd1454ccb02..baff7cffc7d 100644 --- a/src/libmongoc/src/mongoc/mongoc-client.c +++ b/src/libmongoc/src/mongoc/mongoc-client.c @@ -2165,26 +2165,12 @@ _mongoc_client_monitor_op_killcursors (mongoc_cluster_t *cluster, client = cluster->client; - bson_init (&doc); - _mongoc_client_prepare_killcursors_command (cursor_id, collection, &doc); - - mongoc_structured_log ( - cluster->client->topology->structured_log, - MONGOC_STRUCTURED_LOG_LEVEL_DEBUG, - MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND, - "Command started", - int32 ("requestId", cluster->request_id), - server_description (server_stream->sd, SERVER_HOST, SERVER_PORT, SERVER_CONNECTION_ID, SERVICE_ID), - utf8 ("databaseName", db), - utf8 ("commandName", "killCursors"), - int64 ("operationId", operation_id), - bson_as_json ("command", &doc)); - if (!client->apm_callbacks.started) { - bson_destroy (&doc); return; } + bson_init (&doc); + _mongoc_client_prepare_killcursors_command (cursor_id, collection, &doc); mongoc_apm_command_started_init (&event, &doc, db, @@ -2223,6 +2209,10 @@ _mongoc_client_monitor_op_killcursors_succeeded (mongoc_cluster_t *cluster, client = cluster->client; + if (!client->apm_callbacks.succeeded) { + EXIT; + } + /* fake server reply to killCursors command: {ok: 1, cursorsUnknown: [42]} */ bson_init (&doc); bson_append_int32 (&doc, "ok", 2, 1); @@ -2230,24 +2220,6 @@ _mongoc_client_monitor_op_killcursors_succeeded (mongoc_cluster_t *cluster, bson_array_builder_append_int64 (cursors_unknown, cursor_id); bson_append_array_builder_end (&doc, cursors_unknown); - mongoc_structured_log ( - client->topology->structured_log, - MONGOC_STRUCTURED_LOG_LEVEL_DEBUG, - MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND, - "Command succeeded", - int32 ("requestId", cluster->request_id), - server_description (server_stream->sd, SERVER_HOST, SERVER_PORT, SERVER_CONNECTION_ID, SERVICE_ID), - utf8 ("databaseName", db), - utf8 ("commandName", "killCursors"), - int64 ("operationId", operation_id), - monotonic_time_duration (duration), - cmd_name_reply ("killCursors", &doc)); - - if (!client->apm_callbacks.succeeded) { - bson_destroy (&doc); - EXIT; - } - mongoc_apm_command_succeeded_init (&event, duration, &doc, @@ -2285,28 +2257,14 @@ _mongoc_client_monitor_op_killcursors_failed (mongoc_cluster_t *cluster, client = cluster->client; - /* fake server reply to killCursors command: {ok: 0} */ - bson_init (&doc); - bson_append_int32 (&doc, "ok", 2, 0); - - mongoc_structured_log ( - client->topology->structured_log, - MONGOC_STRUCTURED_LOG_LEVEL_DEBUG, - MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND, - "Command failed", - int32 ("requestId", cluster->request_id), - server_description (server_stream->sd, SERVER_HOST, SERVER_PORT, SERVER_CONNECTION_ID, SERVICE_ID), - utf8 ("databaseName", db), - utf8 ("commandName", "killCursors"), - int64 ("operationId", operation_id), - monotonic_time_duration (duration), - bson_as_json ("failure", &doc)); - if (!client->apm_callbacks.failed) { - bson_destroy (&doc); EXIT; } + /* fake server reply to killCursors command: {ok: 0} */ + bson_init (&doc); + bson_append_int32 (&doc, "ok", 2, 0); + mongoc_apm_command_failed_init (&event, duration, "killCursors", From 8610a810350f8b5ce812811f7386dd68e751105c Mon Sep 17 00:00:00 2001 From: Micah Scott Date: Wed, 4 Dec 2024 18:02:07 -0800 Subject: [PATCH 108/139] use BSON_OPTIONAL_PARAM --- src/libmongoc/src/mongoc/mongoc-client-pool.c | 2 +- src/libmongoc/src/mongoc/mongoc-client.c | 2 +- src/libmongoc/src/mongoc/mongoc-topology.c | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/libmongoc/src/mongoc/mongoc-client-pool.c b/src/libmongoc/src/mongoc/mongoc-client-pool.c index a8ab4deb47e..39e019608fa 100644 --- a/src/libmongoc/src/mongoc/mongoc-client-pool.c +++ b/src/libmongoc/src/mongoc/mongoc-client-pool.c @@ -612,7 +612,7 @@ void mongoc_client_pool_set_structured_log_opts (mongoc_client_pool_t *pool, const mongoc_structured_log_opts_t *opts) { BSON_ASSERT_PARAM (pool); - // opts is optional + BSON_OPTIONAL_PARAM (opts); /* The documented restriction for most pool options: They can be set at most once, * and only before the first client is initialized. Structured logging is generally diff --git a/src/libmongoc/src/mongoc/mongoc-client.c b/src/libmongoc/src/mongoc/mongoc-client.c index baff7cffc7d..3fef8d81e9c 100644 --- a/src/libmongoc/src/mongoc/mongoc-client.c +++ b/src/libmongoc/src/mongoc/mongoc-client.c @@ -2615,7 +2615,7 @@ void mongoc_client_set_structured_log_opts (mongoc_client_t *client, const mongoc_structured_log_opts_t *opts) { BSON_ASSERT_PARAM (client); - // opts is optional + BSON_OPTIONAL_PARAM (opts); if (client->topology->single_threaded) { mongoc_topology_set_structured_log_opts (client->topology, opts); diff --git a/src/libmongoc/src/mongoc/mongoc-topology.c b/src/libmongoc/src/mongoc/mongoc-topology.c index c5f9ece53ae..800f1f0fca5 100644 --- a/src/libmongoc/src/mongoc/mongoc-topology.c +++ b/src/libmongoc/src/mongoc/mongoc-topology.c @@ -688,7 +688,7 @@ void mongoc_topology_set_structured_log_opts (mongoc_topology_t *topology, const mongoc_structured_log_opts_t *opts) { BSON_ASSERT_PARAM (topology); - // opts is optional + BSON_OPTIONAL_PARAM (opts); mongoc_structured_log_instance_destroy (topology->structured_log); topology->structured_log = mongoc_structured_log_instance_new (opts); From e216969f1390d584590100d6e97da0d9fc087415 Mon Sep 17 00:00:00 2001 From: Micah Scott Date: Thu, 5 Dec 2024 10:56:17 -0800 Subject: [PATCH 109/139] Comment fix, incorrect param names --- src/libmongoc/src/mongoc/mongoc-structured-log-private.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/libmongoc/src/mongoc/mongoc-structured-log-private.h b/src/libmongoc/src/mongoc/mongoc-structured-log-private.h index 895e5a2e4df..e49e3ca5129 100644 --- a/src/libmongoc/src/mongoc/mongoc-structured-log-private.h +++ b/src/libmongoc/src/mongoc/mongoc-structured-log-private.h @@ -240,7 +240,7 @@ typedef enum { * For cases where a mongo_cmd_t is not available; makes redaction decisions based * on command name but not body, so it's unsuitable for the "hello" reply. * - * @param cmd Command name as a const char * expression. Required. + * @param cmd_name Command name as a const char * expression. Required. * @param reply Borrowed reference to reply document, as a const bson_t * expression. Required. */ #define _mongoc_structured_log_item_cmd_name_reply(_cmd_name, _reply_bson) \ @@ -271,7 +271,7 @@ typedef enum { * The 'error' is examined to determine whether this is a client-side or server-side failure. * The command's name and body may influence the reply's redaction. * - * @param cmd Command name as a const char * expression. Required. + * @param cmd_name Command name as a const char * expression. Required. * @param reply Borrowed reference to reply document, as a const bson_t * expression. Required. * @param error Borrowed reference to a libmongoc error, as a const bson_error_t * expression. Required. */ @@ -285,7 +285,7 @@ typedef enum { * @def server_description(sd, ...) * @brief Structured log item, mongoc_server_description_t fields * - * @param cmd Borrowed server description reference, as a const mongoc_server_description_t * expression. Required. + * @param sd Borrowed server description reference, as a const mongoc_server_description_t * expression. Required. * @param ... Fields to include. Order is not significant. Any of: SERVER_HOST, SERVER_PORT, SERVER_CONNECTION_ID, * SERVICE_ID. * */ From 1a890b1c5a7f44954af39af77f2eb6c4beb26522 Mon Sep 17 00:00:00 2001 From: Micah Scott Date: Thu, 5 Dec 2024 14:08:28 -0800 Subject: [PATCH 110/139] unified test runner: avoid replacing specific error with unspecific error --- src/libmongoc/tests/unified/runner.c | 1 - 1 file changed, 1 deletion(-) diff --git a/src/libmongoc/tests/unified/runner.c b/src/libmongoc/tests/unified/runner.c index 992adcd8abe..607ddc8b277 100644 --- a/src/libmongoc/tests/unified/runner.c +++ b/src/libmongoc/tests/unified/runner.c @@ -1492,7 +1492,6 @@ test_check_expected_log_messages_for_client (test_t *test, mongoc_structured_log_get_level_name (actual_message_iter->level), mongoc_structured_log_get_component_name (actual_message_iter->component), tmp_json (actual_message_iter->message)); - test_set_error (error, "log message does not match expected"); } bson_destroy (&expected_message); if (!is_match) { From f79a216829a1bfd01218ca49b97155b68e057ac2 Mon Sep 17 00:00:00 2001 From: Micah Scott Date: Thu, 5 Dec 2024 14:18:18 -0800 Subject: [PATCH 111/139] unified tests: expected log levels may be aliases rather than canonical names --- src/libmongoc/tests/unified/runner.c | 27 +++++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/src/libmongoc/tests/unified/runner.c b/src/libmongoc/tests/unified/runner.c index 607ddc8b277..69818f22746 100644 --- a/src/libmongoc/tests/unified/runner.c +++ b/src/libmongoc/tests/unified/runner.c @@ -1353,15 +1353,30 @@ test_check_log_message (test_t *test, bson_t *expected, log_message_t *actual, b goto done; } - const char *actual_level_str = mongoc_structured_log_get_level_name (actual->level); - if (0 != bson_strcasecmp (expected_level_str, actual_level_str)) { - test_set_error (error, "expected log level: %s, but got: %s", expected_level_str, actual_level_str); + mongoc_structured_log_level_t expected_level; + if (!mongoc_structured_log_get_named_level (expected_level_str, &expected_level)) { + test_set_error (error, "expected log level '%s' is not recognized", expected_level_str); + goto done; + } + mongoc_structured_log_component_t expected_component; + if (!mongoc_structured_log_get_named_component (expected_component_str, &expected_component)) { + test_set_error (error, "expected log component '%s' is not recognized", expected_component_str); goto done; } - const char *actual_component_str = mongoc_structured_log_get_component_name (actual->component); - if (0 != bson_strcasecmp (expected_component_str, actual_component_str)) { - test_set_error (error, "expected log component: %s, but got: %s", expected_component_str, actual_component_str); + if (expected_level != actual->level) { + test_set_error (error, + "expected log level: %s, but got: %s", + mongoc_structured_log_get_level_name (expected_level), + mongoc_structured_log_get_level_name (actual->level)); + goto done; + } + + if (expected_component != actual->component) { + test_set_error (error, + "expected log component: %s, but got: %s", + mongoc_structured_log_get_component_name (expected_component), + mongoc_structured_log_get_component_name (actual->component)); goto done; } From 61808b39512c9607ebb3edb6c50735b56fa83fa6 Mon Sep 17 00:00:00 2001 From: Micah Scott Date: Fri, 6 Dec 2024 08:08:10 -0800 Subject: [PATCH 112/139] Publish mongoc_structured_log_entry_get_message_string I wasn't sure if it was worth the cost in future API flexibility, to make the log message string accessible publicly as part of the envelope data, but it does seem like a sensible and useful addition. --- ...tructured_log_entry_get_message_string.rst | 31 +++++++++++++++++++ .../doc/mongoc_structured_log_entry_t.rst | 1 + .../examples/example-structured-log.c | 8 +++-- .../src/mongoc/mongoc-structured-log.c | 9 ++++++ .../src/mongoc/mongoc-structured-log.h | 3 ++ .../tests/test-mongoc-structured-log.c | 1 + 6 files changed, 50 insertions(+), 3 deletions(-) create mode 100644 src/libmongoc/doc/mongoc_structured_log_entry_get_message_string.rst diff --git a/src/libmongoc/doc/mongoc_structured_log_entry_get_message_string.rst b/src/libmongoc/doc/mongoc_structured_log_entry_get_message_string.rst new file mode 100644 index 00000000000..e0ae8ce1519 --- /dev/null +++ b/src/libmongoc/doc/mongoc_structured_log_entry_get_message_string.rst @@ -0,0 +1,31 @@ +:man_page: mongoc_structured_log_entry_get_message_string + +mongoc_structured_log_entry_get_message_string() +================================================ + +Synopsis +-------- + +.. code-block:: c + + const char * + mongoc_structured_log_entry_get_message_string (const mongoc_structured_log_entry_t *entry); + +Parameters +---------- + +* ``entry``: A :symbol:`mongoc_structured_log_entry_t` pointer. + +Returns +------- + +A string, guaranteed to be valid only during the lifetime of the structured log handler. +It should not be freed or modified. + +Identical to the value of the ``message`` key in the document returned by :symbol:`mongoc_structured_log_entry_message_as_bson`. + +This is not a complete string representation of the structured log, but rather a standardized identifier for a particular log event. + +.. seealso:: + + | :doc:`structured_log` diff --git a/src/libmongoc/doc/mongoc_structured_log_entry_t.rst b/src/libmongoc/doc/mongoc_structured_log_entry_t.rst index 310a099262f..3cb23f8c3ee 100644 --- a/src/libmongoc/doc/mongoc_structured_log_entry_t.rst +++ b/src/libmongoc/doc/mongoc_structured_log_entry_t.rst @@ -23,6 +23,7 @@ Functions mongoc_structured_log_entry_get_component mongoc_structured_log_entry_get_level + mongoc_structured_log_entry_get_message_string mongoc_structured_log_entry_message_as_bson .. seealso:: diff --git a/src/libmongoc/examples/example-structured-log.c b/src/libmongoc/examples/example-structured-log.c index 78bfd85cbee..5c661b4471a 100644 --- a/src/libmongoc/examples/example-structured-log.c +++ b/src/libmongoc/examples/example-structured-log.c @@ -13,6 +13,7 @@ example_handler (const mongoc_structured_log_entry_t *entry, void *user_data) { mongoc_structured_log_component_t component = mongoc_structured_log_entry_get_component (entry); mongoc_structured_log_level_t level = mongoc_structured_log_entry_get_level (entry); + const char *message_string = mongoc_structured_log_entry_get_message_string (entry); /* * With a single-threaded mongoc_client_t, handlers will always be called @@ -30,9 +31,10 @@ example_handler (const mongoc_structured_log_entry_t *entry, void *user_data) */ pthread_mutex_lock (&handler_mutex); - printf ("Log component=%s level=%s\n", + printf ("Log entry with component=%s level=%s message_string='%s'\n", mongoc_structured_log_get_component_name (component), - mongoc_structured_log_get_level_name (level)); + mongoc_structured_log_get_level_name (level), + message_string); /* * At this point, the handler might make additional filtering decisions @@ -42,7 +44,7 @@ example_handler (const mongoc_structured_log_entry_t *entry, void *user_data) if (component == MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND) { bson_t *message = mongoc_structured_log_entry_message_as_bson (entry); char *json = bson_as_relaxed_extended_json (message, NULL); - printf ("Log body: %s\n", json); + printf ("Full log message, as json: %s\n", json); bson_destroy (message); bson_free (json); } diff --git a/src/libmongoc/src/mongoc/mongoc-structured-log.c b/src/libmongoc/src/mongoc/mongoc-structured-log.c index 348d89b9de0..6cfb45ad267 100644 --- a/src/libmongoc/src/mongoc/mongoc-structured-log.c +++ b/src/libmongoc/src/mongoc/mongoc-structured-log.c @@ -92,6 +92,15 @@ mongoc_structured_log_entry_get_component (const mongoc_structured_log_entry_t * return entry->envelope.component; } +const char * +mongoc_structured_log_entry_get_message_string (const mongoc_structured_log_entry_t *entry) +{ + // Note that 'message' happens to have static lifetime right now (all messages are literals) + // but our API only guarantees a lifetime that matches 'entry'. + BSON_ASSERT_PARAM (entry); + return entry->envelope.message; +} + mongoc_structured_log_level_t mongoc_structured_log_opts_get_max_level_for_component (const mongoc_structured_log_opts_t *opts, mongoc_structured_log_component_t component) diff --git a/src/libmongoc/src/mongoc/mongoc-structured-log.h b/src/libmongoc/src/mongoc/mongoc-structured-log.h index 8ffbb71867a..468e408d415 100644 --- a/src/libmongoc/src/mongoc/mongoc-structured-log.h +++ b/src/libmongoc/src/mongoc/mongoc-structured-log.h @@ -86,6 +86,9 @@ mongoc_structured_log_entry_get_level (const mongoc_structured_log_entry_t *entr MONGOC_EXPORT (mongoc_structured_log_component_t) mongoc_structured_log_entry_get_component (const mongoc_structured_log_entry_t *entry); +MONGOC_EXPORT (const char *) +mongoc_structured_log_entry_get_message_string (const mongoc_structured_log_entry_t *entry); + MONGOC_EXPORT (const char *) mongoc_structured_log_get_level_name (mongoc_structured_log_level_t level); diff --git a/src/libmongoc/tests/test-mongoc-structured-log.c b/src/libmongoc/tests/test-mongoc-structured-log.c index 2a987f95464..c8c7f3ecb75 100644 --- a/src/libmongoc/tests/test-mongoc-structured-log.c +++ b/src/libmongoc/tests/test-mongoc-structured-log.c @@ -38,6 +38,7 @@ structured_log_func (const mongoc_structured_log_entry_t *entry, void *user_data ASSERT_CMPINT (entry->envelope.component, ==, assumption->expected_envelope.component); ASSERT_CMPSTR (entry->envelope.message, assumption->expected_envelope.message); + ASSERT_CMPSTR (entry->envelope.message, mongoc_structured_log_entry_get_message_string(entry)); ASSERT_CMPINT (entry->envelope.level, ==, mongoc_structured_log_entry_get_level (entry)); ASSERT_CMPINT (entry->envelope.component, ==, mongoc_structured_log_entry_get_component (entry)); From d646e6169e3ad27e6ff748bc71765f92c86cc924 Mon Sep 17 00:00:00 2001 From: Micah Scott Date: Fri, 6 Dec 2024 08:19:31 -0800 Subject: [PATCH 113/139] structured_log doc: clean up redundant toc entries --- src/libmongoc/doc/structured_log.rst | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/src/libmongoc/doc/structured_log.rst b/src/libmongoc/doc/structured_log.rst index 151ebff83e1..dd139c0770a 100644 --- a/src/libmongoc/doc/structured_log.rst +++ b/src/libmongoc/doc/structured_log.rst @@ -29,13 +29,6 @@ Structured log messages may be filtered in arbitrary ways by the handler, but as :maxdepth: 1 mongoc_structured_log_opts_t - mongoc_structured_log_opts_new - mongoc_structured_log_opts_destroy - mongoc_structured_log_opts_set_handler - mongoc_structured_log_opts_get_max_level_for_component - mongoc_structured_log_opts_set_max_level_for_component - mongoc_structured_log_opts_set_max_level_for_all_components - mongoc_structured_log_opts_set_max_levels_from_env Levels and Components @@ -97,6 +90,7 @@ Handlers must take care not to re-enter ``libmongoc`` with the same :symbol:`mon mongoc_structured_log_func_t + Log Entries ----------- @@ -112,6 +106,7 @@ This results in a standalone document that may be retained for any amount of tim mongoc_structured_log_entry_t + Example ------- .. literalinclude:: ../examples/example-structured-log.c From 13b04803c770c8b9731487704f7439e5e9e672df Mon Sep 17 00:00:00 2001 From: Micah Scott Date: Fri, 6 Dec 2024 08:22:33 -0800 Subject: [PATCH 114/139] doc: link to log level type --- .../mongoc_structured_log_opts_get_max_level_for_component.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libmongoc/doc/mongoc_structured_log_opts_get_max_level_for_component.rst b/src/libmongoc/doc/mongoc_structured_log_opts_get_max_level_for_component.rst index fb8315f1063..58be3bbc377 100644 --- a/src/libmongoc/doc/mongoc_structured_log_opts_get_max_level_for_component.rst +++ b/src/libmongoc/doc/mongoc_structured_log_opts_get_max_level_for_component.rst @@ -21,7 +21,7 @@ Parameters Returns ------- -Returns the configured maximum log level for a specific component. +Returns the configured maximum log level for a specific component, as a :symbol:`mongoc_structured_log_level_t`. This may be the last value set with :symbol:`mongoc_structured_log_opts_set_max_level_for_component` or :symbol:`mongoc_structured_log_opts_set_max_level_for_all_components`, or it may be the default obtained from environment variables. If an invalid or unknown component enum is given, returns the lowest log level. From 9281586c7257844a21cee960ae960a09ea958168 Mon Sep 17 00:00:00 2001 From: Micah Scott Date: Mon, 9 Dec 2024 09:03:46 -0800 Subject: [PATCH 115/139] clang-format --- src/libmongoc/tests/test-mongoc-structured-log.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libmongoc/tests/test-mongoc-structured-log.c b/src/libmongoc/tests/test-mongoc-structured-log.c index c8c7f3ecb75..60141aa983e 100644 --- a/src/libmongoc/tests/test-mongoc-structured-log.c +++ b/src/libmongoc/tests/test-mongoc-structured-log.c @@ -38,7 +38,7 @@ structured_log_func (const mongoc_structured_log_entry_t *entry, void *user_data ASSERT_CMPINT (entry->envelope.component, ==, assumption->expected_envelope.component); ASSERT_CMPSTR (entry->envelope.message, assumption->expected_envelope.message); - ASSERT_CMPSTR (entry->envelope.message, mongoc_structured_log_entry_get_message_string(entry)); + ASSERT_CMPSTR (entry->envelope.message, mongoc_structured_log_entry_get_message_string (entry)); ASSERT_CMPINT (entry->envelope.level, ==, mongoc_structured_log_entry_get_level (entry)); ASSERT_CMPINT (entry->envelope.component, ==, mongoc_structured_log_entry_get_component (entry)); From a233d7282e790622564ff0d6ebc3c0670cbb85d6 Mon Sep 17 00:00:00 2001 From: micah Date: Mon, 9 Dec 2024 14:49:49 -0800 Subject: [PATCH 116/139] Update src/libmongoc/doc/unstructured_log.rst Co-authored-by: Kevin Albertson --- src/libmongoc/doc/unstructured_log.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libmongoc/doc/unstructured_log.rst b/src/libmongoc/doc/unstructured_log.rst index e84cac1f0fa..1e2eb3c7117 100644 --- a/src/libmongoc/doc/unstructured_log.rst +++ b/src/libmongoc/doc/unstructured_log.rst @@ -69,8 +69,8 @@ Custom Log Handlers ------------------- The default log handler prints a timestamp and the log message to ``stdout``, or to ``stderr`` for warnings, critical messages, and errors. - You can override the handler with ``mongoc_log_set_handler()``. - Your handler function is called in a mutex for thread safety. +You can override the handler with ``mongoc_log_set_handler()``. +Your handler function is called in a mutex for thread safety. For example, you could register a custom handler to suppress messages at INFO level and below: From 33027beb704178df11ddb6392ecec71173f7217b Mon Sep 17 00:00:00 2001 From: micah Date: Mon, 9 Dec 2024 14:52:54 -0800 Subject: [PATCH 117/139] Update src/libmongoc/src/mongoc/mongoc-structured-log.c Co-authored-by: Kevin Albertson --- src/libmongoc/src/mongoc/mongoc-structured-log.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libmongoc/src/mongoc/mongoc-structured-log.c b/src/libmongoc/src/mongoc/mongoc-structured-log.c index 6cfb45ad267..02d43e20504 100644 --- a/src/libmongoc/src/mongoc/mongoc-structured-log.c +++ b/src/libmongoc/src/mongoc/mongoc-structured-log.c @@ -334,7 +334,7 @@ _mongoc_structured_log_default_handler_open_stream (mongoc_structured_log_defaul { // shared->mutex must already be locked - if (!path || !strcmp (path, "stderr")) { + if (!path || !strcasecmp (path, "stderr")) { // Default or explicit stderr shared->stream = stderr; shared->stream_fclose_on_destroy = false; From 191dd4f5272e89111e82d3fedfad22252b8cbecb Mon Sep 17 00:00:00 2001 From: micah Date: Mon, 9 Dec 2024 14:53:12 -0800 Subject: [PATCH 118/139] Update src/libmongoc/src/mongoc/mongoc-structured-log.c Co-authored-by: Kevin Albertson --- src/libmongoc/src/mongoc/mongoc-structured-log.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libmongoc/src/mongoc/mongoc-structured-log.c b/src/libmongoc/src/mongoc/mongoc-structured-log.c index 02d43e20504..ce6022f9f85 100644 --- a/src/libmongoc/src/mongoc/mongoc-structured-log.c +++ b/src/libmongoc/src/mongoc/mongoc-structured-log.c @@ -338,7 +338,7 @@ _mongoc_structured_log_default_handler_open_stream (mongoc_structured_log_defaul // Default or explicit stderr shared->stream = stderr; shared->stream_fclose_on_destroy = false; - } else if (!strcmp (path, "stdout")) { + } else if (!strcasecmp (path, "stdout")) { shared->stream = stdout; shared->stream_fclose_on_destroy = false; } else { From cb53cbbcec68216696511f152340095401980153 Mon Sep 17 00:00:00 2001 From: micah Date: Mon, 9 Dec 2024 14:54:01 -0800 Subject: [PATCH 119/139] Update src/libmongoc/src/mongoc/mongoc-structured-log.c Co-authored-by: Kevin Albertson --- src/libmongoc/src/mongoc/mongoc-structured-log.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/libmongoc/src/mongoc/mongoc-structured-log.c b/src/libmongoc/src/mongoc/mongoc-structured-log.c index ce6022f9f85..a60b1acd145 100644 --- a/src/libmongoc/src/mongoc/mongoc-structured-log.c +++ b/src/libmongoc/src/mongoc/mongoc-structured-log.c @@ -347,7 +347,9 @@ _mongoc_structured_log_default_handler_open_stream (mongoc_structured_log_defaul shared->stream = file; shared->stream_fclose_on_destroy = true; } else { - MONGOC_WARNING ("Cannot open log file '%s' for writing. Logging to stderr instead.", path); + char errmsg_buf[BSON_ERROR_BUFFER_SIZE]; + const char *errmsg = bson_strerror_r (errno, errmsg_buf, sizeof errmsg_buf); + MONGOC_WARNING ("Failed to open log file '%s' with error: '%s'. Logging to stderr instead.", path, errmsg); shared->stream = stderr; shared->stream_fclose_on_destroy = false; } From fa90d6a9268f99b517ffae30622213ef48d969a4 Mon Sep 17 00:00:00 2001 From: micah Date: Mon, 9 Dec 2024 15:06:56 -0800 Subject: [PATCH 120/139] Update src/libmongoc/src/mongoc/mongoc-structured-log-private.h Co-authored-by: Kevin Albertson --- src/libmongoc/src/mongoc/mongoc-structured-log-private.h | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libmongoc/src/mongoc/mongoc-structured-log-private.h b/src/libmongoc/src/mongoc/mongoc-structured-log-private.h index e49e3ca5129..a55064f9f30 100644 --- a/src/libmongoc/src/mongoc/mongoc-structured-log-private.h +++ b/src/libmongoc/src/mongoc/mongoc-structured-log-private.h @@ -131,6 +131,7 @@ mongoc_structured_log_instance_destroy (mongoc_structured_log_instance_t *instan * @brief Structured log item, referencing a utf8 string with explicit key and value lengths. * * @param key Key as a NUL-terminated const char * expression, or NULL to skip this item. + * @param key_len UTF8 value length in bytes, as an int32_t expression. * @param value UTF8 value as a const char * expression, or NULL for a null value. May have embedded NUL bytes. * @param value_len UTF8 value length in bytes, as an int32_t expression. */ From bdd29d49676805a459367ef647a4696c752f9d38 Mon Sep 17 00:00:00 2001 From: Micah Scott Date: Mon, 9 Dec 2024 20:20:57 -0800 Subject: [PATCH 121/139] Confirmed that logs can include both millis and micros for duration --- src/libmongoc/src/mongoc/mongoc-structured-log-private.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libmongoc/src/mongoc/mongoc-structured-log-private.h b/src/libmongoc/src/mongoc/mongoc-structured-log-private.h index a55064f9f30..376a88a97dd 100644 --- a/src/libmongoc/src/mongoc/mongoc-structured-log-private.h +++ b/src/libmongoc/src/mongoc/mongoc-structured-log-private.h @@ -301,8 +301,8 @@ typedef enum { * @brief Structured log item, standard format for a duration in monotonic time. * @param duration Duration in microseconds, as an int64_t expression. * - * @todo Is the (CLAM) spec asking for only the highest resolution available, or that plus milliseconds? - * */ + * Includes milliseconds for consistency across drivers, and microseconds as the highest available resolution. + */ #define _mongoc_structured_log_item_monotonic_time_duration(_duration) \ _mongoc_structured_log_item_int32 ("durationMS", (int32_t) ((_duration) / 1000)) \ _mongoc_structured_log_item_int64 ("durationMicros", (_duration)) From c4847000dd8b95a39d829ec83b093eab9396aa44 Mon Sep 17 00:00:00 2001 From: Micah Scott Date: Mon, 9 Dec 2024 20:22:24 -0800 Subject: [PATCH 122/139] remove incorrect todo comment --- src/libmongoc/tests/unified/entity-map.c | 1 - 1 file changed, 1 deletion(-) diff --git a/src/libmongoc/tests/unified/entity-map.c b/src/libmongoc/tests/unified/entity-map.c index 129db07764c..0b46b8655b7 100644 --- a/src/libmongoc/tests/unified/entity-map.c +++ b/src/libmongoc/tests/unified/entity-map.c @@ -660,7 +660,6 @@ entity_client_new (entity_map_t *em, bson_t *bson, bson_error_t *error) } if (can_reduce_heartbeat && em->reduced_heartbeat) { - // @todo This option is needed for both single-threaded and pooled clients, but only works on pooled mongoc_uri_set_option_as_int32 (uri, MONGOC_URI_HEARTBEATFREQUENCYMS, REDUCED_HEARTBEAT_FREQUENCY_MS); } From da9a4a5300a4deed8b01968248b17654c6bb5c80 Mon Sep 17 00:00:00 2001 From: Micah Scott Date: Mon, 9 Dec 2024 20:26:41 -0800 Subject: [PATCH 123/139] just remove this todo also --- src/libmongoc/tests/unified/entity-map.c | 1 - 1 file changed, 1 deletion(-) diff --git a/src/libmongoc/tests/unified/entity-map.c b/src/libmongoc/tests/unified/entity-map.c index 0b46b8655b7..aaabed23d14 100644 --- a/src/libmongoc/tests/unified/entity-map.c +++ b/src/libmongoc/tests/unified/entity-map.c @@ -671,7 +671,6 @@ entity_client_new (entity_map_t *em, bson_t *bson, bson_error_t *error) mongoc_client_set_structured_log_opts (client, log_opts); if (can_reduce_heartbeat && em->reduced_heartbeat) { - // @todo Examine whether this is needed in addition to the URI param above client->topology->min_heartbeat_frequency_msec = REDUCED_MIN_HEARTBEAT_FREQUENCY_MS; } From 7f0f43c8b0b0fd981b4619383e13176597327176 Mon Sep 17 00:00:00 2001 From: micah Date: Mon, 16 Dec 2024 08:15:25 -0800 Subject: [PATCH 124/139] Update src/libmongoc/src/mongoc/mongoc-structured-log.c Co-authored-by: Kevin Albertson --- src/libmongoc/src/mongoc/mongoc-structured-log.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libmongoc/src/mongoc/mongoc-structured-log.c b/src/libmongoc/src/mongoc/mongoc-structured-log.c index a60b1acd145..097dd550e8e 100644 --- a/src/libmongoc/src/mongoc/mongoc-structured-log.c +++ b/src/libmongoc/src/mongoc/mongoc-structured-log.c @@ -284,7 +284,7 @@ _mongoc_structured_log_get_max_document_length_from_env (void) char *endptr; long int_value = strtol (max_length_str, &endptr, 10); - if (int_value >= 0 && endptr != max_length_str && !*endptr) { + if (int_value >= 0 && int_value <= INT32_MAX && endptr != max_length_str && !*endptr) { return (int32_t) int_value; } From 42a247b8c80750302bdec10cc7116b3c0c420083 Mon Sep 17 00:00:00 2001 From: micah Date: Mon, 16 Dec 2024 09:01:24 -0800 Subject: [PATCH 125/139] Update src/libmongoc/src/mongoc/mongoc-structured-log.c Co-authored-by: Kevin Albertson --- src/libmongoc/src/mongoc/mongoc-structured-log.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libmongoc/src/mongoc/mongoc-structured-log.c b/src/libmongoc/src/mongoc/mongoc-structured-log.c index 097dd550e8e..b95e8373a0c 100644 --- a/src/libmongoc/src/mongoc/mongoc-structured-log.c +++ b/src/libmongoc/src/mongoc/mongoc-structured-log.c @@ -304,7 +304,7 @@ mongoc_structured_log_opts_set_max_levels_from_env (mongoc_structured_log_opts_t bool all_ok = true; mongoc_structured_log_level_t level; - // Errors are not fatal by detault; always reported by return value, and reported the first time only via a log + // Errors are not fatal by default; always reported by return value, and reported the first time only via a log // warning. static int err_count_all_atomic; static int err_count_per_component_atomic[STRUCTURED_LOG_COMPONENT_TABLE_SIZE]; From e990586bf4d111c8c9af6a807e450f57960d9e05 Mon Sep 17 00:00:00 2001 From: Micah Scott Date: Mon, 16 Dec 2024 15:15:33 -0800 Subject: [PATCH 126/139] example-structured-log: motivate the thread safety with a global counter The printf() is meant to stand in for a stream of some sort, but on its own doesn't motivate thread safety because stdio has its own locking. We could use a non-threadsafe file or queue, but Kevin suggested a counter and I like how simple this is. The code now includes a counter, and the comments mention that it could represent something mroe complex like a file or queue. --- src/libmongoc/examples/example-structured-log.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/libmongoc/examples/example-structured-log.c b/src/libmongoc/examples/example-structured-log.c index 5c661b4471a..b6ba56d6c29 100644 --- a/src/libmongoc/examples/example-structured-log.c +++ b/src/libmongoc/examples/example-structured-log.c @@ -28,10 +28,17 @@ example_handler (const mongoc_structured_log_entry_t *entry, void *user_data) * own global mutex. For other apps, this would be a performance bottleneck * and it would be more appropriate for handlers to process their log * messages concurrently. + * + * In this example, our mutex protects access to a global log counter. + * In a real application, you may need to protect access to a shared stream + * or queue. */ pthread_mutex_lock (&handler_mutex); - printf ("Log entry with component=%s level=%s message_string='%s'\n", + static unsigned log_serial_number = 0; + + printf ("%u. Log entry with component=%s level=%s message_string='%s'\n", + ++log_serial_number, mongoc_structured_log_get_component_name (component), mongoc_structured_log_get_level_name (level), message_string); From 3c181180100b57a65ab6ab82884aa87f6e66c439 Mon Sep 17 00:00:00 2001 From: Micah Scott Date: Mon, 16 Dec 2024 15:29:54 -0800 Subject: [PATCH 127/139] Remove leftover includes from attempt at cmd serialization move I had an earlier change which tried to move cmd serialization to mongoc-cmd, but I had to move it back to structured-log due to dependencies on the structured logging options. This reverts an include I missed when reverting the earlier move. --- src/libmongoc/src/mongoc/mongoc-cmd.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/libmongoc/src/mongoc/mongoc-cmd.c b/src/libmongoc/src/mongoc/mongoc-cmd.c index a1d7fb449bb..9817a8d493a 100644 --- a/src/libmongoc/src/mongoc/mongoc-cmd.c +++ b/src/libmongoc/src/mongoc/mongoc-cmd.c @@ -15,14 +15,12 @@ */ -#include "mongoc-apm-private.h" #include "mongoc-cmd-private.h" #include "mongoc-read-prefs-private.h" #include "mongoc-trace-private.h" #include "mongoc-client-private.h" #include "mongoc-read-concern-private.h" #include "mongoc-server-api-private.h" -#include "mongoc-structured-log-private.h" #include "mongoc-write-concern-private.h" /* For strcasecmp on Windows */ #include "mongoc-util-private.h" From 7cbfbae5240b3db7b747e75e85af30897161d597 Mon Sep 17 00:00:00 2001 From: Micah Scott Date: Mon, 16 Dec 2024 17:07:55 -0800 Subject: [PATCH 128/139] Return error on *_set_structured_log_opts misuse --- .../doc/mongoc_client_pool_set_structured_log_opts.rst | 7 ++++++- .../doc/mongoc_client_set_structured_log_opts.rst | 7 ++++++- src/libmongoc/src/mongoc/mongoc-client-pool.c | 5 ++++- src/libmongoc/src/mongoc/mongoc-client-pool.h | 2 +- src/libmongoc/src/mongoc/mongoc-client.c | 4 +++- src/libmongoc/src/mongoc/mongoc-client.h | 2 +- src/libmongoc/tests/unified/entity-map.c | 2 +- 7 files changed, 22 insertions(+), 7 deletions(-) diff --git a/src/libmongoc/doc/mongoc_client_pool_set_structured_log_opts.rst b/src/libmongoc/doc/mongoc_client_pool_set_structured_log_opts.rst index 47e84fcb8c6..4f3c0f8afed 100644 --- a/src/libmongoc/doc/mongoc_client_pool_set_structured_log_opts.rst +++ b/src/libmongoc/doc/mongoc_client_pool_set_structured_log_opts.rst @@ -8,7 +8,7 @@ Synopsis .. code-block:: c - void + bool mongoc_client_pool_set_structured_log_opts (mongoc_client_pool_t *pool, const mongoc_structured_log_opts_t *opts); @@ -28,6 +28,11 @@ Parameters * ``pool``: A :symbol:`mongoc_client_pool_t`. * ``opts``: A :symbol:`mongoc_structured_log_opts_t` allocated with :symbol:`mongoc_structured_log_opts_new`, or NULL to disable structured logging. +Returns +------- + +Returns true when used correctly. If called multiple times per pool or after the first client is initialized, returns false and logs a warning. + .. include:: includes/mongoc_client_pool_call_once.txt Thread safety within the handler is the application's responsibility. Handlers may be invoked concurrently by multiple pool users. diff --git a/src/libmongoc/doc/mongoc_client_set_structured_log_opts.rst b/src/libmongoc/doc/mongoc_client_set_structured_log_opts.rst index 760cf49868a..95002b5c8ee 100644 --- a/src/libmongoc/doc/mongoc_client_set_structured_log_opts.rst +++ b/src/libmongoc/doc/mongoc_client_set_structured_log_opts.rst @@ -8,7 +8,7 @@ Synopsis .. code-block:: c - void + bool mongoc_client_set_structured_log_opts (mongoc_client_t *client, const mongoc_structured_log_opts_t *opts); @@ -31,6 +31,11 @@ Parameters * ``client``: A :symbol:`mongoc_client_t`. * ``opts``: A :symbol:`mongoc_structured_log_opts_t` allocated with :symbol:`mongoc_structured_log_opts_new`, or NULL to disable structured logging. +Returns +------- + +Returns true when used correctly. If called on a pooled client, returns false and logs a warning. + .. seealso:: | :doc:`structured_log` diff --git a/src/libmongoc/src/mongoc/mongoc-client-pool.c b/src/libmongoc/src/mongoc/mongoc-client-pool.c index 39e019608fa..4437798a8e9 100644 --- a/src/libmongoc/src/mongoc/mongoc-client-pool.c +++ b/src/libmongoc/src/mongoc/mongoc-client-pool.c @@ -608,7 +608,7 @@ mongoc_client_pool_set_apm_callbacks (mongoc_client_pool_t *pool, mongoc_apm_cal return true; } -void +bool mongoc_client_pool_set_structured_log_opts (mongoc_client_pool_t *pool, const mongoc_structured_log_opts_t *opts) { BSON_ASSERT_PARAM (pool); @@ -619,12 +619,15 @@ mongoc_client_pool_set_structured_log_opts (mongoc_client_pool_t *pool, const mo * expected to warn but not quit when encountering initialization errors. */ if (pool->structured_log_opts_set) { MONGOC_WARNING ("mongoc_client_pool_set_structured_log_opts can only be called once per pool"); + return false; } else if (pool->client_initialized) { MONGOC_WARNING ("mongoc_client_pool_set_structured_log_opts can only be called before mongoc_client_pool_pop"); + return false; } else { // Now we can be sure no other threads are relying on concurrent access to the instance yet. mongoc_topology_set_structured_log_opts (pool->topology, opts); pool->structured_log_opts_set = true; + return true; } } diff --git a/src/libmongoc/src/mongoc/mongoc-client-pool.h b/src/libmongoc/src/mongoc/mongoc-client-pool.h index ae785841b3d..025d5b60956 100644 --- a/src/libmongoc/src/mongoc/mongoc-client-pool.h +++ b/src/libmongoc/src/mongoc/mongoc-client-pool.h @@ -70,7 +70,7 @@ mongoc_client_pool_enable_auto_encryption (mongoc_client_pool_t *pool, bson_error_t *error); MONGOC_EXPORT (bool) mongoc_client_pool_set_server_api (mongoc_client_pool_t *pool, const mongoc_server_api_t *api, bson_error_t *error); -MONGOC_EXPORT (void) +MONGOC_EXPORT (bool) mongoc_client_pool_set_structured_log_opts (mongoc_client_pool_t *pool, const mongoc_structured_log_opts_t *opts); BSON_END_DECLS diff --git a/src/libmongoc/src/mongoc/mongoc-client.c b/src/libmongoc/src/mongoc/mongoc-client.c index 3fef8d81e9c..b5f75b41116 100644 --- a/src/libmongoc/src/mongoc/mongoc-client.c +++ b/src/libmongoc/src/mongoc/mongoc-client.c @@ -2611,7 +2611,7 @@ mongoc_client_set_apm_callbacks (mongoc_client_t *client, mongoc_apm_callbacks_t } -void +bool mongoc_client_set_structured_log_opts (mongoc_client_t *client, const mongoc_structured_log_opts_t *opts) { BSON_ASSERT_PARAM (client); @@ -2619,9 +2619,11 @@ mongoc_client_set_structured_log_opts (mongoc_client_t *client, const mongoc_str if (client->topology->single_threaded) { mongoc_topology_set_structured_log_opts (client->topology, opts); + return true; } else { MONGOC_WARNING ("Cannot set structured log options on a pooled client, use " "mongoc_client_pool_set_structured_log_opts before the first mongoc_client_pool_pop"); + return false; } } diff --git a/src/libmongoc/src/mongoc/mongoc-client.h b/src/libmongoc/src/mongoc/mongoc-client.h index 7ce456aa6dd..243fe0834e5 100644 --- a/src/libmongoc/src/mongoc/mongoc-client.h +++ b/src/libmongoc/src/mongoc/mongoc-client.h @@ -229,7 +229,7 @@ mongoc_client_set_ssl_opts (mongoc_client_t *client, const mongoc_ssl_opt_t *opt #endif MONGOC_EXPORT (bool) mongoc_client_set_apm_callbacks (mongoc_client_t *client, mongoc_apm_callbacks_t *callbacks, void *context); -MONGOC_EXPORT (void) +MONGOC_EXPORT (bool) mongoc_client_set_structured_log_opts (mongoc_client_t *client, const mongoc_structured_log_opts_t *opts); MONGOC_EXPORT (mongoc_server_description_t *) mongoc_client_get_server_description (mongoc_client_t *client, uint32_t server_id) BSON_GNUC_WARN_UNUSED_RESULT; diff --git a/src/libmongoc/tests/unified/entity-map.c b/src/libmongoc/tests/unified/entity-map.c index aaabed23d14..31091881882 100644 --- a/src/libmongoc/tests/unified/entity-map.c +++ b/src/libmongoc/tests/unified/entity-map.c @@ -668,7 +668,7 @@ entity_client_new (entity_map_t *em, bson_t *bson, bson_error_t *error) mongoc_client_set_error_api (client, MONGOC_ERROR_API_VERSION_2); entity->value = client; mongoc_client_set_apm_callbacks (client, callbacks, entity); - mongoc_client_set_structured_log_opts (client, log_opts); + BSON_ASSERT (mongoc_client_set_structured_log_opts (client, log_opts)); if (can_reduce_heartbeat && em->reduced_heartbeat) { client->topology->min_heartbeat_frequency_msec = REDUCED_MIN_HEARTBEAT_FREQUENCY_MS; From e334612d35688a3a11d1d1d9930724ae431cbf43 Mon Sep 17 00:00:00 2001 From: Micah Scott Date: Mon, 16 Dec 2024 17:26:45 -0800 Subject: [PATCH 129/139] asserts for mongoc_structured_log_opts_set_max_level_for_all_components --- .../src/mongoc/mongoc-structured-log.c | 2 +- .../tests/test-mongoc-structured-log.c | 18 +++++++++--------- src/libmongoc/tests/unified/entity-map.c | 4 ++-- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/src/libmongoc/src/mongoc/mongoc-structured-log.c b/src/libmongoc/src/mongoc/mongoc-structured-log.c index b95e8373a0c..70cebfbe522 100644 --- a/src/libmongoc/src/mongoc/mongoc-structured-log.c +++ b/src/libmongoc/src/mongoc/mongoc-structured-log.c @@ -310,7 +310,7 @@ mongoc_structured_log_opts_set_max_levels_from_env (mongoc_structured_log_opts_t static int err_count_per_component_atomic[STRUCTURED_LOG_COMPONENT_TABLE_SIZE]; if (_mongoc_structured_log_get_log_level_from_env ("MONGODB_LOG_ALL", &level, &err_count_all_atomic)) { - mongoc_structured_log_opts_set_max_level_for_all_components (opts, level); + BSON_ASSERT (mongoc_structured_log_opts_set_max_level_for_all_components (opts, level)); } else { all_ok = false; } diff --git a/src/libmongoc/tests/test-mongoc-structured-log.c b/src/libmongoc/tests/test-mongoc-structured-log.c index 60141aa983e..fa32b1d35c6 100644 --- a/src/libmongoc/tests/test-mongoc-structured-log.c +++ b/src/libmongoc/tests/test-mongoc-structured-log.c @@ -142,7 +142,7 @@ test_structured_log_plain (void) mongoc_structured_log_opts_t *opts = mongoc_structured_log_opts_new (); mongoc_structured_log_opts_set_handler (opts, structured_log_func, &assumption); - mongoc_structured_log_opts_set_max_level_for_all_components (opts, MONGOC_STRUCTURED_LOG_LEVEL_DEBUG); + ASSERT (mongoc_structured_log_opts_set_max_level_for_all_components (opts, MONGOC_STRUCTURED_LOG_LEVEL_DEBUG)); mongoc_structured_log_instance_t *instance = mongoc_structured_log_instance_new (opts); mongoc_structured_log_opts_destroy (opts); @@ -168,7 +168,7 @@ test_structured_log_plain_with_extra_data (void) mongoc_structured_log_opts_t *opts = mongoc_structured_log_opts_new (); mongoc_structured_log_opts_set_handler (opts, structured_log_func, &assumption); - mongoc_structured_log_opts_set_max_level_for_all_components (opts, MONGOC_STRUCTURED_LOG_LEVEL_DEBUG); + ASSERT (mongoc_structured_log_opts_set_max_level_for_all_components (opts, MONGOC_STRUCTURED_LOG_LEVEL_DEBUG)); mongoc_structured_log_instance_t *instance = mongoc_structured_log_instance_new (opts); mongoc_structured_log_opts_destroy (opts); @@ -222,7 +222,7 @@ test_structured_log_basic_data_types (void) mongoc_structured_log_opts_t *opts = mongoc_structured_log_opts_new (); mongoc_structured_log_opts_set_handler (opts, structured_log_func, &assumption); - mongoc_structured_log_opts_set_max_level_for_all_components (opts, MONGOC_STRUCTURED_LOG_LEVEL_DEBUG); + ASSERT (mongoc_structured_log_opts_set_max_level_for_all_components (opts, MONGOC_STRUCTURED_LOG_LEVEL_DEBUG)); mongoc_structured_log_instance_t *instance = mongoc_structured_log_instance_new (opts); mongoc_structured_log_opts_destroy (opts); @@ -274,7 +274,7 @@ test_structured_log_json (void) mongoc_structured_log_opts_t *opts = mongoc_structured_log_opts_new (); mongoc_structured_log_opts_set_handler (opts, structured_log_func, &assumption); - mongoc_structured_log_opts_set_max_level_for_all_components (opts, MONGOC_STRUCTURED_LOG_LEVEL_DEBUG); + ASSERT (mongoc_structured_log_opts_set_max_level_for_all_components (opts, MONGOC_STRUCTURED_LOG_LEVEL_DEBUG)); mongoc_structured_log_instance_t *instance = mongoc_structured_log_instance_new (opts); mongoc_structured_log_opts_destroy (opts); @@ -314,7 +314,7 @@ test_structured_log_oid (void) mongoc_structured_log_opts_t *opts = mongoc_structured_log_opts_new (); mongoc_structured_log_opts_set_handler (opts, structured_log_func, &assumption); - mongoc_structured_log_opts_set_max_level_for_all_components (opts, MONGOC_STRUCTURED_LOG_LEVEL_DEBUG); + ASSERT (mongoc_structured_log_opts_set_max_level_for_all_components (opts, MONGOC_STRUCTURED_LOG_LEVEL_DEBUG)); mongoc_structured_log_instance_t *instance = mongoc_structured_log_instance_new (opts); mongoc_structured_log_opts_destroy (opts); @@ -363,7 +363,7 @@ test_structured_log_error (void) mongoc_structured_log_opts_t *opts = mongoc_structured_log_opts_new (); mongoc_structured_log_opts_set_handler (opts, structured_log_func, &assumption); - mongoc_structured_log_opts_set_max_level_for_all_components (opts, MONGOC_STRUCTURED_LOG_LEVEL_INFO); + ASSERT (mongoc_structured_log_opts_set_max_level_for_all_components (opts, MONGOC_STRUCTURED_LOG_LEVEL_INFO)); mongoc_structured_log_instance_t *instance = mongoc_structured_log_instance_new (opts); mongoc_structured_log_opts_destroy (opts); @@ -435,7 +435,7 @@ test_structured_log_server_description (void) mongoc_structured_log_opts_t *opts = mongoc_structured_log_opts_new (); mongoc_structured_log_opts_set_handler (opts, structured_log_func, &assumption); - mongoc_structured_log_opts_set_max_level_for_all_components (opts, MONGOC_STRUCTURED_LOG_LEVEL_DEBUG); + ASSERT (mongoc_structured_log_opts_set_max_level_for_all_components (opts, MONGOC_STRUCTURED_LOG_LEVEL_DEBUG)); mongoc_structured_log_instance_t *instance = mongoc_structured_log_instance_new (opts); mongoc_structured_log_opts_destroy (opts); @@ -538,7 +538,7 @@ test_structured_log_command (void) mongoc_structured_log_opts_t *opts = mongoc_structured_log_opts_new (); mongoc_structured_log_opts_set_handler (opts, structured_log_func, &assumption); - mongoc_structured_log_opts_set_max_level_for_all_components (opts, MONGOC_STRUCTURED_LOG_LEVEL_DEBUG); + ASSERT (mongoc_structured_log_opts_set_max_level_for_all_components (opts, MONGOC_STRUCTURED_LOG_LEVEL_DEBUG)); mongoc_structured_log_instance_t *instance = mongoc_structured_log_instance_new (opts); mongoc_structured_log_opts_destroy (opts); @@ -590,7 +590,7 @@ test_structured_log_duration (void) mongoc_structured_log_opts_t *opts = mongoc_structured_log_opts_new (); mongoc_structured_log_opts_set_handler (opts, structured_log_func, &assumption); - mongoc_structured_log_opts_set_max_level_for_all_components (opts, MONGOC_STRUCTURED_LOG_LEVEL_DEBUG); + ASSERT (mongoc_structured_log_opts_set_max_level_for_all_components (opts, MONGOC_STRUCTURED_LOG_LEVEL_DEBUG)); mongoc_structured_log_instance_t *instance = mongoc_structured_log_instance_new (opts); mongoc_structured_log_opts_destroy (opts); diff --git a/src/libmongoc/tests/unified/entity-map.c b/src/libmongoc/tests/unified/entity-map.c index 31091881882..47f370d4c07 100644 --- a/src/libmongoc/tests/unified/entity-map.c +++ b/src/libmongoc/tests/unified/entity-map.c @@ -597,8 +597,8 @@ entity_client_new (entity_map_t *em, bson_t *bson, bson_error_t *error) if (not(type (doc)), then (error ("'observeLogMessages' must be a document"))), do ({ // Initialize all components to the lowest available level, and install a handler. - mongoc_structured_log_opts_set_max_level_for_all_components (log_opts, - MONGOC_STRUCTURED_LOG_LEVEL_EMERGENCY); + BSON_ASSERT (mongoc_structured_log_opts_set_max_level_for_all_components ( + log_opts, MONGOC_STRUCTURED_LOG_LEVEL_EMERGENCY)); mongoc_structured_log_opts_set_handler (log_opts, structured_log_cb, entity); }), visitEach ( From 014bc8dfe4faf1d41f5f2597362de43a6c7be110 Mon Sep 17 00:00:00 2001 From: Micah Scott Date: Mon, 16 Dec 2024 17:30:53 -0800 Subject: [PATCH 130/139] asserts for mongoc_structured_log_opts_set_max_level_for_component --- src/libmongoc/src/mongoc/mongoc-structured-log.c | 4 ++-- src/libmongoc/tests/unified/entity-map.c | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/libmongoc/src/mongoc/mongoc-structured-log.c b/src/libmongoc/src/mongoc/mongoc-structured-log.c index 70cebfbe522..e60fed59616 100644 --- a/src/libmongoc/src/mongoc/mongoc-structured-log.c +++ b/src/libmongoc/src/mongoc/mongoc-structured-log.c @@ -318,8 +318,8 @@ mongoc_structured_log_opts_set_max_levels_from_env (mongoc_structured_log_opts_t for (int component = 0; component < STRUCTURED_LOG_COMPONENT_TABLE_SIZE; component++) { if (_mongoc_structured_log_get_log_level_from_env ( gStructuredLogComponentEnvVars[component], &level, &err_count_per_component_atomic[component])) { - mongoc_structured_log_opts_set_max_level_for_component ( - opts, (mongoc_structured_log_component_t) component, level); + BSON_ASSERT (mongoc_structured_log_opts_set_max_level_for_component ( + opts, (mongoc_structured_log_component_t) component, level)); } else { all_ok = false; } diff --git a/src/libmongoc/tests/unified/entity-map.c b/src/libmongoc/tests/unified/entity-map.c index 47f370d4c07..686f6757244 100644 --- a/src/libmongoc/tests/unified/entity-map.c +++ b/src/libmongoc/tests/unified/entity-map.c @@ -614,7 +614,7 @@ entity_client_new (entity_map_t *em, bson_t *bson, bson_error_t *error) if (!mongoc_structured_log_get_named_level (level_name, &level)) { test_error ("Unknown log level '%s' given in 'observeLogMessages'", component_name); } - mongoc_structured_log_opts_set_max_level_for_component (log_opts, component, level); + BSON_ASSERT (mongoc_structured_log_opts_set_max_level_for_component (log_opts, component, level)); }))), visitOthers ( dupPath (errpath), From 20f9890087c9aa0c53beb5bc538a7179b3394564 Mon Sep 17 00:00:00 2001 From: Micah Scott Date: Mon, 16 Dec 2024 18:04:38 -0800 Subject: [PATCH 131/139] Include errorLabels in logged client-side command failures --- src/libmongoc/src/mongoc/mongoc-structured-log.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/libmongoc/src/mongoc/mongoc-structured-log.c b/src/libmongoc/src/mongoc/mongoc-structured-log.c index e60fed59616..a5899a4ab02 100644 --- a/src/libmongoc/src/mongoc/mongoc-structured-log.c +++ b/src/libmongoc/src/mongoc/mongoc-structured-log.c @@ -773,13 +773,19 @@ _mongoc_structured_log_append_redacted_cmd_failure (bson_t *bson, BSON_APPEND_DOCUMENT (bson, "failure", reply); } } else { - // Client-side errors converted directly from bson_error_t, never redacted + /* Client-side errors converted directly from bson_error_t, never redacted. + * In addition to the bson_error_t fields, client side errors may include errorLabels: + * https://mongoc.org/libmongoc/current/errors.html#error-labels */ bson_t failure; if (BSON_APPEND_DOCUMENT_BEGIN (bson, "failure", &failure)) { mongoc_error_append_contents_to_bson (error, &failure, MONGOC_ERROR_CONTENT_FLAG_MESSAGE | MONGOC_ERROR_CONTENT_FLAG_CODE | MONGOC_ERROR_CONTENT_FLAG_DOMAIN); + bson_iter_t iter; + if (bson_iter_init_find (&iter, reply, "errorLabels")) { + bson_append_iter (&failure, bson_iter_key (&iter), bson_iter_key_len (&iter), &iter); + } bson_append_document_end (bson, &failure); } } From 07f49b2c563fdf215bdadf33a6824235dd3fb792 Mon Sep 17 00:00:00 2001 From: Micah Scott Date: Tue, 17 Dec 2024 08:02:42 -0800 Subject: [PATCH 132/139] Refactor failure paths in bson-macros ASSERTs The immediate provocation here is to work around what appears to be a clang static analyzer bug, but it's a nice cleanup anyhow. The noreturn in timegm was unused, but I found it when adding the new BSON_NORETURN needed for the assert implementation. --- src/libbson/src/bson/bson-macros.h | 70 +++++++++++++++++++----------- src/libbson/src/bson/bson-timegm.c | 8 ---- 2 files changed, 45 insertions(+), 33 deletions(-) diff --git a/src/libbson/src/bson/bson-macros.h b/src/libbson/src/bson/bson-macros.h index c207f17cd7c..399b595bb1a 100644 --- a/src/libbson/src/bson/bson-macros.h +++ b/src/libbson/src/bson/bson-macros.h @@ -22,6 +22,7 @@ #include +#include #ifdef __cplusplus #include @@ -190,12 +191,44 @@ #define BSON_FUNC __func__ #endif -#define BSON_ASSERT(test) \ - do { \ - if (!(BSON_LIKELY (test))) { \ - fprintf (stderr, "%s:%d %s(): precondition failed: %s\n", __FILE__, (int) (__LINE__), BSON_FUNC, #test); \ - abort (); \ - } \ + +#if defined(_MSC_VER) +#define BSON_INLINE __inline +#else +#define BSON_INLINE __inline__ +#endif + + +#if defined(__STDC_VERSION__) && __STDC_VERSION__ >= 202311L +#define BSON_NORETURN noreturn +#elif defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L +#define BSON_NORETURN _Noreturn +#elif defined(__GNUC__) && 2 < __GNUC__ + (8 <= __GNUC_MINOR__) +#define BSON_NORETURN __attribute__ ((__noreturn__)) +#else +#define BSON_NORETURN +#endif + + +static BSON_INLINE BSON_NORETURN void +_bson_assert_failed_on_line (const char *file, int line, const char *func, const char *test) +{ + fprintf (stderr, "%s:%d %s(): assertion failed: %s\n", file, line, func, test); + abort (); +} + +static BSON_INLINE BSON_NORETURN void +_bson_assert_failed_on_param (const char *param, const char *func) +{ + fprintf (stderr, "The parameter: %s, in function %s, cannot be NULL\n", param, func); + abort (); +} + +#define BSON_ASSERT(test) \ + do { \ + if (!(BSON_LIKELY (test))) { \ + _bson_assert_failed_on_line (__FILE__, (int) (__LINE__), BSON_FUNC, #test); \ + } \ } while (0) /** @@ -203,12 +236,7 @@ * success. */ #define BSON_ASSERT_INLINE(Assertion, Value) \ - ((void) ((Assertion) \ - ? (0) \ - : ((fprintf ( \ - stderr, "%s:%d %s(): Assertion '%s' failed", __FILE__, (int) (__LINE__), BSON_FUNC, #Assertion), \ - abort ()), \ - 0)), \ + ((void) ((Assertion) ? (0) : (_bson_assert_failed_on_line (__FILE__, (int) (__LINE__), BSON_FUNC, #Assertion), 0)), \ Value) /** @@ -225,12 +253,11 @@ #define BSON_ASSERT_PTR_INLINE(Pointer) BSON_ASSERT_INLINE ((Pointer) != NULL, (Pointer)) /* Used for asserting parameters to provide a more precise error message */ -#define BSON_ASSERT_PARAM(param) \ - do { \ - if ((BSON_UNLIKELY (param == NULL))) { \ - fprintf (stderr, "The parameter: %s, in function %s, cannot be NULL\n", #param, BSON_FUNC); \ - abort (); \ - } \ +#define BSON_ASSERT_PARAM(param) \ + do { \ + if ((BSON_UNLIKELY (param == NULL))) { \ + _bson_assert_failed_on_param (#param, BSON_FUNC); \ + } \ } while (0) // `BSON_OPTIONAL_PARAM` is a documentation-only macro to document X may be NULL. @@ -294,13 +321,6 @@ #endif -#if defined(_MSC_VER) -#define BSON_INLINE __inline -#else -#define BSON_INLINE __inline__ -#endif - - #ifdef _MSC_VER #define BSON_ENSURE_ARRAY_PARAM_SIZE(_n) #define BSON_TYPEOF decltype diff --git a/src/libbson/src/bson/bson-timegm.c b/src/libbson/src/bson/bson-timegm.c index 3753aa9bf6f..0641e856184 100644 --- a/src/libbson/src/bson/bson-timegm.c +++ b/src/libbson/src/bson/bson-timegm.c @@ -31,14 +31,6 @@ #define ATTRIBUTE_FORMAT(spec) /* empty */ #endif -#if !defined _Noreturn && (!defined(__STDC_VERSION__) || __STDC_VERSION__ < 201112) -#if 2 < __GNUC__ + (8 <= __GNUC_MINOR__) -#define _Noreturn __attribute__ ((__noreturn__)) -#else -#define _Noreturn -#endif -#endif - #if !defined(__STDC_VERSION__) && !defined restrict #define restrict /* empty */ #endif From 9338b95af987bffbd5c070e6f60cf5ad9cce7d41 Mon Sep 17 00:00:00 2001 From: Micah Scott Date: Tue, 17 Dec 2024 09:26:58 -0800 Subject: [PATCH 133/139] Fix out-of-date comment about test_is_suppressing_structured_logs --- src/libmongoc/tests/test-libmongoc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libmongoc/tests/test-libmongoc.c b/src/libmongoc/tests/test-libmongoc.c index 58e181f61a9..abd5e9b2f02 100644 --- a/src/libmongoc/tests/test-libmongoc.c +++ b/src/libmongoc/tests/test-libmongoc.c @@ -73,7 +73,7 @@ test_is_suppressing_structured_logs (void) return c > 0; } -// Ignore logs generated by test-internal operations. Can be nested. Structured and traditional logs. +// Ignore logs generated by test-internal operations. Can be nested. Structured logs only; see capturing_logs too. void test_begin_suppressing_structured_logs (void) { From a75f15133c1ad1a85a0dd8f4d54ab60423c8fdce Mon Sep 17 00:00:00 2001 From: Micah Scott Date: Tue, 17 Dec 2024 09:58:53 -0800 Subject: [PATCH 134/139] make ignored error in mongoc_structured_log_opts_new very clear --- src/libmongoc/src/mongoc/mongoc-structured-log.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/libmongoc/src/mongoc/mongoc-structured-log.c b/src/libmongoc/src/mongoc/mongoc-structured-log.c index a5899a4ab02..1516ab3d561 100644 --- a/src/libmongoc/src/mongoc/mongoc-structured-log.c +++ b/src/libmongoc/src/mongoc/mongoc-structured-log.c @@ -414,11 +414,14 @@ mongoc_structured_log_opts_new (void) { mongoc_structured_log_opts_t *opts = (mongoc_structured_log_opts_t *) bson_malloc0 (sizeof *opts); - // Capture default state from the environment now + /* Capture default state from the environment now. + * Note that error return values from mongoc_structured_log_opts_set_* must be ignored here. + * If environment variables can't be parsed, warnings will be logged once but we must, by specification, + * continue to provide structured logging using whatever valid or default settings remain. */ opts->default_handler_path = bson_strdup (getenv ("MONGODB_LOG_PATH")); opts->max_document_length = _mongoc_structured_log_get_max_document_length_from_env (); - mongoc_structured_log_opts_set_max_level_for_all_components (opts, MONGOC_STRUCTURED_LOG_DEFAULT_LEVEL); - mongoc_structured_log_opts_set_max_levels_from_env (opts); + (void) mongoc_structured_log_opts_set_max_level_for_all_components (opts, MONGOC_STRUCTURED_LOG_DEFAULT_LEVEL); + (void) mongoc_structured_log_opts_set_max_levels_from_env (opts); // Set default handler. Its shared state is allocated later, as part of instance_t. mongoc_structured_log_opts_set_handler (opts, _mongoc_structured_log_default_handler, NULL); From 5f13444d0be7cd8680b82438d3af43f70007da7e Mon Sep 17 00:00:00 2001 From: Micah Scott Date: Tue, 17 Dec 2024 10:33:53 -0800 Subject: [PATCH 135/139] More case insensitive: docs, unlimited length --- src/libmongoc/doc/mongoc_structured_log_opts_new.rst | 4 ++-- src/libmongoc/src/mongoc/mongoc-structured-log.c | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/libmongoc/doc/mongoc_structured_log_opts_new.rst b/src/libmongoc/doc/mongoc_structured_log_opts_new.rst index b4f8d0f790f..0859407041f 100644 --- a/src/libmongoc/doc/mongoc_structured_log_opts_new.rst +++ b/src/libmongoc/doc/mongoc_structured_log_opts_new.rst @@ -31,11 +31,11 @@ Environment Variables This is a full list of the captured environment variables. * ``MONGODB_LOG_MAX_DOCUMENT_LENGTH``: Maximum length for JSON-serialized documents that appear within a log message. - It may be a number, in bytes, or ``unlimited``. + It may be a number, in bytes, or ``unlimited`` (case insensitive). By default, the limit is 1000 bytes. This limit affects interior documents like commands and replies, not the total length of a structured log message. -* ``MONGODB_LOG_PATH``: A file path or one of the special strings ``stderr`` or ``stdout``, specifying the destination for structured logs seen by the default handler. +* ``MONGODB_LOG_PATH``: A file path or one of the special strings ``stderr`` or ``stdout`` (case insensitive) specifying the destination for structured logs seen by the default handler. By default, it writes to ``stderr``. This path will be captured during ``mongoc_structured_log_opts_new()``, but it will not immediately be opened. If the file can't be opened, a warning is then written to the unstructured log and the handler writes structured logs to ``stderr`` instead. diff --git a/src/libmongoc/src/mongoc/mongoc-structured-log.c b/src/libmongoc/src/mongoc/mongoc-structured-log.c index 1516ab3d561..aad46d3c349 100644 --- a/src/libmongoc/src/mongoc/mongoc-structured-log.c +++ b/src/libmongoc/src/mongoc/mongoc-structured-log.c @@ -278,7 +278,7 @@ _mongoc_structured_log_get_max_document_length_from_env (void) return MONGOC_STRUCTURED_LOG_DEFAULT_MAX_DOCUMENT_LENGTH; } - if (!strcmp (max_length_str, "unlimited")) { + if (!strcasecmp (max_length_str, "unlimited")) { return BSON_MAX_LEN_UNLIMITED; } From 932b73b027753603979893c3fa37ec69b82dc3b4 Mon Sep 17 00:00:00 2001 From: Micah Scott Date: Wed, 8 Jan 2025 11:55:32 -0800 Subject: [PATCH 136/139] missing bson_t hyperlink in doc --- .../doc/mongoc_structured_log_entry_message_as_bson.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libmongoc/doc/mongoc_structured_log_entry_message_as_bson.rst b/src/libmongoc/doc/mongoc_structured_log_entry_message_as_bson.rst index 9a94e9dd8f1..f83eab380c5 100644 --- a/src/libmongoc/doc/mongoc_structured_log_entry_message_as_bson.rst +++ b/src/libmongoc/doc/mongoc_structured_log_entry_message_as_bson.rst @@ -11,7 +11,7 @@ Synopsis bson_t * mongoc_structured_log_entry_message_as_bson (const mongoc_structured_log_entry_t *entry); -Make a new copy, as a bson_t, of the log entry's standardized BSON representation. +Make a new copy, as a :symbol:`bson_t`, of the log entry's standardized BSON representation. When possible, a log handler should avoid serializing log messages that will be discarded. Each call allocates an independent copy of the message that must be freed. From 4d910a7ec4804ed5f935da1ea1c8f608f8a15b8a Mon Sep 17 00:00:00 2001 From: Micah Scott Date: Wed, 8 Jan 2025 12:25:43 -0800 Subject: [PATCH 137/139] Warning about multiple instances writing to a shared log file --- src/libmongoc/doc/mongoc_structured_log_opts_new.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/libmongoc/doc/mongoc_structured_log_opts_new.rst b/src/libmongoc/doc/mongoc_structured_log_opts_new.rst index 0859407041f..78295e4fb94 100644 --- a/src/libmongoc/doc/mongoc_structured_log_opts_new.rst +++ b/src/libmongoc/doc/mongoc_structured_log_opts_new.rst @@ -40,6 +40,10 @@ This is a full list of the captured environment variables. This path will be captured during ``mongoc_structured_log_opts_new()``, but it will not immediately be opened. If the file can't be opened, a warning is then written to the unstructured log and the handler writes structured logs to ``stderr`` instead. + .. warning:: When a file path is given for ``MONGODB_LOG_PATH``, each log instance (one stand-alone client or pool) will separately open this file for append. + The results are operating system specific. On UNIX-like platforms each instance's output will be interleaved, in most cases without splitting individual log messages. Notably on Windows the file will be opened in exclusive mode by the first instance and subsequent instances will fail, falling back on the default of ``stderr``. + Applications that use multiple processes or multiple client pools will likely want to supply a log handler that annotates each message with information about its originating log instance. + * ``MONGODB_LOG_COMMAND``: A log level name to set as the maximum for ``MONGOC_STRUCTURED_LOG_COMPONENT_COMMAND``. * ``MONGODB_LOG_TOPOLOGY``: A log level name to set as the maximum for ``MONGOC_STRUCTURED_LOG_COMPONENT_TOPOLOGY``. * ``MONGODB_LOG_SERVER_SELECTION``: A log level name to set as the maximum for ``MONGOC_STRUCTURED_LOG_COMPONENT_SERVER_SELECTION``. From 2d853e7b7962cb185c70827bef0364f38be5f65c Mon Sep 17 00:00:00 2001 From: Micah Scott Date: Wed, 8 Jan 2025 17:19:43 -0800 Subject: [PATCH 138/139] typo in comment --- src/libmongoc/tests/test-mongoc-command-monitoring.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libmongoc/tests/test-mongoc-command-monitoring.c b/src/libmongoc/tests/test-mongoc-command-monitoring.c index 8f42a568cd6..e0a491a0179 100644 --- a/src/libmongoc/tests/test-mongoc-command-monitoring.c +++ b/src/libmongoc/tests/test-mongoc-command-monitoring.c @@ -97,7 +97,7 @@ test_command_monitoring_cb (void *scenario) static void test_all_spec_tests (TestSuite *suite) { - // Newer versions of the 'unified' tests have migrated to connection-logging-and-monitoring + // Newer versions of the 'unified' tests have migrated to command-logging-and-monitoring run_unified_tests (suite, JSON_DIR, "command_monitoring/unified"); install_json_test_suite (suite, JSON_DIR, "command_monitoring/legacy", &test_command_monitoring_cb); } From f40496c7a92b7f5ed13b4df4878c14eac8fe19b3 Mon Sep 17 00:00:00 2001 From: Micah Scott Date: Wed, 8 Jan 2025 17:32:18 -0800 Subject: [PATCH 139/139] Typo in test runner error message --- src/libmongoc/tests/unified/runner.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libmongoc/tests/unified/runner.c b/src/libmongoc/tests/unified/runner.c index 69818f22746..f0c5df59b47 100644 --- a/src/libmongoc/tests/unified/runner.c +++ b/src/libmongoc/tests/unified/runner.c @@ -1037,7 +1037,7 @@ test_check_event (test_t *test, bson_t *expected, event_t *actual, bson_error_t goto done; } if (!BSON_ITER_HOLDS_DOCUMENT (&iter)) { - test_set_error (error, "Unexpected type event.command, should be document"); + test_set_error (error, "Unexpected type for event.command, should be document"); goto done; } bson_val_t *expected_val = bson_val_from_bson (expected_command);