CDRIVER-5644 Add option to configure DEK cache lifetime (#1779)

Implement new libmongocrypt option to configure cache expiration time. Sync new spec tests.

src/libmongoc/doc/mongoc_auto_encryption_opts_set_key_expiration.rst

@@ -0,0 +1,27 @@
+:man_page: mongoc_auto_encryption_opts_set_key_expiration
+
+mongoc_auto_encryption_opts_set_key_expiration()
+========================================================
+
+Synopsis
+--------
+
+.. code-block:: c
+
+   void
+   mongoc_auto_encryption_opts_set_key_expiration (
+      mongoc_auto_encryption_opts_t *opts, uint64_t cache_expiration_ms);
+
+
+Parameters
+----------
+
+* ``opts``: The :symbol:`mongoc_auto_encryption_opts_t`
+* ``cache_expiration_ms``: The data encryption key cache expiration time in milliseconds.
+
+.. seealso::
+
+  | :symbol:`mongoc_client_enable_auto_encryption()`
+
+  | `In-Use Encryption <in-use-encryption_>`_
+

src/libmongoc/doc/mongoc_auto_encryption_opts_t.rst

@@ -38,4 +38,5 @@ Synopsis
     mongoc_auto_encryption_opts_set_tls_opts
     mongoc_auto_encryption_opts_set_encrypted_fields_map
     mongoc_auto_encryption_opts_set_bypass_query_analysis
+    mongoc_auto_encryption_opts_set_key_expiration
 

src/libmongoc/doc/mongoc_client_encryption_opts_set_key_expiration.rst

@@ -0,0 +1,27 @@
+:man_page: mongoc_client_encryption_opts_set_key_expiration
+
+mongoc_client_encryption_opts_set_key_expiration()
+========================================================
+
+Synopsis
+--------
+
+.. code-block:: c
+
+   void
+   mongoc_client_encryption_opts_set_key_expiration (
+      mongoc_client_encryption_opts_t *opts, uint64_t cache_expiration_ms);
+
+
+Parameters
+----------
+
+* ``opts``: The :symbol:`mongoc_client_encryption_opts_t`
+* ``cache_expiration_ms``: The data encryption key cache expiration time in milliseconds.
+
+.. seealso::
+
+  | :symbol:`mongoc_client_encryption_new()`
+
+  | `In-Use Encryption <in-use-encryption_>`_
+

src/libmongoc/doc/mongoc_client_encryption_opts_t.rst

@@ -28,6 +28,7 @@ Used to set options for :symbol:`mongoc_client_encryption_new()`.
     mongoc_client_encryption_opts_set_kms_credential_provider_callback
     mongoc_client_encryption_opts_set_kms_providers
     mongoc_client_encryption_opts_set_tls_opts
+    mongoc_client_encryption_opts_set_key_expiration
 
 .. seealso::
 

src/libmongoc/src/mongoc/mongoc-client-side-encryption.c

@@ -52,6 +52,7 @@ struct _mongoc_auto_encryption_opts_t {
    bool bypass_query_analysis;
    mc_kms_credentials_callback creds_cb;
    bson_t *extra;
+   mcd_optional_u64_t cache_expiration_ms;
 };
 
 static void
@@ -134,6 +135,17 @@ mongoc_auto_encryption_opts_set_kms_providers (mongoc_auto_encryption_opts_t *op
    }
 }
 
+void
+mongoc_auto_encryption_opts_set_key_expiration (mongoc_auto_encryption_opts_t *opts, uint64_t expiration)
+{
+   if (!opts) {
+      return;
+   }
+
+   opts->cache_expiration_ms.set = true;
+   opts->cache_expiration_ms.value = expiration;
+}
+
 /* _bson_copy_or_null returns a copy of @bson or NULL if @bson is NULL */
 static bson_t *
 _bson_copy_or_null (const bson_t *bson)
@@ -233,6 +245,7 @@ struct _mongoc_client_encryption_opts_t {
    bson_t *kms_providers;
    bson_t *tls_opts;
    mc_kms_credentials_callback creds_cb;
+   mcd_optional_u64_t cache_expiration_ms;
 };
 
 mongoc_client_encryption_opts_t *
@@ -314,6 +327,14 @@ mongoc_client_encryption_opts_set_kms_credential_provider_callback (mongoc_clien
    opts->creds_cb.userdata = userdata;
 }
 
+void
+mongoc_client_encryption_opts_set_key_expiration (mongoc_client_encryption_opts_t *opts, uint64_t cache_expiration_ms)
+{
+   BSON_ASSERT_PARAM (opts);
+   opts->cache_expiration_ms.set = true;
+   opts->cache_expiration_ms.value = cache_expiration_ms;
+}
+
 /*--------------------------------------------------------------------------
  * Data key options.
  *--------------------------------------------------------------------------
@@ -1794,6 +1815,7 @@ _mongoc_cse_client_enable_auto_encryption (mongoc_client_t *client,
                          opts->bypass_auto_encryption,
                          opts->bypass_query_analysis,
                          opts->creds_cb,
+                         opts->cache_expiration_ms,
                          error);
    if (!client->topology->crypt) {
       GOTO (fail);
@@ -1933,6 +1955,7 @@ _mongoc_cse_client_pool_enable_auto_encryption (mongoc_topology_t *topology,
                                         opts->bypass_auto_encryption,
                                         opts->bypass_query_analysis,
                                         opts->creds_cb,
+                                        opts->cache_expiration_ms,
                                         error);
    if (!topology->crypt) {
       GOTO (fail);
@@ -2031,6 +2054,7 @@ mongoc_client_encryption_new (mongoc_client_encryption_opts_t *opts, bson_error_
                                                  false,
                                                  /* bypass_query_analysis. Not applicable. */
                                                  opts->creds_cb,
+                                                 opts->cache_expiration_ms,
                                                  error);
    if (!client_encryption->crypt) {
       goto fail;

src/libmongoc/src/mongoc/mongoc-client-side-encryption.h

@@ -71,6 +71,9 @@ mongoc_auto_encryption_opts_set_keyvault_namespace (mongoc_auto_encryption_opts_
 MONGOC_EXPORT (void)
 mongoc_auto_encryption_opts_set_kms_providers (mongoc_auto_encryption_opts_t *opts, const bson_t *kms_providers);
 
+MONGOC_EXPORT (void)
+mongoc_auto_encryption_opts_set_key_expiration (mongoc_auto_encryption_opts_t *opts, uint64_t expiration);
+
 MONGOC_EXPORT (void)
 mongoc_auto_encryption_opts_set_tls_opts (mongoc_auto_encryption_opts_t *opts, const bson_t *tls_opts);
 
@@ -130,6 +133,9 @@ mongoc_client_encryption_opts_set_kms_credential_provider_callback (mongoc_clien
                                                                     mongoc_kms_credentials_provider_callback_fn fn,
                                                                     void *userdata);
 
+MONGOC_EXPORT (void)
+mongoc_client_encryption_opts_set_key_expiration (mongoc_client_encryption_opts_t *opts, uint64_t cache_expiration_ms);
+
 MONGOC_EXPORT (mongoc_client_encryption_rewrap_many_datakey_result_t *)
 mongoc_client_encryption_rewrap_many_datakey_result_new (void) BSON_GNUC_WARN_UNUSED_RESULT;
 

src/libmongoc/src/mongoc/mongoc-crypt-private.h

@@ -20,6 +20,7 @@
 #define MONGOC_CRYPT_PRIVATE_H
 
 #include "mongoc-config.h"
+#include "mongoc-util-private.h"
 
 #include "mongoc.h"
 
@@ -48,6 +49,7 @@ _mongoc_crypt_new (const bson_t *kms_providers,
                    bool bypass_auto_encryption,
                    bool bypass_query_analysis,
                    mc_kms_credentials_callback creds_cb,
+                   mcd_optional_u64_t cache_expiration_ms,
                    bson_error_t *error);
 
 void

src/libmongoc/src/mongoc/mongoc-crypt.c

@@ -1380,6 +1380,7 @@ _mongoc_crypt_new (const bson_t *kms_providers,
                    bool bypass_auto_encryption,
                    bool bypass_query_analysis,
                    mc_kms_credentials_callback creds_cb,
+                   mcd_optional_u64_t cache_expiration_ms,
                    bson_error_t *error)
 {
    _mongoc_crypt_t *crypt;
@@ -1457,6 +1458,13 @@ _mongoc_crypt_new (const bson_t *kms_providers,
       goto fail;
    }
 
+   if (cache_expiration_ms.set) {
+      mongocrypt_setopt_key_expiration (crypt->handle, cache_expiration_ms.value);
+      if (!_crypt_check_error (crypt->handle, error, false)) {
+         goto fail;
+      }
+   }
+
    if (!mongocrypt_init (crypt->handle)) {
       _crypt_check_error (crypt->handle, error, true);
       goto fail;

src/libmongoc/src/mongoc/mongoc-util-private.h

@@ -245,6 +245,11 @@ hex_to_bin (const char *hex, uint32_t *len);
 char *
 bin_to_hex (const uint8_t *bin, uint32_t len);
 
+typedef struct {
+   bool set;
+   uint64_t value;
+} mcd_optional_u64_t;
+
 BSON_END_DECLS
 
 #endif /* MONGOC_UTIL_PRIVATE_H */

src/libmongoc/tests/json-test-monitoring.c

@@ -590,13 +590,16 @@ check_json_apm_events (json_test_ctx_t *ctx, const bson_t *expectations)
 
    /* If we do not allow matching against a subset of actual events, check if
     * there are extra "actual" events */
-   if (!allow_subset && bson_iter_next (&actual_iter)) {
+   if (!allow_subset) {
       bson_t extra;
-
-      bson_iter_bson (&actual_iter, &extra);
-      _apm_match_error_context (&ctx->events, expectations);
-      test_error ("extra actual event was not found in expectations: %s\n",
-                  bson_as_canonical_extended_json (&extra, NULL));
+      while (bson_iter_next (&actual_iter)) {
+         bson_iter_bson (&actual_iter, &extra);
+         if (!skip_cse_list_collections (&extra)) {
+            _apm_match_error_context (&ctx->events, expectations);
+            test_error ("extra actual event was not found in expectations: %s\n",
+                        bson_as_canonical_extended_json (&extra, NULL));
+         }
+      }
    }
 
    for (i = 0; i < 2; i++) {

src/libmongoc/tests/json-test.c

@@ -1599,6 +1599,13 @@ set_auto_encryption_opts (mongoc_client_t *client, bson_t *test)
       bson_free (env_cryptSharedLibPath);
    }
 
+   if (bson_iter_init_find (&iter, &opts, "keyExpirationMS")) {
+      BSON_ASSERT (BSON_ITER_HOLDS_INT (&iter));
+      const int expiration = bson_iter_as_int64 (&iter);
+      BSON_ASSERT (expiration > 0);
+      mongoc_auto_encryption_opts_set_key_expiration (auto_encryption_opts, (uint64_t) expiration);
+   }
+
    mongoc_auto_encryption_opts_set_extra (auto_encryption_opts, &extra);
    bson_destroy (&extra);