DRIVERS-3082 add prose tests for $lookup

source/client-side-encryption/etc/data/lookup/key-doc.json

@@ -0,0 +1,30 @@
+{
+ "_id": {
+ "$binary": {
+ "base64": "EjRWeBI0mHYSNBI0VniQEg==",
+ "subType": "04"
+ }
+ },
+ "keyMaterial": {
+ "$binary": {
+ "base64": "sHe0kz57YW7v8g9VP9sf/+K1ex4JqKc5rf/URX3n3p8XdZ6+15uXPaSayC6adWbNxkFskuMCOifDoTT+rkqMtFkDclOy884RuGGtUysq3X7zkAWYTKi8QAfKkajvVbZl2y23UqgVasdQu3OVBQCrH/xY00nNAs/52e958nVjBuzQkSb1T8pKJAyjZsHJ60+FtnfafDZSTAIBJYn7UWBCwQ==",
+ "subType": "00"
+ }
+ },
+ "creationDate": {
+ "$date": {
+ "$numberLong": "1648914851981"
+ }
+ },
+ "updateDate": {
+ "$date": {
+ "$numberLong": "1648914851981"
+ }
+ },
+ "status": {
+ "$numberInt": "0"
+ },
+ "masterKey": {
+ "provider": "local"
+ }
+}

source/client-side-encryption/etc/data/lookup/schema-csfle.json

@@ -0,0 +1,19 @@
+{
+ "properties": {
+ "csfle": {
+ "encrypt": {
+ "keyId": [
+ {
+ "$binary": {
+ "base64": "EjRWeBI0mHYSNBI0VniQEg==",
+ "subType": "04"
+ }
+ }
+ ],
+ "bsonType": "string",
+ "algorithm": "AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic"
+ }
+ }
+ },
+ "bsonType": "object"
+}

source/client-side-encryption/etc/data/lookup/schema-csfle2.json

@@ -0,0 +1,19 @@
+{
+ "properties": {
+ "csfle2": {
+ "encrypt": {
+ "keyId": [
+ {
+ "$binary": {
+ "base64": "EjRWeBI0mHYSNBI0VniQEg==",
+ "subType": "04"
+ }
+ }
+ ],
+ "bsonType": "string",
+ "algorithm": "AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic"
+ }
+ }
+ },
+ "bsonType": "object"
+}

source/client-side-encryption/etc/data/lookup/schema-qe.json

@@ -0,0 +1,20 @@
+{
+ "escCollection": "enxcol_.qe.esc",
+ "ecocCollection": "enxcol_.qe.ecoc",
+ "fields": [
+ {
+ "keyId": {
+ "$binary": {
+ "base64": "EjRWeBI0mHYSNBI0VniQEg==",
+ "subType": "04"
+ }
+ },
+ "path": "qe",
+ "bsonType": "string",
+ "queries": {
+ "queryType": "equality",
+ "contention": 0
+ }
+ }
+ ]
+}

source/client-side-encryption/etc/data/lookup/schema-qe2.json

@@ -0,0 +1,20 @@
+{
+ "escCollection": "enxcol_.qe2.esc",
+ "ecocCollection": "enxcol_.qe2.ecoc",
+ "fields": [
+ {
+ "keyId": {
+ "$binary": {
+ "base64": "EjRWeBI0mHYSNBI0VniQEg==",
+ "subType": "04"
+ }
+ },
+ "path": "qe2",
+ "bsonType": "string",
+ "queries": {
+ "queryType": "equality",
+ "contention": 0
+ }
+ }
+ ]
+}

source/client-side-encryption/tests/README.md

@@ -3413,3 +3413,273 @@ Repeat this test with the `azure` and `gcp` masterKeys.
 2. Call `client_encryption.createDataKey()` with "aws" as the provider. Expect this to fail.
 
 Repeat this test with the `azure` and `gcp` masterKeys.
+
+### 25. Test $lookup
+
+All tests require libmongocrypt 1.13.0, server 7.0+, and must be skipped on standalone. Tests define more constraints.
+
+The syntax `<filename.json>` is used to refer to the content of the corresponding file in `../etc/data/lookup`.
+
+#### Setup
+
+Create an encrypted MongoClient named `encryptedClient` configured with:
+
+```python
+AutoEncryptionOpts(
+ keyVaultNamespace="db.keyvault",
+ kmsProviders={"local": { "key": "<base64 decoding of LOCAL_MASTERKEY>" }}
+)
+```
+
+Use `encryptedClient` to drop `db.keyvault`. Insert `<key-doc.json>` into `db.keyvault` with majority write concern.
+
+Use `encryptedClient` to drop and create the following collections:
+
+- `db.csfle` with options: `{ "validator": { "$jsonSchema": "<schema-csfle.json>"}}`.
+- `db.csfle2` with options: `{ "validator": { "$jsonSchema": "<schema-csfle2.json>"}}`.
+- `db.qe` with options: `{ "encryptedFields": "<schema-qe.json>"}`.
+- `db.qe2` with options: `{ "encryptedFields": "<schema-qe2.json>"}`.
+- `db.no_schema` with no options.
+- `db.no_schema2` with no options.
+
+Create an unencrypted MongoClient named `unencryptedClient`.
+
+Insert documents with `encryptedClient`:
+
+- `{"csfle": "csfle"}` into `db.csfle`
+ - Use `unencryptedClient` to retrieve it. Assert the `csfle` field is BSON binary.
+- `{"csfle2": "csfle2"}` into `db.csfle2`
+ - Use `unencryptedClient` to retrieve it. Assert the `csfle2` field is BSON binary.
+- `{"qe": "qe"}` into `db.qe`
+ - Use `unencryptedClient` to retrieve it. Assert the `qe` field is BSON binary.
+- `{"qe2": "qe2"}` into `db.qe2`
+ - Use `unencryptedClient` to retrieve it. Assert the `qe2` field is BSON binary.
+- `{"no_schema": "no_schema"}` into `db.no_schema`
+- `{"no_schema2": "no_schema2"}` into `db.no_schema2`
+
+#### Case 1: `db.csfle` joins `db.no_schema`
+
+Test requires server 8.1+ and mongocryptd/crypt_shared 8.1+.
+
+Recreate `encryptedClient` with the same `AutoEncryptionOpts` as the setup. (Recreating prevents schema caching from
+impacting the test).
+
+Run an aggregate operation on `db.csfle` with the following pipeline:
+
+```json
+[
+ {"$match" : {"csfle" : "csfle"}},
+ {
+ "$lookup" : {
+ "from" : "no_schema",
+ "as" : "matched",
+ "pipeline" : [ {"$match" : {"no_schema" : "no_schema"}}, {"$project" : {"_id" : 0}} ]
+ }
+ },
+ {"$project" : {"_id" : 0}}
+]
+```
+
+Expect one document to be returned matching: `{"csfle" : "csfle", "matched" : [ {"no_schema" : "no_schema"} ]}`.
+
+#### Case 2: `db.qe` joins `db.no_schema`
+
+Test requires server 8.1+ and mongocryptd/crypt_shared 8.1+.
+
+Recreate `encryptedClient` with the same `AutoEncryptionOpts` as the setup. (Recreating prevents schema caching from
+impacting the test).
+
+Run an aggregate operation on `db.qe` with the following pipeline:
+
+```json
+[
+ {"$match" : {"qe" : "qe"}},
+ {
+ "$lookup" : {
+ "from" : "no_schema",
+ "as" : "matched",
+ "pipeline" :
+ [ {"$match" : {"no_schema" : "no_schema"}}, {"$project" : {"_id" : 0, "__safeContent__" : 0}} ]
+ }
+ },
+ {"$project" : {"_id" : 0, "__safeContent__" : 0}}
+]
+```
+
+Expect one document to be returned matching: `{"qe" : "qe", "matched" : [ {"no_schema" : "no_schema"} ]}`.
+
+#### Case 3: `db.no_schema` joins `db.csfle`
+
+Test requires server 8.1+ and mongocryptd/crypt_shared 8.1+.
+
+Recreate `encryptedClient` with the same `AutoEncryptionOpts` as the setup. (Recreating prevents schema caching from
+impacting the test).
+
+Run an aggregate operation on `db.no_schema` with the following pipeline:
+
+```json
+[
+ {"$match" : {"no_schema" : "no_schema"}},
+ {
+ "$lookup" : {
+ "from" : "csfle",
+ "as" : "matched",
+ "pipeline" : [ {"$match" : {"csfle" : "csfle"}}, {"$project" : {"_id" : 0}} ]
+ }
+ },
+ {"$project" : {"_id" : 0}}
+]
+```
+
+Expect one document to be returned matching: `{"no_schema" : "no_schema", "matched" : [ {"csfle" : "csfle"} ]}`.
+
+#### Case 4: `db.no_schema` joins `db.qe`
+
+Test requires server 8.1+ and mongocryptd/crypt_shared 8.1+.
+
+Recreate `encryptedClient` with the same `AutoEncryptionOpts` as the setup. (Recreating prevents schema caching from
+impacting the test).
+
+Run an aggregate operation on `db.no_schema` with the following pipeline:
+
+```json
+[
+ {"$match" : {"no_schema" : "no_schema"}},
+ {
+ "$lookup" : {
+ "from" : "qe",
+ "as" : "matched",
+ "pipeline" : [ {"$match" : {"qe" : "qe"}}, {"$project" : {"_id" : 0, "__safeContent__" : 0}} ]
+ }
+ },
+ {"$project" : {"_id" : 0}}
+]
+```
+
+Expect one document to be returned matching: `{"no_schema" : "no_schema", "matched" : [ {"qe" : "qe"} ]}`.
+
+#### Case 5: `db.csfle` joins `db.csfle2`
+
+Test requires server 8.1+ and mongocryptd/crypt_shared 8.1+.
+
+Recreate `encryptedClient` with the same `AutoEncryptionOpts` as the setup. (Recreating prevents schema caching from
+impacting the test).
+
+Run an aggregate operation on `db.csfle` with the following pipeline:
+
+```json
+[
+ {"$match" : {"csfle" : "csfle"}},
+ {
+ "$lookup" : {
+ "from" : "csfle2",
+ "as" : "matched",
+ "pipeline" : [ {"$match" : {"csfle2" : "csfle2"}}, {"$project" : {"_id" : 0}} ]
+ }
+ },
+ {"$project" : {"_id" : 0}}
+]
+```
+
+Expect one document to be returned matching: `{"csfle" : "csfle", "matched" : [ {"csfle2" : "csfle2"} ]}`.
+
+#### Case 6: `db.qe` joins `db.qe2`
+
+Test requires server 8.1+ and mongocryptd/crypt_shared 8.1+.
+
+Recreate `encryptedClient` with the same `AutoEncryptionOpts` as the setup. (Recreating prevents schema caching from
+impacting the test).
+
+Run an aggregate operation on `db.qe` with the following pipeline:
+
+```json
+[
+ {"$match" : {"qe" : "qe"}},
+ {
+ "$lookup" : {
+ "from" : "qe2",
+ "as" : "matched",
+ "pipeline" : [ {"$match" : {"qe2" : "qe2"}}, {"$project" : {"_id" : 0, "__safeContent__" : 0}} ]
+ }
+ },
+ {"$project" : {"_id" : 0, "__safeContent__" : 0}}
+]
+```
+
+Expect one document to be returned matching: `{"qe" : "qe", "matched" : [ {"qe2" : "qe2"} ]}`.
+
+#### Case 7: `db.no_schema` joins `db.no_schema2`
+
+Test requires server 8.1+ and mongocryptd/crypt_shared 8.1+.
+
+Recreate `encryptedClient` with the same `AutoEncryptionOpts` as the setup. (Recreating prevents schema caching from
+impacting the test).
+
+Run an aggregate operation on `db.no_schema` with the following pipeline:
+
+```json
+[
+ {"$match" : {"no_schema" : "no_schema"}},
+ {
+ "$lookup" : {
+ "from" : "no_schema2",
+ "as" : "matched",
+ "pipeline" : [ {"$match" : {"no_schema2" : "no_schema2"}}, {"$project" : {"_id" : 0}} ]
+ }
+ },
+ {"$project" : {"_id" : 0}}
+]
+```
+
+Expect one document to be returned matching:
+`{"no_schema" : "no_schema", "matched" : [ {"no_schema2" : "no_schema2"} ]}`.
+
+#### Case 8: `db.csfle` joins `db.qe`
+
+Test requires server 8.1+ and mongocryptd/crypt_shared 8.1+.
+
+Recreate `encryptedClient` with the same `AutoEncryptionOpts` as the setup. (Recreating prevents schema caching from
+impacting the test).
+
+Run an aggregate operation on `db.csfle` with the following pipeline:
+
+```json
+[
+ {"$match" : {"csfle" : "qe"}},
+ {
+ "$lookup" : {
+ "from" : "qe",
+ "as" : "matched",
+ "pipeline" : [ {"$match" : {"qe" : "qe"}}, {"$project" : {"_id" : 0}} ]
+ }
+ },
+ {"$project" : {"_id" : 0}}
+]
+```
+
+Expect an exception to be thrown with a message containing the substring `not supported`.
+
+#### Case 9: test error with \<8.1
+
+This case requires mongocryptd/crypt_shared \<8.1.
+
+Recreate `encryptedClient` with the same `AutoEncryptionOpts` as the setup. (Recreating prevents schema caching from
+impacting the test).
+
+Run an aggregate operation on `db.csfle` with the following pipeline:
+
+```json
+[
+ {"$match" : {"csfle" : "csfle"}},
+ {
+ "$lookup" : {
+ "from" : "no_schema",
+ "as" : "matched",
+ "pipeline" : [ {"$match" : {"no_schema" : "no_schema"}}, {"$project" : {"_id" : 0}} ]
+ }
+ },
+ {"$project" : {"_id" : 0}}
+]
+```
+
+Expect an exception to be thrown with a message containing the substring `Upgrade`.