diff -Nru rust-sequoia-cert-store-0.4.0/.cargo_vcs_info.json rust-sequoia-cert-store-0.5.0/.cargo_vcs_info.json --- rust-sequoia-cert-store-0.4.0/.cargo_vcs_info.json 1970-01-01 00:00:01.000000000 +0000 +++ rust-sequoia-cert-store-0.5.0/.cargo_vcs_info.json 1970-01-01 00:00:01.000000000 +0000 @@ -1,6 +1,6 @@ { "git": { - "sha1": "18a427284ae53e6b1f0475dbb6093318249614d4" + "sha1": "0b8f90b8d933cb2904c4a1524e8977ef0d4b1fee" }, "path_in_vcs": "" } \ No newline at end of file diff -Nru rust-sequoia-cert-store-0.4.0/.gitignore rust-sequoia-cert-store-0.5.0/.gitignore --- rust-sequoia-cert-store-0.4.0/.gitignore 2006-07-24 01:21:28.000000000 +0000 +++ rust-sequoia-cert-store-0.5.0/.gitignore 2006-07-24 01:21:28.000000000 +0000 @@ -1 +1,2 @@ +/target/ *~ diff -Nru rust-sequoia-cert-store-0.4.0/.gitlab-ci.yml rust-sequoia-cert-store-0.5.0/.gitlab-ci.yml --- rust-sequoia-cert-store-0.4.0/.gitlab-ci.yml 2006-07-24 01:21:28.000000000 +0000 +++ rust-sequoia-cert-store-0.5.0/.gitlab-ci.yml 2006-07-24 01:21:28.000000000 +0000 @@ -120,12 +120,6 @@ extends: .rust-stable script: - cargo deny check - rules: - - if: '$CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH' - when: manual - allow_failure: true - - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH' - allow_failure: false cache: [] all_commits: diff -Nru rust-sequoia-cert-store-0.4.0/Cargo.toml rust-sequoia-cert-store-0.5.0/Cargo.toml --- rust-sequoia-cert-store-0.4.0/Cargo.toml 1970-01-01 00:00:01.000000000 +0000 +++ rust-sequoia-cert-store-0.5.0/Cargo.toml 1970-01-01 00:00:01.000000000 +0000 @@ -13,7 +13,7 @@ edition = "2021" rust-version = "1.70" name = "sequoia-cert-store" -version = "0.4.0" +version = "0.5.0" authors = [ "Neal H. Walfield ", "Justus Winter ", @@ -56,13 +56,13 @@ version = "1" [dependencies.openpgp-cert-d] -version = "0.3" +version = "0.3.1" [dependencies.rayon] version = "1" [dependencies.rusqlite] -version = ">=0.29, <0.31" +version = ">=0.29, <0.32" features = [ "collation", "blob", @@ -73,7 +73,7 @@ default-features = false [dependencies.sequoia-openpgp] -version = "1.17.0" +version = "1.19.0" default-features = false [dependencies.smallvec] @@ -97,6 +97,10 @@ ] default-features = false +[target."cfg(windows)".dependencies.rusqlite] +version = ">=0.29, <0.32" +features = ["bundled"] + [target."cfg(windows)".dev-dependencies.sequoia-openpgp] version = "1" features = [ diff -Nru rust-sequoia-cert-store-0.4.0/Cargo.toml.orig rust-sequoia-cert-store-0.5.0/Cargo.toml.orig --- rust-sequoia-cert-store-0.4.0/Cargo.toml.orig 2006-07-24 01:21:28.000000000 +0000 +++ rust-sequoia-cert-store-0.5.0/Cargo.toml.orig 2006-07-24 01:21:28.000000000 +0000 @@ -1,7 +1,7 @@ [package] name = "sequoia-cert-store" description = "A certificate database interface." -version = "0.4.0" +version = "0.5.0" authors = ["Neal H. Walfield ", "Justus Winter "] homepage = "https://sequoia-pgp.org/" @@ -19,25 +19,25 @@ [dependencies] anyhow = "1.0.18" -#chrono = "0.4" -#clap = { version = "4.0", features = [ "derive", "wrap_help" ] } crossbeam = "0.8.1" dirs = "5" -#enumber = "0.3" -#lazy_static = "1.4.0" num_cpus = "1" -openpgp-cert-d = "0.3" +openpgp-cert-d = "0.3.1" rayon = "1" -sequoia-openpgp = { version = "1.17.0", default-features = false } +sequoia-openpgp = { version = "1.19.0", default-features = false } sequoia-net = { version = "0.28", default-features = false } smallvec = "1.1" thiserror = "1.0.2" tokio = { version = "1.13", features = [ "rt" ] } [dependencies.rusqlite] -version = ">=0.29, <0.31" +version = ">=0.29, <0.32" features = ["collation", "blob"] +# Use the bundled sqlite on Windows. +[target.'cfg(windows)'.dependencies] +rusqlite = { version = ">=0.29, <0.32", features = ["bundled"] } + [dev-dependencies] tempfile = "3" diff -Nru rust-sequoia-cert-store-0.4.0/debian/changelog rust-sequoia-cert-store-0.5.0/debian/changelog --- rust-sequoia-cert-store-0.4.0/debian/changelog 2024-01-25 10:08:30.000000000 +0000 +++ rust-sequoia-cert-store-0.5.0/debian/changelog 2024-04-01 05:16:36.000000000 +0000 @@ -1,9 +1,22 @@ +rust-sequoia-cert-store (0.5.0-1build1) noble; urgency=medium + + * No-change rebuild for CVE-2024-3094 + + -- William Grant Mon, 01 Apr 2024 16:16:36 +1100 + +rust-sequoia-cert-store (0.5.0-1) unstable; urgency=medium + + * Team upload. + * Package sequoia-cert-store 0.5.0 from crates.io using debcargo 2.6.1 + + -- Alexander Kjäll Sun, 03 Mar 2024 17:37:44 +0100 + rust-sequoia-cert-store (0.4.0-1) unstable; urgency=medium * Team upload. * Package sequoia-cert-store 0.4.0 from crates.io using debcargo 2.6.1 - -- Alexander Kjäll Thu, 25 Jan 2024 10:08:30 +0000 + -- Alexander Kjäll Sat, 27 Jan 2024 16:51:15 +0100 rust-sequoia-cert-store (0.3.2-2) unstable; urgency=medium diff -Nru rust-sequoia-cert-store-0.4.0/debian/control rust-sequoia-cert-store-0.5.0/debian/control --- rust-sequoia-cert-store-0.4.0/debian/control 2024-01-25 10:08:30.000000000 +0000 +++ rust-sequoia-cert-store-0.5.0/debian/control 2024-04-01 05:16:36.000000000 +0000 @@ -10,18 +10,19 @@ librust-crossbeam-0.8+default-dev (>= 0.8.1-~~) , librust-dirs-5+default-dev , librust-num-cpus-1+default-dev , - librust-openpgp-cert-d-0.3+default-dev , + librust-openpgp-cert-d-0.3+default-dev (>= 0.3.1-~~) , librust-rayon-1+default-dev , - librust-rusqlite+blob-dev (<< 0.31-~~) , - librust-rusqlite+collation-dev (<< 0.31-~~) , - librust-rusqlite+default-dev (<< 0.31-~~) , + librust-rusqlite-0.29+blob-dev , + librust-rusqlite-0.29+collation-dev , + librust-rusqlite-0.29+default-dev , librust-sequoia-net-0.28-dev , - librust-sequoia-openpgp-1-dev (>= 1.17.0-~~) , + librust-sequoia-openpgp-1-dev (>= 1.19.0-~~) , librust-smallvec-1+default-dev (>= 1.1-~~) , librust-thiserror-1+default-dev (>= 1.0.2-~~) , librust-tokio-1+default-dev (>= 1.13-~~) , librust-tokio-1+rt-dev (>= 1.13-~~) -Maintainer: Debian Rust Maintainers +Maintainer: Ubuntu Developers +XSBC-Original-Maintainer: Debian Rust Maintainers Uploaders: Daniel Kahn Gillmor , Holger Levsen @@ -41,13 +42,13 @@ librust-crossbeam-0.8+default-dev (>= 0.8.1-~~), librust-dirs-5+default-dev, librust-num-cpus-1+default-dev, - librust-openpgp-cert-d-0.3+default-dev, + librust-openpgp-cert-d-0.3+default-dev (>= 0.3.1-~~), librust-rayon-1+default-dev, - librust-rusqlite+blob-dev (<< 0.31-~~), - librust-rusqlite+collation-dev (<< 0.31-~~), - librust-rusqlite+default-dev (<< 0.31-~~), + librust-rusqlite-0.29+blob-dev, + librust-rusqlite-0.29+collation-dev, + librust-rusqlite-0.29+default-dev, librust-sequoia-net-0.28-dev, - librust-sequoia-openpgp-1-dev (>= 1.17.0-~~), + librust-sequoia-openpgp-1-dev (>= 1.19.0-~~), librust-smallvec-1+default-dev (>= 1.1-~~), librust-thiserror-1+default-dev (>= 1.0.2-~~), librust-tokio-1+default-dev (>= 1.13-~~), @@ -56,9 +57,9 @@ librust-sequoia-cert-store+default-dev (= ${binary:Version}), librust-sequoia-cert-store-0-dev (= ${binary:Version}), librust-sequoia-cert-store-0+default-dev (= ${binary:Version}), - librust-sequoia-cert-store-0.4-dev (= ${binary:Version}), - librust-sequoia-cert-store-0.4+default-dev (= ${binary:Version}), - librust-sequoia-cert-store-0.4.0-dev (= ${binary:Version}), - librust-sequoia-cert-store-0.4.0+default-dev (= ${binary:Version}) + librust-sequoia-cert-store-0.5-dev (= ${binary:Version}), + librust-sequoia-cert-store-0.5+default-dev (= ${binary:Version}), + librust-sequoia-cert-store-0.5.0-dev (= ${binary:Version}), + librust-sequoia-cert-store-0.5.0+default-dev (= ${binary:Version}) Description: Certificate database interface - Rust source code Source code for Debianized Rust crate "sequoia-cert-store" diff -Nru rust-sequoia-cert-store-0.4.0/debian/patches/drop-windows.patch rust-sequoia-cert-store-0.5.0/debian/patches/drop-windows.patch --- rust-sequoia-cert-store-0.4.0/debian/patches/drop-windows.patch 2024-01-25 10:08:30.000000000 +0000 +++ rust-sequoia-cert-store-0.5.0/debian/patches/drop-windows.patch 2024-03-03 16:37:44.000000000 +0000 @@ -1,11 +1,15 @@ diff --git a/Cargo.toml b/Cargo.toml -index 8cae0e9..a1eccc4 100644 +index b02dc19..9a37877 100644 --- a/Cargo.toml +++ b/Cargo.toml -@@ -101,13 +101,5 @@ features = [ +@@ -97,17 +97,5 @@ features = [ ] default-features = false +-[target."cfg(windows)".dependencies.rusqlite] +-version = ">=0.29, <0.32" +-features = ["bundled"] +- -[target."cfg(windows)".dev-dependencies.sequoia-openpgp] -version = "1" -features = [ diff -Nru rust-sequoia-cert-store-0.4.0/debian/patches/relax-deps.patch rust-sequoia-cert-store-0.5.0/debian/patches/relax-deps.patch --- rust-sequoia-cert-store-0.4.0/debian/patches/relax-deps.patch 2024-01-25 10:08:30.000000000 +0000 +++ rust-sequoia-cert-store-0.5.0/debian/patches/relax-deps.patch 2024-03-03 16:37:44.000000000 +0000 @@ -1,13 +1,13 @@ diff --git a/Cargo.toml b/Cargo.toml -index 8cd6294..d3c7f7d 100644 +index 9a37877..1821f35 100644 --- a/Cargo.toml +++ b/Cargo.toml -@@ -62,7 +62,7 @@ version = "0.3" +@@ -62,7 +62,7 @@ version = "0.3.1" version = "1" [dependencies.rusqlite] --version = ">=0.29, <0.31" -+version = "<0.31" +-version = ">=0.29, <0.32" ++version = "0.29" features = [ "collation", "blob", diff -Nru rust-sequoia-cert-store-0.4.0/debian/tests/control rust-sequoia-cert-store-0.5.0/debian/tests/control --- rust-sequoia-cert-store-0.4.0/debian/tests/control 2024-01-25 10:08:30.000000000 +0000 +++ rust-sequoia-cert-store-0.5.0/debian/tests/control 2024-03-03 16:37:44.000000000 +0000 @@ -1,14 +1,14 @@ -Test-Command: /usr/share/cargo/bin/cargo-auto-test sequoia-cert-store 0.4.0 --all-targets --all-features +Test-Command: /usr/share/cargo/bin/cargo-auto-test sequoia-cert-store 0.5.0 --all-targets --all-features Features: test-name=rust-sequoia-cert-store:@ Depends: dh-cargo (>= 18), librust-sequoia-openpgp-1+--implicit-crypto-backend-for-tests-dev, librust-sequoia-openpgp-1+crypto-nettle-dev, librust-tempfile-3+default-dev, @ Restrictions: allow-stderr, skip-not-installable -Test-Command: /usr/share/cargo/bin/cargo-auto-test sequoia-cert-store 0.4.0 --all-targets +Test-Command: /usr/share/cargo/bin/cargo-auto-test sequoia-cert-store 0.5.0 --all-targets Features: test-name=librust-sequoia-cert-store-dev:default Depends: dh-cargo (>= 18), librust-sequoia-openpgp-1+--implicit-crypto-backend-for-tests-dev, librust-sequoia-openpgp-1+crypto-nettle-dev, librust-tempfile-3+default-dev, @ Restrictions: allow-stderr, skip-not-installable -Test-Command: /usr/share/cargo/bin/cargo-auto-test sequoia-cert-store 0.4.0 --all-targets --no-default-features +Test-Command: /usr/share/cargo/bin/cargo-auto-test sequoia-cert-store 0.5.0 --all-targets --no-default-features Features: test-name=librust-sequoia-cert-store-dev: Depends: dh-cargo (>= 18), librust-sequoia-openpgp-1+--implicit-crypto-backend-for-tests-dev, librust-sequoia-openpgp-1+crypto-nettle-dev, librust-tempfile-3+default-dev, @ Restrictions: allow-stderr, skip-not-installable diff -Nru rust-sequoia-cert-store-0.4.0/src/cert_store.rs rust-sequoia-cert-store-0.5.0/src/cert_store.rs --- rust-sequoia-cert-store-0.4.0/src/cert_store.rs 2006-07-24 01:21:28.000000000 +0000 +++ rust-sequoia-cert-store-0.5.0/src/cert_store.rs 2006-07-24 01:21:28.000000000 +0000 @@ -51,10 +51,11 @@ certd: std::result::Result, store::Certs<'a>>, // Read-only backends. - backends: Vec<(Box + 'a>, AccessMode)>, + backends: Vec<(Box + Send + Sync + 'a>, AccessMode)>, keyserver: Option>>, } +assert_send_and_sync!(CertStore<'_>); impl<'a> CertStore<'a> { /// Returns a CertStore, which does not have any configured backends. @@ -127,7 +128,7 @@ /// Add the specified backend to the CertStore. /// /// The backend is added to the collection of read-only backends. - pub fn add_backend(&mut self, backend: Box + 'a>, + pub fn add_backend(&mut self, backend: Box + Send + Sync + 'a>, mode: AccessMode) -> &mut Self { @@ -178,7 +179,7 @@ where P: AsRef, I: IntoIterator, { - let mut keyring = Certs::empty(); + let keyring = Certs::empty(); let mut error = None; for filename in filenames { let filename = filename.as_ref(); @@ -558,36 +559,36 @@ Box::new(certs.into_iter()) } - fn prefetch_all(&mut self) { - match self.certd.as_mut() { + fn prefetch_all(&self) { + match self.certd.as_ref() { Ok(certd) => certd.prefetch_all(), Err(in_memory) => in_memory.prefetch_all(), }; - for (backend, _mode) in self.backends.iter_mut() { + for (backend, _mode) in self.backends.iter() { backend.prefetch_all(); } } - fn prefetch_some(&mut self, certs: &[KeyHandle]) { - match self.certd.as_mut() { + fn prefetch_some(&self, certs: &[KeyHandle]) { + match self.certd.as_ref() { Ok(certd) => certd.prefetch_some(certs), Err(in_memory) => in_memory.prefetch_some(certs), }; - for (backend, _mode) in self.backends.iter_mut() { + for (backend, _mode) in self.backends.iter() { backend.prefetch_some(certs); } } } impl<'a> store::StoreUpdate<'a> for CertStore<'a> { - fn update_by(&mut self, cert: Arc>, - merge_strategy: &mut dyn MergeCerts<'a>) + fn update_by(&self, cert: Arc>, + merge_strategy: &dyn MergeCerts<'a>) -> Result>> { tracer!(TRACE, "CertStore::update_by"); - match self.certd.as_mut() { + match self.certd.as_ref() { Ok(certd) => { t!("Forwarding to underlying certd"); certd.update_by(cert, merge_strategy) @@ -654,7 +655,7 @@ // Now purge the successfully flushed certs from the keyserver // cache, so that we will not try to flush them again next // time we're called. - flushed_certs.iter().for_each(|c| ks.delete_from_cache(c)); + ks.delete_from_cache(flushed_certs.iter().map(Arc::clone)); t!("Flushed {} certificates", flushed_certs.len()); result diff -Nru rust-sequoia-cert-store-0.4.0/src/lazy_cert.rs rust-sequoia-cert-store-0.5.0/src/lazy_cert.rs --- rust-sequoia-cert-store-0.4.0/src/lazy_cert.rs 2006-07-24 01:21:28.000000000 +0000 +++ rust-sequoia-cert-store-0.5.0/src/lazy_cert.rs 2006-07-24 01:21:28.000000000 +0000 @@ -31,6 +31,7 @@ raw: OnceLock>, cert: OnceLock>, } +assert_send_and_sync!(LazyCert<'_>); impl<'a> std::fmt::Debug for LazyCert<'a> { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { @@ -182,8 +183,10 @@ t!("Resolving {}", raw.fingerprint()); match Cert::try_from(raw) { Ok(cert) => { - self.cert.set(Cow::Owned(cert)) - .expect("just checked that it was empty"); + // The following will fail, if we lost a race to + // parse the raw certificate. That's okay, + // because parsing is deterministic. + let _ = self.cert.set(Cow::Owned(cert)); } Err(err) => { return Err(err); diff -Nru rust-sequoia-cert-store-0.4.0/src/lib.rs rust-sequoia-cert-store-0.5.0/src/lib.rs --- rust-sequoia-cert-store-0.4.0/src/lib.rs 2006-07-24 01:21:28.000000000 +0000 +++ rust-sequoia-cert-store-0.5.0/src/lib.rs 2006-07-24 01:21:28.000000000 +0000 @@ -12,6 +12,8 @@ //! //! # Examples //! +//! We can store certificates in an in-memory store, and query the store: +//! //! ```rust //! use std::sync::Arc; //! use sequoia_openpgp::cert::{Cert, CertBuilder}; @@ -43,6 +45,48 @@ //! let cert: &Cert = cert.to_cert()?; //! # Ok(()) } //! ``` +//! +//! We can use the same API on a persistent certificate store, kept on the +//! filesystem: +//! +//! ```rust +//! use std::fs; +//! # use tempfile; +//! # +//! # use std::sync::Arc; +//! # +//! # use sequoia_openpgp as openpgp; +//! # use openpgp::Result; +//! # use openpgp::cert::{Cert, CertBuilder}; +//! # +//! # use sequoia_cert_store::{CertStore, LazyCert, Store, StoreUpdate}; +//! +//! # fn main() -> Result<()> { +//! // Make a certificate store on the file system, in a fresh empty directory. +//! let directory = "/tmp/test-sequoia-certificate-directory"; +//! # let directory_object = tempfile::tempdir()?; let directory = directory_object.path(); +//! fs::create_dir_all(directory)?; +//! let mut store = CertStore::open(directory)?; +//! +//! // Make a new certificate. +//! let (cert, _rev) = CertBuilder::new().generate()?; +//! let fpr = cert.fingerprint(); +//! +//! // The certificate of course will not be in the store yet. +//! assert!(store.lookup_by_cert_fpr(&fpr).is_err()); +//! +//! // Add it. +//! store.update(Arc::new(LazyCert::from(cert)))?; +//! +//! // Now the certificate can be found. +//! let found = store.lookup_by_cert_fpr(&fpr).expect("present"); +//! assert_eq!(found.fingerprint(), fpr); +//! +//! // Again, we can resolve the `LazyCert` to a `Cert`. +//! let cert: &Cert = found.to_cert()?; +//! # Ok(()) } +//! ``` + use std::str; use sequoia_openpgp as openpgp; @@ -173,7 +217,7 @@ include!("../tests/keyring.rs"); - fn test_backend<'a, B>(backend: B) + fn test_backend<'a, B>(backend: &B) where B: Store<'a> { // Check Store::list. @@ -189,6 +233,8 @@ assert_eq!(got, expected); } + std::thread::yield_now(); + // Check Store::iter. { let mut got: Vec @@ -203,6 +249,8 @@ assert_eq!(got, expected); } + std::thread::yield_now(); + // Iterate over the certificates in the keyring and check that // can look up the certificate by fingerprint, by key, by User // ID, and by email in various ways. @@ -245,6 +293,8 @@ } }, } + + std::thread::yield_now(); } // Check lookup_by_cert using key ids. @@ -278,6 +328,8 @@ } }, } + + std::thread::yield_now(); } // Check lookup_by_cert_or_subkey using fingerprints. @@ -295,6 +347,8 @@ assert!(got.into_iter().any(|c| c.fingerprint() == fpr), "{}, lookup_by_cert_or_subkey({}), with fingerprint, subkey", handle.base, sk.fingerprint()); + + std::thread::yield_now(); } @@ -310,6 +364,8 @@ .expect("present"); assert!(got.into_iter().any(|c| c.fingerprint() == fpr), "{}, lookup_by_cert_or_subkey, with keyid, subkey", handle.base); + + std::thread::yield_now(); } @@ -527,6 +583,8 @@ } } } + + std::thread::yield_now(); } } @@ -556,23 +614,53 @@ .parse::().expect("valid"), ])); + std::thread::yield_now(); // ed's primary is also a subkey on the same certificate. - assert_eq!( - sort_vec(backend.lookup_by_cert_or_subkey( - &"0C346B2B6241263F64E9C7CF1EA300797258A74E" - .parse::().expect("valid")) - .expect("present") - .into_iter() - .map(|c| c.fingerprint()) - .collect::>()), - sort_vec( - vec![ - keyring::ed.fingerprint - .parse::().expect("valid"), - ])); - - + { + let ed_fpr = "0C346B2B6241263F64E9C7CF1EA300797258A74E" + .parse::().expect("valid"); + let ed_kh = KeyHandle::from(&ed_fpr); + + assert_eq!( + sort_vec(backend.lookup_by_cert_or_subkey(&ed_kh) + .expect("present") + .into_iter() + .map(|c| c.fingerprint()) + .collect::>()), + sort_vec( + vec![ + keyring::ed.fingerprint + .parse::().expect("valid"), + ])); + + // Also test that the CertStore implementation is not + // impacted. + let cert = backend.lookup_by_cert_fpr(&ed_fpr) + .expect("found cert"); + assert_eq!(cert.fingerprint(), ed_fpr); + + // Make sure the implementation doesn't return the certificate + // twice, once when matching on the primary key, and once when + // matching on the subkey. + let certs = backend.lookup_by_cert(&ed_kh) + .expect("found cert"); + assert_eq!(certs.len(), 1); + assert_eq!(certs[0].fingerprint(), ed_fpr); + + let certs = backend.lookup_by_cert_or_subkey(&ed_kh) + .expect("found cert"); + assert_eq!(certs.len(), 1); + assert_eq!(certs[0].fingerprint(), ed_fpr); + + let certs = backend.certs(); + assert_eq!(certs.filter(|c| c.fingerprint() == ed_fpr).count(), + 1); + + let fprs = backend.fingerprints(); + assert_eq!(fprs.filter(|fpr| fpr == &ed_fpr).count(), + 1); + } // david has a subkey that doesn't have a binding signature, // but the backend is not supposed to check that. (That @@ -627,6 +715,8 @@ } } + std::thread::yield_now(); + assert!( backend.lookup_by_cert_or_subkey( &"0123 4567 89AB CDEF 0123 4567 89AB CDEF" @@ -677,6 +767,8 @@ .parse::().expect("valid") ]); + std::thread::yield_now(); + // Check that when looking up a subdomain, we don't get back // User IDs in a subdomain. assert_eq!( @@ -732,9 +824,6 @@ fn certd() -> Result<()> { use std::io::Read; - // We expect 8 certificates. - assert_eq!(keyring::certs.len(), 12); - let path = tempfile::tempdir()?; let certd = cert_d::CertD::with_base_dir(&path) .map_err(|err| { @@ -763,7 +852,7 @@ drop (certd); let certd = store::certd::CertD::open(&path).expect("exists"); - test_backend(certd); + test_backend(&certd); Ok(()) } @@ -773,9 +862,6 @@ fn cert_store() -> Result<()> { use std::io::Read; - // Sanity check how many certificates we read. - assert_eq!(keyring::certs.len(), 12); - let path = tempfile::tempdir()?; let certd = cert_d::CertD::with_base_dir(&path) .map_err(|err| { @@ -804,7 +890,7 @@ drop(certd); let cert_store = CertStore::open(&path).expect("exists"); - test_backend(cert_store); + test_backend(&cert_store); Ok(()) } @@ -813,8 +899,6 @@ fn cert_store_layered() -> Result<()> { use std::io::Read; - assert_eq!(keyring::certs.len(), 12); - // A certd for each certificate. let mut paths: Vec = Vec::new(); @@ -852,7 +936,7 @@ paths.push(path); } - test_backend(cert_store); + test_backend(&cert_store); Ok(()) } @@ -861,8 +945,6 @@ fn certs() -> Result<()> { use std::io::Read; - assert_eq!(keyring::certs.len(), 12); - let mut bytes = Vec::new(); for cert in keyring::certs.iter() { let binary = cert.bytes(); @@ -875,7 +957,7 @@ let backend = store::Certs::from_bytes(&bytes) .expect("valid"); - test_backend(backend); + test_backend(&backend); Ok(()) } @@ -884,8 +966,6 @@ fn certd_with_prefetch() -> Result<()> { use std::io::Read; - assert_eq!(keyring::certs.len(), 12); - let path = tempfile::tempdir()?; let certd = cert_d::CertD::with_base_dir(&path) .map_err(|err| { @@ -913,9 +993,9 @@ } drop (certd); - let mut certd = store::CertD::open(&path).expect("exists"); + let certd = store::CertD::open(&path).expect("exists"); certd.prefetch_all(); - test_backend(certd); + test_backend(&certd); Ok(()) } @@ -924,8 +1004,6 @@ fn certs_with_prefetch() -> Result<()> { use std::io::Read; - assert_eq!(keyring::certs.len(), 12); - let mut bytes = Vec::new(); for cert in keyring::certs.iter() { let binary = cert.bytes(); @@ -936,10 +1014,10 @@ .expect(&format!("{}", cert.base)); } - let mut backend = store::Certs::from_bytes(&bytes) + let backend = store::Certs::from_bytes(&bytes) .expect("valid"); backend.prefetch_all(); - test_backend(backend); + test_backend(&backend); Ok(()) } @@ -956,16 +1034,52 @@ PathBuf::from(&base).join(c.filename) }))?; - test_backend(cert_store); + test_backend(&cert_store); Ok(()) } #[test] - fn pep() -> Result<()> { + fn certs_multithreaded() -> Result<()> { use std::io::Read; - assert_eq!(keyring::certs.len(), 12); + let mut bytes = Vec::new(); + for cert in keyring::certs.iter() { + let binary = cert.bytes(); + let mut reader = openpgp::armor::Reader::from_bytes( + &binary, + openpgp::armor::ReaderMode::VeryTolerant); + reader.read_to_end(&mut bytes) + .expect(&format!("{}", cert.base)); + } + + let backend = store::Certs::from_bytes(&bytes) + .expect("valid"); + + std::thread::scope(|s| { + let threads = (0..4usize) + .map(|_| { + s.spawn(|| { + // Really make sure all threads start by sleeping + // for 10ms. + std::thread::sleep(std::time::Duration::new(0, 10)); + + test_backend(&backend); + }) + }) + .collect::>(); + + threads.into_iter().for_each(|h| { + h.join().expect("joined"); + }); + }); + + Ok(()) + } + + #[test] + fn pep() -> Result<()> { + use std::io::Read; let mut bytes = Vec::new(); for cert in keyring::certs.iter() { @@ -977,9 +1091,9 @@ .expect(&format!("{}", cert.base)); } - let mut backend = Pep::from_bytes(&bytes).expect("valid"); + let backend = Pep::from_bytes(&bytes).expect("valid"); backend.prefetch_all(); - test_backend(backend); + test_backend(&backend); Ok(()) } @@ -987,7 +1101,7 @@ // Make sure that when we update a certificate, we are able to // find any new components and we are still able to find the old // components. - fn test_store_update<'a, B>(mut backend: B) -> Result<()> + fn test_store_update<'a, B>(backend: B) -> Result<()> where B: store::StoreUpdate<'a> { let p = &StandardPolicy::new(); @@ -1126,5 +1240,46 @@ let certs = Pep::empty()?; test_store_update(certs) } + + #[test] + fn certs_multithreaded_update() -> Result<()> { + let mut certs = Vec::new(); + for cert in keyring::certs.iter() { + let cert = Cert::from_bytes(&cert.bytes()).expect("valid"); + certs.push(cert); + } + + let mut fprs: Vec + = certs.iter().map(|cert| cert.fingerprint()).collect(); + fprs.sort(); + fprs.dedup(); + + let backend = store::Certs::empty(); + + std::thread::scope(|s| { + let backend = &backend; + let threads = certs.into_iter() + .map(|cert| { + s.spawn(move || { + // Really make sure all threads start by sleeping + // for 10ms. + std::thread::sleep(std::time::Duration::new(0, 10)); + + backend.update(Arc::new(LazyCert::from(cert))) + .expect("ok"); + }) + }) + .collect::>(); + + threads.into_iter().for_each(|h| { + h.join().expect("joined"); + }); + }); + + assert_eq!(backend.certs().count(), fprs.len()); + + Ok(()) + } + } diff -Nru rust-sequoia-cert-store-0.4.0/src/macros.rs rust-sequoia-cert-store-0.5.0/src/macros.rs --- rust-sequoia-cert-store-0.4.0/src/macros.rs 2006-07-24 01:21:28.000000000 +0000 +++ rust-sequoia-cert-store-0.5.0/src/macros.rs 2006-07-24 01:21:28.000000000 +0000 @@ -50,3 +50,74 @@ time_it!("", $body) }; } + +pub(crate) trait Sendable : Send {} +pub(crate) trait Syncable : Sync {} + +/// A simple shortcut for ensuring a type is send and sync. +/// +/// For most types just call it after defining the type: +/// +/// ```ignore +/// pub struct MyStruct {} +/// assert_send_and_sync!(MyStruct); +/// ``` +/// +/// For types with lifetimes, use the anonymous lifetime: +/// +/// ```ignore +/// pub struct WithLifetime<'a> { _p: std::marker::PhantomData<&'a ()> } +/// assert_send_and_sync!(WithLifetime<'_>); +/// ``` +/// +/// For a type generic over another type `W`, +/// pass the type `W` as a where clause +/// including a trait bound when needed: +/// +/// ```ignore +/// pub struct MyWriter { _p: std::marker::PhantomData } +/// assert_send_and_sync!(MyWriter where W: std::io::Write); +/// ``` +/// +/// This will assert that `MyWriterStruct` is `Send` and `Sync` +/// if `W` is `Send` and `Sync`. +/// +/// You can also combine the two and be generic over multiple types. +/// Just make sure to list all the types - even those without additional +/// trait bounds: +/// +/// ```ignore +/// pub struct MyWriterWithLifetime<'a, C, W: std::io::Write> { +/// _p: std::marker::PhantomData<&'a (C, W)>, +/// } +/// assert_send_and_sync!(MyWriterWithLifetime<'_, C, W> where C, W: std::io::Write); +/// ``` +/// +/// If you need multiple additional trait bounds on a single type +/// you can add them separated by `+` like in normal where clauses. +/// However you have to make sure they are `Identifiers` like `Write`. +/// In macro patterns `Paths` (like `std::io::Write`) may not be followed +/// by `+` characters. +// This is copied from sequoia-openpgp/src/macros.rs. +macro_rules! assert_send_and_sync { + ( $x:ty where $( $g:ident$( : $a:path )? $(,)?)*) => { + impl<$( $g ),*> crate::macros::Sendable for $x + where $( $g: Send + Sync $( + $a )? ),* + {} + impl<$( $g ),*> crate::macros::Syncable for $x + where $( $g: Send + Sync $( + $a )? ),* + {} + }; + ( $x:ty where $( $g:ident$( : $a:ident $( + $b:ident )* )? $(,)?)*) => { + impl<$( $g ),*> crate::macros::Sendable for $x + where $( $g: Send + Sync $( + $a $( + $b )* )? ),* + {} + impl<$( $g ),*> crate::macros::Syncable for $x + where $( $g: Send + Sync $( + $a $( + $b )* )? ),* + {} + }; + ( $x:ty ) => { + impl crate::macros::Sendable for $x {} + impl crate::macros::Syncable for $x {} + }; +} diff -Nru rust-sequoia-cert-store-0.4.0/src/store/certd.rs rust-sequoia-cert-store-0.5.0/src/store/certd.rs --- rust-sequoia-cert-store-0.4.0/src/store/certd.rs 2006-07-24 01:21:28.000000000 +0000 +++ rust-sequoia-cert-store-0.5.0/src/store/certd.rs 2006-07-24 01:21:28.000000000 +0000 @@ -1,7 +1,14 @@ +use std::borrow::Cow; +use std::collections::BTreeMap; +use std::collections::btree_map; +use std::iter; use std::path::Path; use std::path::PathBuf; use std::str; use std::sync::Arc; +use std::sync::Mutex; +use std::time::Duration; +use std::time::SystemTime; use anyhow::Context; @@ -9,10 +16,15 @@ use openpgp::Fingerprint; use openpgp::KeyHandle; use openpgp::Result; -use openpgp::cert::raw::RawCertParser; use openpgp::cert::prelude::*; +use openpgp::cert::raw::RawCertParser; +use openpgp::packet::Packet; use openpgp::packet::UserID; +use openpgp::packet::signature::SignatureBuilder; use openpgp::parse::Parse; +use openpgp::serialize::SerializeInto; +use openpgp::types::KeyFlags; +use openpgp::types::SignatureType; use openpgp_cert_d as cert_d; @@ -25,11 +37,33 @@ use crate::TRACE; +/// The creation time for the trust root and intermediate CAs. +/// +/// We use a creation time in the past (Feb 2002) so that it is still +/// possible to use the CA when the reference time is in the past. +fn ca_creation_time() -> SystemTime { + SystemTime::UNIX_EPOCH + Duration::new(1014235320, 0) +} + +struct CA<'a> { + cert: Arc>, + // The tag of the file stored under the special name. + special_tag: cert_d::Tag, + // The tag of the file stored under the fingerprint. + public_tag: cert_d::Tag, +} +assert_send_and_sync!(CA<'_>); + pub struct CertD<'a> { certd: cert_d::CertD, certs: Certs<'a>, + + /// A cache of the trust root, and shadow CAs keyed by the cert-d + /// special name. + cas: Mutex>>, } +assert_send_and_sync!(CertD<'_>); impl<'a> CertD<'a> { /// Returns the canonicalized path. @@ -71,19 +105,19 @@ let mut certd = Self { certd, certs: Certs::empty(), + cas: Default::default(), }; certd.initialize(true)?; Ok(certd) } - /// Returns a reference to the certd, if there is one. + /// Returns a reference to the low-level `CertD`. pub fn certd(&self) -> &cert_d::CertD { &self.certd } - /// Returns a mutable reference to the certd, if there - /// is one. + /// Returns a mutable reference to the low-level `CertD`. pub fn certd_mut(&mut self) -> &mut cert_d::CertD { &mut self.certd } @@ -188,6 +222,812 @@ Ok(()) } + + /// Returns the certificate directory's local trust root, and + /// whether it was just created. + /// + /// If the local trust root does not yet exist, it is created. + pub fn trust_root(&self) -> Result<(Arc>, bool)> { + self.get_ca(cert_d::TRUST_ROOT, + true, + &UserID::from_static_bytes(b"Local Trust Root"), + None) + } + + /// Converts a shadow CA's name to a certificate directory's + /// special name. + /// + /// For instance, this converts "keys.openpgp.org" to its special + /// name. + fn shadow_ca_special(name: &str) -> Cow { + if name == cert_d::TRUST_ROOT { + Cow::Borrowed(name) + } else { + Cow::Owned(format!("_sequoia_ca_{}.pgp", name)) + } + } + + // '/', '\\', ':', and whitespace are invalid. + fn valid_name(name: &str) -> bool { + name.chars().all(Self::valid_char) + } + + /// Returns whether `c` may be used in names. + fn valid_char(c: char) -> bool { + ! (c.is_whitespace() + || c == '/' + || c == '\\' + || c == ':') + } + + /// Returns a shadow CA's key, and whether it was just created. + /// + /// # Introduction + /// + /// Shadow CAs are used to encode evidence of a binding's veracity + /// in standard web of trust data structures so that the + /// information can be automatically incorportated into web of + /// trust calculations. + /// + /// Consider [`keys.openpgp.org`], which is a verifying keyserver. + /// `keys.openpgp.org` checks whether a user ID should be + /// associated with a certificate using a challenge-response + /// authentication scheme. When someone uploads a certificate, + /// `keys.openpgp.org` iterates over the user IDs, and if a user + /// ID contains an email address, sends an email to it containing + /// a secret link. The owner of the email address can visit the + /// link to confirm to `keys.openpgp.org` that the email address + /// should be associated with the certificate. + /// + /// Although `keys.openpgp.org` verifies User IDs, it does not + /// issue third-party certifications attesting to that fact. But, + /// clients can infer this from `keys.openpgp.org` behavior: when + /// `keys.openpgp.org` returns a certificate, it only returns user + /// IDs that it has verified as belonging to the certificate. + /// + /// An OpenPGP client could record the fact that + /// `keys.openpgp.org` returned a user ID in a database. The main + /// question is then how to combine that information with other + /// information in a coherent manner. Alternatively, we can + /// encode the assertion as a certification using a so-called + /// shadow CA. By encoding this information as a normal OpenPGP + /// artifact, it can be directly integrated into web of trust + /// calculations; another, parallel system to combine evidence is + /// not needed. + /// + /// [keys.openpgp.org]: https://keys.openpgp.org + /// + /// # Usage + /// + /// `name` is the shadow CA's name. This is converted to the + /// special name `_sequoia_ca_ + name + .pgp`. Names must not + /// contain white space (` ` or `\n`), path separators (`\` or + /// `/`), or colons (`:`). + /// + /// If the shadow CA doesn't exist and `create` is true, it is + /// created. In this case, the user ID is set to `userid`, and if + /// `trust_amount` is greater than 0, it is designed a trusted + /// introducer by the last `CA` listed in `intermediaries`, or the + /// local trust root, if the list is empty. The certification's + /// trust amount is set to `trust_amount`. + /// + /// `intermediaries` is a list of tuples consisting of a name, a + /// user ID, and a trust amount. Each tuple corresponds to an + /// intermediary shadow CA. It may be empty; the local trust root + /// is always the trust anchor and shouldn't be listed explicitly. + /// + /// This function works from the trust root towards the shadow CA. + /// If an intermediate shadow CA does not exist and `create` is + /// true, it is created, and certified by its parent CA. + /// + /// On success, the shadow CA is returned, and whether the shadow + /// CA was just created. + /// + /// # Examples + /// + /// The following code looks up the shadow CA for + /// `keys.openpgp.org`. It is not directly certified by the trust + /// root, but by a partially trusted intermediary named + /// `public_directories`, which acts as a type of resistor for all + /// public directories. Note: if you are actually looking up the + /// shadow CA for `keys.openpgp.org`, you should use + /// [`CertD::shadow_ca_keys_openpgp_org`] instead. + /// + /// The local key for `keys.openpgp.org` is then used to certify + /// the user IDs on a certificate, which was presumably returned + /// by `keys.openpgp.org`. + /// + /// ```rust + /// use std::sync::Arc; + /// + /// use anyhow::Context; + /// + /// use sequoia_openpgp as openpgp; + /// use openpgp::cert::CertBuilder; + /// use openpgp::Cert; + /// use openpgp::packet::Packet; + /// use openpgp::packet::UserID; + /// use openpgp::packet::signature::SignatureBuilder; + /// use openpgp::types::SignatureType; + /// + /// use sequoia_cert_store as cert_store; + /// use cert_store::LazyCert; + /// use cert_store::store::certd::CertD; + /// use cert_store::store::StoreUpdate; + /// + /// # fn main() -> openpgp::Result<()> { + /// # let certd_path = tempfile::tempdir()?; + /// let mut certd = CertD::open(&certd_path)?; + /// + /// let (koo, _created) = certd.shadow_ca( + /// "keys.openpgp.org", + /// true, + /// "Downloaded from keys.openpgp.org", + /// 1, + /// &[ + /// ( "public_directories", UserID::from("Public Directories"), 40), + /// ])?; + /// + /// // Pretend the following certificate was fetched from keys.openpgp.org. + /// let alice_userid = ""; + /// let (alice, _rev) = CertBuilder::general_purpose(None, Some(alice_userid)) + /// .generate()?; + /// + /// // Certify the binding using the shadow CA. + /// let mut koo_signer = koo.to_cert()?.primary_key().key().parts_as_secret() + /// .context("CA is not certification capable.")? + /// .clone().into_keypair() + /// .context("CA is not certification capable.")?; + /// + /// let sig = SignatureBuilder::new(SignatureType::GenericCertification) + /// .set_exportable_certification(false)? + /// .sign_userid_binding( + /// &mut koo_signer, + /// alice.primary_key().key(), + /// &UserID::from(alice_userid)) + /// .context("Signing binding")?; + /// + /// let alice = alice.insert_packets([ + /// Packet::from(UserID::from(alice_userid)), + /// Packet::from(sig), + /// ])?; + /// + /// certd.update(Arc::new(LazyCert::from(alice))); + /// # Ok(()) } + /// ``` + pub fn shadow_ca(&self, name: &str, + create: bool, userid: U, trust_amount: u8, + intermediaries: &[(&str, UserID, u8)], ) + -> Result<(Arc>, bool)> + where U: Into, + { + let userid = userid.into(); + + tracer!(TRACE, "CertD::shadow_ca"); + t!("{}, create: {}, userid: {:?}, {}, {} intermediaries", + name, create, String::from_utf8_lossy(userid.value()), trust_amount, + intermediaries.len()); + + let name = CertD::shadow_ca_special(name); + + // If the CA is cached in memory, short-circuit everything. + if let Ok(result) = self.get_ca(&name, false, &userid, None) { + t!("Returning cached certificate {} for {}", + result.0.fingerprint(), name); + return Ok(result); + } + + // Iterate from the trust root towards the shadow CA. + let mut ca: (Arc, bool) = self.get_ca( + cert_d::TRUST_ROOT, + create, + &UserID::from_static_bytes(b"Local Trust Root"), + None)?; + + for (name, userid, trust_amount) in + intermediaries + .into_iter() + .map(|(name, userid, amount)| { + (CertD::shadow_ca_special(name), userid, *amount) + }) + .chain(iter::once((name, &userid, trust_amount))) + { + if ! Self::valid_name(&name) { + return Err(cert_d::Error::BadName.into()); + } + + t!("Getting {} (create: {}, userid: {:?}, {})", + name, create, String::from_utf8_lossy(userid.value()), + trust_amount); + + ca = self.get_ca(&name, create, &userid, + Some((ca.0, trust_amount)))?; + + t!("{} -> {}", name, ca.0.fingerprint()); + } + + Ok(ca) + } + + /// Returns the specified CA, optionally creating and certifying + /// it if it doesn't exist. + /// + /// See [`CertD::shadow_ca`] for details about shadow CAs. + /// + /// This function first checks if the specified CA is in the + /// in-memory cache. If so, it checks that the cached version is + /// up to date, and then returns the result. + /// + /// If the CA is not in the in-memory cache, it is loaded from + /// disk. If it doesn't exist on disk an `create` is `true`, it + /// the shadow CA is created. See [`CertD::load_ca`] for details + /// of what is exactly done. + fn get_ca(&self, special_name: &str, + create: bool, userid: &UserID, + parent: Option<(Arc>, u8)>) + -> Result<(Arc>, bool)> + { + tracer!(TRACE, "CertD::get_ca"); + + let mut cas = self.cas.lock().unwrap(); + + let get_if_changed = |tag, name| -> Result> { + match self.certd.get_if_changed(tag, name) { + Ok(None) => { + // The certificate on disk did not change. + return Ok(None); + }, + Ok(Some((new_tag, bytes))) => { + // The certificate on disk changed. Reparse it. + t!("{} changed on disk, reloading.", name); + match Cert::from_bytes(&bytes) { + Ok(cert) => { + return Ok(Some((cert, new_tag))); + } + Err(err) => { + // An error occurred while parsing the + // data. That's not great. We just keep + // using what we have. + t!("While reparsing {}: {}", name, err); + return Ok(None); + } + } + } + // An error occurred while reading the file. That's + // not great. We just keep using what we have. + Err(err) => { + t!("While rereading {}: {}", name, err); + return Ok(None); + } + } + }; + + if let Some(CA { cert: ca, special_tag, public_tag }) + = cas.get_mut(special_name) + { + // The certificate is cached it memory. Make sure the + // in-memory version is up to date. + t!("{} -> {} is cached in memory", + special_name, ca.fingerprint()); + + if let Some((cert, tag)) + = get_if_changed(*special_tag, special_name)? + { + *ca = Arc::new(LazyCert::from( + ca.to_cert()? + .clone() + .merge_public_and_secret(cert)?)); + *special_tag = tag; + } + + if let Some((cert, tag)) + = get_if_changed(*public_tag, &ca.fingerprint().to_string())? + { + *ca = Arc::new(LazyCert::from( + ca.to_cert()? + .clone() + .merge_public_and_secret(cert)?)); + *public_tag = tag; + } + + Ok((Arc::clone(ca), false)) + } else { + // The certificate is not cached in memory; we need to + // read it from disk, or generate it. + + let (ca, created) + = self.load_ca(special_name, create, userid, parent)?; + + let cert = Arc::clone(&ca.cert); + + t!("Loaded {} ({}) from disk, caching in memory", + special_name, cert.fingerprint()); + + match cas.entry(special_name.to_string()) { + btree_map::Entry::Occupied(mut oe) => { + oe.insert(ca); + } + btree_map::Entry::Vacant(ve) => { + ve.insert(ca); + } + } + + Ok((cert, created)) + } + } + + /// Loads a CA's certificate from the certificate directory, or + /// optionally generates one if it does not exist. + /// + /// Loads the certificate stored under the specified special name + /// from the certificate directory. If the special name exists, + /// this also reads and merges in the certificate data stored + /// under the fingerprint. + /// + /// If `create` is `false` and the certificate does not exist, + /// returns `std::io::ErrorKind::NotFound`. + /// + /// If `create` is `true`, and we need to generate a certificate, + /// we are careful to so while we hold the certificate directory's + /// lock to avoid races. + /// + /// If we create a certificate, and `parent` is not `None`, then + /// we certify the certificate using `parent` as a trusted + /// introducer for the specified trust amount. Note: if the CA is + /// not created, this function does **not** check that the parent + /// has certified the CA. + /// + /// If we create a certificate, we write it to disk under both its + /// special name, and its fingerprint. + fn load_ca(&self, name: &str, create: bool, userid: &UserID, + parent: Option<(Arc>, u8)>) + -> Result<(CA<'a>, bool)> + { + tracer!(TRACE, "CertD::load_ca"); + + let certd = self.certd(); + + // Exclusively lock the certificate directory by inserting the + // trust root. We may discover that the trust already exists. + // In that case, we won't generate a new one, but just load + // the existing one. + let mut ca = Err(anyhow::anyhow!("merge callback not invoked")); + let mut created = false; + let (special_tag, _) = certd.insert_special( + name, + (), false, + |(), disk| { + if let Some(disk) = disk { + // We have one. + ca = Cert::from_bytes(&disk); + Ok(cert_d::MergeResult::Keep) + } else if create { + // We don't have one, and we should create one. + let key = Self::generate_ca_key(userid)?; + t!("Created {} for {}", key.fingerprint(), name); + let bytes = key.as_tsk().to_vec()?; + ca = Ok(key); + created = true; + Ok(cert_d::MergeResult::Data(bytes)) + } else { + // We don't have one, and we shouldn't create one. + Err(cert_d::Error::Other( + std::io::Error::new( + std::io::ErrorKind::NotFound, + format!("{} not found", name)).into())) + } + })?; + let mut ca = ca?; + let ca_fpr = format!("{:x}", ca.fingerprint()); + + let mut certify = |parent_cert: Arc, trust_amount| + -> Result<()> + { + t!("Using {} to make {} a trusted introducer", + parent_cert.fingerprint(), ca.fingerprint()); + + let mut signer = parent_cert + .to_cert()? + .primary_key() + .key() + .parts_as_secret() + .context("Trust root can't be used for signing.")? + .clone() + .into_keypair() + .context("Trust root can't be used for signing.")?; + + let sig = SignatureBuilder::new(SignatureType::GenericCertification) + .set_exportable_certification(false)? + .set_trust_signature(255, trust_amount)? + .set_signature_creation_time(ca_creation_time())? + .sign_userid_binding( + &mut signer, + ca.primary_key().key(), + &userid) + .with_context(|| { + format!("Creating certification for {} {:?}", + ca.fingerprint(), + String::from_utf8_lossy(userid.value())) + })?; + + ca = ca.clone().insert_packets([ + Packet::from(userid.clone()), + Packet::from(sig), + ])?; + + Ok(()) + }; + + let public_tag = if created { + // Make the CA a trusted introducer. + if let Some((parent_cert, trust_amount)) = parent { + if let Err(err) = certify(parent_cert, trust_amount) { + // XXX: What should we do with the error? + t!("Failed to authorize CA, {:?}: {}", + userid, err); + } + } + + // Insert the public bits under the fingerprint. + let (public_tag, _) = certd.insert( + &ca_fpr, &ca, false, + |new, _old| { + Ok(cert_d::MergeResult::Data(new.to_vec()?)) + })?; + + public_tag + } else { + // We didn't create it. Read in the public version and + // merge it. + match certd.get(&ca_fpr) { + Ok(Some((public_tag, bytes))) => { + let cert = Cert::from_bytes(&bytes) + .with_context(|| { + format!("Parsing {}'s public certificate ({})", + name, ca.fingerprint()) + })?; + ca = ca.clone().merge_public(cert) + .with_context(|| { + format!("Merging {}'s public certificate ({})", + name, ca.fingerprint()) + })?; + + public_tag + } + Ok(None) => { + // The certificate is saved under the special + // name, but not its fingerprint. Something went + // wrong in the past, but we can fix it. + let (public_tag, _) = certd.insert( + &ca_fpr, &ca, false, + |new, _old| { + Ok(cert_d::MergeResult::Data(new.to_vec()?)) + })?; + + public_tag + } + Err(err) => { + t!("Reading {}'s public certificate: {}", name, err); + return Err(err.into()); + } + } + }; + + let ca = CA { + cert: Arc::new(LazyCert::from(ca)), + special_tag, + public_tag, + }; + + Ok((ca, created)) + } + + /// Returns a new key, which is appropriate for a CA. + /// + /// That is, it has just a certification capable primary and no + /// subkeys. + fn generate_ca_key(userid: &UserID) -> Result { + let (root, _) = CertBuilder::new() + .set_primary_key_flags(KeyFlags::empty().set_certification()) + .set_exportable(false) + .set_creation_time(ca_creation_time()) + // CAs should *not* expire. + .set_validity_period(None) + .add_userid(userid.clone()) + .generate()?; + + Ok(root) + } + + const PUBLIC_DIRECTORY_NAME: &'static str = "public_directories"; + const PUBLIC_DIRECTORY_USERID: UserID + = UserID::from_static_bytes(b"Public Directories"); + const PUBLIC_DIRECTORY_TRUST_AMOUNT: u8 = 40; + + /// A local, intermediate CA used to authorize the shadow CAs for + /// public directories. + /// + /// Shadow CAs for public directories like + /// `hkps://keys.openpgp.org` and WKD are not directly certified + /// by the local trust root, but by a local, intermediate CA. + /// This CA acts as a resistor, which limits the combined trust + /// amount of public directories. + pub fn public_directory_ca(&self) + -> Result<(Arc>, bool)> + { + self.shadow_ca( + Self::PUBLIC_DIRECTORY_NAME, + true, + Self::PUBLIC_DIRECTORY_USERID, + Self::PUBLIC_DIRECTORY_TRUST_AMOUNT, + &[]) + } + + fn via_public_directory_ca(&self, name: &str, userid: &str) + -> Result<(Arc>, bool)> + { + self.shadow_ca( + name, + true, + userid, + // A public directory has the smallest, positive trust + // amount: 1 out of 120. + 1, + &[ + // The public directory CA acts as a resistor and + // limits the combined trust amount of public + // directories. + (Self::PUBLIC_DIRECTORY_NAME, + Self::PUBLIC_DIRECTORY_USERID, + Self::PUBLIC_DIRECTORY_TRUST_AMOUNT), + ]) + } + + /// Returns the shadow CA for keys.openpgp.org. + /// + /// See [`CertD::shadow_ca`] for more information about shadow + /// CAs. + pub fn shadow_ca_keys_openpgp_org(&self) + -> Result<(Arc>, bool)> + { + self.via_public_directory_ca( + "keys.openpgp.org", + "Downloaded from keys.openpgp.org") + } + + /// Returns the shadow CA for Proton. + /// + /// See [`CertD::shadow_ca`] for more information about shadow + /// CAs. + pub fn shadow_ca_proton_me(&self) + -> Result<(Arc>, bool)> + { + self.via_public_directory_ca( + "proton.me", + "Downloaded from Proton Mail") + } + + /// Returns the shadow CA for keys.mailvelope.com. + /// + /// See [`CertD::shadow_ca`] for more information about shadow + /// CAs. + pub fn shadow_ca_keys_mailvelope_com(&self) + -> Result<(Arc>, bool)> + { + self.via_public_directory_ca( + "keys.mailvelope.com", + "Downloaded from keys.mailvelope.com") + } + + /// Returns the shadow CA for WKDs. + /// + /// See [`CertD::shadow_ca`] for more information about shadow + /// CAs. + pub fn shadow_ca_wkd(&self) -> Result<(Arc>, bool)> { + self.via_public_directory_ca( + "wkd", + "Downloaded from a Web Key Directory (WKD)") + } + + /// Returns the shadow CA for DANE. + /// + /// See [`CertD::shadow_ca`] for more information about shadow + /// CAs. + pub fn shadow_ca_dane(&self) -> Result<(Arc>, bool)> { + self.via_public_directory_ca( + "dane", + "Downloaded from DANE") + } + + const WEB_NAME: &'static str = "web"; + const WEB_USERID_STR: &'static str = "Downloaded from the Web"; + const WEB_USERID: UserID + = UserID::from_static_bytes(b"Downloaded from the Web"); + const WEB_TRUST_AMOUNT: u8 = 1; + + /// Returns the shadow CA for the Web. + /// + /// See [`CertD::shadow_ca`] for more information about shadow + /// CAs. + pub fn shadow_ca_web(&self) -> Result<(Arc>, bool)> { + debug_assert_eq!(Self::WEB_USERID_STR.as_bytes(), + Self::WEB_USERID.value()); + + self.via_public_directory_ca( + Self::WEB_NAME, + Self::WEB_USERID_STR) + } + + /// Returns the shadow CA for the give URL. + /// + /// See [`CertD::shadow_ca`] for more information about shadow + /// CAs. + pub fn shadow_ca_for_url(&self, url: &str) + -> Result>, bool)>> + { + debug_assert_eq!(Self::WEB_USERID_STR.as_bytes(), + Self::WEB_USERID.value()); + + // First, parse and sanitize the URL. + let mut url = sequoia_net::reqwest::Url::parse(url)?; + // We only certify the certificate if the transport was + // encrypted and authenticated. + if url.scheme() != "https" { + return Ok(None); + } + // Drop sensitive and irrelevant information. + url.set_username("").expect("https? supports authentication"); + url.set_password(None).expect("https? supports authentication"); + url.set_fragment(None); + + // Percent-encode characters not valid in names. + let mut name = String::from("shadow_of_url_"); + for c in url.as_str().chars() { + if Self::valid_char(c) && c != '%' { + name.push(c); + } else { + // Note: all invalid chars are ASCII characters, so + // u8::try_from will succeed. + debug_assert_eq!(c.len_utf8(), 1); + name.push_str( + &format!("%{:02X}", u8::try_from(c).unwrap_or(0))); + } + } + + self.shadow_ca( + &name, + true, + format!("Downloaded from {}", url.as_str()), + 1, + &[ + (Self::PUBLIC_DIRECTORY_NAME, + Self::PUBLIC_DIRECTORY_USERID, + Self::PUBLIC_DIRECTORY_TRUST_AMOUNT), + (Self::WEB_NAME, + Self::WEB_USERID, + Self::WEB_TRUST_AMOUNT), + ]) + .map(Some) + } + + /// Returns a shadow CA for the specified keyserver. + /// + /// If a keyserver is not known to be a verifying keyserver, then + /// this returns `Ok(None)`. + /// + /// The URI should be of the form `hkps://server.example.com`. + /// Other protocols are not supported. The server name is matched + /// case insensitively. + /// + /// See [`CertD::shadow_ca`] for more information about shadow + /// CAs. + pub fn shadow_ca_keyserver(&self, uri: &str) + -> Result>, bool)>> + { + let uri = uri.to_ascii_lowercase(); + + // We only certify the certificate if the transport was + // encrypted and authenticated. + let server = if let Some(server) = uri.strip_prefix("hkps://") { + server + } else { + return Ok(None); + }; + + let server = server.strip_suffix("/").unwrap_or(server); + + // A basic sanity check on the name, which we are about to use + // as a filename: it can't start with a dot, and no + // whitespace, no slashes, and no colons are allowed. + if server.chars().next() == Some('.') || ! Self::valid_name(server) { + return Ok(None); + } + + // The known verifying key servers. + match &server[..] { + "keys.openpgp.org" | "keys.openpgp.io" => + self.shadow_ca_keys_openpgp_org().map(Some), + "keys.mailvelope.com" => + self.shadow_ca_keys_mailvelope_com().map(Some), + "mail-api.proton.me" | "api.protonmail.ch" => + self.shadow_ca_proton_me().map(Some), + _ => Ok(None), + } + } + + const AUTOCRYPT_NAME: &'static str = "autocrypt"; + const AUTOCRYPT_USERID: UserID + = UserID::from_static_bytes(b"Imported from Autocrypt"); + const AUTOCRYPT_TRUST_AMOUNT: u8 = 40; + + /// Returns the shadow CA for Autocrypt. + /// + /// See [`CertD::shadow_ca`] for more information about shadow + /// CAs. + pub fn shadow_ca_autocrypt(&self) -> Result<(Arc>, bool)> { + self.shadow_ca( + Self::AUTOCRYPT_NAME, + true, + Self::AUTOCRYPT_USERID, + Self::AUTOCRYPT_TRUST_AMOUNT, + &[]) + } + + const AUTOCRYPT_GOSSIP_NAME: &'static str = "autocrypt-gossip"; + const AUTOCRYPT_GOSSIP_USERID: UserID + = UserID::from_static_bytes(b"Imported from Autocrypt Gossip"); + const AUTOCRYPT_GOSSIP_TRUST_AMOUNT: u8 = 40; + + /// Returns the shadow CA for Autocrypt Gossip. + /// + /// See [`CertD::shadow_ca`] for more information about shadow + /// CAs. + pub fn shadow_ca_autocrypt_gossip(&self) + -> Result<(Arc>, bool)> + { + self.shadow_ca( + Self::AUTOCRYPT_GOSSIP_NAME, + true, + Self::AUTOCRYPT_GOSSIP_USERID, + Self::AUTOCRYPT_GOSSIP_TRUST_AMOUNT, + &[ + (Self::AUTOCRYPT_NAME, + Self::AUTOCRYPT_USERID, + Self::AUTOCRYPT_TRUST_AMOUNT), + ]) + } + + /// Returns the shadow CA for recording Autocrypt Gossip in the + /// name of the given cert and sender address. + /// + /// If you observe an Autocrypt mail from `addr` signed by `cert`, + /// and the message contains Autocrypt Gossip headers in the + /// encrypted payload, use this function to get a shadow CA to + /// turn the Autocrypt Gossip into OpenPGP certifications. + /// + /// See [`CertD::shadow_ca`] for more information about shadow + /// CAs. + pub fn shadow_ca_autocrypt_gossip_for(&self, cert: &Cert, addr: &str) + -> Result<(Arc>, bool)> + { + // Sanity-check the address. + let u = UserID::from_address( + "Autocrypt Gossip from".into(), None, addr)?; + + self.shadow_ca( + &format!("shadow_of_{:x}", cert.fingerprint()), + true, + u, + 1, + &[ + (Self::AUTOCRYPT_NAME, + Self::AUTOCRYPT_USERID, + Self::AUTOCRYPT_TRUST_AMOUNT), + (Self::AUTOCRYPT_GOSSIP_NAME, + Self::AUTOCRYPT_GOSSIP_USERID, + Self::AUTOCRYPT_GOSSIP_TRUST_AMOUNT), + ]) + } } impl<'a> Store<'a> for CertD<'a> { @@ -242,18 +1082,18 @@ self.certs.certs() } - fn prefetch_all(&mut self) { + fn prefetch_all(&self) { self.certs.prefetch_all() } - fn prefetch_some(&mut self, certs: &[KeyHandle]) { + fn prefetch_some(&self, certs: &[KeyHandle]) { self.certs.prefetch_some(certs) } } impl<'a> StoreUpdate<'a> for CertD<'a> { - fn update_by(&mut self, cert: Arc>, - merge_strategy: &mut dyn MergeCerts<'a>) + fn update_by(&self, cert: Arc>, + merge_strategy: &dyn MergeCerts<'a>) -> Result>> { tracer!(TRACE, "CertD::update_by"); @@ -327,6 +1167,7 @@ use anyhow::Context; use openpgp::packet::UserID; + use openpgp::policy::StandardPolicy; use openpgp::serialize::Serialize; use crate::store::StoreError; @@ -463,4 +1304,397 @@ Ok(()) } + + #[test] + fn shadow_ca() -> Result<()> { + tracer!(true, "shadow_ca"); + + let path = tempfile::tempdir()?; + let certd = CertD::open(&path)?; + + // Check that the special exists, and includes secret key + // material, and that there is a corresponding entry under its + // fingerprint, which doesn't include any secret key material. + let check = |name: &str| { + t!("check({})", name); + assert!(cert_d::CertD::is_special(name).is_ok()); + let (_tag, ca) = certd.certd() + .get(name) + .unwrap() + .expect("exists"); + let ca = Cert::from_bytes(&ca).expect("valid"); + assert!(ca.is_tsk()); + + let (_tag, cert) = certd.certd() + .get(&ca.fingerprint().to_string()) + .unwrap() + .expect("exists"); + let cert = Cert::from_bytes(&cert).expect("valid"); + assert!(! cert.is_tsk()); + + assert_eq!(ca.fingerprint(), cert.fingerprint()); + + ca + }; + + let (koo, _tag) = certd.shadow_ca( + "keys.openpgp.org", + true, + UserID::from_static_bytes(b"Downloaded from keys.openpgp.org"), + 1, + &[ + ( "public_directories", + UserID::from_static_bytes(b"Public Directories"), + 10, + ), + ])?; + + let cert = check(&CertD::shadow_ca_special("keys.openpgp.org")); + assert_eq!(cert.fingerprint(), koo.fingerprint()); + let public_directories + = check(&CertD::shadow_ca_special("public_directories")); + let trust_root + = check(&CertD::shadow_ca_special(cert_d::TRUST_ROOT)); + + let (dane, _tag) = certd.shadow_ca( + "dane", + true, + UserID::from_static_bytes(b"Downloaded from DANE"), + 1, + &[ + ( "public_directories", + UserID::from_static_bytes(b"Public Directories"), + 10), + ])?; + + let cert = check(&CertD::shadow_ca_special("dane")); + assert_eq!(cert.fingerprint(), dane.fingerprint()); + let cert = check(&CertD::shadow_ca_special("keys.openpgp.org")); + assert_eq!(cert.fingerprint(), koo.fingerprint()); + let cert = check(&CertD::shadow_ca_special("public_directories")); + assert_eq!(cert, public_directories); + let cert = check(&CertD::shadow_ca_special(cert_d::TRUST_ROOT)); + assert_eq!(cert, trust_root); + + let (tofu, _tag) = certd.shadow_ca( + "tofu", + true, + UserID::from_static_bytes(b"Trust on First Use (TOFU)"), + 120, + &[])?; + + let cert = check(&CertD::shadow_ca_special("tofu")); + assert_eq!(cert.fingerprint(), tofu.fingerprint()); + let cert = check(&CertD::shadow_ca_special("dane")); + assert_eq!(cert.fingerprint(), dane.fingerprint()); + let cert = check(&CertD::shadow_ca_special("keys.openpgp.org")); + assert_eq!(cert.fingerprint(), koo.fingerprint()); + let cert = check(&CertD::shadow_ca_special("public_directories")); + assert_eq!(cert, public_directories); + let cert = check(&CertD::shadow_ca_special(cert_d::TRUST_ROOT)); + assert_eq!(cert, trust_root); + + Ok(()) + } + + #[test] + fn shadow_ca_keyserver() -> Result<()> { + tracer!(true, "keyserver_shadow_ca"); + + let path = tempfile::tempdir()?; + + let test = |certd: &CertD, may_create: bool| { + let (koo, created) = certd.shadow_ca_keys_openpgp_org().unwrap(); + if ! may_create { + assert!(! created); + } + let koo = koo.fingerprint(); + + let (mailvelope, created) + = certd.shadow_ca_keys_mailvelope_com().unwrap(); + if ! may_create { + assert!(! created); + } + let mailvelope = mailvelope.fingerprint(); + + let lookup = |uri, expected: Option<&Fingerprint>| { + let result = certd.shadow_ca_keyserver(uri).unwrap(); + if let Some(fingerprint) = expected { + let (ca, created) = result.expect("valid URI"); + if ! may_create { + assert!(! created); + } + assert_eq!(fingerprint, &ca.fingerprint()); + } else { + assert!(result.is_none()); + } + }; + + lookup("hkps://keys.openpgp.org", Some(&koo)); + lookup("HKPS://KEYS.OPENPGP.ORG", Some(&koo)); + lookup("HKPS://KEYS.OPENPGP.io", Some(&koo)); + // hkp is not considered secure. + lookup("hkp://keys.openpgp.org", None); + // https is not the right protocol. + lookup("https://keys.openpgp.org", None); + // Not a verifying keyserver. + lookup("hkps://keyserver.ubuntu.com", None); + + lookup("hkps://keys.mailvelope.com", Some(&mailvelope)); + }; + + // The first time through we create the trust root and CAs. + t!("Creating CAs"); + let certd = CertD::open(&path)?; + test(&certd, true); + + // The second time through we load them from disk. + t!("Loading CAs"); + let certd2 = CertD::open(&path)?; + test(&certd2, false); + + Ok(()) + } + + fn check_certifiation(certd: &CertD, certifier: &Cert, cert: Fingerprint, + count: usize) + { + const P: &StandardPolicy = &StandardPolicy::new(); + let cert: Arc + = certd.lookup_by_cert_fpr(&cert).expect("exists"); + let cert: &Cert = cert.to_cert().expect("valid cert"); + + let userids: Vec = cert.userids().collect(); + assert_eq!(userids.len(), 1); + + let certifications = userids[0].valid_certifications_by_key( + P, None, certifier.primary_key().key()); + assert_eq!(certifications.count(), count); + } + + #[test] + fn shadow_ca_cerified() -> Result<()> { + + // Check that the shadow CA and intermediate CA are actually + // certified. + tracer!(true, "keyserver_shadow_certified"); + + let path = tempfile::tempdir()?; + + // The first time through we create the trust root and CAs. + t!("Creating CAs"); + let certd = CertD::open(&path)?; + + let (koo, created) = certd.shadow_ca_keys_openpgp_org().unwrap(); + assert!(created); + let koo = koo.to_cert().expect("valid cert"); + + let (pd, created) = certd.public_directory_ca().unwrap(); + // This should have been created in the last step. + assert!(! created); + let pd = pd.to_cert().expect("valid cert"); + + let (tr, created) = certd.trust_root().unwrap(); + // This should have been created in the last step. + assert!(! created); + let tr = tr.to_cert().expect("valid cert"); + + let (kmc, created) = certd.shadow_ca_keys_mailvelope_com().unwrap(); + assert!(created); + let kmc = kmc.to_cert().expect("valid cert"); + + // Avoid the in-memory cache. + let certd = CertD::open(&path)?; + + t!("Checking that the PD CA certified the KOO shadow CA."); + check_certifiation(&certd, &pd, koo.fingerprint(), 1); + t!("Checking that the PD CA certified the KMC shadow CA."); + check_certifiation(&certd, &pd, kmc.fingerprint(), 1); + t!("Checking that the trust root certified the PD CA."); + check_certifiation(&certd, &tr, pd.fingerprint(), 1); + t!("Checking that the trust root didn't the KOO shadow CA."); + check_certifiation(&certd, &tr, koo.fingerprint(), 0); + t!("Checking that the trust root didn't the KMC shadow CA."); + check_certifiation(&certd, &tr, kmc.fingerprint(), 0); + + Ok(()) + } + + #[test] + fn shadow_ca_for_url() -> Result<()> { + tracer!(true, "shadow_ca_for_url"); + + let path = tempfile::tempdir()?; + + let test = |certd: &CertD, may_create: bool| { + let (alice, created) = certd + .shadow_ca_for_url("https://example.org/alice.pgp") + .unwrap().unwrap(); + if ! may_create { + assert!(! created); + } + let alice = alice.fingerprint(); + + #[derive(Debug)] + enum Expectation { Alice, Other, NoCA } + use Expectation::*; + let lookup = |uri, expectation| { + t!("looking up {} expecting {:?}", uri, expectation); + let result = certd.shadow_ca_for_url(uri).unwrap(); + match expectation { + Alice => { + let (ca, created) = result.expect("valid URI"); + if ! may_create { + assert!(! created); + } + assert_eq!(alice, ca.fingerprint()); + }, + Other => { + let (_ca, created) = result.expect("valid URI"); + if ! may_create { + assert!(! created); + } + }, + NoCA => { + assert!(result.is_none()); + }, + } + }; + + lookup("https://example.org/alice.pgp", Alice); + lookup("HTTPS://EXAMPLE.ORG/alice.pgp", Alice); + lookup("https://foo@example.org/alice.pgp", Alice); + lookup("https://foo:bar@example.org/alice.pgp", Alice); + lookup("https://foo:bar@example.org/alice.pgp#fragment", Alice); + // http is not considered secure. + lookup("http://example.org/alice.pgp", NoCA); + // hkps is not the right protocol. + lookup("hkps://example.org/alice.pgp", NoCA); + // Different URLs. + lookup("https://example.org/bob.pgp", Other); + lookup("https://example.net/alice.pgp", Other); + lookup("https://example.org/alice.pgp?some=query", Other); + lookup("https://example.org/alice.pgp/other/path", Other); + }; + + // The first time through we create the trust root and CAs. + t!("Creating CAs"); + let certd = CertD::open(&path)?; + test(&certd, true); + + // The second time through we load them from disk. + t!("Loading CAs"); + let certd2 = CertD::open(&path)?; + test(&certd2, false); + + Ok(()) + } + + #[test] + fn shadow_ca_for_web_cerified() -> Result<()> { + // Check that the shadow CA and intermediate CA are actually + // certified. + tracer!(true, "shadow_ca_for_web_cerified"); + + let path = tempfile::tempdir()?; + + // The first time through we create the trust root and CAs. + t!("Creating CAs"); + let certd = CertD::open(&path)?; + + let (alice, created) = certd + .shadow_ca_for_url("https://example.org/alice.pgp") + .unwrap().unwrap(); + assert!(created); + let alice = alice.to_cert().expect("valid cert"); + + let (tr, created) = certd.trust_root().unwrap(); + // This should have been created in the first step. + assert!(! created); + let tr = tr.to_cert().expect("valid cert"); + + let (pd, created) = certd.public_directory_ca().unwrap(); + // This should have been created in the first step. + assert!(! created); + let pd = pd.to_cert().expect("valid cert"); + + let (web, created) = certd.shadow_ca_web().unwrap(); + // This should have been created in the first step. + assert!(! created); + let web = web.to_cert().expect("valid cert"); + + // Avoid the in-memory cache. + let certd = CertD::open(&path)?; + + t!("Checking that the trust root certified the PD CA."); + check_certifiation(&certd, &tr, pd.fingerprint(), 1); + t!("Checking that the PD CA certified the web CA."); + check_certifiation(&certd, &pd, web.fingerprint(), 1); + t!("Checking that the web CA certified the ALICE shadow CA."); + check_certifiation(&certd, &web, alice.fingerprint(), 1); + t!("Checking that the trust root didn't the web CA."); + check_certifiation(&certd, &tr, web.fingerprint(), 0); + t!("Checking that the trust root didn't the ALICE shadow CA."); + check_certifiation(&certd, &tr, alice.fingerprint(), 0); + t!("Checking that the PD CA didn't the ALICE shadow CA."); + check_certifiation(&certd, &pd, alice.fingerprint(), 0); + + Ok(()) + } + + /// Test that we can create and sanitize, or reject CAs for funny + /// URLs. + #[test] + fn shadow_ca_for_url_escapes() -> Result<()> { + tracer!(true, "shadow_ca_for_url_escapes"); + + let path = tempfile::tempdir()?; + let test = |certd: &CertD, url, stripped| { + t!("trying {:?}", url); + let (alice, _created) = + certd.shadow_ca_for_url(url).unwrap().unwrap(); + if let Some(c) = stripped { + // Make sure the character is stripped. + assert!(alice.userids().all( + |uid| ! str::from_utf8(uid.value()).unwrap().contains(c))); + } else { + // Make sure the odd character is percent-encoded. + assert!(alice.userids().all( + |uid| str::from_utf8(uid.value()).unwrap().contains("%"))); + } + }; + + let test_fail = |certd: &CertD, url| { + t!("trying {:?}", url); + certd.shadow_ca_for_url(url).unwrap_err(); + }; + + let certd = CertD::open(&path)?; + + // In the path: + test(&certd, "https://example.org/alice\u{0a}bob.pgp", Some("\u{0a}")); + test(&certd, "https://example.org/alice\u{0d}bob.pgp", Some("\u{0d}")); + test(&certd, "https://example.org/alice\u{09}bob.pgp", Some("\u{09}")); + test(&certd, "https://example.org/alice\u{0c}bob.pgp", Some("\u{0c}")); + test(&certd, "https://example.org/alice\u{08}\u{08}\u{08}\u{08}\u{08}bob.pgp", None); + test(&certd, "https://example.org/alice\u{200f}bob.pgp", None); // RTL mark + + // In the query: + test(&certd, "https://example.org/alice.pgp?q=\u{0a}bob.pgp", Some("\u{0a}")); + test(&certd, "https://example.org/alice.pgp?q=\u{0d}bob.pgp", Some("\u{0d}")); + test(&certd, "https://example.org/alice.pgp?q=\u{09}bob.pgp", Some("\u{09}")); + test(&certd, "https://example.org/alice.pgp?q=\u{0c}bob.pgp", Some("\u{0c}")); + test(&certd, "https://example.org/alice.pgp?q=\u{08}\u{08}\u{08}\u{08}\u{08}bob.pgp", None); + test(&certd, "https://example.org/alice.pgp?q=\u{200f}bob.pgp", None); // RTL mark + + // In the host: + test(&certd, "https://example.\u{0a}bob.pgp.org/alice.pgp", Some("\u{0a}")); + test(&certd, "https://example.\u{0d}bob.pgp.org/alice.pgp", Some("\u{0d}")); + test(&certd, "https://example.\u{09}bob.pgp.org/alice.pgp", Some("\u{09}")); + test_fail(&certd, "https://example.\u{0c}bob.pgp.org/alice.pgp"); + test_fail(&certd, "https://example.\u{08}\u{08}\u{08}\u{08}\u{08}bob.pgp.org/alice.pgp"); + test_fail(&certd, "https://example.\u{200f}bob.pgp.org/alice.pgp"); // RTL mark + + Ok(()) + } } diff -Nru rust-sequoia-cert-store-0.4.0/src/store/certs.rs rust-sequoia-cert-store-0.5.0/src/store/certs.rs --- rust-sequoia-cert-store-0.4.0/src/store/certs.rs 2006-07-24 01:21:28.000000000 +0000 +++ rust-sequoia-cert-store-0.5.0/src/store/certs.rs 2006-07-24 01:21:28.000000000 +0000 @@ -1,6 +1,7 @@ use std::collections::BTreeMap; use std::collections::btree_map::Entry; use std::sync::Arc; +use std::sync::RwLock; use smallvec::SmallVec; use smallvec::smallvec; @@ -14,6 +15,7 @@ use openpgp::Fingerprint; use openpgp::KeyID; use openpgp::KeyHandle; +use openpgp::packet::UserID; use openpgp::parse::Parse; use openpgp::Result; @@ -32,6 +34,11 @@ /// `Cert`s implements `StoreUpdate`, but it does not write the /// updates to disk; they are only updated in memory. pub struct Certs<'a> { + inner: RwLock>, +} +assert_send_and_sync!(Certs<'_>); + +struct CertsInner<'a> { // Indexed by primary key fingerprint. certs: BTreeMap>>, // Indexed by a key's KeyID (primary key or subkey) and maps to @@ -49,9 +56,11 @@ /// be added to it using the [`StoreUpdate`] interface. pub fn empty() -> Self { Certs { - certs: Default::default(), - keys: Default::default(), - userid_index: UserIDIndex::new(), + inner: RwLock::new(CertsInner { + certs: Default::default(), + keys: Default::default(), + userid_index: UserIDIndex::new(), + }), } } @@ -77,7 +86,7 @@ -> Result where I: Into> { - let mut r = Self::empty(); + let r = Self::empty(); for cert in certs { r.update(Arc::new(cert.into())).expect("implementation doesn't fail") } @@ -86,7 +95,7 @@ } } -impl<'a> Store<'a> for Certs<'a> +impl<'a> Store<'a> for CertsInner<'a> { fn lookup_by_cert(&self, kh: &KeyHandle) -> Result>>> { tracer!(TRACE, "Certs::lookup_by_cert"); @@ -182,16 +191,101 @@ Box::new(self.certs.values().cloned()) } - fn prefetch_all(&mut self) { - self.prefetch(true, &[]) + fn prefetch_all(&self) { + // Because the interior mutability is implemented via Certs, + // not CertInner, and we need a mutable reference here, but we + // only have a normal, non-mutable reference, the prefetch + // implementation is dispatched via `Certs`. + } + + fn prefetch_some(&self, _certs: &[KeyHandle]) { + // Because the interior mutability is implemented via Certs, + // not CertInner, and we need a mutable reference here, but we + // only have a normal, non-mutable reference, the prefetch + // implementation is dispatched via `Certs`. + } +} + +impl<'a> Store<'a> for Certs<'a> +{ + fn lookup_by_cert(&self, kh: &KeyHandle) -> Result>>> { + let inner = self.inner.read().unwrap(); + inner.lookup_by_cert(kh) + } + + fn lookup_by_cert_fpr(&self, fingerprint: &Fingerprint) + -> Result>> + { + let inner = self.inner.read().unwrap(); + inner.lookup_by_cert_fpr(fingerprint) + } + + fn lookup_by_cert_or_subkey(&self, kh: &KeyHandle) -> Result>>> { + let inner = self.inner.read().unwrap(); + inner.lookup_by_cert_or_subkey(kh) + } + + fn select_userid(&self, query: &UserIDQueryParams, pattern: &str) + -> Result>>> + { + let inner = self.inner.read().unwrap(); + inner.select_userid(query, pattern) + } + + fn lookup_by_userid(&self, userid: &UserID) -> Result>>> { + let inner = self.inner.read().unwrap(); + inner.lookup_by_userid(userid) } - fn prefetch_some(&mut self, certs: &[KeyHandle]) { - self.prefetch(false, certs) + fn grep_userid(&self, pattern: &str) -> Result>>> { + let inner = self.inner.read().unwrap(); + inner.grep_userid(pattern) + } + + fn lookup_by_email(&self, email: &str) -> Result>>> { + let inner = self.inner.read().unwrap(); + inner.lookup_by_email(email) + } + + fn grep_email(&self, pattern: &str) -> Result>>> { + let inner = self.inner.read().unwrap(); + inner.grep_email(pattern) + } + + fn lookup_by_email_domain(&self, domain: &str) -> Result>>> { + let inner = self.inner.read().unwrap(); + inner.lookup_by_email_domain(domain) + } + + fn fingerprints<'b>(&'b self) -> Box + 'b> { + // We aren't lazy to avoid holding the lock. + let inner = self.inner.read().unwrap(); + let fprs: Vec<_> = inner.fingerprints().collect(); + Box::new(fprs.into_iter()) + } + + fn certs<'b>(&'b self) + -> Box>> + 'b> + where 'a: 'b + { + // We aren't lazy to avoid holding the lock. + let inner = self.inner.read().unwrap(); + let certs: Vec<_> = inner.certs().collect(); + Box::new(certs.into_iter()) + } + + fn prefetch_all(&self) { + let mut inner = self.inner.write().unwrap(); + inner.prefetch(true, &[]) + } + + fn prefetch_some(&self, certs: &[KeyHandle]) { + let mut inner = self.inner.write().unwrap(); + inner.prefetch(false, certs) } } -impl Certs<'_> { +impl CertsInner<'_> { /// Prefetches the certs identified by `khs`, or `all`. fn prefetch(&mut self, all: bool, khs: &[KeyHandle]) { // LazyCert is Sync and Send, but keeping a reference to the @@ -325,17 +419,19 @@ } impl<'a> StoreUpdate<'a> for Certs<'a> { - fn update_by(&mut self, cert: Arc>, - merge_strategy: &mut dyn MergeCerts<'a>) + fn update_by(&self, cert: Arc>, + merge_strategy: &dyn MergeCerts<'a>) -> Result>> { tracer!(TRACE, "Certs::update_by"); + let mut inner = self.inner.write().unwrap(); + let fpr = cert.fingerprint(); // Add the cert fingerprint -> cert entry. let merged: Arc; - match self.certs.entry(fpr.clone()) { + match inner.certs.entry(fpr.clone()) { Entry::Occupied(mut oe) => { t!("Updating {}", fpr); @@ -359,7 +455,7 @@ // Populate the key map. This is a merge so we are not // removing anything. for k in merged.keys() { - match self.keys.entry(k.keyid()) { + match inner.keys.entry(k.keyid()) { Entry::Occupied(mut oe) => { let fprs = oe.get_mut(); if ! fprs.contains(&fpr) { @@ -372,7 +468,7 @@ } } - self.userid_index.insert(&fpr, merged.userids()); + inner.userid_index.insert(&fpr, merged.userids()); Ok(merged) } diff -Nru rust-sequoia-cert-store-0.4.0/src/store/keyserver.rs rust-sequoia-cert-store-0.5.0/src/store/keyserver.rs --- rust-sequoia-cert-store-0.4.0/src/store/keyserver.rs 2006-07-24 01:21:28.000000000 +0000 +++ rust-sequoia-cert-store-0.5.0/src/store/keyserver.rs 2006-07-24 01:21:28.000000000 +0000 @@ -1,7 +1,7 @@ -use std::cell::RefCell; use std::collections::BTreeMap; use std::collections::BTreeSet; use std::sync::Arc; +use std::sync::Mutex; use sequoia_openpgp as openpgp; use openpgp::Cert; @@ -38,40 +38,47 @@ /// A keyserver backend. pub struct KeyServer<'a> { - keyserver: RefCell, + inner: Mutex>, +} +assert_send_and_sync!(KeyServer<'_>); + +struct KeyServerInner<'a> { + keyserver: net::KeyServer, id: String, - tx: RefCell, - listeners: Vec>, + tx: usize, + listeners: Vec>, // A cache. We only cache certificates; we don't cache User ID // searches. // Primary keys and subkeys. - hits_fpr: RefCell>>>, - hits_keyid: RefCell>, + hits_fpr: BTreeMap>>, + hits_keyid: BTreeMap, // What we failed to look up. - misses_fpr: RefCell>, - misses_keyid: RefCell>, + misses_fpr: BTreeSet, + misses_keyid: BTreeSet, } impl KeyServer<'_> { /// Returns a new key server instance. pub fn new(url: &str) -> Result { Ok(Self { - keyserver: RefCell::new(net::KeyServer::new(url)?), - id: if url.len() <= 10 { - // Only prefix "key server" if the URL is short. - format!("key server {}", url) - } else { - url.to_string() - }, - tx: RefCell::new(0), - listeners: Vec::new(), - hits_fpr: Default::default(), - hits_keyid: Default::default(), - misses_fpr: Default::default(), - misses_keyid: Default::default(), + inner: Mutex::new(KeyServerInner { + keyserver: net::KeyServer::new(url)?, + id: if url.len() <= 10 { + // Only prefix "key server" if the URL is short. + format!("key server {}", url) + } else { + url.to_string() + }, + tx: 0, + listeners: Vec::new(), + hits_fpr: Default::default(), + hits_keyid: Default::default(), + misses_fpr: Default::default(), + misses_keyid: Default::default(), + }), }) } @@ -98,22 +105,22 @@ } /// Sends status updates to the listener. - pub fn add_listener(&mut self, listener: Box) { - self.listeners.push(listener); + pub fn add_listener(&mut self, listener: Box) { + self.inner.lock().unwrap().listeners.push(listener); } } -impl<'a> KeyServer<'a> { +impl<'a> KeyServerInner<'a> { // Looks for a certificate in the cache. fn check_cache(&self, kh: &KeyHandle) -> Option>>>> { let kh_; let kh = if let KeyHandle::KeyID(keyid) = kh { - if let Some(fpr) = self.hits_keyid.borrow().get(keyid) { + if let Some(fpr) = self.hits_keyid.get(keyid) { kh_ = KeyHandle::Fingerprint(fpr.clone()); &kh_ - } else if self.misses_keyid.borrow().get(keyid).is_some() { + } else if self.misses_keyid.get(keyid).is_some() { return Some(Err(StoreError::NotFound( KeyHandle::from(kh.clone())).into())); } else { @@ -123,10 +130,10 @@ kh }; if let KeyHandle::Fingerprint(fpr) = kh { - if let Some(cert) = self.hits_fpr.borrow().get(fpr) { + if let Some(cert) = self.hits_fpr.get(fpr) { return Some(Ok(vec![ cert.clone() ])); } - if self.misses_fpr.borrow().get(fpr).is_some() { + if self.misses_fpr.get(fpr).is_some() { return Some(Err(StoreError::NotFound( KeyHandle::from(kh.clone())).into())); } @@ -136,24 +143,18 @@ } // Adds the cert to the in-memory cache. - fn cache(&self, cert: &Arc>) { - let mut hits_fpr = self.hits_fpr.borrow_mut(); - let mut hits_keyid = self.hits_keyid.borrow_mut(); - + fn cache(&mut self, cert: &Arc>) { for k in cert.keys() { - hits_fpr.insert(k.fingerprint(), cert.clone()); - hits_keyid.insert(k.keyid(), k.fingerprint()); + self.hits_fpr.insert(k.fingerprint(), cert.clone()); + self.hits_keyid.insert(k.keyid(), k.fingerprint()); } } /// Deletes the key with the given fingerprint from the cache. - pub(crate) fn delete_from_cache(&mut self, cert: &LazyCert<'a>) { - let mut hits_fpr = self.hits_fpr.borrow_mut(); - let mut hits_keyid = self.hits_keyid.borrow_mut(); - + fn delete_from_cache(&mut self, cert: &LazyCert<'_>) { for key in cert.keys() { let fp = key.fingerprint(); - hits_fpr.remove(&fp); + self.hits_fpr.remove(&fp); let keyid = KeyID::from(fp); // XXX: Technically, a key ID could map to multiple @@ -161,7 +162,19 @@ // But, by using a BTreeMap we do accept // this kind of loss in the first place, as later cache // insertions will overwrite any existing mappings. - hits_keyid.remove(&keyid); + self.hits_keyid.remove(&keyid); + } + } +} + +impl<'a> KeyServer<'a> { + /// Deletes the key with the given fingerprint from the cache. + pub(crate) fn delete_from_cache(&self, certs: I) + where I: Iterator>>, + { + let mut inner = self.inner.lock().unwrap(); + for cert in certs { + inner.delete_from_cache(&cert); } } } @@ -171,7 +184,7 @@ ( $self:expr, $update:ident, $($args:expr),* ) => {{ if ! $self.listeners.is_empty() { let update = StatusUpdate::$update( - *$self.tx.borrow(), &$self.id, $($args),*); + $self.tx, &$self.id, $($args),*); for sub in $self.listeners.iter() { sub.update(&update); @@ -202,21 +215,21 @@ // Check the cache. t!("Looking up {} on keyserver...", kh); - if ! self.listeners.is_empty() { - let tx = *self.tx.borrow(); - *self.tx.borrow_mut() = tx + 1; - update!(self, LookupStarted, kh, None); + let mut inner = self.inner.lock().unwrap(); + if ! inner.listeners.is_empty() { + inner.tx += 1; + update!(inner, LookupStarted, kh, None); }; - if let Some(r) = self.check_cache(kh) { + if let Some(r) = inner.check_cache(kh) { t!("Found in in-memory cache"); match r.as_ref() { Ok(certs) => { - update!(self, LookupFinished, kh, + update!(inner, LookupFinished, kh, &certs[..], Some("Found in in-memory cache")); } Err(err) => { - update!(self, LookupFailed, kh, Some(&err)); + update!(inner, LookupFailed, kh, Some(&err)); } } return r; @@ -228,7 +241,7 @@ // The keyserver interface currently only returns a single // result. let r = rt.block_on(async { - self.keyserver.borrow_mut().get(kh.clone()).await + inner.keyserver.get(kh.clone()).await }); match r { @@ -242,7 +255,7 @@ t!("keyserver returned {}", cert.fingerprint()); // Add the result to the cache. - self.cache(cert); + inner.cache(cert); } // Make sure the key server gave us the right @@ -254,7 +267,7 @@ .collect(); if ! requested.is_empty() { - update!(self, LookupFinished, kh, &requested[..], None); + update!(inner, LookupFinished, kh, &requested[..], None); Ok(requested) } else { let err = KeyServerError::UnexpectedResults( @@ -262,7 +275,7 @@ certs.iter().map(|c| c.fingerprint()).collect()) .into(); t!("{}", err); - update!(self, LookupFailed, kh, Some(&err)); + update!(inner, LookupFailed, kh, Some(&err)); Err(StoreError::NotFound( KeyHandle::from(kh.clone())).into()) } @@ -273,11 +286,11 @@ if let Some(net::Error::NotFound) = err.downcast_ref::() { - update!(self, LookupFailed, kh, None); + update!(inner, LookupFailed, kh, None); Err(StoreError::NotFound( KeyHandle::from(kh.clone())).into()) } else { - update!(self, LookupFailed, kh, Some(&err)); + update!(inner, LookupFailed, kh, Some(&err)); Err(err) } } @@ -291,10 +304,10 @@ t!("{}", pattern); t!("Looking {:?} up on the keyserver... ", pattern); - if ! self.listeners.is_empty() { - let tx = *self.tx.borrow(); - *self.tx.borrow_mut() = tx + 1; - update!(self, SearchStarted, pattern, None); + let mut inner = self.inner.lock().unwrap(); + if ! inner.listeners.is_empty() { + inner.tx += 1; + update!(inner, SearchStarted, pattern, None); } let email = if query.email && query.anchor_start && query.anchor_end { @@ -318,8 +331,7 @@ let rt = tokio::runtime::Runtime::new().unwrap(); let (ks, wkd) = rt.block_on(async { // Query the keyserver. - let ks = self.keyserver.borrow_mut(); - let ks = ks.search(pattern); + let ks = inner.keyserver.search(pattern); // And the WKD (if it appears to be an email address). let wkd = async { @@ -338,9 +350,9 @@ match ks { Ok(c) => { t!("Key server returned {} results", c.len()); - if ! self.listeners.is_empty() { + if ! inner.listeners.is_empty() { let msg = format!("Key server returned {} results", c.len()); - update!(self, SearchStatus, pattern, &msg); + update!(inner, SearchStatus, pattern, &msg); } certs.extend(c); }, @@ -349,10 +361,10 @@ match wkd { Ok(c) => { t!("WKD server returned {} results", c.len()); - if ! self.listeners.is_empty() { + if ! inner.listeners.is_empty() { let msg = format!("WKD server returned {} results", c.len()); - update!(self, SearchStatus, pattern, &msg); + update!(inner, SearchStatus, pattern, &msg); } certs.extend(c); }, @@ -383,7 +395,7 @@ certs.into_iter().map(|c| Arc::new(LazyCert::from(c))).collect(); // Add the results to the cache. - certs.iter().for_each(|cert| self.cache(cert)); + certs.iter().for_each(|cert| inner.cache(cert)); // Only keep the certificates that actually satisfy the // constraints. @@ -394,10 +406,10 @@ }); if certs.is_empty() { - update!(self, SearchFailed, pattern, None); + update!(inner, SearchFailed, pattern, None); Err(StoreError::NoMatches(pattern.to_string()).into()) } else { - if TRACE || ! self.listeners.is_empty() { + if TRACE || ! inner.listeners.is_empty() { let msg = format!( "Got {} results:\n {}", certs.len(), @@ -424,7 +436,7 @@ .collect::>() .join("\n ")); t!("{}", msg); - update!(self, SearchFinished, pattern, &certs[..], Some(&msg)); + update!(inner, SearchFinished, pattern, &certs[..], Some(&msg)); } Ok(certs) } @@ -433,8 +445,8 @@ fn fingerprints<'b>(&'b self) -> Box + 'b> { // Return the entries in our cache. Box::new(self + .inner.lock().unwrap() .hits_fpr - .borrow() .keys() .cloned() .collect::>() diff -Nru rust-sequoia-cert-store-0.4.0/src/store/pep.rs rust-sequoia-cert-store-0.5.0/src/store/pep.rs --- rust-sequoia-cert-store-0.4.0/src/store/pep.rs 2006-07-24 01:21:28.000000000 +0000 +++ rust-sequoia-cert-store-0.5.0/src/store/pep.rs 2006-07-24 01:21:28.000000000 +0000 @@ -1,6 +1,7 @@ use std::path::Path; use std::path::PathBuf; use std::sync::Arc; +use std::sync::Mutex; use anyhow::Context; @@ -75,8 +76,9 @@ /// /// [pEp]: https://gitea.pep.foundation/pEp.foundation/pEpEngine pub struct Pep { - conn: rusqlite::Connection, + conn: Mutex, } +assert_send_and_sync!(Pep); // Generates a convenience method that returns a prepared statement // for the specified sql. If preparing the statement results in an @@ -204,7 +206,7 @@ -> Result where I: Into> { - let mut r = Self::init_in_memory()?; + let r = Self::init_in_memory()?; for cert in certs { r.update(Arc::new(cert.into())).expect("implementation doesn't fail") } @@ -314,7 +316,7 @@ keys_db.display()))?; Ok(Pep { - conn, + conn: Mutex::new(conn), }) } @@ -445,7 +447,8 @@ { tracer!(TRACE, "Pep::tsk_lookup_by_cert_fpr"); - let mut stmt = Self::tsk_find_stmt(&self.conn)?; + let conn = self.conn.lock().unwrap(); + let mut stmt = Self::tsk_find_stmt(&conn)?; let r = cert_query!(stmt, [ fpr.to_hex() ], StoreError::NotFound(KeyHandle::from(fpr)))?; @@ -464,7 +467,8 @@ { tracer!(TRACE, "Pep::tsk_lookup_by_cert_or_subkey"); - let mut stmt = Self::tsk_find_with_key_stmt(&self.conn)?; + let conn = self.conn.lock().unwrap(); + let mut stmt = Self::tsk_find_with_key_stmt(&conn)?; let keyid = KeyID::from(kh).to_hex(); t!("({})", keyid); @@ -482,7 +486,8 @@ tracer!(TRACE, "Pep::tsks"); let inner = || -> Result> { - let mut stmt = Self::tsk_all_stmt(&self.conn)?; + let conn = self.conn.lock().unwrap(); + let mut stmt = Self::tsk_all_stmt(&conn)?; cert_query!(stmt, [ ], StoreError::NoMatches("EOF".into())) }; @@ -502,8 +507,10 @@ /// /// Returns an error if the specified certificate is not found. pub fn cert_delete(&mut self, fpr: Fingerprint) -> Result<()> { + let conn = self.conn.lock().unwrap(); + let changes = wrap_err!( - Self::cert_delete_stmt(&self.conn)? + Self::cert_delete_stmt(&conn)? .execute(params![ fpr.to_hex() ]), CannotDeleteKey, format!("Deleting {}", fpr))?; @@ -526,12 +533,14 @@ fn lookup_by_cert(&self, kh: &KeyHandle) -> Result>>> { tracer!(TRACE, "Pep::lookup_by_cert"); + let conn = self.conn.lock().unwrap(); + let mut stmt = match kh { KeyHandle::Fingerprint(_) => { - Self::cert_find_stmt(&self.conn)? + Self::cert_find_stmt(&conn)? } KeyHandle::KeyID(_) => { - Self::cert_find_by_keyid_stmt(&self.conn)? + Self::cert_find_by_keyid_stmt(&conn)? } }; @@ -549,7 +558,9 @@ fn lookup_by_cert_or_subkey(&self, kh: &KeyHandle) -> Result>>> { tracer!(TRACE, "Pep::lookup_by_cert_or_subkey"); - let mut stmt = Self::cert_find_with_key_stmt(&self.conn)?; + let conn = self.conn.lock().unwrap(); + + let mut stmt = Self::cert_find_with_key_stmt(&conn)?; let keyid = KeyID::from(kh).to_hex(); t!("({})", keyid); @@ -630,7 +641,9 @@ let userid = crate::email_to_userid(&email)?; let email = userid.email_normalized()?.expect("have one"); - let mut stmt = Self::cert_find_by_email_stmt(&self.conn)?; + let conn = self.conn.lock().unwrap(); + + let mut stmt = Self::cert_find_by_email_stmt(&conn)?; cert_query!(stmt, [ &email ], StoreError::NoMatches(email.into())) } @@ -647,7 +660,8 @@ tracer!(TRACE, "Pep::fingerprints"); let inner = || -> Result> { - let mut stmt = Self::cert_list_stmt(&self.conn)?; + let conn = self.conn.lock().unwrap(); + let mut stmt = Self::cert_list_stmt(&conn)?; let rows = wrap_err!( stmt.query_map([ ], |row: &Row| { @@ -693,7 +707,8 @@ tracer!(TRACE, "Pep::certs"); let inner = || -> Result> { - let mut stmt = Self::cert_all_stmt(&self.conn)?; + let conn = self.conn.lock().unwrap(); + let mut stmt = Self::cert_all_stmt(&conn)?; cert_query!(stmt, [ ], StoreError::NoMatches("EOF".into())) }; @@ -708,8 +723,8 @@ } impl<'a> StoreUpdate<'a> for Pep { - fn update_by(&mut self, cert: Arc>, - merge_strategy: &mut dyn MergeCerts<'a>) + fn update_by(&self, cert: Arc>, + merge_strategy: &dyn MergeCerts<'a>) -> Result>> { tracer!(TRACE, "Pep::update_by"); @@ -717,8 +732,9 @@ let fpr = cert.fingerprint(); t!("Updating {}", fpr); + let mut conn = self.conn.lock().unwrap(); let tx = wrap_err!( - self.conn.transaction(), + conn.transaction(), UnknownDbError, "starting transaction" )?; @@ -1311,7 +1327,7 @@ std::fs::write(&filename, data)?; } - let mut pep = Pep::open(Some(filename)).expect("can open"); + let pep = Pep::open(Some(filename)).expect("can open"); // Certs. assert_eq!(records.len(), pep.certs().count()); diff -Nru rust-sequoia-cert-store-0.4.0/src/store/userid_index.rs rust-sequoia-cert-store-0.5.0/src/store/userid_index.rs --- rust-sequoia-cert-store-0.4.0/src/store/userid_index.rs 2006-07-24 01:21:28.000000000 +0000 +++ rust-sequoia-cert-store-0.5.0/src/store/userid_index.rs 2006-07-24 01:21:28.000000000 +0000 @@ -1,8 +1,9 @@ -use std::cell::RefCell; use std::collections::BTreeMap; use std::collections::BTreeSet; use std::collections::VecDeque; use std::str; +use std::sync::RwLock; +use std::sync::RwLockReadGuard; use sequoia_openpgp as openpgp; use openpgp::Fingerprint; @@ -28,6 +29,11 @@ /// email address in `O(log n)` time. Substring and case insensitive /// matching, however, currently requires `O(n)` time. pub struct UserIDIndex { + inner: RwLock, +} +assert_send_and_sync!(UserIDIndex); + +struct UserIDIndexInner { by_userid: BTreeMap>, // The *normalized* email. @@ -38,19 +44,21 @@ // use a `BTreeMap::range`. That requires a bit of gymnastics. // Alternatively we could use a trie, which is keyed on (domain, // localpart). - by_email: RefCell>>, + by_email: BTreeMap>, // Because extracting the email address is expensive, we wait // until a caller actually needs `by_email`. In particular, when // used in a one-shot context, `by_email` is not access at all. - by_email_pending: RefCell>, + by_email_pending: VecDeque<(UserID, Fingerprint)>, } impl Default for UserIDIndex { fn default() -> Self { UserIDIndex { - by_userid: Default::default(), - by_email: RefCell::new(Default::default()), - by_email_pending: RefCell::new(VecDeque::new()), + inner: RwLock::new(UserIDIndexInner { + by_userid: Default::default(), + by_email: Default::default(), + by_email_pending: VecDeque::new(), + }), } } } @@ -71,12 +79,15 @@ pub fn insert(&mut self, fpr: &Fingerprint, userids: I) where I: Iterator { + let mut inner = self.inner.write().unwrap(); + for userid in userids { - self.by_userid.entry(userid.clone()) + inner.by_userid + .entry(userid.clone()) .or_default() .insert(fpr.clone()); - self.by_email_pending.borrow_mut() + inner.by_email_pending .push_back((userid, fpr.clone())); } } @@ -84,14 +95,30 @@ /// Executes any pending insertions. /// /// This needs to be called before accessing by_email. - fn execute_pending_insertions(&self) { - for (userid, fpr) in self.by_email_pending.borrow_mut().drain(..) { - if let Ok(Some(email)) = userid.email_normalized() { - self.by_email.borrow_mut() - .entry(email) - .or_default() - .insert(fpr.clone()); + fn execute_pending_insertions(&self) -> RwLockReadGuard { + loop { + // First, check if we have anything to do. + let inner = self.inner.read().unwrap(); + if inner.by_email_pending.is_empty() { + return inner; } + drop(inner); + + // Looks like we do. Get the writer lock. + let mut inner = self.inner.write().unwrap(); + let mut pending = std::mem::take(&mut inner.by_email_pending); + for (userid, fpr) in pending.drain(..) { + if let Ok(Some(email)) = userid.email_normalized() { + inner.by_email + .entry(email) + .or_default() + .insert(fpr.clone()); + } + } + + // std::sync::RwLock doesn't implement downgrading a + // writer lock to a reader lock, so we loop. + drop(inner); } } @@ -116,7 +143,8 @@ } => { // Exact User ID match. let userid = UserID::from(pattern); - self.by_userid.get(&userid) + let inner = self.inner.read().unwrap(); + inner.by_userid.get(&userid) .ok_or_else(|| { StoreError::NoMatches(pattern.into()) })? @@ -132,8 +160,8 @@ ignore_case: false, } => { // Exact email match. - self.execute_pending_insertions(); - self.by_email.borrow() + let inner = self.execute_pending_insertions(); + inner.by_email .get(pattern) .ok_or_else(|| { StoreError::NoMatches(pattern.into()) @@ -197,9 +225,8 @@ }; if *email { - self.execute_pending_insertions(); - self.by_email - .borrow() + let inner = self.execute_pending_insertions(); + inner.by_email .iter() .filter_map(|(email, matches)| { if check(email) { @@ -212,7 +239,8 @@ .cloned() .collect::>() } else { - self.by_userid + let inner = self.inner.read().unwrap(); + inner.by_userid .iter() .filter_map(|(userid, matches)| { // If it is not UTF-8 encoded. Ignore it. diff -Nru rust-sequoia-cert-store-0.4.0/src/store.rs rust-sequoia-cert-store-0.5.0/src/store.rs --- rust-sequoia-cert-store-0.4.0/src/store.rs 2006-07-24 01:21:28.000000000 +0000 +++ rust-sequoia-cert-store-0.5.0/src/store.rs 2006-07-24 01:21:28.000000000 +0000 @@ -1,5 +1,7 @@ use std::str; use std::sync::Arc; +use std::sync::atomic::AtomicUsize; +use std::sync::atomic::Ordering; use anyhow::Context; @@ -39,6 +41,7 @@ email: bool, ignore_case: bool, } +assert_send_and_sync!(UserIDQueryParams); impl UserIDQueryParams { /// Returns a new `UserIDQueryParams`. @@ -552,7 +555,15 @@ self.lookup_by_cert(&kh) .and_then(|v| { - assert!(v.len() <= 1); + assert!(v.len() <= 1, + "Looking up {} returned multiple certificates: {}", + fingerprint, + v.iter() + .map(|c| { + c.fingerprint().to_string() + }) + .collect::>() + .join(", ")); v.into_iter().next() .ok_or(StoreError::NotFound(kh).into()) }) @@ -697,7 +708,7 @@ /// /// Errors should be silently ignored and propagated when the /// operation in question is executed directly. - fn prefetch_all(&mut self) { + fn prefetch_all(&self) { } /// Prefetches some certificates. @@ -711,7 +722,7 @@ /// /// Errors should be silently ignored and propagated when the /// operation in question is executed directly. - fn prefetch_some(&mut self, certs: &[KeyHandle]) { + fn prefetch_some(&self, certs: &[KeyHandle]) { let _ = certs; } } @@ -773,11 +784,11 @@ self.as_ref().certs() } - fn prefetch_all(&mut self) { + fn prefetch_all(&self) { self.as_ref().prefetch_all() } - fn prefetch_some(&mut self, certs: &[KeyHandle]) { + fn prefetch_some(&self, certs: &[KeyHandle]) { self.as_ref().prefetch_some(certs) } } @@ -891,11 +902,11 @@ (**self).certs() } - fn prefetch_all(&mut self) { + fn prefetch_all(&self) { (**self).prefetch_all() } - fn prefetch_some(&mut self, certs: &[KeyHandle]) { + fn prefetch_some(&self, certs: &[KeyHandle]) { (**self).prefetch_some(certs) } } @@ -914,7 +925,12 @@ /// [`Cert::merge_public`]. This means that any secret key /// material in `disk` is preserved, any secret key material in /// `new` is ignored, and unhashed subpacket areas are merged. - fn merge_public<'b>(&mut self, + /// + /// Note: `self` is a normal reference, not a mutable reference. + /// This means that the underlying implementation has to use + /// interior mutability. The advantage is that multiple threads + /// can have a reference to the store, and update it in parallel. + fn merge_public<'b>(&self, new: Arc>, disk: Option>>) -> Result>> @@ -943,7 +959,12 @@ /// /// This uses the default implementation of [`MergeCerts`] to /// merge the certificate with any existing certificate. - fn update(&mut self, cert: Arc>) -> Result<()> { + /// + /// Note: `self` is a normal reference, not a mutable reference. + /// This means that the underlying implementation has to use + /// interior mutability. The advantage is that multiple threads + /// can have a reference to the store, and update it in parallel. + fn update(&self, cert: Arc>) -> Result<()> { self.update_by(cert, &mut ())?; Ok(()) @@ -963,35 +984,41 @@ /// /// To use the default merge strategy, either call /// [`StoreUpdate::update`] directly, or pass `&mut ()`. - fn update_by(&mut self, cert: Arc>, - merge_strategy: &mut dyn MergeCerts<'a>) + /// + /// Note: `self` and `merge_strategy` are normal references, not a + /// mutable reference. This means that the underlying + /// implementation has to use interior mutability. The advantage + /// is that multiple threads can have a reference to the store, + /// and update it in parallel. + fn update_by(&self, cert: Arc>, + merge_strategy: &dyn MergeCerts<'a>) -> Result>>; } impl<'a: 't, 't, T> StoreUpdate<'a> for Box where T: StoreUpdate<'a> + ?Sized + 't { - fn update(&mut self, cert: Arc>) -> Result<()> { - self.as_mut().update(cert) + fn update(&self, cert: Arc>) -> Result<()> { + self.as_ref().update(cert) } - fn update_by(&mut self, cert: Arc>, - merge_strategy: &mut dyn MergeCerts<'a>) + fn update_by(&self, cert: Arc>, + merge_strategy: &dyn MergeCerts<'a>) -> Result>> { - self.as_mut().update_by(cert, merge_strategy) + self.as_ref().update_by(cert, merge_strategy) } } -impl<'a: 't, 't, T> StoreUpdate<'a> for &'t mut T +impl<'a: 't, 't, T> StoreUpdate<'a> for &'t T where T: StoreUpdate<'a> + ?Sized { - fn update(&mut self, cert: Arc>) -> Result<()> { + fn update(&self, cert: Arc>) -> Result<()> { (*self).update(cert) } - fn update_by(&mut self, cert: Arc>, - merge_strategy: &mut dyn MergeCerts<'a>) + fn update_by(&self, cert: Arc>, + merge_strategy: &dyn MergeCerts<'a>) -> Result>> { (*self).update_by(cert, merge_strategy) @@ -1002,36 +1029,77 @@ /// /// This is primarily useful as the `merge_strategy` callback to /// [`StoreUpdate::update_by`]. -#[derive(Debug, Clone)] +#[derive(Debug)] #[non_exhaustive] pub struct MergePublicCollectStats { /// Number of new certificates. - pub new: usize, + new_certs: AtomicUsize, /// Number of unchanged certificates. /// /// Note: there may be false negative. That is some certificates /// may be unchanged, but the heuristic thinks that they have been /// updated. - pub unchanged: usize, + unchanged_certs: AtomicUsize, /// Number of update certificates. - pub updated: usize, + updated_certs: AtomicUsize, /// Number of errors. - pub errors: usize, + errors: AtomicUsize, } +assert_send_and_sync!(MergePublicCollectStats); impl MergePublicCollectStats { /// Returns a new `MergePublicCollectStats` with all stats set to 0. pub fn new() -> Self { Self { - new: 0, - unchanged: 0, - updated: 0, - errors: 0, + new_certs: AtomicUsize::new(0), + unchanged_certs: AtomicUsize::new(0), + updated_certs: AtomicUsize::new(0), + errors: AtomicUsize::new(0), } } + + /// Returns the number of new certificates. + pub fn new_certs(&self) -> usize { + self.new_certs.load(Ordering::Relaxed) + } + + /// Returns the number of unchanged certificates. + pub fn unchanged_certs(&self) -> usize { + self.unchanged_certs.load(Ordering::Relaxed) + } + + /// Returns the number of updated certificates. + pub fn updated_certs(&self) -> usize { + self.updated_certs.load(Ordering::Relaxed) + } + + /// Returns the number of errors. + pub fn errors(&self) -> usize { + self.errors.load(Ordering::Relaxed) + } + + /// Increments the number of new certificates by 1. + pub fn inc_new_certs(&self) { + self.new_certs.fetch_add(1, Ordering::Relaxed); + } + + /// Increments the number of unchanged certificates by 1. + pub fn inc_unchanged_certs(&self) { + self.unchanged_certs.fetch_add(1, Ordering::Relaxed); + } + + /// Increments the number of updated certificates by 1. + pub fn inc_updated_certs(&self) { + self.updated_certs.fetch_add(1, Ordering::Relaxed); + } + + /// Increments the number of errors by 1. + pub fn inc_errors(&self) { + self.errors.fetch_add(1, Ordering::Relaxed); + } } impl<'a> MergeCerts<'a> for MergePublicCollectStats { @@ -1071,10 +1139,10 @@ /// certs.update_by(Arc::new(LazyCert::from(cert)), &mut stats) /// .expect("valid"); /// - /// assert_eq!(stats.new, 1); + /// assert_eq!(stats.new_certs(), 1); /// # Ok(()) } /// ``` - fn merge_public<'b, 'rb>(&mut self, + fn merge_public<'b, 'rb>(&self, new: Arc>, disk: Option>>) -> Result>> @@ -1082,7 +1150,7 @@ let disk = if let Some(disk) = disk { disk } else { - self.new += 1; + self.inc_new_certs(); if new.is_tsk() { return Ok(Arc::new(LazyCert::from( @@ -1109,18 +1177,22 @@ .clone(); if disk == new { - self.unchanged += 1; + self.inc_unchanged_certs(); Ok(Arc::new(LazyCert::from(new))) } else { // If the on-disk version has secrets, we preserve them. // If new has secrets, we ignore them. - match disk.merge_public(new) { + match disk.clone().merge_public(new) { Ok(merged) => { - self.updated += 1; + if merged == disk { + self.inc_unchanged_certs(); + } else { + self.inc_updated_certs(); + } Ok(Arc::new(LazyCert::from(merged))) } Err(err) => { - self.errors += 1; + self.inc_errors(); Err(err.into()) } } @@ -1199,8 +1271,8 @@ } let backend = store::Certs::empty(); - let mut backend: Box = Box::new(backend); - let foo = Foo::new(&mut backend); + let backend: Box = Box::new(backend); + let foo = Foo::new(&backend); // Do something (anything) with the backend. assert_eq!(foo.count(), 0); @@ -1258,9 +1330,7 @@ use crate::CertStore; use crate::store::MergePublicCollectStats; - assert_eq!(keyring::certs.len(), 12); - - let mut certs = CertStore::empty(); + let certs = CertStore::empty(); let mut stats = MergePublicCollectStats::new(); @@ -1277,13 +1347,16 @@ eprintln!("After inserting {} ({}), stats: {:?}", i, fpr, stats); - assert_eq!(stats.new, seen.len()); - assert_eq!(stats.new + stats.updated + stats.unchanged, i + 1); + assert_eq!(stats.new_certs(), seen.len()); + assert_eq!(stats.new_certs() + + stats.updated_certs() + + stats.unchanged_certs(), + i + 1); } - let new = stats.new; - let updated = stats.updated; - let unchanged = stats.unchanged; + let new = stats.new_certs(); + let updated = stats.updated_certs(); + let unchanged = stats.unchanged_certs(); // Insert again. This time nothing should change. for (i, cert) in keyring::certs.iter().enumerate() { @@ -1297,10 +1370,10 @@ i, fpr, stats); // These should not change: - assert_eq!(stats.new, new); + assert_eq!(stats.new_certs(), new); // Update should also not change, but there may be false // positives. - assert_eq!(stats.unchanged + stats.updated, + assert_eq!(stats.unchanged_certs() + stats.updated_certs(), updated + unchanged + i + 1); } } diff -Nru rust-sequoia-cert-store-0.4.0/tests/Makefile rust-sequoia-cert-store-0.5.0/tests/Makefile --- rust-sequoia-cert-store-0.4.0/tests/Makefile 2006-07-24 01:21:28.000000000 +0000 +++ rust-sequoia-cert-store-0.5.0/tests/Makefile 2006-07-24 01:21:28.000000000 +0000 @@ -14,4 +14,4 @@ keyring.rs keyring.pgp: $(certs) cert2rust.sh ./cert2rust.sh $(certs) > keyring.rs - sq keyring join $(certs) > keyring.pgp + sq toolbox keyring merge $(certs) > keyring.pgp diff -Nru rust-sequoia-cert-store-0.4.0/tests/cert2rust.sh rust-sequoia-cert-store-0.5.0/tests/cert2rust.sh --- rust-sequoia-cert-store-0.4.0/tests/cert2rust.sh 2006-07-24 01:21:28.000000000 +0000 +++ rust-sequoia-cert-store-0.5.0/tests/cert2rust.sh 2006-07-24 01:21:28.000000000 +0000 @@ -38,7 +38,7 @@ echo "FILE:$file:$base" if test "x$file" != "xEOF!" then - sq packet dump "$file" + sq toolbox packet dump "$file" fi done | awk -F '[ \t]*:[ \t]*' ' BEGIN {