diff -Nru certspotter-0.14.0/auditing.go certspotter-0.15.1/auditing.go --- certspotter-0.14.0/auditing.go 2022-06-13 15:23:35.000000000 +0000 +++ certspotter-0.15.1/auditing.go 1970-01-01 00:00:00.000000000 +0000 @@ -1,213 +0,0 @@ -// Copyright (C) 2016 Opsmate, Inc. -// -// This Source Code Form is subject to the terms of the Mozilla -// Public License, v. 2.0. If a copy of the MPL was not distributed -// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. -// -// This software is distributed WITHOUT A WARRANTY OF ANY KIND. -// See the Mozilla Public License for details. - -package certspotter - -import ( - "bytes" - "crypto/sha256" - "encoding/json" - "errors" - "software.sslmate.com/src/certspotter/ct" -) - -func reverseHashes(hashes []ct.MerkleTreeNode) { - for i := 0; i < len(hashes)/2; i++ { - j := len(hashes) - i - 1 - hashes[i], hashes[j] = hashes[j], hashes[i] - } -} - -func VerifyConsistencyProof(proof ct.ConsistencyProof, first *ct.SignedTreeHead, second *ct.SignedTreeHead) bool { - // TODO: make sure every hash in proof is right length? otherwise input to hashChildren is ambiguous - if second.TreeSize < first.TreeSize { - // Can't be consistent if tree got smaller - return false - } - if first.TreeSize == second.TreeSize { - if !(bytes.Equal(first.SHA256RootHash[:], second.SHA256RootHash[:]) && len(proof) == 0) { - return false - } - return true - } - if first.TreeSize == 0 { - // The purpose of the consistency proof is to ensure the append-only - // nature of the tree; i.e. that the first tree is a "prefix" of the - // second tree. If the first tree is empty, then it's trivially a prefix - // of the second tree, so no proof is needed. - if len(proof) != 0 { - return false - } - return true - } - // Guaranteed that 0 < first.TreeSize < second.TreeSize - - node := first.TreeSize - 1 - lastNode := second.TreeSize - 1 - - // While we're the right child, everything is in both trees, so move one level up. - for node%2 == 1 { - node /= 2 - lastNode /= 2 - } - - var newHash ct.MerkleTreeNode - var oldHash ct.MerkleTreeNode - if node > 0 { - if len(proof) == 0 { - return false - } - newHash = proof[0] - proof = proof[1:] - } else { - // The old tree was balanced, so we already know the first hash to use - newHash = first.SHA256RootHash[:] - } - oldHash = newHash - - for node > 0 { - if node%2 == 1 { - // node is a right child; left sibling exists in both trees - if len(proof) == 0 { - return false - } - newHash = hashChildren(proof[0], newHash) - oldHash = hashChildren(proof[0], oldHash) - proof = proof[1:] - } else if node < lastNode { - // node is a left child; rigth sibling only exists in the new tree - if len(proof) == 0 { - return false - } - newHash = hashChildren(newHash, proof[0]) - proof = proof[1:] - } // else node == lastNode: node is a left child with no sibling in either tree - node /= 2 - lastNode /= 2 - } - - if !bytes.Equal(oldHash, first.SHA256RootHash[:]) { - return false - } - - // If trees have different height, continue up the path to reach the new root - for lastNode > 0 { - if len(proof) == 0 { - return false - } - newHash = hashChildren(newHash, proof[0]) - proof = proof[1:] - lastNode /= 2 - } - - if !bytes.Equal(newHash, second.SHA256RootHash[:]) { - return false - } - - return true -} - -func hashNothing() ct.MerkleTreeNode { - return sha256.New().Sum(nil) -} - -func hashLeaf(leafBytes []byte) ct.MerkleTreeNode { - hasher := sha256.New() - hasher.Write([]byte{0x00}) - hasher.Write(leafBytes) - return hasher.Sum(nil) -} - -func hashChildren(left ct.MerkleTreeNode, right ct.MerkleTreeNode) ct.MerkleTreeNode { - hasher := sha256.New() - hasher.Write([]byte{0x01}) - hasher.Write(left) - hasher.Write(right) - return hasher.Sum(nil) -} - -type CollapsedMerkleTree struct { - nodes []ct.MerkleTreeNode - size uint64 -} - -func calculateNumNodes(size uint64) int { - numNodes := 0 - for size > 0 { - numNodes += int(size & 1) - size >>= 1 - } - return numNodes -} -func EmptyCollapsedMerkleTree() *CollapsedMerkleTree { - return &CollapsedMerkleTree{} -} -func NewCollapsedMerkleTree(nodes []ct.MerkleTreeNode, size uint64) (*CollapsedMerkleTree, error) { - if len(nodes) != calculateNumNodes(size) { - return nil, errors.New("NewCollapsedMerkleTree: nodes has incorrect size") - } - return &CollapsedMerkleTree{nodes: nodes, size: size}, nil -} -func CloneCollapsedMerkleTree(source *CollapsedMerkleTree) *CollapsedMerkleTree { - nodes := make([]ct.MerkleTreeNode, len(source.nodes)) - copy(nodes, source.nodes) - return &CollapsedMerkleTree{nodes: nodes, size: source.size} -} - -func (tree *CollapsedMerkleTree) Add(hash ct.MerkleTreeNode) { - tree.nodes = append(tree.nodes, hash) - tree.size++ - size := tree.size - for size%2 == 0 { - left, right := tree.nodes[len(tree.nodes)-2], tree.nodes[len(tree.nodes)-1] - tree.nodes = tree.nodes[:len(tree.nodes)-2] - tree.nodes = append(tree.nodes, hashChildren(left, right)) - size /= 2 - } -} - -func (tree *CollapsedMerkleTree) CalculateRoot() ct.MerkleTreeNode { - if len(tree.nodes) == 0 { - return hashNothing() - } - i := len(tree.nodes) - 1 - hash := tree.nodes[i] - for i > 0 { - i -= 1 - hash = hashChildren(tree.nodes[i], hash) - } - return hash -} - -func (tree *CollapsedMerkleTree) GetSize() uint64 { - return tree.size -} - -func (tree *CollapsedMerkleTree) MarshalJSON() ([]byte, error) { - return json.Marshal(map[string]interface{}{ - "nodes": tree.nodes, - "size": tree.size, - }) -} - -func (tree *CollapsedMerkleTree) UnmarshalJSON(b []byte) error { - var rawTree struct { - Nodes []ct.MerkleTreeNode `json:"nodes"` - Size uint64 `json:"size"` - } - if err := json.Unmarshal(b, &rawTree); err != nil { - return errors.New("Failed to unmarshal CollapsedMerkleTree: " + err.Error()) - } - if len(rawTree.Nodes) != calculateNumNodes(rawTree.Size) { - return errors.New("Failed to unmarshal CollapsedMerkleTree: nodes has incorrect length") - } - tree.size = rawTree.Size - tree.nodes = rawTree.Nodes - return nil -} diff -Nru certspotter-0.14.0/CHANGELOG.md certspotter-0.15.1/CHANGELOG.md --- certspotter-0.14.0/CHANGELOG.md 1970-01-01 00:00:00.000000000 +0000 +++ certspotter-0.15.1/CHANGELOG.md 2023-02-09 18:44:06.000000000 +0000 @@ -0,0 +1,179 @@ +# Change Log + +## v0.15.1 (2023-02-09) +- Fix some typos in help and error messages. +- Allow version to be set via linker flag, to facilitate distro package building. + +## v0.15.0 (2023-02-08) +- **Significant behavior change**: certspotter is now intended to run as + a daemon instead of a cron job. Specifically, certspotter no longer + terminates unless it receives SIGTERM or SIGINT or there is a serious error. + You should remove certspotter from your crontab and arrange to run it as a + daemon, passing either the `-email` option or `-script` option to configure + how you want to be notified about certificates. + + Reason for this change: although using cron made sense in the early days of + Certificate Transparency, certspotter now needs to run continuously to reliably + keep up with the high growth rate of contemporary CT logs, and to gracefully + handle the many transient errors that can arise when monitoring CT. + See for background. + +- `-script` is now officially supported and can be used to execute + a command when a certificate is discovered or there is an error. For details, + see the [certspotter-script(8) man page](man/certspotter-script.md). + + Note the following changes from the experimental, undocumented `-script` + option found in previous versions: + - The script is also executed when there is an error. Consult the `$EVENT` + variable to determine why the script was executed. + - The `$DNS_NAMES` and `$IP_ADDRESSES` variables have been removed because + the OS limits the size of environment variables and some certificates have + too many identifiers. To determine a certificate's identifiers, you can + read the JSON file specified by the `$JSON_FILENAME` variable, as explained + in the [certspotter-script(8) man page](man/certspotter-script.md). + - The `$CERT_TYPE` variable has been removed because it is almost always + a serious mistake (that can make you miss malicious certificates) to treat + certificates and precertificates differently. If you are currently + using this variable to skip precertificates, stop doing that because + precertificates imply the existence of a corresponding certificate that you + **might not** be separately notified about. For more details, see + . + - New variable `$WATCH_ITEM` contains the first watch list item which + matched the certificate. + +- New `-email` option can be used to send an email when a certificate is + discovered or there is an error. Your system must have a working `sendmail` + command. + +- (Behavior change) You must specify the `-stdout` option if you want discovered + certificates to be written to stdout. This only makes sense when running + certspotter from the terminal; when running as a daemon you probably want to + use `-email` or `-script` instead. + +- Once a day, certspotter will send you a notification (per `-email` or + `-script`) if any problems are preventing it from detecting all certificates. + As in previous versions of certspotter, errors are written to stderr when they + occur, but since most errors are transient, you can now ignore stderr and rely + on the daily health check to notify you about any persistent problems that + require your attention. + +- certspotter now saves `.json` and `.txt` files alongside the `.pem` files + containing parsed representations of the certificate. + +- `.pem` files no longer have `.cert` or `.precert` in the filename. + +- certspotter will save its state periodically, and before terminating due to + SIGTERM or SIGINT, meaning it can resume monitoring without having to + re-download entries it has already processed. + +- The experimental "BygoneSSL" feature has been removed due to limited utility. + +- The `-num_workers` option has been removed. + +- The `-all_time` option has been removed. You can remove the certspotter state + directory if you want to re-download all entries. + +- The minimum supported Go version is now 1.19. + +## v0.14.0 (2022-06-13) +- Switch to Go module versioning conventions. + +## v0.13 (2022-06-13) +- Reduce minimum Go version to 1.17. +- Update install instructions. + +## v0.12 (2022-06-07) +- Retry failed log requests. This should make certspotter resilient + to rate limiting by logs. +- Add `-version` flag. +- Eliminate unnecessary dependency. certspotter now depends only on + golang.org/x packages. +- Switch to Go modules. + +## v0.11 (2021-08-17) +- Add support for contacting logs via HTTP proxies; + just set the appropriate environment variable as documented at + . +- Work around RFC 6962 ambiguity related to consistency proofs + for empty trees. + +## v0.10 (2020-04-29) +- Improve speed by processing logs in parallel +- Add `-start_at_end` option to begin monitoring new logs at the end, + which significantly speeds up Cert Spotter, at the cost of missing + certificates that were added to a log before Cert Spotter starts + monitoring it +- (Behavior change) Scan logs in their entirety the first time Cert + Spotter is run, unless `-start_at_end` specified (behavior change) +- The log list is now retrieved from certspotter.org at startup instead + of being embedded in the source. This will allow Cert Spotter to react + more quickly to the frequent changes in logs. +- (Behavior change) the `-logs` option now expects a JSON file in the v2 + log list format. See + and . +- `-logs` now accepts an HTTPS URL in addition to a file path. +- (Behavior change) the `-underwater` option has been removed. If you want + its behavior, specify `https://loglist.certspotter.org/underwater.json` to + the `-logs` option. + +## v0.9 (2018-04-19) +- Add Cloudflare Nimbus logs +- Remove Google Argon 2017 log +- Remove WoSign and StartCom logs due to disqualification by Chromium + and extended downtime + +## v0.8 (2017-12-08) +- Add Symantec Sirius log +- Add DigiCert 2 log + +## v0.7 (2017-11-13) +- Add Google Argon logs +- Fix bug that caused crash on 32 bit architectures + +## v0.6 (2017-10-19) +- Add Comodo Mammoth and Comodo Sabre logs +- Minor bug fixes and improvements + +## v0.5 (2017-05-18) +- Remove PuChuangSiDa 1 log due to excessive downtime and presumptive + disqualification from Chrome +- Add Venafi Gen2 log +- Improve monitoring robustness under certain pathological behavior + by logs +- Minor documentation improvements + +## v0.4 (2017-04-03) +- Add PuChuangSiDa 1 log +- Remove Venafi log due to fork and disqualification from Chrome + +## v0.3 (2017-02-20) +- Revise `-all_time` flag (behavior change): + - If `-all_time` is specified, scan the entirety of all logs, even + existing logs + - When a new log is added, scan it in its entirety even if `-all_time` + is not specified +- Add new logs: + - Google Icarus + - Google Skydiver + - StartCom + - WoSign +- Overhaul log processing and auditing logic: + - STHs are never deleted unless they can be verified + - Multiple unverified STHs can be queued per log, laying groundwork + for STH pollination support + - New state directory layout; current state directories will be + migrated, but migration will be removed in a future version + - Persist condensed Merkle Tree state between runs, instead of + reconstructing it from consistency proof every time +- Use a lock file to prevent multiple instances of Cert Spotter from + running concurrently (which could clobber the state directory). +- Minor bug fixes and improvements + +## v0.2 (2016-08-25) +- Suppress duplicate identifiers in output. +- Fix "EOF" error when running under Go 1.7. +- Fix bug where hook script could fail silently. +- Fix compilation under Go 1.5. + +## v0.1 (2016-07-27) +- Initial release. diff -Nru certspotter-0.14.0/cmd/certspotter/main.go certspotter-0.15.1/cmd/certspotter/main.go --- certspotter-0.14.0/cmd/certspotter/main.go 2022-06-13 15:23:35.000000000 +0000 +++ certspotter-0.15.1/cmd/certspotter/main.go 2023-02-09 18:44:06.000000000 +0000 @@ -1,4 +1,4 @@ -// Copyright (C) 2016 Opsmate, Inc. +// Copyright (C) 2016, 2023 Opsmate, Inc. // // This Source Code Form is subject to the terms of the Mozilla // Public License, v. 2.0. If a copy of the MPL was not distributed @@ -10,224 +10,178 @@ package main import ( - "bufio" + "context" + "errors" "flag" "fmt" - "io" + "io/fs" + insecurerand "math/rand" "os" + "os/signal" "path/filepath" + "runtime" + "runtime/debug" "strings" + "syscall" "time" - "golang.org/x/net/idna" - - "software.sslmate.com/src/certspotter" - "software.sslmate.com/src/certspotter/cmd" - "software.sslmate.com/src/certspotter/ct" + "software.sslmate.com/src/certspotter/loglist" + "software.sslmate.com/src/certspotter/monitor" ) +var programName = os.Args[0] +var Version = "" + +const defaultLogList = "https://loglist.certspotter.org/monitor.json" + +func certspotterVersion() string { + if Version != "" { + return Version + "?" + } + info, ok := debug.ReadBuildInfo() + if !ok { + return "unknown" + } + if strings.HasPrefix(info.Main.Version, "v") { + return info.Main.Version + } + var vcs, vcsRevision, vcsModified string + for _, s := range info.Settings { + switch s.Key { + case "vcs": + vcs = s.Value + case "vcs.revision": + vcsRevision = s.Value + case "vcs.modified": + vcsModified = s.Value + } + } + if vcs == "git" && vcsRevision != "" && vcsModified == "true" { + return vcsRevision + "+" + } else if vcs == "git" && vcsRevision != "" { + return vcsRevision + } + return "unknown" +} + +func homedir() string { + homedir, err := os.UserHomeDir() + if err != nil { + panic(fmt.Errorf("unable to determine home directory: %w", err)) + } + return homedir +} func defaultStateDir() string { if envVar := os.Getenv("CERTSPOTTER_STATE_DIR"); envVar != "" { return envVar } else { - return cmd.DefaultStateDir("certspotter") + return filepath.Join(homedir(), ".certspotter") } } func defaultConfigDir() string { if envVar := os.Getenv("CERTSPOTTER_CONFIG_DIR"); envVar != "" { return envVar } else { - return cmd.DefaultConfigDir("certspotter") + return filepath.Join(homedir(), ".certspotter") } } -func trimTrailingDots(value string) string { - length := len(value) - for length > 0 && value[length-1] == '.' { - length-- - } - return value[0:length] -} - -var stateDir = flag.String("state_dir", defaultStateDir(), "Directory for storing state") -var watchlistFilename = flag.String("watchlist", filepath.Join(defaultConfigDir(), "watchlist"), "File containing identifiers to watch (- for stdin)") - -type watchlistItem struct { - Domain []string - AcceptSuffix bool - ValidAt *time.Time // optional -} - -var watchlist []watchlistItem - -func parseWatchlistItem(str string) (watchlistItem, error) { - fields := strings.Fields(str) - if len(fields) == 0 { - return watchlistItem{}, fmt.Errorf("Empty domain") - } - domain := fields[0] - var validAt *time.Time = nil - - // parse options - for i := 1; i < len(fields); i++ { - chunks := strings.SplitN(fields[i], ":", 2) - if len(chunks) != 2 { - return watchlistItem{}, fmt.Errorf("Missing Value `%s'", fields[i]) - } - switch chunks[0] { - case "valid_at": - validAtTime, err := time.Parse("2006-01-02", chunks[1]) - if err != nil { - return watchlistItem{}, fmt.Errorf("Invalid Date `%s': %s", chunks[1], err) - } - validAt = &validAtTime - default: - return watchlistItem{}, fmt.Errorf("Unknown Option `%s'", fields[i]) - } - } - - // parse domain - // "." as in root zone (matches everything) - if domain == "." { - return watchlistItem{ - Domain: []string{}, - AcceptSuffix: true, - ValidAt: validAt, - }, nil - } - - acceptSuffix := false - if strings.HasPrefix(domain, ".") { - acceptSuffix = true - domain = domain[1:] - } - - asciiDomain, err := idna.ToASCII(strings.ToLower(trimTrailingDots(domain))) +func readWatchListFile(filename string) (monitor.WatchList, error) { + file, err := os.Open(filename) if err != nil { - return watchlistItem{}, fmt.Errorf("Invalid domain `%s': %s", domain, err) - } - return watchlistItem{ - Domain: strings.Split(asciiDomain, "."), - AcceptSuffix: acceptSuffix, - ValidAt: validAt, - }, nil -} - -func readWatchlist(reader io.Reader) ([]watchlistItem, error) { - items := []watchlistItem{} - scanner := bufio.NewScanner(reader) - for scanner.Scan() { - line := scanner.Text() - if line == "" || strings.HasPrefix(line, "#") { - continue - } - item, err := parseWatchlistItem(line) - if err != nil { - return nil, err + var pathErr *fs.PathError + if errors.As(err, &pathErr) { + err = pathErr.Err } - items = append(items, item) + return nil, err } - return items, scanner.Err() -} - -func dnsLabelMatches(certLabel string, watchLabel string) bool { - // For fail-safe behavior, if a label was unparsable, it matches everything. - // Similarly, redacted labels match everything, since the label _might_ be - // for a name we're interested in. - - return certLabel == "*" || - certLabel == "?" || - certLabel == certspotter.UnparsableDNSLabelPlaceholder || - certspotter.MatchesWildcard(watchLabel, certLabel) + defer file.Close() + return monitor.ReadWatchList(file) } -func dnsNameMatches(dnsName []string, watchDomain []string, acceptSuffix bool) bool { - for len(dnsName) > 0 && len(watchDomain) > 0 { - certLabel := dnsName[len(dnsName)-1] - watchLabel := watchDomain[len(watchDomain)-1] - - if !dnsLabelMatches(certLabel, watchLabel) { - return false - } - - dnsName = dnsName[:len(dnsName)-1] - watchDomain = watchDomain[:len(watchDomain)-1] +func appendFunc(slice *[]string) func(string) error { + return func(value string) error { + *slice = append(*slice, value) + return nil } - - return len(watchDomain) == 0 && (acceptSuffix || len(dnsName) == 0) } -func anyDnsNameIsWatched(info *certspotter.EntryInfo) bool { - dnsNames := info.Identifiers.DNSNames - matched := false - for _, dnsName := range dnsNames { - labels := strings.Split(dnsName, ".") - for _, item := range watchlist { - if dnsNameMatches(labels, item.Domain, item.AcceptSuffix) { - if item.ValidAt != nil { - // BygoneSSL Check - // was the SSL certificate issued before the domain was registered - // and valid after - if item.ValidAt.Before(*info.CertInfo.NotAfter()) && - item.ValidAt.After(*info.CertInfo.NotBefore()) { - info.Bygone = true - return true - } - } - // keep iterating in case another domain watched matches valid_at - matched = true - } - } - } - return matched -} - -func processEntry(scanner *certspotter.Scanner, entry *ct.LogEntry) { - info := certspotter.EntryInfo{ - LogUri: scanner.LogUri, - Entry: entry, - IsPrecert: certspotter.IsPrecert(entry), - FullChain: certspotter.GetFullChain(entry), - } +func main() { + insecurerand.Seed(time.Now().UnixNano()) // TODO: remove after upgrading to Go 1.20 - info.CertInfo, info.ParseError = certspotter.MakeCertInfoFromLogEntry(entry) + loglist.UserAgent = fmt.Sprintf("certspotter/%s (%s; %s; %s)", certspotterVersion(), runtime.Version(), runtime.GOOS, runtime.GOARCH) - if info.CertInfo != nil { - info.Identifiers, info.IdentifiersParseError = info.CertInfo.ParseIdentifiers() + var flags struct { + batchSize int // TODO-4: respect this option + email []string + healthcheck time.Duration + logs string + noSave bool + script string + startAtEnd bool + stateDir string + stdout bool + verbose bool + version bool + watchlist string + } + flag.IntVar(&flags.batchSize, "batch_size", 1000, "Max number of entries to request per call to get-entries (advanced)") + flag.Func("email", "Email address to contact when matching certificate is discovered (repeatable)", appendFunc(&flags.email)) + flag.DurationVar(&flags.healthcheck, "healthcheck", 24*time.Hour, "How frequently to perform a health check") + flag.StringVar(&flags.logs, "logs", defaultLogList, "File path or URL of JSON list of logs to monitor") + flag.BoolVar(&flags.noSave, "no_save", false, "Do not save a copy of matching certificates in state directory") + flag.StringVar(&flags.script, "script", "", "Program to execute when a matching certificate is discovered") + flag.BoolVar(&flags.startAtEnd, "start_at_end", false, "Start monitoring logs from the end rather than the beginning (saves considerable bandwidth)") + flag.StringVar(&flags.stateDir, "state_dir", defaultStateDir(), "Directory for storing log position and discovered certificates") + flag.BoolVar(&flags.stdout, "stdout", false, "Write matching certificates to stdout") + flag.BoolVar(&flags.verbose, "verbose", false, "Be verbose") + flag.BoolVar(&flags.version, "version", false, "Print version and exit") + flag.StringVar(&flags.watchlist, "watchlist", filepath.Join(defaultConfigDir(), "watchlist"), "File containing domain names to watch") + flag.Parse() + + if flags.version { + fmt.Fprintf(os.Stdout, "certspotter version %s\n", certspotterVersion()) + os.Exit(0) + } + + if len(flags.email) == 0 && len(flags.script) == 0 && flags.stdout == false { + fmt.Fprintf(os.Stderr, "%s: at least one of -email, -script, or -stdout must be specified (see -help for details)\n", programName) + os.Exit(2) + } + + config := &monitor.Config{ + LogListSource: flags.logs, + StateDir: flags.stateDir, + SaveCerts: !flags.noSave, + StartAtEnd: flags.startAtEnd, + Verbose: flags.verbose, + Script: flags.script, + Email: flags.email, + Stdout: flags.stdout, + HealthCheckInterval: flags.healthcheck, } - // Fail safe behavior: if info.Identifiers is nil (which is caused by a - // parse error), report the certificate because we can't say for sure it - // doesn't match a domain we care about. We try very hard to make sure - // parsing identifiers always succeeds, so false alarms should be rare. - if info.Identifiers == nil || anyDnsNameIsWatched(&info) { - cmd.LogEntry(&info) - } -} - -func main() { - cmd.ParseFlags() - - if *watchlistFilename == "-" { - var err error - watchlist, err = readWatchlist(os.Stdin) + if flags.watchlist == "-" { + watchlist, err := monitor.ReadWatchList(os.Stdin) if err != nil { - fmt.Fprintf(os.Stderr, "%s: (stdin): %s\n", os.Args[0], err) + fmt.Fprintf(os.Stderr, "%s: error reading watchlist from standard in: %s\n", programName, err) os.Exit(1) } + config.WatchList = watchlist } else { - file, err := os.Open(*watchlistFilename) - if err != nil { - fmt.Fprintf(os.Stderr, "%s: %s: %s\n", os.Args[0], *watchlistFilename, err) - os.Exit(1) - } - watchlist, err = readWatchlist(file) - file.Close() + watchlist, err := readWatchListFile(flags.watchlist) if err != nil { - fmt.Fprintf(os.Stderr, "%s: %s: %s\n", os.Args[0], *watchlistFilename, err) + fmt.Fprintf(os.Stderr, "%s: error reading watchlist from %q: %s\n", programName, flags.watchlist, err) os.Exit(1) } + config.WatchList = watchlist } - os.Exit(cmd.Main(*stateDir, processEntry)) + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stop() + + if err := monitor.Run(ctx, config); err != nil && !errors.Is(err, context.Canceled) { + fmt.Fprintf(os.Stderr, "%s: %s\n", programName, err) + os.Exit(1) + } } diff -Nru certspotter-0.14.0/cmd/common.go certspotter-0.15.1/cmd/common.go --- certspotter-0.14.0/cmd/common.go 2022-06-13 15:23:35.000000000 +0000 +++ certspotter-0.15.1/cmd/common.go 1970-01-01 00:00:00.000000000 +0000 @@ -1,363 +0,0 @@ -// Copyright (C) 2016-2017 Opsmate, Inc. -// -// This Source Code Form is subject to the terms of the Mozilla -// Public License, v. 2.0. If a copy of the MPL was not distributed -// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. -// -// This software is distributed WITHOUT A WARRANTY OF ANY KIND. -// See the Mozilla Public License for details. - -package cmd - -import ( - "bytes" - "crypto/x509" - "flag" - "fmt" - "log" - "os" - "os/user" - "path/filepath" - "sync" - - "software.sslmate.com/src/certspotter" - "software.sslmate.com/src/certspotter/ct" - "software.sslmate.com/src/certspotter/loglist" -) - -const defaultLogList = "https://loglist.certspotter.org/monitor.json" - -var batchSize = flag.Int("batch_size", 1000, "Max number of entries to request at per call to get-entries (advanced)") -var numWorkers = flag.Int("num_workers", 2, "Number of concurrent matchers (advanced)") -var script = flag.String("script", "", "Script to execute when a matching certificate is found") -var logsURL = flag.String("logs", defaultLogList, "File path or URL of JSON list of logs to monitor") -var noSave = flag.Bool("no_save", false, "Do not save a copy of matching certificates") -var verbose = flag.Bool("verbose", false, "Be verbose") -var showVersion = flag.Bool("version", false, "Print version and exit") -var startAtEnd = flag.Bool("start_at_end", false, "Start monitoring logs from the end rather than the beginning") -var allTime = flag.Bool("all_time", false, "Scan certs from all time, not just since last scan") -var state *State - -var printMutex sync.Mutex - -func homedir() string { - home := os.Getenv("HOME") - if home != "" { - return home - } - user, err := user.Current() - if err == nil { - return user.HomeDir - } - panic("Unable to determine home directory") -} - -func DefaultStateDir(programName string) string { - return filepath.Join(homedir(), "."+programName) -} - -func DefaultConfigDir(programName string) string { - return filepath.Join(homedir(), "."+programName) -} - -func LogEntry(info *certspotter.EntryInfo) { - if !*noSave { - var alreadyPresent bool - var err error - alreadyPresent, info.Filename, err = state.SaveCert(info.IsPrecert, info.FullChain) - if err != nil { - log.Print(err) - } - if alreadyPresent { - return - } - } - - if *script != "" { - if err := info.InvokeHookScript(*script); err != nil { - log.Print(err) - } - } else { - printMutex.Lock() - info.Write(os.Stdout) - fmt.Fprintf(os.Stdout, "\n") - printMutex.Unlock() - } -} - -func loadLogList() ([]*loglist.Log, error) { - list, err := loglist.Load(*logsURL) - if err != nil { - return nil, fmt.Errorf("Error loading log list: %s", err) - } - return list.AllLogs(), nil -} - -type logHandle struct { - scanner *certspotter.Scanner - state *LogState - tree *certspotter.CollapsedMerkleTree - verifiedSTH *ct.SignedTreeHead -} - -func makeLogHandle(logInfo *loglist.Log) (*logHandle, error) { - ctlog := new(logHandle) - - logKey, err := x509.ParsePKIXPublicKey(logInfo.Key) - if err != nil { - return nil, fmt.Errorf("Bad public key: %s", err) - } - ctlog.scanner = certspotter.NewScanner(logInfo.URL, logInfo.LogID, logKey, &certspotter.ScannerOptions{ - BatchSize: *batchSize, - NumWorkers: *numWorkers, - Quiet: !*verbose, - }) - - ctlog.state, err = state.OpenLogState(logInfo) - if err != nil { - return nil, fmt.Errorf("Error opening state directory: %s", err) - } - ctlog.tree, err = ctlog.state.GetTree() - if err != nil { - return nil, fmt.Errorf("Error loading tree: %s", err) - } - ctlog.verifiedSTH, err = ctlog.state.GetVerifiedSTH() - if err != nil { - return nil, fmt.Errorf("Error loading verified STH: %s", err) - } - - if ctlog.tree == nil && ctlog.verifiedSTH == nil { // This branch can be removed eventually - legacySTH, err := state.GetLegacySTH(logInfo) - if err != nil { - return nil, fmt.Errorf("Error loading legacy STH: %s", err) - } - if legacySTH != nil { - log.Print(logInfo.URL, ": Initializing log state from legacy state directory") - ctlog.tree, err = ctlog.scanner.MakeCollapsedMerkleTree(legacySTH) - if err != nil { - return nil, fmt.Errorf("Error reconstructing Merkle Tree for legacy STH: %s", err) - } - if err := ctlog.state.StoreTree(ctlog.tree); err != nil { - return nil, fmt.Errorf("Error storing tree: %s", err) - } - if err := ctlog.state.StoreVerifiedSTH(legacySTH); err != nil { - return nil, fmt.Errorf("Error storing verified STH: %s", err) - } - state.RemoveLegacySTH(logInfo) - } - } - - return ctlog, nil -} - -func (ctlog *logHandle) refresh() error { - if *verbose { - log.Print(ctlog.scanner.LogUri, ": Retrieving latest STH from log") - } - latestSTH, err := ctlog.scanner.GetSTH() - if err != nil { - return fmt.Errorf("Error retrieving STH from log: %s", err) - } - if ctlog.verifiedSTH == nil { - if *verbose { - log.Printf("%s: No existing STH is known; presuming latest STH (%d) is valid", ctlog.scanner.LogUri, latestSTH.TreeSize) - } - ctlog.verifiedSTH = latestSTH - if err := ctlog.state.StoreVerifiedSTH(ctlog.verifiedSTH); err != nil { - return fmt.Errorf("Error storing verified STH: %s", err) - } - } else { - if err := ctlog.state.StoreUnverifiedSTH(latestSTH); err != nil { - return fmt.Errorf("Error storing unverified STH: %s", err) - } - } - return nil -} - -func (ctlog *logHandle) verifySTH(sth *ct.SignedTreeHead) error { - isValid, err := ctlog.scanner.CheckConsistency(ctlog.verifiedSTH, sth) - if err != nil { - return fmt.Errorf("Error fetching consistency proof: %s", err) - } - if !isValid { - return fmt.Errorf("Consistency proof between %d and %d is invalid", ctlog.verifiedSTH.TreeSize, sth.TreeSize) - } - return nil -} - -func (ctlog *logHandle) audit() error { - sths, err := ctlog.state.GetUnverifiedSTHs() - if err != nil { - return fmt.Errorf("Error loading unverified STHs: %s", err) - } - - for _, sth := range sths { - if *verbose { - log.Printf("%s: Verifying consistency of STH %d (%x) with previously-verified STH %d (%x)", ctlog.scanner.LogUri, sth.TreeSize, sth.SHA256RootHash, ctlog.verifiedSTH.TreeSize, ctlog.verifiedSTH.SHA256RootHash) - } - if err := ctlog.verifySTH(sth); err != nil { - log.Printf("%s: Unable to verify consistency of STH %d (%s) (if this error persists, it should be construed as misbehavior by the log): %s", ctlog.scanner.LogUri, sth.TreeSize, ctlog.state.UnverifiedSTHFilename(sth), err) - continue - } - if sth.TreeSize > ctlog.verifiedSTH.TreeSize { - if *verbose { - log.Printf("%s: STH %d (%x) is now the latest verified STH", ctlog.scanner.LogUri, sth.TreeSize, sth.SHA256RootHash) - } - ctlog.verifiedSTH = sth - if err := ctlog.state.StoreVerifiedSTH(ctlog.verifiedSTH); err != nil { - return fmt.Errorf("Error storing verified STH: %s", err) - } - } - if err := ctlog.state.RemoveUnverifiedSTH(sth); err != nil { - return fmt.Errorf("Error removing redundant STH: %s", err) - } - } - - return nil -} - -func (ctlog *logHandle) scan(processCallback certspotter.ProcessCallback) error { - startIndex := int64(ctlog.tree.GetSize()) - endIndex := int64(ctlog.verifiedSTH.TreeSize) - - if endIndex > startIndex { - tree := certspotter.CloneCollapsedMerkleTree(ctlog.tree) - - if err := ctlog.scanner.Scan(startIndex, endIndex, processCallback, tree); err != nil { - return fmt.Errorf("Error scanning log (if this error persists, it should be construed as misbehavior by the log): %s", err) - } - - rootHash := tree.CalculateRoot() - if !bytes.Equal(rootHash, ctlog.verifiedSTH.SHA256RootHash[:]) { - return fmt.Errorf("Log has misbehaved: log entries at tree size %d do not correspond to signed tree root", ctlog.verifiedSTH.TreeSize) - } - - ctlog.tree = tree - if err := ctlog.state.StoreTree(ctlog.tree); err != nil { - return fmt.Errorf("Error storing tree: %s", err) - } - } - - return nil -} - -func processLog(logInfo *loglist.Log, processCallback certspotter.ProcessCallback) int { - ctlog, err := makeLogHandle(logInfo) - if err != nil { - log.Print(logInfo.URL, ": ", err) - return 1 - } - - if err := ctlog.refresh(); err != nil { - log.Print(logInfo.URL, ": ", err) - return 1 - } - - if err := ctlog.audit(); err != nil { - log.Print(logInfo.URL, ": ", err) - return 1 - } - - if *allTime { - ctlog.tree = certspotter.EmptyCollapsedMerkleTree() - if *verbose { - log.Printf("%s: Scanning all %d entries in the log because -all_time option specified", logInfo.URL, ctlog.verifiedSTH.TreeSize) - } - } else if ctlog.tree != nil { - if *verbose { - log.Printf("%s: Existing log; scanning %d new entries since previous scan", logInfo.URL, ctlog.verifiedSTH.TreeSize-ctlog.tree.GetSize()) - } - } else if *startAtEnd { - ctlog.tree, err = ctlog.scanner.MakeCollapsedMerkleTree(ctlog.verifiedSTH) - if err != nil { - log.Printf("%s: Error reconstructing Merkle Tree: %s", logInfo.URL, err) - return 1 - } - if *verbose { - log.Printf("%s: New log; not scanning %d existing entries because -start_at_end option was specified", logInfo.URL, ctlog.verifiedSTH.TreeSize) - } - } else { - ctlog.tree = certspotter.EmptyCollapsedMerkleTree() - if *verbose { - log.Printf("%s: New log; scanning all %d entries in the log (use the -start_at_end option to scan new logs from the end rather than the beginning)", logInfo.URL, ctlog.verifiedSTH.TreeSize) - } - } - if err := ctlog.state.StoreTree(ctlog.tree); err != nil { - log.Printf("%s: Error storing tree: %s\n", logInfo.URL, err) - return 1 - } - - if err := ctlog.scan(processCallback); err != nil { - log.Print(logInfo.URL, ": ", err) - return 1 - } - - if *verbose { - log.Printf("%s: Final log size = %d, final root hash = %x", logInfo.URL, ctlog.verifiedSTH.TreeSize, ctlog.verifiedSTH.SHA256RootHash) - } - - return 0 -} - -func ParseFlags() { - flag.Parse() - if *showVersion { - fmt.Fprintf(os.Stdout, "Cert Spotter %s\n", certspotter.Version) - os.Exit(0) - } -} - -func Main(statePath string, processCallback certspotter.ProcessCallback) int { - var err error - - logs, err := loadLogList() - if err != nil { - fmt.Fprintf(os.Stderr, "%s: %s\n", os.Args[0], err) - return 1 - } - - state, err = OpenState(statePath) - if err != nil { - fmt.Fprintf(os.Stderr, "%s: %s\n", os.Args[0], err) - return 1 - } - locked, err := state.Lock() - if err != nil { - fmt.Fprintf(os.Stderr, "%s: Error locking state directory: %s\n", os.Args[0], err) - return 1 - } - if !locked { - var otherPidInfo string - if otherPid := state.LockingPid(); otherPid != 0 { - otherPidInfo = fmt.Sprintf(" (as process ID %d)", otherPid) - } - fmt.Fprintf(os.Stderr, "%s: Another instance of %s is already running%s; remove the file %s if this is not the case\n", os.Args[0], os.Args[0], otherPidInfo, state.LockFilename()) - return 1 - } - - processLogResults := make(chan int) - for _, logInfo := range logs { - go func(logInfo *loglist.Log) { - processLogResults <- processLog(logInfo, processCallback) - }(logInfo) - } - - exitCode := 0 - for range logs { - exitCode |= <-processLogResults - } - - if state.IsFirstRun() && exitCode == 0 { - if err := state.WriteOnceFile(); err != nil { - fmt.Fprintf(os.Stderr, "%s: Error writing once file: %s\n", os.Args[0], err) - exitCode |= 1 - } - } - - if err := state.Unlock(); err != nil { - fmt.Fprintf(os.Stderr, "%s: Error unlocking state directory: %s\n", os.Args[0], err) - exitCode |= 1 - } - - return exitCode -} diff -Nru certspotter-0.14.0/cmd/ctparsewatch/.gitignore certspotter-0.15.1/cmd/ctparsewatch/.gitignore --- certspotter-0.14.0/cmd/ctparsewatch/.gitignore 2022-06-13 15:23:35.000000000 +0000 +++ certspotter-0.15.1/cmd/ctparsewatch/.gitignore 1970-01-01 00:00:00.000000000 +0000 @@ -1 +0,0 @@ -/ctparsewatch diff -Nru certspotter-0.14.0/cmd/ctparsewatch/main.go certspotter-0.15.1/cmd/ctparsewatch/main.go --- certspotter-0.14.0/cmd/ctparsewatch/main.go 2022-06-13 15:23:35.000000000 +0000 +++ certspotter-0.15.1/cmd/ctparsewatch/main.go 1970-01-01 00:00:00.000000000 +0000 @@ -1,52 +0,0 @@ -// Copyright (C) 2016 Opsmate, Inc. -// -// This Source Code Form is subject to the terms of the Mozilla -// Public License, v. 2.0. If a copy of the MPL was not distributed -// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. -// -// This software is distributed WITHOUT A WARRANTY OF ANY KIND. -// See the Mozilla Public License for details. - -package main - -import ( - "flag" - "os" - - "software.sslmate.com/src/certspotter" - "software.sslmate.com/src/certspotter/cmd" - "software.sslmate.com/src/certspotter/ct" -) - -func DefaultStateDir() string { - if envVar := os.Getenv("CTPARSEWATCH_STATE_DIR"); envVar != "" { - return envVar - } else { - return cmd.DefaultStateDir("ctparsewatch") - } -} - -var stateDir = flag.String("state_dir", DefaultStateDir(), "Directory for storing state") - -func processEntry(scanner *certspotter.Scanner, entry *ct.LogEntry) { - info := certspotter.EntryInfo{ - LogUri: scanner.LogUri, - Entry: entry, - IsPrecert: certspotter.IsPrecert(entry), - FullChain: certspotter.GetFullChain(entry), - } - - info.CertInfo, info.ParseError = certspotter.MakeCertInfoFromLogEntry(entry) - if info.CertInfo != nil { - info.Identifiers, info.IdentifiersParseError = info.CertInfo.ParseIdentifiers() - } - - if info.HasParseErrors() { - cmd.LogEntry(&info) - } -} - -func main() { - cmd.ParseFlags() - os.Exit(cmd.Main(*stateDir, processEntry)) -} diff -Nru certspotter-0.14.0/cmd/helpers.go certspotter-0.15.1/cmd/helpers.go --- certspotter-0.14.0/cmd/helpers.go 2022-06-13 15:23:35.000000000 +0000 +++ certspotter-0.15.1/cmd/helpers.go 1970-01-01 00:00:00.000000000 +0000 @@ -1,87 +0,0 @@ -// Copyright (C) 2017 Opsmate, Inc. -// -// This Source Code Form is subject to the terms of the Mozilla -// Public License, v. 2.0. If a copy of the MPL was not distributed -// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. -// -// This software is distributed WITHOUT A WARRANTY OF ANY KIND. -// See the Mozilla Public License for details. - -package cmd - -import ( - "crypto/sha256" - "encoding/hex" - "encoding/json" - "io/ioutil" - "os" - - "software.sslmate.com/src/certspotter/ct" -) - -func fileExists(path string) bool { - _, err := os.Lstat(path) - return err == nil -} - -func writeFile(filename string, data []byte, perm os.FileMode) error { - tempname := filename + ".new" - if err := ioutil.WriteFile(tempname, data, perm); err != nil { - return err - } - if err := os.Rename(tempname, filename); err != nil { - os.Remove(tempname) - return err - } - return nil -} - -func writeJSONFile(filename string, obj interface{}, perm os.FileMode) error { - tempname := filename + ".new" - f, err := os.OpenFile(tempname, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, perm) - if err != nil { - return err - } - if err := json.NewEncoder(f).Encode(obj); err != nil { - f.Close() - os.Remove(tempname) - return err - } - if err := f.Close(); err != nil { - os.Remove(tempname) - return err - } - if err := os.Rename(tempname, filename); err != nil { - os.Remove(tempname) - return err - } - return nil -} - -func readJSONFile(filename string, obj interface{}) error { - bytes, err := ioutil.ReadFile(filename) - if err != nil { - return err - } - if err = json.Unmarshal(bytes, obj); err != nil { - return err - } - return nil -} - -func readSTHFile(filename string) (*ct.SignedTreeHead, error) { - sth := new(ct.SignedTreeHead) - if err := readJSONFile(filename, sth); err != nil { - return nil, err - } - return sth, nil -} - -func sha256sum(data []byte) []byte { - sum := sha256.Sum256(data) - return sum[:] -} - -func sha256hex(data []byte) string { - return hex.EncodeToString(sha256sum(data)) -} diff -Nru certspotter-0.14.0/cmd/log_state.go certspotter-0.15.1/cmd/log_state.go --- certspotter-0.14.0/cmd/log_state.go 2022-06-13 15:23:35.000000000 +0000 +++ certspotter-0.15.1/cmd/log_state.go 1970-01-01 00:00:00.000000000 +0000 @@ -1,145 +0,0 @@ -// Copyright (C) 2017 Opsmate, Inc. -// -// This Source Code Form is subject to the terms of the Mozilla -// Public License, v. 2.0. If a copy of the MPL was not distributed -// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. -// -// This software is distributed WITHOUT A WARRANTY OF ANY KIND. -// See the Mozilla Public License for details. - -package cmd - -import ( - "crypto/sha256" - "encoding/base64" - "encoding/binary" - "fmt" - "os" - "path/filepath" - "strconv" - "strings" - - "software.sslmate.com/src/certspotter" - "software.sslmate.com/src/certspotter/ct" -) - -type LogState struct { - path string -} - -// generate a filename that uniquely identifies the STH (within the context of a particular log) -func sthFilename(sth *ct.SignedTreeHead) string { - hasher := sha256.New() - switch sth.Version { - case ct.V1: - binary.Write(hasher, binary.LittleEndian, sth.Timestamp) - binary.Write(hasher, binary.LittleEndian, sth.SHA256RootHash) - default: - panic(fmt.Sprintf("Unsupported STH version %d", sth.Version)) - } - // For 6962-bis, we will need to handle a variable-length root hash, and include the signature in the filename hash (since signatures must be deterministic) - return strconv.FormatUint(sth.TreeSize, 10) + "-" + base64.RawURLEncoding.EncodeToString(hasher.Sum(nil)) + ".json" -} - -func makeLogStateDir(logStatePath string) error { - if err := os.Mkdir(logStatePath, 0777); err != nil && !os.IsExist(err) { - return fmt.Errorf("%s: %s", logStatePath, err) - } - for _, subdir := range []string{"unverified_sths"} { - path := filepath.Join(logStatePath, subdir) - if err := os.Mkdir(path, 0777); err != nil && !os.IsExist(err) { - return fmt.Errorf("%s: %s", path, err) - } - } - return nil -} - -func OpenLogState(logStatePath string) (*LogState, error) { - if err := makeLogStateDir(logStatePath); err != nil { - return nil, fmt.Errorf("Error creating log state directory: %s", err) - } - return &LogState{path: logStatePath}, nil -} - -func (logState *LogState) VerifiedSTHFilename() string { - return filepath.Join(logState.path, "sth.json") -} - -func (logState *LogState) GetVerifiedSTH() (*ct.SignedTreeHead, error) { - sth, err := readSTHFile(logState.VerifiedSTHFilename()) - if err != nil { - if os.IsNotExist(err) { - return nil, nil - } else { - return nil, err - } - } - return sth, nil -} - -func (logState *LogState) StoreVerifiedSTH(sth *ct.SignedTreeHead) error { - return writeJSONFile(logState.VerifiedSTHFilename(), sth, 0666) -} - -func (logState *LogState) GetUnverifiedSTHs() ([]*ct.SignedTreeHead, error) { - dir, err := os.Open(filepath.Join(logState.path, "unverified_sths")) - if err != nil { - if os.IsNotExist(err) { - return []*ct.SignedTreeHead{}, nil - } else { - return nil, err - } - } - filenames, err := dir.Readdirnames(0) - if err != nil { - return nil, err - } - - sths := make([]*ct.SignedTreeHead, 0, len(filenames)) - for _, filename := range filenames { - if !strings.HasPrefix(filename, ".") { - sth, _ := readSTHFile(filepath.Join(dir.Name(), filename)) - if sth != nil { - sths = append(sths, sth) - } - } - } - return sths, nil -} - -func (logState *LogState) UnverifiedSTHFilename(sth *ct.SignedTreeHead) string { - return filepath.Join(logState.path, "unverified_sths", sthFilename(sth)) -} - -func (logState *LogState) StoreUnverifiedSTH(sth *ct.SignedTreeHead) error { - filename := logState.UnverifiedSTHFilename(sth) - if fileExists(filename) { - return nil - } - return writeJSONFile(filename, sth, 0666) -} - -func (logState *LogState) RemoveUnverifiedSTH(sth *ct.SignedTreeHead) error { - filename := logState.UnverifiedSTHFilename(sth) - err := os.Remove(filepath.Join(filename)) - if err != nil && !os.IsNotExist(err) { - return err - } - return nil -} - -func (logState *LogState) GetTree() (*certspotter.CollapsedMerkleTree, error) { - tree := new(certspotter.CollapsedMerkleTree) - if err := readJSONFile(filepath.Join(logState.path, "tree.json"), tree); err != nil { - if os.IsNotExist(err) { - return nil, nil - } else { - return nil, err - } - } - return tree, nil -} - -func (logState *LogState) StoreTree(tree *certspotter.CollapsedMerkleTree) error { - return writeJSONFile(filepath.Join(logState.path, "tree.json"), tree, 0666) -} diff -Nru certspotter-0.14.0/cmd/state.go certspotter-0.15.1/cmd/state.go --- certspotter-0.14.0/cmd/state.go 2022-06-13 15:23:35.000000000 +0000 +++ certspotter-0.15.1/cmd/state.go 1970-01-01 00:00:00.000000000 +0000 @@ -1,220 +0,0 @@ -// Copyright (C) 2017 Opsmate, Inc. -// -// This Source Code Form is subject to the terms of the Mozilla -// Public License, v. 2.0. If a copy of the MPL was not distributed -// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. -// -// This software is distributed WITHOUT A WARRANTY OF ANY KIND. -// See the Mozilla Public License for details. - -package cmd - -import ( - "bytes" - "encoding/base64" - "encoding/pem" - "fmt" - "io/ioutil" - "log" - "os" - "path/filepath" - "strconv" - "strings" - - "software.sslmate.com/src/certspotter/ct" - "software.sslmate.com/src/certspotter/loglist" -) - -type State struct { - path string -} - -func legacySTHFilename(logInfo *loglist.Log) string { - return strings.Replace(strings.Replace(logInfo.URL, "://", "_", 1), "/", "_", -1) -} - -func readVersionFile(statePath string) (int, error) { - versionFilePath := filepath.Join(statePath, "version") - versionBytes, err := ioutil.ReadFile(versionFilePath) - if err == nil { - version, err := strconv.Atoi(string(bytes.TrimSpace(versionBytes))) - if err != nil { - return -1, fmt.Errorf("%s: contains invalid integer: %s", versionFilePath, err) - } - if version < 0 { - return -1, fmt.Errorf("%s: contains negative integer", versionFilePath) - } - return version, nil - } else if os.IsNotExist(err) { - if fileExists(filepath.Join(statePath, "sths")) { - // Original version of certspotter had no version file. - // Infer version 0 if "sths" directory is present. - return 0, nil - } - return -1, nil - } else { - return -1, fmt.Errorf("%s: %s", versionFilePath, err) - } -} - -func writeVersionFile(statePath string) error { - version := 1 - versionString := fmt.Sprintf("%d\n", version) - versionFilePath := filepath.Join(statePath, "version") - if err := ioutil.WriteFile(versionFilePath, []byte(versionString), 0666); err != nil { - return fmt.Errorf("%s: %s\n", versionFilePath, err) - } - return nil -} - -func makeStateDir(statePath string) error { - if err := os.Mkdir(statePath, 0777); err != nil && !os.IsExist(err) { - return fmt.Errorf("%s: %s", statePath, err) - } - for _, subdir := range []string{"certs", "logs"} { - path := filepath.Join(statePath, subdir) - if err := os.Mkdir(path, 0777); err != nil && !os.IsExist(err) { - return fmt.Errorf("%s: %s", path, err) - } - } - return nil -} - -func OpenState(statePath string) (*State, error) { - version, err := readVersionFile(statePath) - if err != nil { - return nil, fmt.Errorf("Error reading version file: %s", err) - } - - if version < 1 { - if err := makeStateDir(statePath); err != nil { - return nil, fmt.Errorf("Error creating state directory: %s", err) - } - if version == 0 { - log.Printf("Migrating state directory (%s) to new layout...", statePath) - if err := os.Rename(filepath.Join(statePath, "sths"), filepath.Join(statePath, "legacy_sths")); err != nil { - return nil, fmt.Errorf("Error migrating STHs directory: %s", err) - } - for _, subdir := range []string{"evidence", "legacy_sths"} { - os.Remove(filepath.Join(statePath, subdir)) - } - if err := ioutil.WriteFile(filepath.Join(statePath, "once"), []byte{}, 0666); err != nil { - return nil, fmt.Errorf("Error creating once file: %s", err) - } - } - if err := writeVersionFile(statePath); err != nil { - return nil, fmt.Errorf("Error writing version file: %s", err) - } - } else if version > 1 { - return nil, fmt.Errorf("%s was created by a newer version of Cert Spotter; please remove this directory or upgrade Cert Spotter", statePath) - } - - return &State{path: statePath}, nil -} - -func (state *State) IsFirstRun() bool { - return !fileExists(filepath.Join(state.path, "once")) -} - -func (state *State) WriteOnceFile() error { - if err := ioutil.WriteFile(filepath.Join(state.path, "once"), []byte{}, 0666); err != nil { - return fmt.Errorf("Error writing once file: %s", err) - } - return nil -} - -func (state *State) SaveCert(isPrecert bool, certs [][]byte) (bool, string, error) { - if len(certs) == 0 { - return false, "", fmt.Errorf("Cannot write an empty certificate chain") - } - - fingerprint := sha256hex(certs[0]) - prefixPath := filepath.Join(state.path, "certs", fingerprint[0:2]) - var filenameSuffix string - if isPrecert { - filenameSuffix = ".precert.pem" - } else { - filenameSuffix = ".cert.pem" - } - if err := os.Mkdir(prefixPath, 0777); err != nil && !os.IsExist(err) { - return false, "", fmt.Errorf("Failed to create prefix directory %s: %s", prefixPath, err) - } - path := filepath.Join(prefixPath, fingerprint+filenameSuffix) - file, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0666) - if err != nil { - if os.IsExist(err) { - return true, path, nil - } else { - return false, path, fmt.Errorf("Failed to open %s for writing: %s", path, err) - } - } - for _, cert := range certs { - if err := pem.Encode(file, &pem.Block{Type: "CERTIFICATE", Bytes: cert}); err != nil { - file.Close() - return false, path, fmt.Errorf("Error writing to %s: %s", path, err) - } - } - if err := file.Close(); err != nil { - return false, path, fmt.Errorf("Error writing to %s: %s", path, err) - } - - return false, path, nil -} - -func (state *State) OpenLogState(logInfo *loglist.Log) (*LogState, error) { - return OpenLogState(filepath.Join(state.path, "logs", base64.RawURLEncoding.EncodeToString(logInfo.LogID[:]))) -} - -func (state *State) GetLegacySTH(logInfo *loglist.Log) (*ct.SignedTreeHead, error) { - sth, err := readSTHFile(filepath.Join(state.path, "legacy_sths", legacySTHFilename(logInfo))) - if err != nil { - if os.IsNotExist(err) { - return nil, nil - } else { - return nil, err - } - } - return sth, nil -} -func (state *State) RemoveLegacySTH(logInfo *loglist.Log) error { - err := os.Remove(filepath.Join(state.path, "legacy_sths", legacySTHFilename(logInfo))) - os.Remove(filepath.Join(state.path, "legacy_sths")) - return err -} -func (state *State) LockFilename() string { - return filepath.Join(state.path, "lock") -} -func (state *State) Lock() (bool, error) { - file, err := os.OpenFile(state.LockFilename(), os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0666) - if err != nil { - if os.IsExist(err) { - return false, nil - } else { - return false, err - } - } - if _, err := fmt.Fprintf(file, "%d\n", os.Getpid()); err != nil { - file.Close() - os.Remove(state.LockFilename()) - return false, err - } - if err := file.Close(); err != nil { - os.Remove(state.LockFilename()) - return false, err - } - return true, nil -} -func (state *State) Unlock() error { - return os.Remove(state.LockFilename()) -} -func (state *State) LockingPid() int { - pidBytes, err := ioutil.ReadFile(state.LockFilename()) - if err != nil { - return 0 - } - pid, err := strconv.Atoi(string(bytes.TrimSpace(pidBytes))) - if err != nil { - return 0 - } - return pid -} diff -Nru certspotter-0.14.0/cmd/submitct/main.go certspotter-0.15.1/cmd/submitct/main.go --- certspotter-0.14.0/cmd/submitct/main.go 2022-06-13 15:23:35.000000000 +0000 +++ certspotter-0.15.1/cmd/submitct/main.go 2023-02-09 18:44:06.000000000 +0000 @@ -151,7 +151,7 @@ log.Fatalf("Error reading stdin: %s", err) } - list, err := loglist.Load(*logsURL) + list, err := loglist.Load(context.Background(), *logsURL) if err != nil { log.Fatalf("Error loading log list: %s", err) } diff -Nru certspotter-0.14.0/COPYING certspotter-0.15.1/COPYING --- certspotter-0.14.0/COPYING 2022-06-13 15:23:35.000000000 +0000 +++ certspotter-0.15.1/COPYING 1970-01-01 00:00:00.000000000 +0000 @@ -1,373 +0,0 @@ -Mozilla Public License Version 2.0 -================================== - -1. Definitions --------------- - -1.1. "Contributor" - means each individual or legal entity that creates, contributes to - the creation of, or owns Covered Software. - -1.2. "Contributor Version" - means the combination of the Contributions of others (if any) used - by a Contributor and that particular Contributor's Contribution. - -1.3. "Contribution" - means Covered Software of a particular Contributor. - -1.4. "Covered Software" - means Source Code Form to which the initial Contributor has attached - the notice in Exhibit A, the Executable Form of such Source Code - Form, and Modifications of such Source Code Form, in each case - including portions thereof. - -1.5. "Incompatible With Secondary Licenses" - means - - (a) that the initial Contributor has attached the notice described - in Exhibit B to the Covered Software; or - - (b) that the Covered Software was made available under the terms of - version 1.1 or earlier of the License, but not also under the - terms of a Secondary License. - -1.6. "Executable Form" - means any form of the work other than Source Code Form. - -1.7. "Larger Work" - means a work that combines Covered Software with other material, in - a separate file or files, that is not Covered Software. - -1.8. "License" - means this document. - -1.9. "Licensable" - means having the right to grant, to the maximum extent possible, - whether at the time of the initial grant or subsequently, any and - all of the rights conveyed by this License. - -1.10. "Modifications" - means any of the following: - - (a) any file in Source Code Form that results from an addition to, - deletion from, or modification of the contents of Covered - Software; or - - (b) any new file in Source Code Form that contains any Covered - Software. - -1.11. "Patent Claims" of a Contributor - means any patent claim(s), including without limitation, method, - process, and apparatus claims, in any patent Licensable by such - Contributor that would be infringed, but for the grant of the - License, by the making, using, selling, offering for sale, having - made, import, or transfer of either its Contributions or its - Contributor Version. - -1.12. "Secondary License" - means either the GNU General Public License, Version 2.0, the GNU - Lesser General Public License, Version 2.1, the GNU Affero General - Public License, Version 3.0, or any later versions of those - licenses. - -1.13. "Source Code Form" - means the form of the work preferred for making modifications. - -1.14. "You" (or "Your") - means an individual or a legal entity exercising rights under this - License. For legal entities, "You" includes any entity that - controls, is controlled by, or is under common control with You. For - purposes of this definition, "control" means (a) the power, direct - or indirect, to cause the direction or management of such entity, - whether by contract or otherwise, or (b) ownership of more than - fifty percent (50%) of the outstanding shares or beneficial - ownership of such entity. - -2. License Grants and Conditions --------------------------------- - -2.1. Grants - -Each Contributor hereby grants You a world-wide, royalty-free, -non-exclusive license: - -(a) under intellectual property rights (other than patent or trademark) - Licensable by such Contributor to use, reproduce, make available, - modify, display, perform, distribute, and otherwise exploit its - Contributions, either on an unmodified basis, with Modifications, or - as part of a Larger Work; and - -(b) under Patent Claims of such Contributor to make, use, sell, offer - for sale, have made, import, and otherwise transfer either its - Contributions or its Contributor Version. - -2.2. Effective Date - -The licenses granted in Section 2.1 with respect to any Contribution -become effective for each Contribution on the date the Contributor first -distributes such Contribution. - -2.3. Limitations on Grant Scope - -The licenses granted in this Section 2 are the only rights granted under -this License. No additional rights or licenses will be implied from the -distribution or licensing of Covered Software under this License. -Notwithstanding Section 2.1(b) above, no patent license is granted by a -Contributor: - -(a) for any code that a Contributor has removed from Covered Software; - or - -(b) for infringements caused by: (i) Your and any other third party's - modifications of Covered Software, or (ii) the combination of its - Contributions with other software (except as part of its Contributor - Version); or - -(c) under Patent Claims infringed by Covered Software in the absence of - its Contributions. - -This License does not grant any rights in the trademarks, service marks, -or logos of any Contributor (except as may be necessary to comply with -the notice requirements in Section 3.4). - -2.4. Subsequent Licenses - -No Contributor makes additional grants as a result of Your choice to -distribute the Covered Software under a subsequent version of this -License (see Section 10.2) or under the terms of a Secondary License (if -permitted under the terms of Section 3.3). - -2.5. Representation - -Each Contributor represents that the Contributor believes its -Contributions are its original creation(s) or it has sufficient rights -to grant the rights to its Contributions conveyed by this License. - -2.6. Fair Use - -This License is not intended to limit any rights You have under -applicable copyright doctrines of fair use, fair dealing, or other -equivalents. - -2.7. Conditions - -Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted -in Section 2.1. - -3. Responsibilities -------------------- - -3.1. Distribution of Source Form - -All distribution of Covered Software in Source Code Form, including any -Modifications that You create or to which You contribute, must be under -the terms of this License. You must inform recipients that the Source -Code Form of the Covered Software is governed by the terms of this -License, and how they can obtain a copy of this License. You may not -attempt to alter or restrict the recipients' rights in the Source Code -Form. - -3.2. Distribution of Executable Form - -If You distribute Covered Software in Executable Form then: - -(a) such Covered Software must also be made available in Source Code - Form, as described in Section 3.1, and You must inform recipients of - the Executable Form how they can obtain a copy of such Source Code - Form by reasonable means in a timely manner, at a charge no more - than the cost of distribution to the recipient; and - -(b) You may distribute such Executable Form under the terms of this - License, or sublicense it under different terms, provided that the - license for the Executable Form does not attempt to limit or alter - the recipients' rights in the Source Code Form under this License. - -3.3. Distribution of a Larger Work - -You may create and distribute a Larger Work under terms of Your choice, -provided that You also comply with the requirements of this License for -the Covered Software. If the Larger Work is a combination of Covered -Software with a work governed by one or more Secondary Licenses, and the -Covered Software is not Incompatible With Secondary Licenses, this -License permits You to additionally distribute such Covered Software -under the terms of such Secondary License(s), so that the recipient of -the Larger Work may, at their option, further distribute the Covered -Software under the terms of either this License or such Secondary -License(s). - -3.4. Notices - -You may not remove or alter the substance of any license notices -(including copyright notices, patent notices, disclaimers of warranty, -or limitations of liability) contained within the Source Code Form of -the Covered Software, except that You may alter any license notices to -the extent required to remedy known factual inaccuracies. - -3.5. Application of Additional Terms - -You may choose to offer, and to charge a fee for, warranty, support, -indemnity or liability obligations to one or more recipients of Covered -Software. However, You may do so only on Your own behalf, and not on -behalf of any Contributor. You must make it absolutely clear that any -such warranty, support, indemnity, or liability obligation is offered by -You alone, and You hereby agree to indemnify every Contributor for any -liability incurred by such Contributor as a result of warranty, support, -indemnity or liability terms You offer. You may include additional -disclaimers of warranty and limitations of liability specific to any -jurisdiction. - -4. Inability to Comply Due to Statute or Regulation ---------------------------------------------------- - -If it is impossible for You to comply with any of the terms of this -License with respect to some or all of the Covered Software due to -statute, judicial order, or regulation then You must: (a) comply with -the terms of this License to the maximum extent possible; and (b) -describe the limitations and the code they affect. Such description must -be placed in a text file included with all distributions of the Covered -Software under this License. Except to the extent prohibited by statute -or regulation, such description must be sufficiently detailed for a -recipient of ordinary skill to be able to understand it. - -5. Termination --------------- - -5.1. The rights granted under this License will terminate automatically -if You fail to comply with any of its terms. However, if You become -compliant, then the rights granted under this License from a particular -Contributor are reinstated (a) provisionally, unless and until such -Contributor explicitly and finally terminates Your grants, and (b) on an -ongoing basis, if such Contributor fails to notify You of the -non-compliance by some reasonable means prior to 60 days after You have -come back into compliance. Moreover, Your grants from a particular -Contributor are reinstated on an ongoing basis if such Contributor -notifies You of the non-compliance by some reasonable means, this is the -first time You have received notice of non-compliance with this License -from such Contributor, and You become compliant prior to 30 days after -Your receipt of the notice. - -5.2. If You initiate litigation against any entity by asserting a patent -infringement claim (excluding declaratory judgment actions, -counter-claims, and cross-claims) alleging that a Contributor Version -directly or indirectly infringes any patent, then the rights granted to -You by any and all Contributors for the Covered Software under Section -2.1 of this License shall terminate. - -5.3. In the event of termination under Sections 5.1 or 5.2 above, all -end user license agreements (excluding distributors and resellers) which -have been validly granted by You or Your distributors under this License -prior to termination shall survive termination. - -************************************************************************ -* * -* 6. Disclaimer of Warranty * -* ------------------------- * -* * -* Covered Software is provided under this License on an "as is" * -* basis, without warranty of any kind, either expressed, implied, or * -* statutory, including, without limitation, warranties that the * -* Covered Software is free of defects, merchantable, fit for a * -* particular purpose or non-infringing. The entire risk as to the * -* quality and performance of the Covered Software is with You. * -* Should any Covered Software prove defective in any respect, You * -* (not any Contributor) assume the cost of any necessary servicing, * -* repair, or correction. This disclaimer of warranty constitutes an * -* essential part of this License. No use of any Covered Software is * -* authorized under this License except under this disclaimer. * -* * -************************************************************************ - -************************************************************************ -* * -* 7. Limitation of Liability * -* -------------------------- * -* * -* Under no circumstances and under no legal theory, whether tort * -* (including negligence), contract, or otherwise, shall any * -* Contributor, or anyone who distributes Covered Software as * -* permitted above, be liable to You for any direct, indirect, * -* special, incidental, or consequential damages of any character * -* including, without limitation, damages for lost profits, loss of * -* goodwill, work stoppage, computer failure or malfunction, or any * -* and all other commercial damages or losses, even if such party * -* shall have been informed of the possibility of such damages. This * -* limitation of liability shall not apply to liability for death or * -* personal injury resulting from such party's negligence to the * -* extent applicable law prohibits such limitation. Some * -* jurisdictions do not allow the exclusion or limitation of * -* incidental or consequential damages, so this exclusion and * -* limitation may not apply to You. * -* * -************************************************************************ - -8. Litigation -------------- - -Any litigation relating to this License may be brought only in the -courts of a jurisdiction where the defendant maintains its principal -place of business and such litigation shall be governed by laws of that -jurisdiction, without reference to its conflict-of-law provisions. -Nothing in this Section shall prevent a party's ability to bring -cross-claims or counter-claims. - -9. Miscellaneous ----------------- - -This License represents the complete agreement concerning the subject -matter hereof. If any provision of this License is held to be -unenforceable, such provision shall be reformed only to the extent -necessary to make it enforceable. Any law or regulation which provides -that the language of a contract shall be construed against the drafter -shall not be used to construe this License against a Contributor. - -10. Versions of the License ---------------------------- - -10.1. New Versions - -Mozilla Foundation is the license steward. Except as provided in Section -10.3, no one other than the license steward has the right to modify or -publish new versions of this License. Each version will be given a -distinguishing version number. - -10.2. Effect of New Versions - -You may distribute the Covered Software under the terms of the version -of the License under which You originally received the Covered Software, -or under the terms of any subsequent version published by the license -steward. - -10.3. Modified Versions - -If you create software not governed by this License, and you want to -create a new license for such software, you may create and use a -modified version of this License if you rename the license and remove -any references to the name of the license steward (except to note that -such modified license differs from this License). - -10.4. Distributing Source Code Form that is Incompatible With Secondary -Licenses - -If You choose to distribute Source Code Form that is Incompatible With -Secondary Licenses under the terms of this version of the License, the -notice described in Exhibit B of this License must be attached. - -Exhibit A - Source Code Form License Notice -------------------------------------------- - - This Source Code Form is subject to the terms of the Mozilla Public - License, v. 2.0. If a copy of the MPL was not distributed with this - file, You can obtain one at http://mozilla.org/MPL/2.0/. - -If it is not possible or desirable to put the notice in a particular -file, then You may include the notice in a location (such as a LICENSE -file in a relevant directory) where a recipient would be likely to look -for such a notice. - -You may add additional accurate notices of copyright ownership. - -Exhibit B - "Incompatible With Secondary Licenses" Notice ---------------------------------------------------------- - - This Source Code Form is "Incompatible With Secondary Licenses", as - defined by the Mozilla Public License, v. 2.0. diff -Nru certspotter-0.14.0/ct/client/logclient.go certspotter-0.15.1/ct/client/logclient.go --- certspotter-0.14.0/ct/client/logclient.go 2022-06-13 15:23:35.000000000 +0000 +++ certspotter-0.15.1/ct/client/logclient.go 2023-02-09 18:44:06.000000000 +0000 @@ -47,14 +47,12 @@ return time.Duration(seconds) * time.Second, true } -func sleep(ctx context.Context, duration time.Duration) bool { +func sleep(ctx context.Context, duration time.Duration) { timer := time.NewTimer(duration) defer timer.Stop() select { case <-ctx.Done(): - return false case <-timer.C: - return true } } @@ -71,6 +69,7 @@ type LogClient struct { uri string // the base URI of the log. e.g. http://ct.googleapis/pilot httpClient *http.Client // used to interact with the log via HTTP + verifier *ct.SignatureVerifier // if non-nil, used to verify STH signatures } ////////////////////////////////////////////////////////////////////////////////// @@ -78,7 +77,7 @@ // These represent the structures returned by the CT Log server. ////////////////////////////////////////////////////////////////////////////////// -// getSTHResponse respresents the JSON response to the get-sth CT method +// getSTHResponse represents the JSON response to the get-sth CT method type getSTHResponse struct { TreeSize uint64 `json:"tree_size"` // Number of certs in the current tree Timestamp uint64 `json:"timestamp"` // Time that the tree was created @@ -86,13 +85,13 @@ TreeHeadSignature []byte `json:"tree_head_signature"` // Log signature for this STH } -// base64LeafEntry respresents a Base64 encoded leaf entry +// base64LeafEntry represents a Base64 encoded leaf entry type base64LeafEntry struct { LeafInput []byte `json:"leaf_input"` ExtraData []byte `json:"extra_data"` } -// getEntriesReponse respresents the JSON response to the CT get-entries method +// getEntriesReponse represents the JSON response to the CT get-entries method type getEntriesResponse struct { Entries []base64LeafEntry `json:"entries"` // the list of returned entries } @@ -124,8 +123,13 @@ // |uri| is the base URI of the CT log instance to interact with, e.g. // http://ct.googleapis.com/pilot func New(uri string) *LogClient { + return NewWithVerifier(uri, nil) +} + +func NewWithVerifier(uri string, verifier *ct.SignatureVerifier) *LogClient { var c LogClient c.uri = uri + c.verifier = verifier transport := &http.Transport{ Proxy: http.ProxyFromEnvironment, TLSHandshakeTimeout: 15 * time.Second, @@ -178,10 +182,14 @@ func (c *LogClient) doAndParse(ctx context.Context, method string, uri string, reqBody interface{}, respBody interface{}) error { numRetries := 0 retry: + if ctx.Err() != nil { + return ctx.Err() + } req, err := c.makeRequest(ctx, method, uri, reqBody) if err != nil { return fmt.Errorf("%s %s: error creating request: %w", method, uri, err) } + req.Header.Set("User-Agent", "") // Don't send a User-Agent to make life harder for malicious logs resp, err := c.httpClient.Do(req) if err != nil { if c.shouldRetry(ctx, numRetries, nil) { @@ -213,10 +221,6 @@ } func (c *LogClient) shouldRetry(ctx context.Context, numRetries int, resp *http.Response) bool { - if ctx.Err() != nil { - return false - } - if numRetries == maxRetries { return false } @@ -240,7 +244,8 @@ return false } - return sleep(ctx, delay) + sleep(ctx, delay) + return true } // GetSTH retrieves the current STH from the log. @@ -264,11 +269,44 @@ if err != nil { return nil, err } - // TODO(alcutter): Verify signature sth.TreeHeadSignature = *ds + if c.verifier != nil { + if err := c.verifier.VerifySTHSignature(*sth); err != nil { + return nil, fmt.Errorf("STH returned by server has invalid signature: %w", err) + } + } return } +type GetEntriesItem struct { + LeafInput []byte `json:"leaf_input"` + ExtraData []byte `json:"extra_data"` +} + +// Retrieve the entries in the sequence [start, end] from the CT log server. +// If error is nil, at least one entry is returned, and no excess entries are returned. +// Fewer entries than requested may be returned. +func (c *LogClient) GetRawEntries(ctx context.Context, start, end uint64) ([]GetEntriesItem, error) { + if end < start { + panic("LogClient.GetRawEntries: end < start") + } + var response struct { + Entries []GetEntriesItem `json:"entries"` + } + uri := fmt.Sprintf("%s%s?start=%d&end=%d", c.uri, GetEntriesPath, start, end) + err := c.fetchAndParse(ctx, uri, &response) + if err != nil { + return nil, err + } + if len(response.Entries) == 0 { + return nil, fmt.Errorf("GET %s: log server returned an empty get-entries response", uri) + } + if uint64(len(response.Entries)) > end-start+1 { + return nil, fmt.Errorf("GET %s: log server returned a get-entries response with extraneous entries", uri) + } + return response.Entries, nil +} + // GetEntries attempts to retrieve the entries in the sequence [|start|, |end|] from the CT // log server. (see section 4.6.) // Returns a slice of LeafInputs or a non-nil error. diff -Nru certspotter-0.14.0/ct/types.go certspotter-0.15.1/ct/types.go --- certspotter-0.14.0/ct/types.go 2022-06-13 15:23:35.000000000 +0000 +++ certspotter-0.15.1/ct/types.go 2023-02-09 18:44:06.000000000 +0000 @@ -6,6 +6,7 @@ "encoding/base64" "encoding/json" "fmt" + "time" ) const ( @@ -155,7 +156,7 @@ } } -// SignatureAlgorithm from the the DigitallySigned struct +// SignatureAlgorithm from the DigitallySigned struct type SignatureAlgorithm byte // SignatureAlgorithm constants @@ -259,6 +260,11 @@ return base64.StdEncoding.EncodeToString(s[:]) } +// Returns the raw base64url representation of this SHA256Hash. +func (s SHA256Hash) Base64URLString() string { + return base64.RawURLEncoding.EncodeToString(s[:]) +} + // MarshalJSON implements the json.Marshaller interface for SHA256Hash. func (s SHA256Hash) MarshalJSON() ([]byte, error) { return []byte(`"` + s.Base64String() + `"`), nil @@ -284,6 +290,10 @@ LogID SHA256Hash `json:"log_id"` // The SHA256 hash of the log's public key } +func (sth *SignedTreeHead) TimestampTime() time.Time { + return time.Unix(int64(sth.Timestamp/1000), int64(sth.Timestamp%1000)*1_000_000).UTC() +} + // SignedCertificateTimestamp represents the structure returned by the // add-chain and add-pre-chain methods after base64 decoding. (see RFC sections // 3.2 ,4.1 and 4.2) @@ -291,7 +301,7 @@ SCTVersion Version `json:"sct_version"` // The version of the protocol to which the SCT conforms LogID SHA256Hash `json:"id"` // the SHA-256 hash of the log's public key, calculated over // the DER encoding of the key represented as SubjectPublicKeyInfo. - Timestamp uint64 `json:"timestamp"` // Timestamp (in ms since unix epoc) at which the SCT was issued + Timestamp uint64 `json:"timestamp"` // Timestamp (in ms since unix epoch) at which the SCT was issued Extensions CTExtensions `json:"extensions"` // For future extensions to the protocol Signature DigitallySigned `json:"signature"` // The Log's signature for this SCT } @@ -314,7 +324,7 @@ Extensions CTExtensions } -// MerkleTreeLeaf represents the deserialized sructure of the hash input for the +// MerkleTreeLeaf represents the deserialized structure of the hash input for the // leaves of a log's Merkle tree. See RFC section 3.4 type MerkleTreeLeaf struct { Version Version // the version of the protocol to which the MerkleTreeLeaf corresponds diff -Nru certspotter-0.14.0/debian/changelog certspotter-0.15.1/debian/changelog --- certspotter-0.14.0/debian/changelog 2023-01-08 17:38:06.000000000 +0000 +++ certspotter-0.15.1/debian/changelog 2023-02-14 15:08:50.000000000 +0000 @@ -1,3 +1,27 @@ +certspotter (0.15.1-1) unstable; urgency=medium + + * New upstream release. + - Drop patch -Xmain.Version patch, merged upstream. + - No other upstream changes. + + -- Faidon Liambotis Tue, 14 Feb 2023 17:08:50 +0200 + +certspotter (0.15.0-1) unstable; urgency=medium + + * New upstream release. + - certspotter is now a daemon; drop the systemd timer. + - The experimental -script interface has been reworked, and different + variables are now present. Notably, this resolves some security + considerations, as described by upstream in upstream issue #63. + - Upstream changes broke --version in the Debian package, so add upstream + patch to allow debian/rules to set the version through Go's ldflags. + - Remove debian/man, as the manpages are now merged upstream. + * Add new build dependencies golang-golang-x-exp-dev and + golang-golang-x-sync-dev as required by the new upstream release. + * Bump Standards-Version to 4.6.2, no changes needed. + + -- Faidon Liambotis Thu, 09 Feb 2023 15:28:24 +0200 + certspotter (0.14.0-1) unstable; urgency=medium * New upstream release. diff -Nru certspotter-0.14.0/debian/control certspotter-0.15.1/debian/control --- certspotter-0.14.0/debian/control 2023-01-08 17:38:06.000000000 +0000 +++ certspotter-0.15.1/debian/control 2023-02-09 03:30:03.000000000 +0000 @@ -8,9 +8,11 @@ dh-sequence-golang, dh-sequence-installsysusers, golang-any, + golang-golang-x-exp-dev, golang-golang-x-net-dev, + golang-golang-x-sync-dev, lowdown , -Standards-Version: 4.6.1 +Standards-Version: 4.6.2 Homepage: https://github.com/SSLMate/certspotter Vcs-Browser: https://salsa.debian.org/go-team/packages/certspotter Vcs-Git: https://salsa.debian.org/go-team/packages/certspotter.git diff -Nru certspotter-0.14.0/debian/copyright certspotter-0.15.1/debian/copyright --- certspotter-0.14.0/debian/copyright 2023-01-08 07:11:38.000000000 +0000 +++ certspotter-0.15.1/debian/copyright 2023-02-09 03:29:30.000000000 +0000 @@ -3,7 +3,7 @@ Source: https://github.com/SSLMate/certspotter Files: * -Copyright: 2016-2022 Opsmate, Inc. +Copyright: 2016-2023 Opsmate, Inc. License: MPL-2.0 Files: ct/* diff -Nru certspotter-0.14.0/debian/docs certspotter-0.15.1/debian/docs --- certspotter-0.14.0/debian/docs 2021-01-10 01:09:56.000000000 +0000 +++ certspotter-0.15.1/debian/docs 2023-02-09 03:30:28.000000000 +0000 @@ -1 +1 @@ -README +README.md diff -Nru certspotter-0.14.0/debian/man/certspotter.md certspotter-0.15.1/debian/man/certspotter.md --- certspotter-0.14.0/debian/man/certspotter.md 2023-01-08 16:38:16.000000000 +0000 +++ certspotter-0.15.1/debian/man/certspotter.md 1970-01-01 00:00:00.000000000 +0000 @@ -1,207 +0,0 @@ -# NAME - -certspotter - Certificate Transparency Log Monitor - -# SYNOPSIS - -**certspotter** [`-verbose`] [`-start_at_end`] [`-watchlist` *WATCHLIST*] `...` - -**certspotter** -version - -# DESCRIPTION - -**Cert Spotter** is a Certificate Transparency log monitor from SSLMate that -alerts you when a SSL/TLS certificate is issued for one of your domains. Cert -Spotter is easier than other open source CT monitors, since it does not require -a database. It's also more robust, since it uses a special certificate parser -that ensures it won't miss certificates. - -Cert Spotter is also available as a hosted service by SSLMate, -. - -You can use Cert Spotter to detect: - - * Certificates issued to attackers who have compromised your DNS and - are redirecting your visitors to their malicious site. - * Certificates issued to attackers who have taken over an abandoned - sub-domain in order to serve malware under your name. - * Certificates issued to attackers who have compromised a certificate - authority and want to impersonate your site. - * Certificates issued in violation of your corporate policy - or outside of your centralized certificate procurement process. - -# OPTIONS - -`-all_time` - -: Scan certs from all time, not just those logged since the previous run of - Cert Spotter. - -`-batch_size int` - -: Max number of entries to request at per call to get-entries. This is - advanced option. Defaults to 1000. - -`-logs string` - -: Filename or HTTPS URL of a JSON file containing logs to monitor, in the - format documented at . - Defaults to , which includes - the union of active logs recognized by Chrome and Apple. - -`-no_save` - -: Do not save a copy of matching certificates. - -`-num_workers int` - -: Number of concurrent matchers. Default 2. - -`-script string` - -: Script to execute when a matching certificate is found. See - certspotter-script(8) for information about the interface to scripts. - -`-start_at_end` - -: Start monitoring logs from the end rather than the beginning. - - **WARNING**: mnitoring from the beginning guarantees detection of all - certificates, but requires downloading hundreds of millions of - certificates, which takes days. - -`-state_dir string` - -: Directory for storing state. Defaults to "~/.certspotter". - -`-verbose` - -: Be verbose. - -`-version` - -: Print version and exit. - -`-watchlist string` - -: File containing identifiers to watch. Use `-` for stdin. - Defaults to "~/.certspotter/watchlist". - -# NOTES - -## Method of operation - -Every time you run Cert Spotter, it scans all browser-recognized -Certificate Transparency logs for certificates matching domains on -your watch list. When Cert Spotter detects a matching certificate, it -writes a report to standard out. - -Cert Spotter also saves a copy of matching certificates in -`~/.certspotter/certs` (unless you specify the `-no_save` option). - -When Cert Spotter has previously monitored a log, it scans the log -from the previous position, to avoid downloading the same log entry -more than once. (To override this behavior and scan all logs from the -beginning, specify the `-all_time` option.) - -When Cert Spotter has not previously monitored a log, it can either start -monitoring the log from the beginning, or seek to the end of the log and -start monitoring from there. Monitoring from the beginning guarantees -detection of all certificates, but requires downloading hundreds of -millions of certificates, which takes days. The default behavior is to -monitor from the beginning. To start monitoring new logs from the end, -specify the `-start_at_end` option. - -You can add and remove domains on your watchlist at any time. However, -the certspotter command only notifies you of certificates that were -logged since adding a domain to the watchlist, unless you specify the -`-all_time` option, which requires scanning the entirety of every log -and takes many days to complete with a fast Internet connection. -To examine preexisting certificates, it's better to use the Cert -Spotter service , the Cert Spotter -API , or a CT search engine such -as . - -## Coverage - -Any certificate that is logged to a Certificate Transparency log trusted by -Chromium will be detected by Cert Spotter. All certificates issued after April -30, 2018 must be logged to such a log to be trusted by Chromium. - -Generally, certificate authorities will automatically submit certificates -to logs so that they will work in Chromium. In addition, certificates -that are discovered during Internet-wide scans are submitted to Certificate -Transparency logs. - -## Bygone certificates - -Cert Spotter can also notify users of bygone SSL certificates, which are SSL -certificates that outlived their prior domain owner's registration into the -next owners registration. To detect these certificates add a `valid_at` argument -to each domain in the watchlist followed by the date the domain was registered -in the following format YYYY-MM-DD. For example: - -``` -example.com valid_at:2014-05-02 -``` - -## Security considerations - -Cert Spotter assumes an adversarial model in which an attacker produces a -certificate that is accepted by at least some clients but goes undetected -because of an encoding error that prevents CT monitors from understanding it. -To defend against this attack, Cert Spotter uses a special certificate parser -that keeps the certificate unparsed except for the identifiers. If one of the -identifiers matches a domain on your watchlist, you will be notified, even if -other parts of the certificate are unparsable. - -Cert Spotter takes special precautions to ensure identifiers are parsed -correctly, and implements defenses against identifier-based attacks. For -instance, if a DNS identifier contains a null byte, Cert Spotter interprets it -as two identifiers: the complete identifier, and the identifier formed by -truncating at the first null byte. For example, a certificate for -*example.org\0.example.com* will alert the owners of both `example.org` and -*example.com*. This defends against null prefix attacks -. - -SSLMate continuously monitors CT logs to make sure every certificate's -identifiers can be successfully parsed, and will release updates to Cert -Spotter as necessary to fix parsing failures. - -Cert Spotter understands wildcard and redacted DNS names, and will alert you if -a wildcard or redacted certificate might match an identifier on your watchlist. -For example, a watchlist entry for *sub.example.com* would match certificates for -*\*.example.com* or *?.example.com*. - -Cert Spotter is not just a log monitor, but also a log auditor which checks -that the log is obeying its append-only property. A future release of Cert -Spotter will support gossiping with other log monitors to ensure the log is -presenting a single view. - -# EXIT STATUS - -certspotter exits 0 on success, 1 on any error. - -# ENVIRONMENT - -`CERTSPOTTER_STATE_DIR` - -: Directory for storing state. Overridden by `-state_dir`. Defaults to - `~/.certspotter`. - -`CERTSPOTTER_CONFIG_DIR` - -: Directory from which any configuration, such as the watchlist, is read. - Defaults to `~/.certspotter`. - -# SEE ALSO - -certspotter-script(8) - -# COPYRIGHT - -Copyright (c) 2016-2022 Opsmate, Inc. - -# BUGS - -Report bugs to . diff -Nru certspotter-0.14.0/debian/man/certspotter-script.md certspotter-0.15.1/debian/man/certspotter-script.md --- certspotter-0.14.0/debian/man/certspotter-script.md 2023-01-08 17:02:04.000000000 +0000 +++ certspotter-0.15.1/debian/man/certspotter-script.md 1970-01-01 00:00:00.000000000 +0000 @@ -1,125 +0,0 @@ -# NAME - -certspotter-script - Certificate Transparency Log Monitor (hook script) - -# SYNOPSIS - -**certspotter-script** - -# DESCRIPTION - -**certspotter-script** is *any* program that is called from **certspotter**'s -*-script* argument. **certspotter** executes this script when a file from the -CT log matches against the watchlist. - -# ENVIRONMENT - -The script will have the following in its environment: - -## Entry information - -`CERT_FILENAME` - -: The path of the saved certificate on the local filesystem, if one exists. - -`CERT_TYPE` - -: The certificate's type ("cert" or "precert"). - -`FINGERPRINT` - -: The certificate's fingerprint. - -`LOG_URI` - -: The URI of the log the certificate was found on. - -`ENTRY_INDEX` - -: The entry's index in the log. - -`CERT_PARSEABLE` - -: Whether the certificate could be parsed. - -## Identifiers - -`DNS_NAMES` - -: A comma-separated list of the certificate's dnsNames. - -`IP_ADDRESSES` - -: A comma-separated list of the certificate's IP addresses. - -## Certificate information - -`PUBKEY_HASH` - -: The certificate public key's hash. - -`SERIAL` - -: The certificate's serial. - -`NOT_BEFORE`, `NOT_AFTER` - -: The certificate's validity information, as a string. - -`NOT_BEFORE_UNIXTIME`, `NOT_AFTER_UNIXTIME` - -: The certificate's validity information, as UNIX time. - -`SUBJECT_DN` - -: The certificate's subject distinguished name (DN). - -`ISSUER_DN` - -: the certificate issuer distinguished name (DN). - -## Errors - -`PARSE_ERROR` - -: Set to the error that occurred when attempting to extract information about - the certificate. In this case, `CERT_PARSEABLE` will also be set to "no" - and information such as `PUBKEY_HASH`, `SERIAL`, as well as validity and - subject, will not be present. - -`SERIAL_PARSE_ERROR` - -: Set to the error that occurred when attempting to extract the certificate's - serial. Emitted instead of `SERIAL`. - -`IDENTIFIERS_PARSE_ERROR` - -: Set to the error that occurred when attempting to extract the certificate's - identifiers. Emitted instead of `DNS_NAMES`, `IP_ADDRESSES`. - -`VALIDITY_PARSE_ERROR` - -: Set to the error that occurred when attempting to extract the certificate's - validity information. Emitted instead of `NOT_BEFORE`, `NOT_AFTER`. - -`SUBJECT_PARSE_ERROR` - -: Set to the error that occurred when attempting to extract the certificate's - subject information. Emitted instead of `SUBJECT_DN`. - -`ISSUER_PARSE_ERROR` - -: Set to the error that occurred when attempting to extract the certificate's - issuer information. Emitted instead of `ISSUER_DN`. - -# SEE ALSO - -certspotter(8) - -# COPYRIGHT - -Copyright (c) 2016-2022 Opsmate, Inc. - -# BUGS - -Report bugs to . diff -Nru certspotter-0.14.0/debian/rules certspotter-0.15.1/debian/rules --- certspotter-0.14.0/debian/rules 2023-01-08 17:02:04.000000000 +0000 +++ certspotter-0.15.1/debian/rules 2023-02-09 14:02:03.000000000 +0000 @@ -1,13 +1,13 @@ #!/usr/bin/make -f +include /usr/share/dpkg/pkg-info.mk +export GOFLAGS = -ldflags=-X=main.Version=$(DEB_VERSION_UPSTREAM) + export DEB_BUILD_MAINT_OPTIONS = hardening=+all %: dh $@ --buildsystem=golang -override_dh_installchangelogs: - dh_installchangelogs NEWS - # the source's root is a library that includes subdirectories that are # sub-packages, such as "ct" (Google's CT code) and "cmd/certspotter" and # "cmd/ctparsewatch" for the actual resulting binaries. We actually want to @@ -22,7 +22,7 @@ ifeq (,$(findstring nodoc,$(DEB_BUILD_OPTIONS) $(DEB_BUILD_PROFILES))) MANPAGES = debian/certspotter.8 debian/certspotter-script.8 -debian/%.8: debian/man/%.md debian/changelog +debian/%.8: man/%.md debian/changelog lowdown -s -Tman \ -M title:$(basename $(notdir $@)) \ -M section:$(subst .,,$(suffix $@)) \ diff -Nru certspotter-0.14.0/debian/service certspotter-0.15.1/debian/service --- certspotter-0.14.0/debian/service 2023-01-08 17:02:04.000000000 +0000 +++ certspotter-0.15.1/debian/service 2023-02-09 14:02:03.000000000 +0000 @@ -6,7 +6,7 @@ ConditionPathExists=/etc/certspotter/watchlist [Service] -Type=oneshot +Type=simple User=_certspotter Group=_certspotter ExecCondition=grep -q -E -v '^\s*(#|$)' /etc/certspotter/watchlist @@ -16,3 +16,6 @@ CacheDirectory=certspotter # not strict, because we want to allow some flexibility to hooks ProtectSystem=full + +[Install] +WantedBy=multi-user.target diff -Nru certspotter-0.14.0/debian/tests/control certspotter-0.15.1/debian/tests/control --- certspotter-0.14.0/debian/tests/control 2023-01-08 17:38:06.000000000 +0000 +++ certspotter-0.15.1/debian/tests/control 2023-02-09 14:02:03.000000000 +0000 @@ -2,6 +2,7 @@ Test-Command: certspotter --version Depends: @ Restrictions: superficial +Features: test-name=version-check # Do a sample run against the (production) CT logs Tests: run-with-root-domain diff -Nru certspotter-0.14.0/debian/tests/run-with-root-domain certspotter-0.15.1/debian/tests/run-with-root-domain --- certspotter-0.14.0/debian/tests/run-with-root-domain 2023-01-08 17:38:06.000000000 +0000 +++ certspotter-0.15.1/debian/tests/run-with-root-domain 2023-02-09 14:02:03.000000000 +0000 @@ -6,6 +6,9 @@ if [ -z "${AUTOPKGTEST_TMP:-}" ]; then AUTOPKGTEST_TMP="$(mktemp -d --suffix=.autopkgtest)" fi +if [ -z "${AUTOPKGTEST_ARTIFACTS:-}" ]; then + AUTOPKGTEST_ARTIFACTS=$AUTOPKGTEST_TMP +fi # certspotter uses these to override the default ~/.certspotter path. # We use the environment variables (rather than -watchlist and -state_dir) @@ -18,18 +21,9 @@ # monitor all domains - this should always have traffic echo '.' > $CERTSPOTTER_CONFIG_DIR/watchlist -# initialize the certspotter database, starting at the end -# -# pass -no_save, as otherwise the output can be several hundred megabytes -# (depending on the timing) -certspotter -no_save -start_at_end - -# flaky, likely due to piping -rm -f $CERTSPOTTER_STATE_DIR/lock - -# give some time for the CT logs to advance; typically a second's worth of logs -# contains multiple hundred entries -sleep 1 +# start the log from the beginning, and fetch some lines of output, which +# should have plenty of certificates in it +certspotter -no_save -stdout | head -n 500 > $AUTOPKGTEST_ARTIFACTS/stdout # now check for at least one logged certificate -certspotter -no_save -start_at_end | grep -q Pubkey +grep -q Pubkey $AUTOPKGTEST_ARTIFACTS/stdout diff -Nru certspotter-0.14.0/debian/timer certspotter-0.15.1/debian/timer --- certspotter-0.14.0/debian/timer 2023-01-08 17:02:04.000000000 +0000 +++ certspotter-0.15.1/debian/timer 1970-01-01 00:00:00.000000000 +0000 @@ -1,10 +0,0 @@ -[Unit] -Description=Certificate Transparency Log Monitor -Documentation=man:certspotter(8) - -[Timer] -OnCalendar=hourly -Persistent=true - -[Install] -WantedBy=timers.target diff -Nru certspotter-0.14.0/go.mod certspotter-0.15.1/go.mod --- certspotter-0.14.0/go.mod 2022-06-13 15:23:35.000000000 +0000 +++ certspotter-0.15.1/go.mod 2023-02-09 18:44:06.000000000 +0000 @@ -1,8 +1,11 @@ module software.sslmate.com/src/certspotter -go 1.17 +go 1.19 require ( - golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4 // indirect - golang.org/x/text v0.3.7 // indirect + golang.org/x/exp v0.0.0-20230203172020-98cc5a0785f9 + golang.org/x/net v0.5.0 + golang.org/x/sync v0.1.0 ) + +require golang.org/x/text v0.6.0 // indirect diff -Nru certspotter-0.14.0/go.sum certspotter-0.15.1/go.sum --- certspotter-0.14.0/go.sum 2022-06-13 15:23:35.000000000 +0000 +++ certspotter-0.15.1/go.sum 2023-02-09 18:44:06.000000000 +0000 @@ -1,4 +1,8 @@ -golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4 h1:HVyaeDAYux4pnY+D/SiwmLOR36ewZ4iGQIIrtnuCjFA= -golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= -golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/exp v0.0.0-20230203172020-98cc5a0785f9 h1:frX3nT9RkKybPnjyI+yvZh6ZucTZatCCEm9D47sZ2zo= +golang.org/x/exp v0.0.0-20230203172020-98cc5a0785f9/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= +golang.org/x/net v0.5.0 h1:GyT4nK/YDHSqa1c4753ouYCDajOYKTja9Xb/OHtgvSw= +golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= +golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/text v0.6.0 h1:3XmdazWV+ubf7QgHSTWeykHOci5oeekaGJBLkrkaw4k= +golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= diff -Nru certspotter-0.14.0/helpers.go certspotter-0.15.1/helpers.go --- certspotter-0.14.0/helpers.go 2022-06-13 15:23:35.000000000 +0000 +++ certspotter-0.15.1/helpers.go 2023-02-09 18:44:06.000000000 +0000 @@ -10,105 +10,16 @@ package certspotter import ( - "bytes" - "crypto/sha256" - "encoding/hex" - "encoding/json" "fmt" - "io" - "io/ioutil" "math/big" - "os" - "os/exec" - "strconv" - "strings" - "time" "software.sslmate.com/src/certspotter/ct" ) -func ReadSTHFile(path string) (*ct.SignedTreeHead, error) { - content, err := ioutil.ReadFile(path) - if err != nil { - if os.IsNotExist(err) { - return nil, nil - } - return nil, err - } - - var sth ct.SignedTreeHead - if err := json.Unmarshal(content, &sth); err != nil { - return nil, err - } - - return &sth, nil -} - -func WriteSTHFile(path string, sth *ct.SignedTreeHead) error { - sthJson, err := json.MarshalIndent(sth, "", "\t") - if err != nil { - return err - } - sthJson = append(sthJson, byte('\n')) - return ioutil.WriteFile(path, sthJson, 0666) -} - -func WriteProofFile(path string, proof ct.ConsistencyProof) error { - proofJson, err := json.MarshalIndent(proof, "", "\t") - if err != nil { - return err - } - proofJson = append(proofJson, byte('\n')) - return ioutil.WriteFile(path, proofJson, 0666) -} - func IsPrecert(entry *ct.LogEntry) bool { return entry.Leaf.TimestampedEntry.EntryType == ct.PrecertLogEntryType } -func GetFullChain(entry *ct.LogEntry) [][]byte { - certs := make([][]byte, 0, len(entry.Chain)+1) - - if entry.Leaf.TimestampedEntry.EntryType == ct.X509LogEntryType { - certs = append(certs, entry.Leaf.TimestampedEntry.X509Entry) - } - for _, cert := range entry.Chain { - certs = append(certs, cert) - } - - return certs -} - -func formatSerialNumber(serial *big.Int) string { - if serial != nil { - return fmt.Sprintf("%x", serial) - } else { - return "" - } -} - -func sha256sum(data []byte) []byte { - sum := sha256.Sum256(data) - return sum[:] -} - -func sha256hex(data []byte) string { - return hex.EncodeToString(sha256sum(data)) -} - -type EntryInfo struct { - LogUri string - Entry *ct.LogEntry - IsPrecert bool - FullChain [][]byte // first entry is logged X509 cert or pre-cert - CertInfo *CertInfo - ParseError error // set iff CertInfo is nil - Identifiers *Identifiers - IdentifiersParseError error - Filename string - Bygone bool -} - type CertInfo struct { TBS *TBSCertificate @@ -170,202 +81,6 @@ } } -func (info *CertInfo) NotBefore() *time.Time { - if info.ValidityParseError == nil { - return &info.Validity.NotBefore - } else { - return nil - } -} - -func (info *CertInfo) NotAfter() *time.Time { - if info.ValidityParseError == nil { - return &info.Validity.NotAfter - } else { - return nil - } -} - -func (info *CertInfo) PubkeyHash() string { - return sha256hex(info.TBS.GetRawPublicKey()) -} - -func (info *CertInfo) PubkeyHashBytes() []byte { - return sha256sum(info.TBS.GetRawPublicKey()) -} - -func (info *CertInfo) Environ() []string { - env := make([]string, 0, 10) - - env = append(env, "PUBKEY_HASH="+info.PubkeyHash()) - - if info.SerialNumberParseError != nil { - env = append(env, "SERIAL_PARSE_ERROR="+info.SerialNumberParseError.Error()) - } else { - env = append(env, "SERIAL="+formatSerialNumber(info.SerialNumber)) - } - - if info.ValidityParseError != nil { - env = append(env, "VALIDITY_PARSE_ERROR="+info.ValidityParseError.Error()) - } else { - env = append(env, "NOT_BEFORE="+info.Validity.NotBefore.String()) - env = append(env, "NOT_BEFORE_UNIXTIME="+strconv.FormatInt(info.Validity.NotBefore.Unix(), 10)) - env = append(env, "NOT_AFTER="+info.Validity.NotAfter.String()) - env = append(env, "NOT_AFTER_UNIXTIME="+strconv.FormatInt(info.Validity.NotAfter.Unix(), 10)) - } - - if info.SubjectParseError != nil { - env = append(env, "SUBJECT_PARSE_ERROR="+info.SubjectParseError.Error()) - } else { - env = append(env, "SUBJECT_DN="+info.Subject.String()) - } - - if info.IssuerParseError != nil { - env = append(env, "ISSUER_PARSE_ERROR="+info.IssuerParseError.Error()) - } else { - env = append(env, "ISSUER_DN="+info.Issuer.String()) - } - - // TODO: include SANs in environment - - return env -} - -func (info *EntryInfo) HasParseErrors() bool { - return info.ParseError != nil || - info.IdentifiersParseError != nil || - info.CertInfo.SubjectParseError != nil || - info.CertInfo.IssuerParseError != nil || - info.CertInfo.SANsParseError != nil || - info.CertInfo.SerialNumberParseError != nil || - info.CertInfo.ValidityParseError != nil || - info.CertInfo.IsCAParseError != nil -} - -func (info *EntryInfo) Fingerprint() string { - if len(info.FullChain) > 0 { - return sha256hex(info.FullChain[0]) - } else { - return "" - } -} - -func (info *EntryInfo) FingerprintBytes() []byte { - if len(info.FullChain) > 0 { - return sha256sum(info.FullChain[0]) - } else { - return []byte{} - } -} - -func (info *EntryInfo) typeString() string { - if info.IsPrecert { - return "precert" - } else { - return "cert" - } -} - -func (info *EntryInfo) typeFriendlyString() string { - if info.IsPrecert { - return "Pre-certificate" - } else { - return "Certificate" - } -} - -func yesnoString(value bool) string { - if value { - return "yes" - } else { - return "no" - } -} - -func (info *EntryInfo) Environ() []string { - env := []string{ - "FINGERPRINT=" + info.Fingerprint(), - "CERT_TYPE=" + info.typeString(), - "CERT_PARSEABLE=" + yesnoString(info.ParseError == nil), - "LOG_URI=" + info.LogUri, - "ENTRY_INDEX=" + strconv.FormatInt(info.Entry.Index, 10), - } - - if info.Filename != "" { - env = append(env, "CERT_FILENAME="+info.Filename) - } - if info.ParseError != nil { - env = append(env, "PARSE_ERROR="+info.ParseError.Error()) - } else if info.CertInfo != nil { - certEnv := info.CertInfo.Environ() - env = append(env, certEnv...) - } - if info.IdentifiersParseError != nil { - env = append(env, "IDENTIFIERS_PARSE_ERROR="+info.IdentifiersParseError.Error()) - } else if info.Identifiers != nil { - env = append(env, "DNS_NAMES="+info.Identifiers.dnsNamesString(",")) - env = append(env, "IP_ADDRESSES="+info.Identifiers.ipAddrsString(",")) - } - - return env -} - -func writeField(out io.Writer, name string, value interface{}, err error) { - if err == nil { - fmt.Fprintf(out, "\t%13s = %s\n", name, value) - } else { - fmt.Fprintf(out, "\t%13s = *** UNKNOWN (%s) ***\n", name, err) - } -} - -func (info *EntryInfo) Write(out io.Writer) { - fingerprint := info.Fingerprint() - fmt.Fprintf(out, "%s:\n", fingerprint) - if info.IdentifiersParseError != nil { - writeField(out, "Identifiers", nil, info.IdentifiersParseError) - } else if info.Identifiers != nil { - for _, dnsName := range info.Identifiers.DNSNames { - writeField(out, "DNS Name", dnsName, nil) - } - for _, ipaddr := range info.Identifiers.IPAddrs { - writeField(out, "IP Address", ipaddr, nil) - } - } - if info.ParseError != nil { - writeField(out, "Parse Error", "*** "+info.ParseError.Error()+" ***", nil) - } else if info.CertInfo != nil { - writeField(out, "Pubkey", info.CertInfo.PubkeyHash(), nil) - writeField(out, "Issuer", info.CertInfo.Issuer, info.CertInfo.IssuerParseError) - writeField(out, "Not Before", info.CertInfo.NotBefore(), info.CertInfo.ValidityParseError) - writeField(out, "Not After", info.CertInfo.NotAfter(), info.CertInfo.ValidityParseError) - if info.Bygone { - writeField(out, "BygoneSSL", "True", info.CertInfo.ValidityParseError) - } - } - writeField(out, "Log Entry", fmt.Sprintf("%d @ %s (%s)", info.Entry.Index, info.LogUri, info.typeFriendlyString()), nil) - writeField(out, "crt.sh", "https://crt.sh/?sha256="+fingerprint, nil) - if info.Filename != "" { - writeField(out, "Filename", info.Filename, nil) - } -} - -func (info *EntryInfo) InvokeHookScript(command string) error { - cmd := exec.Command(command) - cmd.Env = os.Environ() - infoEnv := info.Environ() - cmd.Env = append(cmd.Env, infoEnv...) - stderrBuffer := bytes.Buffer{} - cmd.Stderr = &stderrBuffer - if err := cmd.Run(); err != nil { - if _, isExitError := err.(*exec.ExitError); isExitError { - return fmt.Errorf("Script failed: %s: %s", command, strings.TrimSpace(stderrBuffer.String())) - } else { - return fmt.Errorf("Failed to execute script: %s: %s", command, err) - } - } - return nil -} - func MatchesWildcard(dnsName string, pattern string) bool { for len(pattern) > 0 { if pattern[0] == '*' { diff -Nru certspotter-0.14.0/LICENSE certspotter-0.15.1/LICENSE --- certspotter-0.14.0/LICENSE 1970-01-01 00:00:00.000000000 +0000 +++ certspotter-0.15.1/LICENSE 2023-02-09 18:44:06.000000000 +0000 @@ -0,0 +1,373 @@ +Mozilla Public License Version 2.0 +================================== + +1. Definitions +-------------- + +1.1. "Contributor" + means each individual or legal entity that creates, contributes to + the creation of, or owns Covered Software. + +1.2. "Contributor Version" + means the combination of the Contributions of others (if any) used + by a Contributor and that particular Contributor's Contribution. + +1.3. "Contribution" + means Covered Software of a particular Contributor. + +1.4. "Covered Software" + means Source Code Form to which the initial Contributor has attached + the notice in Exhibit A, the Executable Form of such Source Code + Form, and Modifications of such Source Code Form, in each case + including portions thereof. + +1.5. "Incompatible With Secondary Licenses" + means + + (a) that the initial Contributor has attached the notice described + in Exhibit B to the Covered Software; or + + (b) that the Covered Software was made available under the terms of + version 1.1 or earlier of the License, but not also under the + terms of a Secondary License. + +1.6. "Executable Form" + means any form of the work other than Source Code Form. + +1.7. "Larger Work" + means a work that combines Covered Software with other material, in + a separate file or files, that is not Covered Software. + +1.8. "License" + means this document. + +1.9. "Licensable" + means having the right to grant, to the maximum extent possible, + whether at the time of the initial grant or subsequently, any and + all of the rights conveyed by this License. + +1.10. "Modifications" + means any of the following: + + (a) any file in Source Code Form that results from an addition to, + deletion from, or modification of the contents of Covered + Software; or + + (b) any new file in Source Code Form that contains any Covered + Software. + +1.11. "Patent Claims" of a Contributor + means any patent claim(s), including without limitation, method, + process, and apparatus claims, in any patent Licensable by such + Contributor that would be infringed, but for the grant of the + License, by the making, using, selling, offering for sale, having + made, import, or transfer of either its Contributions or its + Contributor Version. + +1.12. "Secondary License" + means either the GNU General Public License, Version 2.0, the GNU + Lesser General Public License, Version 2.1, the GNU Affero General + Public License, Version 3.0, or any later versions of those + licenses. + +1.13. "Source Code Form" + means the form of the work preferred for making modifications. + +1.14. "You" (or "Your") + means an individual or a legal entity exercising rights under this + License. For legal entities, "You" includes any entity that + controls, is controlled by, or is under common control with You. For + purposes of this definition, "control" means (a) the power, direct + or indirect, to cause the direction or management of such entity, + whether by contract or otherwise, or (b) ownership of more than + fifty percent (50%) of the outstanding shares or beneficial + ownership of such entity. + +2. License Grants and Conditions +-------------------------------- + +2.1. Grants + +Each Contributor hereby grants You a world-wide, royalty-free, +non-exclusive license: + +(a) under intellectual property rights (other than patent or trademark) + Licensable by such Contributor to use, reproduce, make available, + modify, display, perform, distribute, and otherwise exploit its + Contributions, either on an unmodified basis, with Modifications, or + as part of a Larger Work; and + +(b) under Patent Claims of such Contributor to make, use, sell, offer + for sale, have made, import, and otherwise transfer either its + Contributions or its Contributor Version. + +2.2. Effective Date + +The licenses granted in Section 2.1 with respect to any Contribution +become effective for each Contribution on the date the Contributor first +distributes such Contribution. + +2.3. Limitations on Grant Scope + +The licenses granted in this Section 2 are the only rights granted under +this License. No additional rights or licenses will be implied from the +distribution or licensing of Covered Software under this License. +Notwithstanding Section 2.1(b) above, no patent license is granted by a +Contributor: + +(a) for any code that a Contributor has removed from Covered Software; + or + +(b) for infringements caused by: (i) Your and any other third party's + modifications of Covered Software, or (ii) the combination of its + Contributions with other software (except as part of its Contributor + Version); or + +(c) under Patent Claims infringed by Covered Software in the absence of + its Contributions. + +This License does not grant any rights in the trademarks, service marks, +or logos of any Contributor (except as may be necessary to comply with +the notice requirements in Section 3.4). + +2.4. Subsequent Licenses + +No Contributor makes additional grants as a result of Your choice to +distribute the Covered Software under a subsequent version of this +License (see Section 10.2) or under the terms of a Secondary License (if +permitted under the terms of Section 3.3). + +2.5. Representation + +Each Contributor represents that the Contributor believes its +Contributions are its original creation(s) or it has sufficient rights +to grant the rights to its Contributions conveyed by this License. + +2.6. Fair Use + +This License is not intended to limit any rights You have under +applicable copyright doctrines of fair use, fair dealing, or other +equivalents. + +2.7. Conditions + +Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted +in Section 2.1. + +3. Responsibilities +------------------- + +3.1. Distribution of Source Form + +All distribution of Covered Software in Source Code Form, including any +Modifications that You create or to which You contribute, must be under +the terms of this License. You must inform recipients that the Source +Code Form of the Covered Software is governed by the terms of this +License, and how they can obtain a copy of this License. You may not +attempt to alter or restrict the recipients' rights in the Source Code +Form. + +3.2. Distribution of Executable Form + +If You distribute Covered Software in Executable Form then: + +(a) such Covered Software must also be made available in Source Code + Form, as described in Section 3.1, and You must inform recipients of + the Executable Form how they can obtain a copy of such Source Code + Form by reasonable means in a timely manner, at a charge no more + than the cost of distribution to the recipient; and + +(b) You may distribute such Executable Form under the terms of this + License, or sublicense it under different terms, provided that the + license for the Executable Form does not attempt to limit or alter + the recipients' rights in the Source Code Form under this License. + +3.3. Distribution of a Larger Work + +You may create and distribute a Larger Work under terms of Your choice, +provided that You also comply with the requirements of this License for +the Covered Software. If the Larger Work is a combination of Covered +Software with a work governed by one or more Secondary Licenses, and the +Covered Software is not Incompatible With Secondary Licenses, this +License permits You to additionally distribute such Covered Software +under the terms of such Secondary License(s), so that the recipient of +the Larger Work may, at their option, further distribute the Covered +Software under the terms of either this License or such Secondary +License(s). + +3.4. Notices + +You may not remove or alter the substance of any license notices +(including copyright notices, patent notices, disclaimers of warranty, +or limitations of liability) contained within the Source Code Form of +the Covered Software, except that You may alter any license notices to +the extent required to remedy known factual inaccuracies. + +3.5. Application of Additional Terms + +You may choose to offer, and to charge a fee for, warranty, support, +indemnity or liability obligations to one or more recipients of Covered +Software. However, You may do so only on Your own behalf, and not on +behalf of any Contributor. You must make it absolutely clear that any +such warranty, support, indemnity, or liability obligation is offered by +You alone, and You hereby agree to indemnify every Contributor for any +liability incurred by such Contributor as a result of warranty, support, +indemnity or liability terms You offer. You may include additional +disclaimers of warranty and limitations of liability specific to any +jurisdiction. + +4. Inability to Comply Due to Statute or Regulation +--------------------------------------------------- + +If it is impossible for You to comply with any of the terms of this +License with respect to some or all of the Covered Software due to +statute, judicial order, or regulation then You must: (a) comply with +the terms of this License to the maximum extent possible; and (b) +describe the limitations and the code they affect. Such description must +be placed in a text file included with all distributions of the Covered +Software under this License. Except to the extent prohibited by statute +or regulation, such description must be sufficiently detailed for a +recipient of ordinary skill to be able to understand it. + +5. Termination +-------------- + +5.1. The rights granted under this License will terminate automatically +if You fail to comply with any of its terms. However, if You become +compliant, then the rights granted under this License from a particular +Contributor are reinstated (a) provisionally, unless and until such +Contributor explicitly and finally terminates Your grants, and (b) on an +ongoing basis, if such Contributor fails to notify You of the +non-compliance by some reasonable means prior to 60 days after You have +come back into compliance. Moreover, Your grants from a particular +Contributor are reinstated on an ongoing basis if such Contributor +notifies You of the non-compliance by some reasonable means, this is the +first time You have received notice of non-compliance with this License +from such Contributor, and You become compliant prior to 30 days after +Your receipt of the notice. + +5.2. If You initiate litigation against any entity by asserting a patent +infringement claim (excluding declaratory judgment actions, +counter-claims, and cross-claims) alleging that a Contributor Version +directly or indirectly infringes any patent, then the rights granted to +You by any and all Contributors for the Covered Software under Section +2.1 of this License shall terminate. + +5.3. In the event of termination under Sections 5.1 or 5.2 above, all +end user license agreements (excluding distributors and resellers) which +have been validly granted by You or Your distributors under this License +prior to termination shall survive termination. + +************************************************************************ +* * +* 6. Disclaimer of Warranty * +* ------------------------- * +* * +* Covered Software is provided under this License on an "as is" * +* basis, without warranty of any kind, either expressed, implied, or * +* statutory, including, without limitation, warranties that the * +* Covered Software is free of defects, merchantable, fit for a * +* particular purpose or non-infringing. The entire risk as to the * +* quality and performance of the Covered Software is with You. * +* Should any Covered Software prove defective in any respect, You * +* (not any Contributor) assume the cost of any necessary servicing, * +* repair, or correction. This disclaimer of warranty constitutes an * +* essential part of this License. No use of any Covered Software is * +* authorized under this License except under this disclaimer. * +* * +************************************************************************ + +************************************************************************ +* * +* 7. Limitation of Liability * +* -------------------------- * +* * +* Under no circumstances and under no legal theory, whether tort * +* (including negligence), contract, or otherwise, shall any * +* Contributor, or anyone who distributes Covered Software as * +* permitted above, be liable to You for any direct, indirect, * +* special, incidental, or consequential damages of any character * +* including, without limitation, damages for lost profits, loss of * +* goodwill, work stoppage, computer failure or malfunction, or any * +* and all other commercial damages or losses, even if such party * +* shall have been informed of the possibility of such damages. This * +* limitation of liability shall not apply to liability for death or * +* personal injury resulting from such party's negligence to the * +* extent applicable law prohibits such limitation. Some * +* jurisdictions do not allow the exclusion or limitation of * +* incidental or consequential damages, so this exclusion and * +* limitation may not apply to You. * +* * +************************************************************************ + +8. Litigation +------------- + +Any litigation relating to this License may be brought only in the +courts of a jurisdiction where the defendant maintains its principal +place of business and such litigation shall be governed by laws of that +jurisdiction, without reference to its conflict-of-law provisions. +Nothing in this Section shall prevent a party's ability to bring +cross-claims or counter-claims. + +9. Miscellaneous +---------------- + +This License represents the complete agreement concerning the subject +matter hereof. If any provision of this License is held to be +unenforceable, such provision shall be reformed only to the extent +necessary to make it enforceable. Any law or regulation which provides +that the language of a contract shall be construed against the drafter +shall not be used to construe this License against a Contributor. + +10. Versions of the License +--------------------------- + +10.1. New Versions + +Mozilla Foundation is the license steward. Except as provided in Section +10.3, no one other than the license steward has the right to modify or +publish new versions of this License. Each version will be given a +distinguishing version number. + +10.2. Effect of New Versions + +You may distribute the Covered Software under the terms of the version +of the License under which You originally received the Covered Software, +or under the terms of any subsequent version published by the license +steward. + +10.3. Modified Versions + +If you create software not governed by this License, and you want to +create a new license for such software, you may create and use a +modified version of this License if you rename the license and remove +any references to the name of the license steward (except to note that +such modified license differs from this License). + +10.4. Distributing Source Code Form that is Incompatible With Secondary +Licenses + +If You choose to distribute Source Code Form that is Incompatible With +Secondary Licenses under the terms of this version of the License, the +notice described in Exhibit B of this License must be attached. + +Exhibit A - Source Code Form License Notice +------------------------------------------- + + This Source Code Form is subject to the terms of the Mozilla Public + License, v. 2.0. If a copy of the MPL was not distributed with this + file, You can obtain one at http://mozilla.org/MPL/2.0/. + +If it is not possible or desirable to put the notice in a particular +file, then You may include the notice in a location (such as a LICENSE +file in a relevant directory) where a recipient would be likely to look +for such a notice. + +You may add additional accurate notices of copyright ownership. + +Exhibit B - "Incompatible With Secondary Licenses" Notice +--------------------------------------------------------- + + This Source Code Form is "Incompatible With Secondary Licenses", as + defined by the Mozilla Public License, v. 2.0. diff -Nru certspotter-0.14.0/loglist/load.go certspotter-0.15.1/loglist/load.go --- certspotter-0.14.0/loglist/load.go 2022-06-13 15:23:35.000000000 +0000 +++ certspotter-0.15.1/loglist/load.go 2023-02-09 18:44:06.000000000 +0000 @@ -1,4 +1,4 @@ -// Copyright (C) 2020 Opsmate, Inc. +// Copyright (C) 2020, 2023 Opsmate, Inc. // // This Source Code Form is subject to the terms of the Mozilla // Public License, v. 2.0. If a copy of the MPL was not distributed @@ -10,39 +10,96 @@ package loglist import ( + "context" "encoding/json" + "errors" "fmt" + "io" "net/http" - "io/ioutil" + "os" "strings" + "time" ) -func Load(urlOrFile string) (*List, error) { +var UserAgent = "certspotter" + +type ModificationToken struct { + etag string + modified time.Time +} + +var ErrNotModified = errors.New("loglist has not been modified") + +func newModificationToken(response *http.Response) *ModificationToken { + token := &ModificationToken{ + etag: response.Header.Get("ETag"), + } + if t, err := time.Parse(http.TimeFormat, response.Header.Get("Last-Modified")); err == nil { + token.modified = t + } + return token +} + +func (token *ModificationToken) setRequestHeaders(request *http.Request) { + if token.etag != "" { + request.Header.Set("If-None-Match", token.etag) + } else if !token.modified.IsZero() { + request.Header.Set("If-Modified-Since", token.modified.Format(http.TimeFormat)) + } +} + +func Load(ctx context.Context, urlOrFile string) (*List, error) { + list, _, err := LoadIfModified(ctx, urlOrFile, nil) + return list, err +} + +func LoadIfModified(ctx context.Context, urlOrFile string, token *ModificationToken) (*List, *ModificationToken, error) { if strings.HasPrefix(urlOrFile, "https://") { - return Fetch(urlOrFile) + return FetchIfModified(ctx, urlOrFile, token) } else { - return ReadFile(urlOrFile) + list, err := ReadFile(urlOrFile) + return list, nil, err } } -func Fetch(url string) (*List, error) { - response, err := http.Get(url) +func Fetch(ctx context.Context, url string) (*List, error) { + list, _, err := FetchIfModified(ctx, url, nil) + return list, err +} + +func FetchIfModified(ctx context.Context, url string, token *ModificationToken) (*List, *ModificationToken, error) { + request, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) if err != nil { - return nil, err + return nil, nil, err + } + request.Header.Set("User-Agent", UserAgent) + if token != nil { + token.setRequestHeaders(request) } - content, err := ioutil.ReadAll(response.Body) + response, err := http.DefaultClient.Do(request) + if err != nil { + return nil, nil, err + } + content, err := io.ReadAll(response.Body) response.Body.Close() if err != nil { - return nil, err + return nil, nil, err + } + if token != nil && response.StatusCode == http.StatusNotModified { + return nil, nil, ErrNotModified } if response.StatusCode != 200 { - return nil, fmt.Errorf("%s: %s", url, response.Status) + return nil, nil, fmt.Errorf("%s: %s", url, response.Status) + } + list, err := Unmarshal(content) + if err != nil { + return nil, nil, fmt.Errorf("error parsing %s: %w", url, err) } - return Unmarshal(content) + return list, newModificationToken(response), err } func ReadFile(filename string) (*List, error) { - content, err := ioutil.ReadFile(filename) + content, err := os.ReadFile(filename) if err != nil { return nil, err } diff -Nru certspotter-0.14.0/man/certspotter.md certspotter-0.15.1/man/certspotter.md --- certspotter-0.14.0/man/certspotter.md 1970-01-01 00:00:00.000000000 +0000 +++ certspotter-0.15.1/man/certspotter.md 2023-02-09 18:44:06.000000000 +0000 @@ -0,0 +1,200 @@ +# NAME + +**certspotter** - Certificate Transparency Log Monitor + +# SYNOPSIS + +**certspotter** [`-start_at_end`] [`-watchlist` *FILENAME*] [`-email` *ADDRESS*] `...` + +# DESCRIPTION + +**Cert Spotter** is a Certificate Transparency log monitor from SSLMate that +alerts you when a SSL/TLS certificate is issued for one of your domains. +Cert Spotter is easier to use than other open source CT monitors, since it does not require +a database. It's also more robust, since it uses a special certificate parser +that ensures it won't miss certificates. + +Cert Spotter is also available as a hosted service by SSLMate, +. + +You can use Cert Spotter to detect: + + * Certificates issued to attackers who have compromised your DNS and + are redirecting your visitors to their malicious site. + * Certificates issued to attackers who have taken over an abandoned + sub-domain in order to serve malware under your name. + * Certificates issued to attackers who have compromised a certificate + authority and want to impersonate your site. + * Certificates issued in violation of your corporate policy + or outside of your centralized certificate procurement process. + +# OPTIONS + +-batch_size *NUMBER* + +: Maximum number of entries to request per call to get-entries. + You should not generally need to change this. Defaults to 1000. + +-email *ADDRESS* + +: Email address to contact when a matching certificate is discovered, or + an error occurs. You can specify this option more than once to email + multiple addresses. Your system must have a working sendmail(1) command. + +-healthcheck *INTERVAL* + +: Perform a health check at the given interval (default: "24h") as described + below. *INTERVAL* must be a decimal number followed by "h" for hours or + "m" for minutes. + +-logs *ADDRESS* + +: Filename or HTTPS URL of a v2 or v3 JSON log list containing logs to monitor. + The schema for this file can be found at . + Defaults to , which includes + the union of active logs recognized by Chrome and Apple. certspotter periodically + reloads the log list in case it has changed. + +-no\_save + +: Do not save a copy of matching certificates. + +-script *COMMAND* + +: Command to execute when a matching certificate is found or an error occurs. See + certspotter-script(8) for information about the interface to scripts. + +-start\_at\_end + +: Start monitoring logs from the end rather than the beginning. + + **WARNING**: monitoring from the beginning guarantees detection of all + certificates, but requires downloading hundreds of millions of + certificates, which takes days. + +-state\_dir *PATH* + +: Directory for storing state. Defaults to `$CERTSPOTTER_STATE_DIR`, which is + "~/.certspotter" by default. + +-stdout + +: Write matching certificates and errors to stdout. + +-verbose + +: Be verbose. + +-version + +: Print version and exit. + +-watchlist *PATH* + +: File containing DNS names to monitor, one per line. To monitor an entire + domain namespace (including the domain itself and all sub-domains) prefix + the domain name with a dot (e.g. ".example.com"). To monitor a single DNS + name only, do not prefix the name with a dot. + + Defaults to `$CERTSPOTTER_CONFIG_DIR/watchlist`, which is + "~/.certspotter/watchlist" by default. + Specify `-` to read the watch list from stdin. + + certspotter reads the watch list only when starting up, so you must restart + certspotter if you change it. + +# OPERATION + +certspotter continuously monitors all browser-recognized Certificate +Transparency logs looking for certificates which are valid for any domain +on your watch list. When certspotter detects a matching certificate, it +emails you (if `-email` is specified), executes a script (if `-script` +is specified), and/or writes a report to standard out (if `-stdout` +is specified). + +certspotter also saves a copy of matching certificates in +`$CERTSPOTTER_STATE_DIR/certs` ("~/.certspotter/certs" by default) +unless you specify the `-no_save` option. + +When certspotter has not previously monitored a log, it can either start +monitoring the log from the beginning, or seek to the end of the log and +start monitoring from there. Monitoring from the beginning guarantees +detection of all certificates, but requires downloading hundreds of +millions of certificates, which takes days. The default behavior is to +monitor from the beginning. To start monitoring new logs from the end, +specify the `-start_at_end` option. + +If certspotter has previously monitored a log, it resumes monitoring +the log from the previous position. This means that if you add +a domain to your watch list, certspotter will not detect any certificates +that were logged prior to the addition. To detect such certificates, +you must delete `$CERTSPOTTER_STATE_DIR/logs`, which will cause certspotter +to restart monitoring from the very beginning of each log (provided +`-start_at_end` is not specified). This will cause certspotter to download +hundreds of millions of certificates, which takes days. To find preexisting +certificates, it's faster to use the Cert Spotter service +, SSLMate's Certificate Transparency Search +API , or a CT search engine such as +. + +# ERROR HANDLING + +When certspotter encounters a problem with the local system (e.g. failure +to write a file or execute a script), it prints a message to stderr and +exits with a non-zero status. + +When certspotter encounters a problem monitoring a log, it prints a message +to stderr and continues running. It will try monitoring the log again later; +most log errors are transient. + +Every 24 hours (unless overridden by `-healthcheck`), certspotter performs the +following health checks: + + * Ensure that the log list has been successfully retrieved at least once + since the previous health check. + * Ensure that every log has been successfully contacted at least once + since the previous health check. + * Ensure that certspotter is not falling behind monitoring any logs. + +If any health check fails, certspotter notifies you by email (if `-email` +is specified), script (if `-script` is specified), and/or standard out +(if `-stdout` is specified). + +Health check failures should be rare, and you should take them seriously because it means +certspotter might not detect all certificates. It might also be an indication +of CT log misbehavior. Consult certspotter's stderr output for details, and if +you need help, file an issue at . + +# EXIT STATUS + +certspotter exits 0 when it receives `SIGTERM` or `SIGINT`, +and non-zero when a serious error occurs. + +# ENVIRONMENT + +`CERTSPOTTER_STATE_DIR` + +: Directory for storing state. Overridden by `-state_dir`. Defaults to + `~/.certspotter`. + +`CERTSPOTTER_CONFIG_DIR` + +: Directory from which any configuration, such as the watch list, is read. + Defaults to `~/.certspotter`. + +`HTTPS_PROXY` + +: URL of proxy server for making HTTPS requests. `http://`, `https://`, and + `socks5://` URLs are supported. By default, no proxy server is used. + +# SEE ALSO + +certspotter-script(8) + +# COPYRIGHT + +Copyright (c) 2016-2023 Opsmate, Inc. + +# BUGS + +Report bugs to . diff -Nru certspotter-0.14.0/man/certspotter-script.md certspotter-0.15.1/man/certspotter-script.md --- certspotter-0.14.0/man/certspotter-script.md 1970-01-01 00:00:00.000000000 +0000 +++ certspotter-0.15.1/man/certspotter-script.md 2023-02-09 18:44:06.000000000 +0000 @@ -0,0 +1,237 @@ +# NAME + +**certspotter-script** - Certificate Transparency Log Monitor (hook script) + +# DESCRIPTION + +**certspotter-script** is *any* program that is called using **certspotter(8)**'s +*-script* argument. **certspotter** executes this program when it needs to notify +you about an event, such as detecting a certificate for a domain on your watch list. + +# ENVIRONMENT + +## Event information + +The following environment variables are set for all types of events: + +`EVENT` + +: One of the following values, indicating the type of event: + + * `discovered_cert` - certspotter has discovered a certificate for a + domain on your watch list. + + * `malformed_cert` - certspotter can't determine if a certificate + matches your watch list because the certificate or the log entry + is malformed. + + * `error` - a problem is preventing certspotter from monitoring all + logs. + + Additional event types may be defined in the future, so your script should + be able to handle unknown values. + +`SUMMARY` + +: A short human-readable string describing the event. + + +## Discovered certificate information + +The following environment variables are set for `discovered_cert` events: + +`WATCH_ITEM` + +: The item from your watch list which matches this certificate. + (If more than one item matches, the first one is used.) + +`LOG_URI` + +: The URI of the log containing the certificate. + +`ENTRY_INDEX` + +: The index of the log entry containing the certificate. + +`TBS_SHA256` + +: The hex-encoded SHA-256 digest of the TBSCertificate, as defined in RFC 6962 Section 3.2. + Certificates and their corresponding precertificates have the same `TBS_SHA256` value. + +`CERT_SHA256` + +: The hex-encoded SHA-256 digest (sometimes called fingerprint) of the certificate. + The digest is computed over the ASN.1 DER encoding. + +`PUBKEY_SHA256` + +: The hex-encoded SHA-256 digest of the certificate's Subject Public Key Info. + +`CERT_FILENAME` + +: Path to a file containing the PEM-encoded certificate chain. Not set if `-no_save` was used. + +`JSON_FILENAME` + +: Path to a JSON containing additional information about the certificate. See below for the format of the JSON file. + Not set if `-no_save` was used. + +`TEXT_FILENAME` + +: Path to a file containing a text representation of the certificate. This file contains the same text that + certspotter uses in emails. You should not attempt to parse this file because its format may change in the future. + Not set if `-no_save` was used. + +`NOT_BEFORE`, `NOT_BEFORE_UNIXTIME`, `NOT_BEFORE_RFC3339` + +: The not before time of the certificate, in a human-readable format, seconds since the UNIX epoch, and RFC3339, respectively. These variables may be unset if there was a parse error, in which case `VALIDITY_PARSE_ERROR` is set. + +`NOT_AFTER`, `NOT_AFTER_UNIXTIME`, `NOT_AFTER_RFC3339` + +: The not after (expiration) time of the certificate, in a human-readable format, seconds since the UNIX epoch, and RFC3339, respectively. These variables may be unset if there was a parse error, in which case `VALIDITY_PARSE_ERROR` is set. + +`VALIDITY_PARSE_ERROR` + +: Error parsing not before and not after, if any. If this variable is set, then the `NOT_BEFORE` and `NOT_AFTER` family of variables are unset. + +`SUBJECT_DN` + +: The distinguished name of the certificate's subject. This variable may be unset if there was a parse error, in which case `SUBJECT_PARSE_ERROR` is set. + +`SUBJECT_PARSE_ERROR` + +: Error parsing the subject, if any. If this variable is set, then `SUBJECT_DN` is unset. + +`ISSUER_DN` + +: The distinguished name of the certificate's issuer. This variable may be unset if there was a parse error, in which case `ISSUER_PARSE_ERROR` is set. + +`ISSUER_PARSE_ERROR` + +: Error parsing the issuer, if any. If this variable is set, then `ISSUER_DN` is unset. + +`SERIAL` + +: The hex-encoded serial number of the certificate. Prefixed with a minus (-) sign if negative. This variable may be unset if there was a parse error, in which case `SERIAL_PARSE_ERROR` is set. + +`SERIAL_PARSE_ERROR` + +: Error parsing the serial number, if any. If this variable is set, then `SERIAL` is unset. + +## Malformed certificate information + +The following environment variables are set for `malformed_cert` events: + +`LOG_URI` + +: The URI of the log containing the malformed certificate. + +`ENTRY_INDEX` + +: The index of the log entry containing the malformed certificate. + +`LEAF_HASH` + +: The base64-encoded Merkle hash of the leaf containing the malformed certificate. + +`PARSE_ERROR` + +: A human-readable string describing why the certificate is malformed. + +# JSON FILE FORMAT + +Unless `-no_save` is used, certspotter saves a JSON file for every discovered certificate +under `$CERTSPOTTER_STATE_DIR`, and puts the path to the file in `$JSON_FILENAME`. Your +script can read the JSON file, such as with the jq(1) command, to get additional information +about the certificate which isn't appropriate for environment variables. + +The JSON file contains an object with the following fields: + +`tbs_sha256` + +: A string containing the hex-encoded SHA-256 digest of the TBSCertificate, as defined in RFC 6962 Section 3.2. + Certificates and their corresponding precertificates have the same `tbs_sha256` value. + +`pubkey_sha256` + +: A string containing the hex-encoded SHA-256 digest of the certificate's Subject Public Key Info. + +`dns_names` + +: An array of strings containing the DNS names for which the + certificate is valid, taken from both the DNS subject alternative names + (SANs) and the subject common name (CN). Internationalized domain names + are encoded in Punycode. + +`ip_addresses` + +: An array of strings containing the IP addresses for which the certificate is valid, + taken from both the IP subject alternative names (SANs) and the subject common name (CN). + +`not_before` + +: A string containing the not before time of the certificate in RFC3339 format. + Null if there was an error parsing the certificate's validity. + +`not_after` + +: A string containing the not after (expiration) time of the certificate in RFC3339 format. + Null if there was an error parsing the certificate's validity. + +Additional fields will be added in the future based on user feedback. Please open +an issue at if you have a use case for another field. + +# EXAMPLES + +Example environment variables for a `discovered_cert` event: + +``` +CERT_FILENAME=/home/andrew/.certspotter/certs/3c/3cdc83b3932c194fcdf17aa2bf1abc34e8438b293c3d5c70693e175b38ff128a.pem +CERT_SHA256=3cdc83b3932c194fcdf17aa2bf1abc34e8438b293c3d5c70693e175b38ff128a +ENTRY_INDEX=6464843 +EVENT=discovered_cert +ISSUER_DN=C=GB, ST=Greater Manchester, L=Salford, O=Sectigo Limited, CN=Sectigo RSA Domain Validation Secure Server CA +JSON_FILENAME=/usr2/andrew/.certspotter/certs/3c/3cdc83b3932c194fcdf17aa2bf1abc34e8438b293c3d5c70693e175b38ff128a.v1.json +LOG_URI=https://ct.cloudflare.com/logs/nimbus2024/ +NOT_AFTER='2024-01-26 03:47:26 +0000 UTC' +NOT_AFTER_RFC3339=2024-01-26T03:47:26Z +NOT_AFTER_UNIXTIME=1706240846 +NOT_BEFORE='2023-01-31 03:47:26 +0000 UTC' +NOT_BEFORE_RFC3339=2023-01-31T03:47:26Z +NOT_BEFORE_UNIXTIME=1675136846 +PUBKEY_SHA256=33ac1d9b9e56005ccac045eac2398b3e9dd6b3f5b66ae6260f2d478c7c0d82c8 +SERIAL=c170fbf3bf27481e5c351a4db6f2dc5f +SUBJECT_DN=CN=sslmate.com +SUMMARY='certificate discovered for .sslmate.com' +TBS_SHA256=2388ee81c6f45cffc73e68a35fa8921e839e20acc9a98e8e6dcaea07cbfbdef8 +TEXT_FILENAME=/usr2/andrew/.certspotter/certs/3c/3cdc83b3932c194fcdf17aa2bf1abc34e8438b293c3d5c70693e175b38ff128a.txt +WATCH_ITEM=.sslmate.com +``` + +Example JSON file for a discovered certificate: + +``` +{ + "dns_names": [ + "sslmate.com", + "www.sslmate.com" + ], + "ip_addresses": [], + "not_after": "2024-01-26T03:47:26Z", + "not_before": "2023-01-31T03:47:26Z", + "pubkey_sha256": "33ac1d9b9e56005ccac045eac2398b3e9dd6b3f5b66ae6260f2d478c7c0d82c8", + "tbs_sha256": "2388ee81c6f45cffc73e68a35fa8921e839e20acc9a98e8e6dcaea07cbfbdef8" +} +``` + +# SEE ALSO + +certspotter(8) + +# COPYRIGHT + +Copyright (c) 2016-2023 Opsmate, Inc. + +# BUGS + +Report bugs to . diff -Nru certspotter-0.14.0/man/.gitignore certspotter-0.15.1/man/.gitignore --- certspotter-0.14.0/man/.gitignore 1970-01-01 00:00:00.000000000 +0000 +++ certspotter-0.15.1/man/.gitignore 2023-02-09 18:44:06.000000000 +0000 @@ -0,0 +1 @@ +*.8 diff -Nru certspotter-0.14.0/man/Makefile certspotter-0.15.1/man/Makefile --- certspotter-0.14.0/man/Makefile 1970-01-01 00:00:00.000000000 +0000 +++ certspotter-0.15.1/man/Makefile 2023-02-09 18:44:06.000000000 +0000 @@ -0,0 +1,9 @@ +all: certspotter-script.8 certspotter.8 + +%.8: %.md + lowdown -s -Tman \ + -M title:$(basename $(notdir $@)) \ + -M section:$(subst .,,$(suffix $@)) \ + -M date:$(if $(SOURCE_DATE_EPOCH),$(shell date -I -u -d "@$(SOURCE_DATE_EPOCH)"),$(shell date -I -u)) \ + -o $@ $< + diff -Nru certspotter-0.14.0/merkletree/collapsed_tree.go certspotter-0.15.1/merkletree/collapsed_tree.go --- certspotter-0.14.0/merkletree/collapsed_tree.go 1970-01-01 00:00:00.000000000 +0000 +++ certspotter-0.15.1/merkletree/collapsed_tree.go 2023-02-09 18:44:06.000000000 +0000 @@ -0,0 +1,98 @@ +// Copyright (C) 2022 Opsmate, Inc. +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License, v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. +// +// This software is distributed WITHOUT A WARRANTY OF ANY KIND. +// See the Mozilla Public License for details. + +package merkletree + +import ( + "encoding/json" + "fmt" +) + +type CollapsedTree struct { + nodes []Hash + size uint64 +} + +func calculateNumNodes(size uint64) int { + numNodes := 0 + for size > 0 { + numNodes += int(size & 1) + size >>= 1 + } + return numNodes +} + +func EmptyCollapsedTree() *CollapsedTree { + return &CollapsedTree{nodes: []Hash{}, size: 0} +} + +func NewCollapsedTree(nodes []Hash, size uint64) (*CollapsedTree, error) { + if len(nodes) != calculateNumNodes(size) { + return nil, fmt.Errorf("nodes has wrong length (should be %d, not %d)", calculateNumNodes(size), len(nodes)) + } + return &CollapsedTree{nodes: nodes, size: size}, nil +} + +func CloneCollapsedTree(source *CollapsedTree) *CollapsedTree { + nodes := make([]Hash, len(source.nodes)) + copy(nodes, source.nodes) + return &CollapsedTree{nodes: nodes, size: source.size} +} + +func (tree *CollapsedTree) Add(hash Hash) { + tree.nodes = append(tree.nodes, hash) + tree.size++ + size := tree.size + for size%2 == 0 { + left, right := tree.nodes[len(tree.nodes)-2], tree.nodes[len(tree.nodes)-1] + tree.nodes = tree.nodes[:len(tree.nodes)-2] + tree.nodes = append(tree.nodes, HashChildren(left, right)) + size /= 2 + } +} + +func (tree *CollapsedTree) CalculateRoot() Hash { + if len(tree.nodes) == 0 { + return HashNothing() + } + i := len(tree.nodes) - 1 + hash := tree.nodes[i] + for i > 0 { + i -= 1 + hash = HashChildren(tree.nodes[i], hash) + } + return hash +} + +func (tree *CollapsedTree) Size() uint64 { + return tree.size +} + +func (tree *CollapsedTree) MarshalJSON() ([]byte, error) { + return json.Marshal(map[string]interface{}{ + "nodes": tree.nodes, + "size": tree.size, + }) +} + +func (tree *CollapsedTree) UnmarshalJSON(b []byte) error { + var rawTree struct { + Nodes []Hash `json:"nodes"` + Size uint64 `json:"size"` + } + if err := json.Unmarshal(b, &rawTree); err != nil { + return fmt.Errorf("error unmarshalling Collapsed Merkle Tree: %w", err) + } + if len(rawTree.Nodes) != calculateNumNodes(rawTree.Size) { + return fmt.Errorf("error unmarshalling Collapsed Merkle Tree: nodes has wrong length (should be %d, not %d)", calculateNumNodes(rawTree.Size), len(rawTree.Nodes)) + } + tree.size = rawTree.Size + tree.nodes = rawTree.Nodes + return nil +} diff -Nru certspotter-0.14.0/merkletree/hash.go certspotter-0.15.1/merkletree/hash.go --- certspotter-0.14.0/merkletree/hash.go 1970-01-01 00:00:00.000000000 +0000 +++ certspotter-0.15.1/merkletree/hash.go 2023-02-09 18:44:06.000000000 +0000 @@ -0,0 +1,64 @@ +// Copyright (C) 2022 Opsmate, Inc. +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License, v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. +// +// This software is distributed WITHOUT A WARRANTY OF ANY KIND. +// See the Mozilla Public License for details. + +package merkletree + +import ( + "crypto/sha256" + "encoding/base64" + "encoding/json" + "fmt" +) + +const HashLen = 32 + +type Hash [HashLen]byte + +func (h Hash) Base64String() string { + return base64.StdEncoding.EncodeToString(h[:]) +} + +func (h Hash) MarshalJSON() ([]byte, error) { + return json.Marshal(h[:]) +} + +func (h *Hash) UnmarshalJSON(b []byte) error { + var hashBytes []byte + if err := json.Unmarshal(b, &hashBytes); err != nil { + return err + } + if len(hashBytes) != HashLen { + return fmt.Errorf("Merkle Tree hash has wrong length (should be %d bytes long, not %d)", HashLen, len(hashBytes)) + } + copy(h[:], hashBytes) + return nil +} + +func HashNothing() Hash { + return sha256.Sum256(nil) +} + +func HashLeaf(leafBytes []byte) Hash { + var hash Hash + hasher := sha256.New() + hasher.Write([]byte{0x00}) + hasher.Write(leafBytes) + hasher.Sum(hash[:0]) + return hash +} + +func HashChildren(left Hash, right Hash) Hash { + var hash Hash + hasher := sha256.New() + hasher.Write([]byte{0x01}) + hasher.Write(left[:]) + hasher.Write(right[:]) + hasher.Sum(hash[:0]) + return hash +} diff -Nru certspotter-0.14.0/monitor/config.go certspotter-0.15.1/monitor/config.go --- certspotter-0.14.0/monitor/config.go 1970-01-01 00:00:00.000000000 +0000 +++ certspotter-0.15.1/monitor/config.go 2023-02-09 18:44:06.000000000 +0000 @@ -0,0 +1,27 @@ +// Copyright (C) 2023 Opsmate, Inc. +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License, v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. +// +// This software is distributed WITHOUT A WARRANTY OF ANY KIND. +// See the Mozilla Public License for details. + +package monitor + +import ( + "time" +) + +type Config struct { + LogListSource string + StateDir string + StartAtEnd bool + WatchList WatchList + Verbose bool + SaveCerts bool + Script string + Email []string + Stdout bool + HealthCheckInterval time.Duration +} diff -Nru certspotter-0.14.0/monitor/daemon.go certspotter-0.15.1/monitor/daemon.go --- certspotter-0.14.0/monitor/daemon.go 1970-01-01 00:00:00.000000000 +0000 +++ certspotter-0.15.1/monitor/daemon.go 2023-02-09 18:44:06.000000000 +0000 @@ -0,0 +1,167 @@ +// Copyright (C) 2023 Opsmate, Inc. +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License, v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. +// +// This software is distributed WITHOUT A WARRANTY OF ANY KIND. +// See the Mozilla Public License for details. + +package monitor + +import ( + "context" + "errors" + "fmt" + "golang.org/x/sync/errgroup" + "log" + insecurerand "math/rand" + "software.sslmate.com/src/certspotter/loglist" + "time" +) + +const ( + reloadLogListIntervalMin = 30 * time.Minute + reloadLogListIntervalMax = 90 * time.Minute +) + +func randomDuration(min, max time.Duration) time.Duration { + return min + time.Duration(insecurerand.Int63n(int64(max-min+1))) +} + +func reloadLogListInterval() time.Duration { + return randomDuration(reloadLogListIntervalMin, reloadLogListIntervalMax) +} + +type task struct { + log *loglist.Log + stop context.CancelFunc +} + +type daemon struct { + config *Config + taskgroup *errgroup.Group + tasks map[LogID]task + logsLoadedAt time.Time + logListToken *loglist.ModificationToken + logListError string + logListErrorAt time.Time +} + +func (daemon *daemon) healthCheck(ctx context.Context) error { + if time.Since(daemon.logsLoadedAt) >= daemon.config.HealthCheckInterval { + if err := notify(ctx, daemon.config, &staleLogListEvent{ + Source: daemon.config.LogListSource, + LastSuccess: daemon.logsLoadedAt, + LastError: daemon.logListError, + LastErrorTime: daemon.logListErrorAt, + }); err != nil { + return fmt.Errorf("error notifying about stale log list: %w", err) + } + } + + for _, task := range daemon.tasks { + if err := healthCheckLog(ctx, daemon.config, task.log); err != nil { + return fmt.Errorf("error checking health of log %q: %w", task.log.URL, err) + } + } + return nil +} + +func (daemon *daemon) startTask(ctx context.Context, ctlog *loglist.Log) task { + ctx, cancel := context.WithCancel(ctx) + daemon.taskgroup.Go(func() error { + defer cancel() + err := monitorLogContinously(ctx, daemon.config, ctlog) + if daemon.config.Verbose { + log.Printf("task for log %s stopped with error %s", ctlog.URL, err) + } + if ctx.Err() == context.Canceled && errors.Is(err, context.Canceled) { + return nil + } else { + return fmt.Errorf("error while monitoring %s: %w", ctlog.URL, err) + } + }) + return task{log: ctlog, stop: cancel} +} + +func (daemon *daemon) loadLogList(ctx context.Context) error { + newLogList, newToken, err := getLogList(ctx, daemon.config.LogListSource, daemon.logListToken) + if errors.Is(err, loglist.ErrNotModified) { + return nil + } else if err != nil { + return err + } + + if daemon.config.Verbose { + log.Printf("fetched %d logs from %q", len(newLogList), daemon.config.LogListSource) + } + + for logID, task := range daemon.tasks { + if _, exists := newLogList[logID]; exists { + continue + } + if daemon.config.Verbose { + log.Printf("stopping task for log %s", logID.Base64String()) + } + task.stop() + delete(daemon.tasks, logID) + } + for logID, ctlog := range newLogList { + if _, isRunning := daemon.tasks[logID]; isRunning { + continue + } + if daemon.config.Verbose { + log.Printf("starting task for log %s (%s)", logID.Base64String(), ctlog.URL) + } + daemon.tasks[logID] = daemon.startTask(ctx, ctlog) + } + daemon.logsLoadedAt = time.Now() + daemon.logListToken = newToken + return nil +} + +func (daemon *daemon) run(ctx context.Context) error { + if err := prepareStateDir(daemon.config.StateDir); err != nil { + return fmt.Errorf("error preparing state directory: %w", err) + } + + if err := daemon.loadLogList(ctx); err != nil { + return fmt.Errorf("error loading log list: %w", err) + } + + reloadLogListTicker := time.NewTicker(reloadLogListInterval()) + defer reloadLogListTicker.Stop() + + healthCheckTicker := time.NewTicker(daemon.config.HealthCheckInterval) + defer healthCheckTicker.Stop() + + for ctx.Err() == nil { + select { + case <-ctx.Done(): + case <-reloadLogListTicker.C: + if err := daemon.loadLogList(ctx); err != nil { + daemon.logListError = err.Error() + daemon.logListErrorAt = time.Now() + recordError(fmt.Errorf("error reloading log list (will try again later): %w", err)) + } + reloadLogListTicker.Reset(reloadLogListInterval()) + case <-healthCheckTicker.C: + if err := daemon.healthCheck(ctx); err != nil { + return err + } + } + } + return ctx.Err() +} + +func Run(ctx context.Context, config *Config) error { + group, ctx := errgroup.WithContext(ctx) + daemon := &daemon{ + config: config, + taskgroup: group, + tasks: make(map[LogID]task), + } + group.Go(func() error { return daemon.run(ctx) }) + return group.Wait() +} diff -Nru certspotter-0.14.0/monitor/discoveredcert.go certspotter-0.15.1/monitor/discoveredcert.go --- certspotter-0.14.0/monitor/discoveredcert.go 1970-01-01 00:00:00.000000000 +0000 +++ certspotter-0.15.1/monitor/discoveredcert.go 2023-02-09 18:44:06.000000000 +0000 @@ -0,0 +1,175 @@ +// Copyright (C) 2023 Opsmate, Inc. +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License, v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. +// +// This software is distributed WITHOUT A WARRANTY OF ANY KIND. +// See the Mozilla Public License for details. + +package monitor + +import ( + "bytes" + "encoding/hex" + "encoding/json" + "encoding/pem" + "fmt" + "strings" + "time" + + "software.sslmate.com/src/certspotter" + "software.sslmate.com/src/certspotter/ct" +) + +type discoveredCert struct { + WatchItem WatchItem + LogEntry *logEntry + Info *certspotter.CertInfo + Chain []ct.ASN1Cert // first entry is the leaf certificate or precertificate + TBSSHA256 [32]byte // computed over Info.TBS.Raw + SHA256 [32]byte // computed over Chain[0] + PubkeySHA256 [32]byte // computed over Info.TBS.PublicKey.FullBytes + Identifiers *certspotter.Identifiers + CertPath string // empty if not saved on the filesystem + JSONPath string // empty if not saved on the filesystem + TextPath string // empty if not saved on the filesystem +} + +func (cert *discoveredCert) pemChain() []byte { + var buffer bytes.Buffer + for _, certBytes := range cert.Chain { + if err := pem.Encode(&buffer, &pem.Block{ + Type: "CERTIFICATE", + Bytes: certBytes, + }); err != nil { + panic(fmt.Errorf("encoding certificate as PEM failed unexpectedly: %w", err)) + } + } + return buffer.Bytes() +} + +func (cert *discoveredCert) json() []byte { + object := map[string]any{ + "tbs_sha256": hex.EncodeToString(cert.TBSSHA256[:]), + "pubkey_sha256": hex.EncodeToString(cert.PubkeySHA256[:]), + "dns_names": cert.Identifiers.DNSNames, + "ip_addresses": cert.Identifiers.IPAddrs, + } + + if cert.Info.ValidityParseError == nil { + object["not_before"] = cert.Info.Validity.NotBefore + object["not_after"] = cert.Info.Validity.NotAfter + } else { + object["not_before"] = nil + object["not_after"] = nil + } + + jsonBytes, err := json.Marshal(object) + if err != nil { + panic(fmt.Errorf("encoding certificate as JSON failed unexpectedly: %w", err)) + } + return jsonBytes +} + +func (cert *discoveredCert) save() error { + if err := writeFile(cert.CertPath, cert.pemChain(), 0666); err != nil { + return err + } + if err := writeFile(cert.JSONPath, cert.json(), 0666); err != nil { + return err + } + if err := writeFile(cert.TextPath, []byte(cert.Text()), 0666); err != nil { + return err + } + return nil +} + +func (cert *discoveredCert) Environ() []string { + env := []string{ + "EVENT=discovered_cert", + "SUMMARY=certificate discovered for " + cert.WatchItem.String(), + "CERT_PARSEABLE=yes", // backwards compat with pre-0.15.0; not documented + "LOG_URI=" + cert.LogEntry.Log.URL, + "ENTRY_INDEX=" + fmt.Sprint(cert.LogEntry.Index), + "WATCH_ITEM=" + cert.WatchItem.String(), + "TBS_SHA256=" + hex.EncodeToString(cert.TBSSHA256[:]), + "CERT_SHA256=" + hex.EncodeToString(cert.SHA256[:]), + "FINGERPRINT=" + hex.EncodeToString(cert.SHA256[:]), // backwards compat with pre-0.15.0; not documented + "PUBKEY_SHA256=" + hex.EncodeToString(cert.PubkeySHA256[:]), + "PUBKEY_HASH=" + hex.EncodeToString(cert.PubkeySHA256[:]), // backwards compat with pre-0.15.0; not documented + "CERT_FILENAME=" + cert.CertPath, + "JSON_FILENAME=" + cert.JSONPath, + "TEXT_FILENAME=" + cert.TextPath, + } + + if cert.Info.ValidityParseError == nil { + env = append(env, "NOT_BEFORE="+cert.Info.Validity.NotBefore.String()) + env = append(env, "NOT_BEFORE_UNIXTIME="+fmt.Sprint(cert.Info.Validity.NotBefore.Unix())) + env = append(env, "NOT_BEFORE_RFC3339="+cert.Info.Validity.NotBefore.Format(time.RFC3339)) + env = append(env, "NOT_AFTER="+cert.Info.Validity.NotAfter.String()) + env = append(env, "NOT_AFTER_UNIXTIME="+fmt.Sprint(cert.Info.Validity.NotAfter.Unix())) + env = append(env, "NOT_AFTER_RFC3339="+cert.Info.Validity.NotAfter.Format(time.RFC3339)) + } else { + env = append(env, "VALIDITY_PARSE_ERROR="+cert.Info.ValidityParseError.Error()) + } + + if cert.Info.SubjectParseError == nil { + env = append(env, "SUBJECT_DN="+cert.Info.Subject.String()) + } else { + env = append(env, "SUBJECT_PARSE_ERROR="+cert.Info.SubjectParseError.Error()) + } + + if cert.Info.IssuerParseError == nil { + env = append(env, "ISSUER_DN="+cert.Info.Issuer.String()) + } else { + env = append(env, "ISSUER_PARSE_ERROR="+cert.Info.IssuerParseError.Error()) + } + + if cert.Info.SerialNumberParseError == nil { + env = append(env, "SERIAL="+fmt.Sprintf("%x", cert.Info.SerialNumber)) + } else { + env = append(env, "SERIAL_PARSE_ERROR="+cert.Info.SerialNumberParseError.Error()) + } + + return env +} + +func (cert *discoveredCert) Text() string { + // TODO-4: improve the output: include WatchItem, indicate hash algorithm used for fingerprints, ... (look at SSLMate email for inspiration) + + text := new(strings.Builder) + writeField := func(name string, value any) { fmt.Fprintf(text, "\t%13s = %s\n", name, value) } + + fmt.Fprintf(text, "%x:\n", cert.SHA256) + for _, dnsName := range cert.Identifiers.DNSNames { + writeField("DNS Name", dnsName) + } + for _, ipaddr := range cert.Identifiers.IPAddrs { + writeField("IP Address", ipaddr) + } + writeField("Pubkey", hex.EncodeToString(cert.PubkeySHA256[:])) + if cert.Info.IssuerParseError == nil { + writeField("Issuer", cert.Info.Issuer) + } else { + writeField("Issuer", fmt.Sprintf("[unable to parse: %s]", cert.Info.IssuerParseError)) + } + if cert.Info.ValidityParseError == nil { + writeField("Not Before", cert.Info.Validity.NotBefore) + writeField("Not After", cert.Info.Validity.NotAfter) + } else { + writeField("Not Before", fmt.Sprintf("[unable to parse: %s]", cert.Info.ValidityParseError)) + writeField("Not After", fmt.Sprintf("[unable to parse: %s]", cert.Info.ValidityParseError)) + } + writeField("Log Entry", fmt.Sprintf("%d @ %s", cert.LogEntry.Index, cert.LogEntry.Log.URL)) + writeField("crt.sh", "https://crt.sh/?sha256="+hex.EncodeToString(cert.SHA256[:])) + if cert.CertPath != "" { + writeField("Filename", cert.CertPath) + } + + return text.String() +} + +func (cert *discoveredCert) EmailSubject() string { + return fmt.Sprintf("[certspotter] Certificate Discovered for %s", cert.WatchItem) +} diff -Nru certspotter-0.14.0/monitor/errors.go certspotter-0.15.1/monitor/errors.go --- certspotter-0.14.0/monitor/errors.go 1970-01-01 00:00:00.000000000 +0000 +++ certspotter-0.15.1/monitor/errors.go 2023-02-09 18:44:06.000000000 +0000 @@ -0,0 +1,18 @@ +// Copyright (C) 2023 Opsmate, Inc. +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License, v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. +// +// This software is distributed WITHOUT A WARRANTY OF ANY KIND. +// See the Mozilla Public License for details. + +package monitor + +import ( + "log" +) + +func recordError(err error) { + log.Print(err) +} diff -Nru certspotter-0.14.0/monitor/fileutils.go certspotter-0.15.1/monitor/fileutils.go --- certspotter-0.14.0/monitor/fileutils.go 1970-01-01 00:00:00.000000000 +0000 +++ certspotter-0.15.1/monitor/fileutils.go 2023-02-09 18:44:06.000000000 +0000 @@ -0,0 +1,42 @@ +// Copyright (C) 2017, 2023 Opsmate, Inc. +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License, v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. +// +// This software is distributed WITHOUT A WARRANTY OF ANY KIND. +// See the Mozilla Public License for details. + +package monitor + +import ( + "crypto/rand" + "encoding/hex" + "fmt" + "os" +) + +func randomFileSuffix() string { + var randomBytes [12]byte + if _, err := rand.Read(randomBytes[:]); err != nil { + panic(err) + } + return hex.EncodeToString(randomBytes[:]) +} + +func writeFile(filename string, data []byte, perm os.FileMode) error { + tempname := filename + ".tmp." + randomFileSuffix() + if err := os.WriteFile(tempname, data, perm); err != nil { + return fmt.Errorf("error writing %s: %w", filename, err) + } + if err := os.Rename(tempname, filename); err != nil { + os.Remove(tempname) + return fmt.Errorf("error writing %s: %w", filename, err) + } + return nil +} + +func fileExists(filename string) bool { + _, err := os.Lstat(filename) + return err == nil +} diff -Nru certspotter-0.14.0/monitor/healthcheck.go certspotter-0.15.1/monitor/healthcheck.go --- certspotter-0.14.0/monitor/healthcheck.go 1970-01-01 00:00:00.000000000 +0000 +++ certspotter-0.15.1/monitor/healthcheck.go 2023-02-09 18:44:06.000000000 +0000 @@ -0,0 +1,152 @@ +// Copyright (C) 2023 Opsmate, Inc. +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License, v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. +// +// This software is distributed WITHOUT A WARRANTY OF ANY KIND. +// See the Mozilla Public License for details. + +package monitor + +import ( + "context" + "errors" + "fmt" + "io/fs" + "path/filepath" + "strings" + "time" + + "software.sslmate.com/src/certspotter/ct" + "software.sslmate.com/src/certspotter/loglist" +) + +func healthCheckLog(ctx context.Context, config *Config, ctlog *loglist.Log) error { + var ( + stateDirPath = filepath.Join(config.StateDir, "logs", ctlog.LogID.Base64URLString()) + stateFilePath = filepath.Join(stateDirPath, "state.json") + sthsDirPath = filepath.Join(stateDirPath, "unverified_sths") + ) + state, err := loadStateFile(stateFilePath) + if errors.Is(err, fs.ErrNotExist) { + return nil + } else if err != nil { + return fmt.Errorf("error loading state file: %w", err) + } + + if time.Since(state.LastSuccess) < config.HealthCheckInterval { + return nil + } + + sths, err := loadSTHsFromDir(sthsDirPath) + if err != nil { + return fmt.Errorf("error loading STHs directory: %w", err) + } + + if len(sths) == 0 { + if err := notify(ctx, config, &staleSTHEvent{ + Log: ctlog, + LastSuccess: state.LastSuccess, + LatestSTH: state.VerifiedSTH, + }); err != nil { + return fmt.Errorf("error notifying about stale STH: %w", err) + } + } else { + if err := notify(ctx, config, &backlogEvent{ + Log: ctlog, + LatestSTH: sths[len(sths)-1], + Position: state.DownloadPosition.Size(), + }); err != nil { + return fmt.Errorf("error notifying about backlog: %w", err) + } + } + + return nil +} + +type staleSTHEvent struct { + Log *loglist.Log + LastSuccess time.Time + LatestSTH *ct.SignedTreeHead // may be nil +} +type backlogEvent struct { + Log *loglist.Log + LatestSTH *ct.SignedTreeHead + Position uint64 +} +type staleLogListEvent struct { + Source string + LastSuccess time.Time + LastError string + LastErrorTime time.Time +} + +func (e *backlogEvent) Backlog() uint64 { + return e.LatestSTH.TreeSize - e.Position +} + +func (e *staleSTHEvent) Environ() []string { + return []string{ + "EVENT=error", + "SUMMARY=" + fmt.Sprintf("unable to contact %s since %s", e.Log.URL, e.LastSuccess), + } +} +func (e *backlogEvent) Environ() []string { + return []string{ + "EVENT=error", + "SUMMARY=" + fmt.Sprintf("backlog of size %d from %s", e.Backlog(), e.Log.URL), + } +} +func (e *staleLogListEvent) Environ() []string { + return []string{ + "EVENT=error", + "SUMMARY=" + fmt.Sprintf("unable to retrieve log list since %s: %s", e.LastSuccess, e.LastError), + } +} + +func (e *staleSTHEvent) EmailSubject() string { + return fmt.Sprintf("[certspotter] Unable to contact %s since %s", e.Log.URL, e.LastSuccess) +} +func (e *backlogEvent) EmailSubject() string { + return fmt.Sprintf("[certspotter] Backlog of size %d from %s", e.Backlog(), e.Log.URL) +} +func (e *staleLogListEvent) EmailSubject() string { + return fmt.Sprintf("[certspotter] Unable to retrieve log list since %s", e.LastSuccess) +} + +func (e *staleSTHEvent) Text() string { + text := new(strings.Builder) + fmt.Fprintf(text, "certspotter has been unable to contact %s since %s. Consequentially, certspotter may fail to notify you about certificates in this log.\n", e.Log.URL, e.LastSuccess) + fmt.Fprintf(text, "\n") + fmt.Fprintf(text, "For details, see certspotter's stderr output.\n") + fmt.Fprintf(text, "\n") + if e.LatestSTH != nil { + fmt.Fprintf(text, "Latest known log size = %d (as of %s)\n", e.LatestSTH.TreeSize, e.LatestSTH.TimestampTime()) + } else { + fmt.Fprintf(text, "Latest known log size = none\n") + } + return text.String() +} +func (e *backlogEvent) Text() string { + text := new(strings.Builder) + fmt.Fprintf(text, "certspotter has been unable to download entries from %s in a timely manner. Consequentially, certspotter may be slow to notify you about certificates in this log.\n", e.Log.URL) + fmt.Fprintf(text, "\n") + fmt.Fprintf(text, "For more details, see certspotter's stderr output.\n") + fmt.Fprintf(text, "\n") + fmt.Fprintf(text, "Current log size = %d (as of %s)\n", e.LatestSTH.TreeSize, e.LatestSTH.TimestampTime()) + fmt.Fprintf(text, "Current position = %d\n", e.Position) + fmt.Fprintf(text, " Backlog = %d\n", e.Backlog()) + return text.String() +} +func (e *staleLogListEvent) Text() string { + text := new(strings.Builder) + fmt.Fprintf(text, "certspotter has been unable to retrieve the log list from %s since %s.\n", e.Source, e.LastSuccess) + fmt.Fprintf(text, "\n") + fmt.Fprintf(text, "Last error (at %s): %s\n", e.LastErrorTime, e.LastError) + fmt.Fprintf(text, "\n") + fmt.Fprintf(text, "Consequentially, certspotter may not be monitoring all logs, and might fail to detect certificates.\n") + return text.String() +} + +// TODO-3: make the errors more actionable diff -Nru certspotter-0.14.0/monitor/loglist.go certspotter-0.15.1/monitor/loglist.go --- certspotter-0.14.0/monitor/loglist.go 1970-01-01 00:00:00.000000000 +0000 +++ certspotter-0.15.1/monitor/loglist.go 2023-02-09 18:44:06.000000000 +0000 @@ -0,0 +1,38 @@ +// Copyright (C) 2023 Opsmate, Inc. +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License, v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. +// +// This software is distributed WITHOUT A WARRANTY OF ANY KIND. +// See the Mozilla Public License for details. + +package monitor + +import ( + "context" + "fmt" + "software.sslmate.com/src/certspotter/ct" + "software.sslmate.com/src/certspotter/loglist" +) + +type LogID = ct.SHA256Hash + +func getLogList(ctx context.Context, source string, token *loglist.ModificationToken) (map[LogID]*loglist.Log, *loglist.ModificationToken, error) { + list, newToken, err := loglist.LoadIfModified(ctx, source, token) + if err != nil { + return nil, nil, err + } + + logs := make(map[LogID]*loglist.Log) + for operatorIndex := range list.Operators { + for logIndex := range list.Operators[operatorIndex].Logs { + log := &list.Operators[operatorIndex].Logs[logIndex] + if _, exists := logs[log.LogID]; exists { + return nil, nil, fmt.Errorf("log list contains more than one entry with ID %s", log.LogID.Base64String()) + } + logs[log.LogID] = log + } + } + return logs, newToken, nil +} diff -Nru certspotter-0.14.0/monitor/malformed.go certspotter-0.15.1/monitor/malformed.go --- certspotter-0.14.0/monitor/malformed.go 1970-01-01 00:00:00.000000000 +0000 +++ certspotter-0.15.1/monitor/malformed.go 2023-02-09 18:44:06.000000000 +0000 @@ -0,0 +1,48 @@ +// Copyright (C) 2023 Opsmate, Inc. +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License, v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. +// +// This software is distributed WITHOUT A WARRANTY OF ANY KIND. +// See the Mozilla Public License for details. + +package monitor + +import ( + "fmt" + "strings" +) + +type malformedLogEntry struct { + Entry *logEntry + Error string +} + +func (malformed *malformedLogEntry) Environ() []string { + return []string{ + "EVENT=malformed_cert", + "SUMMARY=" + fmt.Sprintf("unable to parse entry %d in %s", malformed.Entry.Index, malformed.Entry.Log.URL), + "LOG_URI=" + malformed.Entry.Log.URL, + "ENTRY_INDEX=" + fmt.Sprint(malformed.Entry.Index), + "LEAF_HASH=" + malformed.Entry.LeafHash.Base64String(), + "PARSE_ERROR=" + malformed.Error, + "CERT_PARSEABLE=no", // backwards compat with pre-0.15.0; not documented + } +} + +func (malformed *malformedLogEntry) Text() string { + text := new(strings.Builder) + writeField := func(name string, value any) { fmt.Fprintf(text, "\t%13s = %s\n", name, value) } + + fmt.Fprintf(text, "Unable to determine if log entry matches your watchlist. Please file a bug report at https://github.com/SSLMate/certspotter/issues/new with the following details:\n") + writeField("Log Entry", fmt.Sprintf("%d @ %s", malformed.Entry.Index, malformed.Entry.Log.URL)) + writeField("Leaf Hash", malformed.Entry.LeafHash.Base64String()) + writeField("Error", malformed.Error) + + return text.String() +} + +func (malformed *malformedLogEntry) EmailSubject() string { + return fmt.Sprintf("[certspotter] Unable to Parse Entry %d in %s", malformed.Entry.Index, malformed.Entry.Log.URL) +} diff -Nru certspotter-0.14.0/monitor/monitor.go certspotter-0.15.1/monitor/monitor.go --- certspotter-0.14.0/monitor/monitor.go 1970-01-01 00:00:00.000000000 +0000 +++ certspotter-0.15.1/monitor/monitor.go 2023-02-09 18:44:06.000000000 +0000 @@ -0,0 +1,293 @@ +// Copyright (C) 2023 Opsmate, Inc. +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License, v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. +// +// This software is distributed WITHOUT A WARRANTY OF ANY KIND. +// See the Mozilla Public License for details. + +package monitor + +import ( + "context" + "crypto/x509" + "errors" + "fmt" + "io/fs" + "log" + "os" + "path/filepath" + "strings" + "time" + + "software.sslmate.com/src/certspotter/ct" + "software.sslmate.com/src/certspotter/ct/client" + "software.sslmate.com/src/certspotter/loglist" + "software.sslmate.com/src/certspotter/merkletree" +) + +const ( + maxGetEntriesSize = 1000 + monitorLogInterval = 5 * time.Minute +) + +func isFatalLogError(err error) bool { + return errors.Is(err, context.Canceled) +} + +func newLogClient(ctlog *loglist.Log) (*client.LogClient, error) { + logKey, err := x509.ParsePKIXPublicKey(ctlog.Key) + if err != nil { + return nil, fmt.Errorf("error parsing log key: %w", err) + } + verifier, err := ct.NewSignatureVerifier(logKey) + if err != nil { + return nil, fmt.Errorf("error with log key: %w", err) + } + return client.NewWithVerifier(strings.TrimRight(ctlog.URL, "/"), verifier), nil +} + +func monitorLogContinously(ctx context.Context, config *Config, ctlog *loglist.Log) error { + logClient, err := newLogClient(ctlog) + if err != nil { + return err + } + + ticker := time.NewTicker(monitorLogInterval) + defer ticker.Stop() + + for ctx.Err() == nil { + if err := monitorLog(ctx, config, ctlog, logClient); err != nil { + return err + } + select { + case <-ctx.Done(): + case <-ticker.C: + } + } + return ctx.Err() +} + +func monitorLog(ctx context.Context, config *Config, ctlog *loglist.Log, logClient *client.LogClient) (returnedErr error) { + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + var ( + stateDirPath = filepath.Join(config.StateDir, "logs", ctlog.LogID.Base64URLString()) + stateFilePath = filepath.Join(stateDirPath, "state.json") + sthsDirPath = filepath.Join(stateDirPath, "unverified_sths") + ) + for _, dirPath := range []string{stateDirPath, sthsDirPath} { + if err := os.Mkdir(dirPath, 0777); err != nil && !errors.Is(err, fs.ErrExist) { + return fmt.Errorf("error creating state directory: %w", err) + } + } + + startTime := time.Now() + latestSTH, err := logClient.GetSTH(ctx) + if isFatalLogError(err) { + return err + } else if err != nil { + recordError(fmt.Errorf("error fetching latest STH for %s: %w", ctlog.URL, err)) + return nil + } + latestSTH.LogID = ctlog.LogID + if err := storeSTHInDir(sthsDirPath, latestSTH); err != nil { + return fmt.Errorf("error storing latest STH: %w", err) + } + + state, err := loadStateFile(stateFilePath) + if errors.Is(err, fs.ErrNotExist) { + if config.StartAtEnd { + tree, err := reconstructTree(ctx, logClient, latestSTH) + if isFatalLogError(err) { + return err + } else if err != nil { + recordError(fmt.Errorf("error reconstructing tree of size %d for %s: %w", latestSTH.TreeSize, ctlog.URL, err)) + return nil + } + state = &stateFile{ + DownloadPosition: tree, + VerifiedPosition: tree, + VerifiedSTH: latestSTH, + LastSuccess: startTime.UTC(), + } + } else { + state = &stateFile{ + DownloadPosition: merkletree.EmptyCollapsedTree(), + VerifiedPosition: merkletree.EmptyCollapsedTree(), + VerifiedSTH: nil, + LastSuccess: startTime.UTC(), + } + } + if config.Verbose { + log.Printf("brand new log %s (starting from %d)", ctlog.URL, state.DownloadPosition.Size()) + } + if err := state.store(stateFilePath); err != nil { + return fmt.Errorf("error storing state file: %w", err) + } + } else if err != nil { + return fmt.Errorf("error loading state file: %w", err) + } + + sths, err := loadSTHsFromDir(sthsDirPath) + if err != nil { + return fmt.Errorf("error loading STHs directory: %w", err) + } + + for len(sths) > 0 && sths[0].TreeSize <= state.DownloadPosition.Size() { + // TODO-4: audit sths[0] against state.VerifiedSTH + if err := removeSTHFromDir(sthsDirPath, sths[0]); err != nil { + return fmt.Errorf("error removing STH: %w", err) + } + sths = sths[1:] + } + + defer func() { + if config.Verbose { + log.Printf("saving state in defer for %s", ctlog.URL) + } + if err := state.store(stateFilePath); err != nil && returnedErr == nil { + returnedErr = fmt.Errorf("error storing state file: %w", err) + } + }() + + if len(sths) == 0 { + state.LastSuccess = startTime.UTC() + return nil + } + + var ( + downloadBegin = state.DownloadPosition.Size() + downloadEnd = sths[len(sths)-1].TreeSize + entries = make(chan client.GetEntriesItem, maxGetEntriesSize) + downloadErr error + ) + if config.Verbose { + log.Printf("downloading entries from %s in range [%d, %d)", ctlog.URL, downloadBegin, downloadEnd) + } + go func() { + defer close(entries) + downloadErr = downloadEntries(ctx, logClient, entries, downloadBegin, downloadEnd) + }() + for rawEntry := range entries { + entry := &logEntry{ + Log: ctlog, + Index: state.DownloadPosition.Size(), + LeafInput: rawEntry.LeafInput, + ExtraData: rawEntry.ExtraData, + LeafHash: merkletree.HashLeaf(rawEntry.LeafInput), + } + if err := processLogEntry(ctx, config, entry); err != nil { + return fmt.Errorf("error processing entry %d: %w", entry.Index, err) + } + + state.DownloadPosition.Add(entry.LeafHash) + rootHash := state.DownloadPosition.CalculateRoot() + shouldSaveState := state.DownloadPosition.Size()%10000 == 0 + + for len(sths) > 0 && state.DownloadPosition.Size() == sths[0].TreeSize { + if merkletree.Hash(sths[0].SHA256RootHash) != rootHash { + recordError(fmt.Errorf("error verifying %s at tree size %d: the STH root hash (%x) does not match the entries returned by the log (%x)", ctlog.URL, sths[0].TreeSize, sths[0].SHA256RootHash, rootHash)) + + state.DownloadPosition = state.VerifiedPosition + if err := state.store(stateFilePath); err != nil { + return fmt.Errorf("error storing state file: %w", err) + } + return nil + } + + state.VerifiedPosition = state.DownloadPosition + state.VerifiedSTH = sths[0] + shouldSaveState = true + if err := removeSTHFromDir(sthsDirPath, sths[0]); err != nil { + return fmt.Errorf("error removing verified STH: %w", err) + } + + sths = sths[1:] + } + + if shouldSaveState { + if err := state.store(stateFilePath); err != nil { + return fmt.Errorf("error storing state file: %w", err) + } + } + } + + if isFatalLogError(downloadErr) { + return downloadErr + } else if downloadErr != nil { + recordError(fmt.Errorf("error downloading entries from %s: %w", ctlog.URL, downloadErr)) + return nil + } + + if config.Verbose { + log.Printf("finished downloading entries from %s", ctlog.URL) + } + + state.LastSuccess = startTime.UTC() + return nil +} + +func downloadEntries(ctx context.Context, logClient *client.LogClient, entriesChan chan<- client.GetEntriesItem, begin, end uint64) error { + for begin < end && ctx.Err() == nil { + size := begin - end + if size > maxGetEntriesSize { + size = maxGetEntriesSize + } + entries, err := logClient.GetRawEntries(ctx, begin, begin+size-1) + if err != nil { + return err + } + for _, entry := range entries { + if ctx.Err() != nil { + return ctx.Err() + } + select { + case <-ctx.Done(): + return ctx.Err() + case entriesChan <- entry: + } + } + begin += uint64(len(entries)) + } + return ctx.Err() +} + +func reconstructTree(ctx context.Context, logClient *client.LogClient, sth *ct.SignedTreeHead) (*merkletree.CollapsedTree, error) { + if sth.TreeSize == 0 { + return merkletree.EmptyCollapsedTree(), nil + } + entries, err := logClient.GetRawEntries(ctx, sth.TreeSize-1, sth.TreeSize-1) + if err != nil { + return nil, err + } + leafHash := merkletree.HashLeaf(entries[0].LeafInput) + + var tree *merkletree.CollapsedTree + if sth.TreeSize > 1 { + auditPath, _, err := logClient.GetAuditProof(ctx, leafHash[:], sth.TreeSize) + if err != nil { + return nil, err + } + hashes := make([]merkletree.Hash, len(auditPath)) + for i := range hashes { + copy(hashes[i][:], auditPath[len(auditPath)-i-1]) + } + tree, err = merkletree.NewCollapsedTree(hashes, sth.TreeSize-1) + if err != nil { + return nil, fmt.Errorf("log returned invalid audit proof for %x to %d: %w", leafHash, sth.TreeSize, err) + } + } else { + tree = merkletree.EmptyCollapsedTree() + } + + tree.Add(leafHash) + rootHash := tree.CalculateRoot() + if rootHash != merkletree.Hash(sth.SHA256RootHash) { + return nil, fmt.Errorf("calculated root hash (%x) does not match signed tree head (%x) at size %d", rootHash, sth.SHA256RootHash, sth.TreeSize) + } + + return tree, nil +} diff -Nru certspotter-0.14.0/monitor/notify.go certspotter-0.15.1/monitor/notify.go --- certspotter-0.14.0/monitor/notify.go 1970-01-01 00:00:00.000000000 +0000 +++ certspotter-0.15.1/monitor/notify.go 2023-02-09 18:44:06.000000000 +0000 @@ -0,0 +1,109 @@ +// Copyright (C) 2023 Opsmate, Inc. +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License, v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. +// +// This software is distributed WITHOUT A WARRANTY OF ANY KIND. +// See the Mozilla Public License for details. + +package monitor + +import ( + "bytes" + "context" + "fmt" + "os" + "os/exec" + "strings" + "sync" +) + +var stdoutMu sync.Mutex + +type notification interface { + Environ() []string + EmailSubject() string + Text() string +} + +func notify(ctx context.Context, config *Config, notif notification) error { + if config.Stdout { + writeToStdout(notif) + } + + if len(config.Email) > 0 { + if err := sendEmail(ctx, config.Email, notif); err != nil { + return err + } + } + + if config.Script != "" { + if err := execScript(ctx, config.Script, notif); err != nil { + return err + } + } + + return nil +} + +func writeToStdout(notif notification) { + stdoutMu.Lock() + defer stdoutMu.Unlock() + os.Stdout.WriteString(notif.Text() + "\n") +} + +func sendEmail(ctx context.Context, to []string, notif notification) error { + stdin := new(bytes.Buffer) + stderr := new(bytes.Buffer) + + fmt.Fprintf(stdin, "To: %s\n", strings.Join(to, ", ")) + fmt.Fprintf(stdin, "Subject: %s\n", notif.EmailSubject()) + fmt.Fprintf(stdin, "Mime-Version: 1.0\n") + fmt.Fprintf(stdin, "Content-Type: text/plain; charset=US-ASCII\n") + fmt.Fprintf(stdin, "X-Mailer: certspotter\n") + fmt.Fprintf(stdin, "\n") + fmt.Fprint(stdin, notif.Text()) + + args := []string{"-i", "--"} + args = append(args, to...) + + sendmail := exec.CommandContext(ctx, "/usr/sbin/sendmail", args...) + sendmail.Stdin = stdin + sendmail.Stderr = stderr + + if err := sendmail.Run(); err == nil { + return nil + } else if ctx.Err() != nil { + return ctx.Err() + } else if exitErr, isExitError := err.(*exec.ExitError); isExitError && exitErr.Exited() { + return fmt.Errorf("error sending email to %v: sendmail failed with exit code %d and error %q", to, exitErr.ExitCode(), strings.TrimSpace(stderr.String())) + } else { + return fmt.Errorf("error sending email to %v: %w", to, err) + } +} + +func execScript(ctx context.Context, scriptName string, notif notification) error { + stderr := new(bytes.Buffer) + + cmd := exec.CommandContext(ctx, scriptName) + cmd.Env = os.Environ() + cmd.Env = append(cmd.Env, notif.Environ()...) + cmd.Stderr = stderr + + if err := cmd.Run(); err == nil { + return nil + } else if ctx.Err() != nil { + return ctx.Err() + } else if exitErr, isExitError := err.(*exec.ExitError); isExitError && exitErr.Exited() { + return fmt.Errorf("script %q exited with code %d and error %q", scriptName, exitErr.ExitCode(), strings.TrimSpace(stderr.String())) + } else if isExitError { + return fmt.Errorf("script %q terminated by signal with error %q", scriptName, strings.TrimSpace(stderr.String())) + } else { + return fmt.Errorf("error executing script: %w", err) + } +} + +func isExecutable(mode os.FileMode) bool { + return mode&0111 != 0 +} diff -Nru certspotter-0.14.0/monitor/process.go certspotter-0.15.1/monitor/process.go --- certspotter-0.14.0/monitor/process.go 1970-01-01 00:00:00.000000000 +0000 +++ certspotter-0.15.1/monitor/process.go 2023-02-09 18:44:06.000000000 +0000 @@ -0,0 +1,170 @@ +// Copyright (C) 2023 Opsmate, Inc. +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License, v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. +// +// This software is distributed WITHOUT A WARRANTY OF ANY KIND. +// See the Mozilla Public License for details. + +package monitor + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + "software.sslmate.com/src/certspotter" + "software.sslmate.com/src/certspotter/ct" + "software.sslmate.com/src/certspotter/loglist" + "software.sslmate.com/src/certspotter/merkletree" +) + +type logEntry struct { + Log *loglist.Log + Index uint64 + LeafInput []byte + ExtraData []byte + LeafHash merkletree.Hash +} + +func processLogEntry(ctx context.Context, config *Config, entry *logEntry) error { + leaf, err := ct.ReadMerkleTreeLeaf(bytes.NewReader(entry.LeafInput)) + if err != nil { + return processMalformedLogEntry(ctx, config, entry, fmt.Errorf("error parsing Merkle Tree Leaf: %w", err)) + } + switch leaf.TimestampedEntry.EntryType { + case ct.X509LogEntryType: + return processX509LogEntry(ctx, config, entry, leaf.TimestampedEntry.X509Entry) + case ct.PrecertLogEntryType: + return processPrecertLogEntry(ctx, config, entry, leaf.TimestampedEntry.PrecertEntry) + default: + return processMalformedLogEntry(ctx, config, entry, fmt.Errorf("unknown log entry type %d", leaf.TimestampedEntry.EntryType)) + } +} + +func processX509LogEntry(ctx context.Context, config *Config, entry *logEntry, cert ct.ASN1Cert) error { + certInfo, err := certspotter.MakeCertInfoFromRawCert(cert) + if err != nil { + return processMalformedLogEntry(ctx, config, entry, fmt.Errorf("error parsing X.509 certificate: %w", err)) + } + + chain, err := ct.UnmarshalX509ChainArray(entry.ExtraData) + if err != nil { + return processMalformedLogEntry(ctx, config, entry, fmt.Errorf("error parsing extra_data for X.509 entry: %w", err)) + } + chain = append([]ct.ASN1Cert{cert}, chain...) + + if precertTBS, err := certspotter.ReconstructPrecertTBS(certInfo.TBS); err == nil { + certInfo.TBS = precertTBS + } else { + return processMalformedLogEntry(ctx, config, entry, fmt.Errorf("error reconstructing precertificate TBSCertificate: %w", err)) + } + + return processCertificate(ctx, config, entry, certInfo, chain) +} + +func processPrecertLogEntry(ctx context.Context, config *Config, entry *logEntry, precert ct.PreCert) error { + certInfo, err := certspotter.MakeCertInfoFromRawTBS(precert.TBSCertificate) + if err != nil { + return processMalformedLogEntry(ctx, config, entry, fmt.Errorf("error parsing precert TBSCertificate: %w", err)) + } + + chain, err := ct.UnmarshalPrecertChainArray(entry.ExtraData) + if err != nil { + return processMalformedLogEntry(ctx, config, entry, fmt.Errorf("error parsing extra_data for precert entry: %w", err)) + } + + if _, err := certspotter.ValidatePrecert(chain[0], precert.TBSCertificate); err != nil { + return processMalformedLogEntry(ctx, config, entry, fmt.Errorf("precertificate in extra_data does not match TBSCertificate in leaf_input: %w", err)) + } + + return processCertificate(ctx, config, entry, certInfo, chain) +} + +func processCertificate(ctx context.Context, config *Config, entry *logEntry, certInfo *certspotter.CertInfo, chain []ct.ASN1Cert) error { + identifiers, err := certInfo.ParseIdentifiers() + if err != nil { + return processMalformedLogEntry(ctx, config, entry, err) + } + matched, watchItem := config.WatchList.Matches(identifiers) + if !matched { + return nil + } + + cert := &discoveredCert{ + WatchItem: watchItem, + LogEntry: entry, + Info: certInfo, + Chain: chain, + TBSSHA256: sha256.Sum256(certInfo.TBS.Raw), + SHA256: sha256.Sum256(chain[0]), + PubkeySHA256: sha256.Sum256(certInfo.TBS.PublicKey.FullBytes), + Identifiers: identifiers, + } + + var notifiedPath string + if config.SaveCerts { + hexFingerprint := hex.EncodeToString(cert.SHA256[:]) + prefixPath := filepath.Join(config.StateDir, "certs", hexFingerprint[0:2]) + var ( + notifiedFilename = "." + hexFingerprint + ".notified" + certFilename = hexFingerprint + ".pem" + jsonFilename = hexFingerprint + ".v1.json" + textFilename = hexFingerprint + ".txt" + legacyCertFilename = hexFingerprint + ".cert.pem" + legacyPrecertFilename = hexFingerprint + ".precert.pem" + ) + + for _, filename := range []string{notifiedFilename, legacyCertFilename, legacyPrecertFilename} { + if fileExists(filepath.Join(prefixPath, filename)) { + return nil + } + } + + if err := os.Mkdir(prefixPath, 0777); err != nil && !errors.Is(err, fs.ErrExist) { + return fmt.Errorf("error creating directory in which to save certificate %x: %w", cert.SHA256, err) + } + + notifiedPath = filepath.Join(prefixPath, notifiedFilename) + cert.CertPath = filepath.Join(prefixPath, certFilename) + cert.JSONPath = filepath.Join(prefixPath, jsonFilename) + cert.TextPath = filepath.Join(prefixPath, textFilename) + + if err := cert.save(); err != nil { + return fmt.Errorf("error saving certificate %x: %w", cert.SHA256, err) + } + } else { + // TODO-4: save cert to temporary files, and defer their unlinking + } + + if err := notify(ctx, config, cert); err != nil { + return fmt.Errorf("error notifying about discovered certificate for %s (%x): %w", cert.WatchItem, cert.SHA256, err) + } + + if notifiedPath != "" { + if err := os.WriteFile(notifiedPath, nil, 0666); err != nil { + return fmt.Errorf("error saving certificate %x: %w", cert.SHA256, err) + } + } + + return nil +} + +func processMalformedLogEntry(ctx context.Context, config *Config, entry *logEntry, parseError error) error { + // TODO-4: save the malformed entry (in get-entries format) in the state directory so user can inspect it + + malformed := &malformedLogEntry{ + Entry: entry, + Error: parseError.Error(), + } + if err := notify(ctx, config, malformed); err != nil { + return fmt.Errorf("error notifying about malformed log entry %d in %s (%q): %w", entry.Index, entry.Log.URL, parseError, err) + } + return nil +} diff -Nru certspotter-0.14.0/monitor/statedir.go certspotter-0.15.1/monitor/statedir.go --- certspotter-0.14.0/monitor/statedir.go 1970-01-01 00:00:00.000000000 +0000 +++ certspotter-0.15.1/monitor/statedir.go 2023-02-09 18:44:06.000000000 +0000 @@ -0,0 +1,155 @@ +// Copyright (C) 2023 Opsmate, Inc. +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License, v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. +// +// This software is distributed WITHOUT A WARRANTY OF ANY KIND. +// See the Mozilla Public License for details. + +package monitor + +import ( + "encoding/json" + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + "software.sslmate.com/src/certspotter/ct" + "software.sslmate.com/src/certspotter/merkletree" + "strconv" + "strings" + "time" +) + +func readVersion(stateDir string) (int, error) { + path := filepath.Join(stateDir, "version") + + fileBytes, err := os.ReadFile(path) + if errors.Is(err, fs.ErrNotExist) { + if fileExists(filepath.Join(stateDir, "evidence")) { + return 0, nil + } else { + return -1, nil + } + } else if err != nil { + return -1, err + } + + version, err := strconv.Atoi(strings.TrimSpace(string(fileBytes))) + if err != nil { + return -1, fmt.Errorf("version file %q is malformed: %w", path, err) + } + + return version, nil +} + +func writeVersion(stateDir string) error { + return writeFile(filepath.Join(stateDir, "version"), []byte{'2', '\n'}, 0666) +} + +func migrateLogStateDirV1(dir string) error { + var sth ct.SignedTreeHead + var tree merkletree.CollapsedTree + + sthPath := filepath.Join(dir, "sth.json") + sthData, err := os.ReadFile(sthPath) + if errors.Is(err, fs.ErrNotExist) { + return nil + } else if err != nil { + return err + } + + treePath := filepath.Join(dir, "tree.json") + treeData, err := os.ReadFile(treePath) + if errors.Is(err, fs.ErrNotExist) { + return nil + } else if err != nil { + return err + } + + if err := json.Unmarshal(sthData, &sth); err != nil { + return fmt.Errorf("error unmarshaling %s: %w", sthPath, err) + } + if err := json.Unmarshal(treeData, &tree); err != nil { + return fmt.Errorf("error unmarshaling %s: %w", treePath, err) + } + + stateFile := stateFile{ + DownloadPosition: &tree, + VerifiedPosition: &tree, + VerifiedSTH: &sth, + LastSuccess: time.Now().UTC(), + } + if stateFile.store(filepath.Join(dir, "state.json")); err != nil { + return err + } + + if err := os.Remove(sthPath); err != nil { + return err + } + if err := os.Remove(treePath); err != nil { + return err + } + return nil +} + +func migrateStateDirV1(stateDir string) error { + if lockfile := filepath.Join(stateDir, "lock"); fileExists(lockfile) { + return fmt.Errorf("directory is locked by another instance of certspotter; remove %s if this is not the case", lockfile) + } + + if logDirs, err := os.ReadDir(filepath.Join(stateDir, "logs")); err == nil { + for _, logDir := range logDirs { + if strings.HasPrefix(logDir.Name(), ".") || !logDir.IsDir() { + continue + } + if err := migrateLogStateDirV1(filepath.Join(stateDir, "logs", logDir.Name())); err != nil { + return fmt.Errorf("error migrating log state: %w", err) + } + } + } else if !errors.Is(err, fs.ErrNotExist) { + return err + } + + if err := writeVersion(stateDir); err != nil { + return err + } + + if err := os.Remove(filepath.Join(stateDir, "once")); err != nil && !errors.Is(err, fs.ErrNotExist) { + return err + } + + return nil +} + +func prepareStateDir(stateDir string) error { + if err := os.Mkdir(stateDir, 0777); err != nil && !errors.Is(err, fs.ErrExist) { + return err + } + + if version, err := readVersion(stateDir); err != nil { + return err + } else if version == -1 { + if err := writeVersion(stateDir); err != nil { + return err + } + } else if version == 0 { + return fmt.Errorf("%s was created by a very old version of certspotter; run any version of certspotter after 0.2 and before 0.15.0 to upgrade this directory, or remove it to start from scratch", stateDir) + } else if version == 1 { + if err := migrateStateDirV1(stateDir); err != nil { + return err + } + } else if version > 2 { + return fmt.Errorf("%s was created by a newer version of certspotter; upgrade to the latest version of certspotter or remove this directory to start from scratch", stateDir) + } + + for _, subdir := range []string{"certs", "logs"} { + if err := os.Mkdir(filepath.Join(stateDir, subdir), 0777); err != nil && !errors.Is(err, fs.ErrExist) { + return err + } + } + + return nil +} diff -Nru certspotter-0.14.0/monitor/statefile.go certspotter-0.15.1/monitor/statefile.go --- certspotter-0.14.0/monitor/statefile.go 1970-01-01 00:00:00.000000000 +0000 +++ certspotter-0.15.1/monitor/statefile.go 2023-02-09 18:44:06.000000000 +0000 @@ -0,0 +1,47 @@ +// Copyright (C) 2023 Opsmate, Inc. +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License, v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. +// +// This software is distributed WITHOUT A WARRANTY OF ANY KIND. +// See the Mozilla Public License for details. + +package monitor + +import ( + "encoding/json" + "fmt" + "os" + "software.sslmate.com/src/certspotter/ct" + "software.sslmate.com/src/certspotter/merkletree" + "time" +) + +type stateFile struct { + DownloadPosition *merkletree.CollapsedTree `json:"download_position"` + VerifiedPosition *merkletree.CollapsedTree `json:"verified_position"` + VerifiedSTH *ct.SignedTreeHead `json:"verified_sth"` + LastSuccess time.Time `json:"last_success"` +} + +func loadStateFile(filePath string) (*stateFile, error) { + fileBytes, err := os.ReadFile(filePath) + if err != nil { + return nil, err + } + file := new(stateFile) + if err := json.Unmarshal(fileBytes, file); err != nil { + return nil, fmt.Errorf("error parsing %s: %w", filePath, err) + } + return file, nil +} + +func (file *stateFile) store(filePath string) error { + fileBytes, err := json.Marshal(file) + if err != nil { + return err + } + fileBytes = append(fileBytes, '\n') + return writeFile(filePath, fileBytes, 0666) +} diff -Nru certspotter-0.14.0/monitor/sthdir.go certspotter-0.15.1/monitor/sthdir.go --- certspotter-0.14.0/monitor/sthdir.go 1970-01-01 00:00:00.000000000 +0000 +++ certspotter-0.15.1/monitor/sthdir.go 2023-02-09 18:44:06.000000000 +0000 @@ -0,0 +1,96 @@ +// Copyright (C) 2017, 2023 Opsmate, Inc. +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License, v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. +// +// This software is distributed WITHOUT A WARRANTY OF ANY KIND. +// See the Mozilla Public License for details. + +package monitor + +import ( + "crypto/sha256" + "encoding/base64" + "encoding/binary" + "encoding/json" + "errors" + "fmt" + "golang.org/x/exp/slices" + "io/fs" + "os" + "path/filepath" + "software.sslmate.com/src/certspotter/ct" + "strconv" + "strings" +) + +func loadSTHsFromDir(dirPath string) ([]*ct.SignedTreeHead, error) { + entries, err := os.ReadDir(dirPath) + if errors.Is(err, fs.ErrNotExist) { + return []*ct.SignedTreeHead{}, nil + } else if err != nil { + return nil, err + } + sths := make([]*ct.SignedTreeHead, 0, len(entries)) + for _, entry := range entries { + filename := entry.Name() + if strings.HasPrefix(filename, ".") || !strings.HasSuffix(filename, ".json") { + continue + } + sth, err := readSTHFile(filepath.Join(dirPath, filename)) + if err != nil { + return nil, err + } + sths = append(sths, sth) + } + slices.SortFunc(sths, func(a, b *ct.SignedTreeHead) bool { return a.TreeSize < b.TreeSize }) + return sths, nil +} + +func readSTHFile(filePath string) (*ct.SignedTreeHead, error) { + fileBytes, err := os.ReadFile(filePath) + if err != nil { + return nil, err + } + sth := new(ct.SignedTreeHead) + if err := json.Unmarshal(fileBytes, sth); err != nil { + return nil, fmt.Errorf("error parsing %s: %w", filePath, err) + } + return sth, nil +} + +func storeSTHInDir(dirPath string, sth *ct.SignedTreeHead) error { + filePath := filepath.Join(dirPath, sthFilename(sth)) + if fileExists(filePath) { + return nil + } + fileBytes, err := json.Marshal(sth) + if err != nil { + return err + } + return writeFile(filePath, fileBytes, 0666) +} + +func removeSTHFromDir(dirPath string, sth *ct.SignedTreeHead) error { + filePath := filepath.Join(dirPath, sthFilename(sth)) + err := os.Remove(filePath) + if err != nil && !errors.Is(err, fs.ErrNotExist) { + return err + } + return nil +} + +// generate a filename that uniquely identifies the STH (within the context of a particular log) +func sthFilename(sth *ct.SignedTreeHead) string { + hasher := sha256.New() + switch sth.Version { + case ct.V1: + binary.Write(hasher, binary.LittleEndian, sth.Timestamp) + binary.Write(hasher, binary.LittleEndian, sth.SHA256RootHash) + default: + panic(fmt.Errorf("sthFilename: invalid STH version %d", sth.Version)) + } + // For 6962-bis, we will need to handle a variable-length root hash, and include the signature in the filename hash (since signatures must be deterministic) + return strconv.FormatUint(sth.TreeSize, 10) + "-" + base64.RawURLEncoding.EncodeToString(hasher.Sum(nil)) + ".json" +} diff -Nru certspotter-0.14.0/monitor/watchlist.go certspotter-0.15.1/monitor/watchlist.go --- certspotter-0.14.0/monitor/watchlist.go 1970-01-01 00:00:00.000000000 +0000 +++ certspotter-0.15.1/monitor/watchlist.go 2023-02-09 18:44:06.000000000 +0000 @@ -0,0 +1,135 @@ +// Copyright (C) 2016, 2023 Opsmate, Inc. +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License, v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. +// +// This software is distributed WITHOUT A WARRANTY OF ANY KIND. +// See the Mozilla Public License for details. + +package monitor + +import ( + "bufio" + "fmt" + "golang.org/x/net/idna" + "io" + "software.sslmate.com/src/certspotter" + "strings" +) + +type WatchItem struct { + domain []string + acceptSuffix bool +} + +type WatchList []WatchItem + +func ParseWatchItem(str string) (WatchItem, error) { + fields := strings.Fields(str) + if len(fields) == 0 { + return WatchItem{}, fmt.Errorf("empty domain") + } + domain := fields[0] + + for _, field := range fields[1:] { + switch { + case strings.HasPrefix(field, "valid_at:"): + // Ignore for backwards compatibility + default: + return WatchItem{}, fmt.Errorf("unknown parameter %q", field) + } + } + + if domain == "." { + // "." as in root zone -> matches everything + return WatchItem{ + domain: []string{}, + acceptSuffix: true, + }, nil + } + + acceptSuffix := false + if strings.HasPrefix(domain, ".") { + acceptSuffix = true + domain = domain[1:] + } + + asciiDomain, err := idna.ToASCII(strings.ToLower(strings.TrimRight(domain, "."))) + if err != nil { + return WatchItem{}, fmt.Errorf("invalid domain %q (%w)", domain, err) + } + return WatchItem{ + domain: strings.Split(asciiDomain, "."), + acceptSuffix: acceptSuffix, + }, nil +} + +func ReadWatchList(reader io.Reader) (WatchList, error) { + items := make(WatchList, 0, 50) + scanner := bufio.NewScanner(reader) + lineNo := 0 + for scanner.Scan() { + line := scanner.Text() + lineNo++ + if line == "" || strings.HasPrefix(line, "#") { + continue + } + item, err := ParseWatchItem(line) + if err != nil { + return nil, fmt.Errorf("%w on line %d", err, lineNo) + } + items = append(items, item) + } + return items, scanner.Err() +} + +func (item WatchItem) String() string { + if item.acceptSuffix { + return "." + strings.Join(item.domain, ".") + } else { + return strings.Join(item.domain, ".") + } +} + +func (item WatchItem) matchesDNSName(dnsName []string) bool { + watchDomain := item.domain + for len(dnsName) > 0 && len(watchDomain) > 0 { + certLabel := dnsName[len(dnsName)-1] + watchLabel := watchDomain[len(watchDomain)-1] + + if !dnsLabelMatches(certLabel, watchLabel) { + return false + } + + dnsName = dnsName[:len(dnsName)-1] + watchDomain = watchDomain[:len(watchDomain)-1] + } + return len(watchDomain) == 0 && (item.acceptSuffix || len(dnsName) == 0) +} + +func dnsLabelMatches(certLabel string, watchLabel string) bool { + // For fail-safe behavior, if a label was unparsable, it matches everything. + // Similarly, redacted labels match everything, since the label _might_ be + // for a name we're interested in. + + return certLabel == "*" || + certLabel == "?" || + certLabel == certspotter.UnparsableDNSLabelPlaceholder || + certspotter.MatchesWildcard(watchLabel, certLabel) +} + +func (list WatchList) Matches(identifiers *certspotter.Identifiers) (bool, WatchItem) { + dnsNames := make([][]string, len(identifiers.DNSNames)) + for i, dnsName := range identifiers.DNSNames { + dnsNames[i] = strings.Split(dnsName, ".") + } + for _, item := range list { + for _, dnsName := range dnsNames { + if item.matchesDNSName(dnsName) { + return true, item + } + } + } + return false, WatchItem{} +} diff -Nru certspotter-0.14.0/NEWS certspotter-0.15.1/NEWS --- certspotter-0.14.0/NEWS 2022-06-13 15:23:35.000000000 +0000 +++ certspotter-0.15.1/NEWS 1970-01-01 00:00:00.000000000 +0000 @@ -1,102 +0,0 @@ -v0.14.0 (2022-06-13) - * Switch to Go module versioning conventions. - -v0.13 (2022-06-13) - * Reduce minimum Go version to 1.17. - * Update install instructions. - -v0.12 (2022-06-07) - * Retry failed log requests. This should make certspotter resilient - to rate limiting by logs. - * Add -version flag. - * Eliminate unnecessary dependency. certspotter now depends only on - golang.org/x packages. - * Switch to Go modules. - -v0.11 (2021-08-17) - * Add support for contacting logs via HTTP proxies; - just set the appropriate environment variable as documented at - https://golang.org/pkg/net/http/#ProxyFromEnvironment - * Work around RFC 6962 ambiguity related to consistency proofs - for empty trees. - -v0.10 (2020-04-29) - * Improve speed by processing logs in parallel - * Add -start_at_end option to begin monitoring new logs at the end, - which significantly speeds up Cert Spotter, at the cost of missing - certificates that were added to a log before Cert Spotter starts - monitoring it - * (Behavior change) Scan logs in their entirety the first time Cert - Spotter is run, unless -start_at_end specified (behavior change) - * The log list is now retrieved from certspotter.org at startup instead - of being embedded in the source. This will allow Cert Spotter to react - more quickly to the frequent changes in logs. - * (Behavior change) the -logs option now expects a JSON file in the v2 - log list format. See - and . - * -logs now accepts an HTTPS URL in addition to a file path. - * (Behavior change) the -underwater option has been removed. If you want - its behavior, specify https://loglist.certspotter.org/underwater.json to - the -logs option. - -v0.9 (2018-04-19) - * Add Cloudflare Nimbus logs - * Remove Google Argon 2017 log - * Remove WoSign and StartCom logs due to disqualification by Chromium - and extended downtime - -v0.8 (2017-12-08) - * Add Symantec Sirius log - * Add DigiCert 2 log - -v0.7 (2017-11-13) - * Add Google Argon logs - * Fix bug that caused crash on 32 bit architectures - -v0.6 (2017-10-19) - * Add Comodo Mammoth and Comodo Sabre logs - * Minor bug fixes and improvements - -v0.5 (2017-05-18) - * Remove PuChuangSiDa 1 log due to excessive downtime and presumptive - disqualification from Chrome - * Add Venafi Gen2 log - * Improve monitoring robustness under certain pathological behavior - by logs - * Minor documentation improvements - -v0.4 (2017-04-03) - * Add PuChuangSiDa 1 log - * Remove Venafi log due to fork and disqualification from Chrome - -v0.3 (2017-02-20) - * Revise -all_time flag (behavior change): - - If -all_time is specified, scan the entirety of all logs, even - existing logs - - When a new log is added, scan it in its entirety even if -all_time - is not specified - * Add new logs: - - Google Icarus - - Google Skydiver - - StartCom - - WoSign - * Overhaul log processing and auditing logic: - - STHs are never deleted unless they can be verified - - Multiple unverified STHs can be queued per log, laying groundwork - for STH pollination support - - New state directory layout; current state directories will be - migrated, but migration will be removed in a future version - - Persist condensed Merkle Tree state between runs, instead of - reconstructing it from consistency proof every time - * Use a lock file to prevent multiple instances of Cert Spotter from - running concurrently (which could clobber the state directory). - * Minor bug fixes and improvements - -v0.2 (2016-08-25) - * Suppress duplicate identifiers in output. - * Fix "EOF" error when running under Go 1.7. - * Fix bug where hook script could fail silently. - * Fix compilation under Go 1.5. - -v0.1 (2016-07-27) - * Initial release. diff -Nru certspotter-0.14.0/README certspotter-0.15.1/README --- certspotter-0.14.0/README 2022-06-13 15:23:35.000000000 +0000 +++ certspotter-0.15.1/README 1970-01-01 00:00:00.000000000 +0000 @@ -1,162 +0,0 @@ -Cert Spotter is a Certificate Transparency log monitor from SSLMate that -alerts you when a SSL/TLS certificate is issued for one of your domains. -Cert Spotter is easier than other open source CT monitors, since it does -not require a database. It's also more robust, since it uses a special -certificate parser that ensures it won't miss certificates. - -Cert Spotter is also available as a hosted service by SSLMate that -requires zero setup and provides an easy web dashboard to centrally -manage your certificates. Visit -to sign up. - -You can use Cert Spotter to detect: - -* Certificates issued to attackers who have compromised your DNS and - are redirecting your visitors to their malicious site. - -* Certificates issued to attackers who have taken over an abandoned - sub-domain in order to serve malware under your name. - -* Certificates issued to attackers who have compromised a certificate - authority and want to impersonate your site. - -* Certificates issued in violation of your corporate policy - or outside of your centralized certificate procurement process. - - -USING CERT SPOTTER - -The easiest way to use Cert Spotter is to sign up for an account at -. If you want to run Cert Spotter on -your own server, follow these instructions. - -Cert Spotter requires Go version 1.17 or higher. - -1. Install Cert Spotter using the `go` command: - - go install software.sslmate.com/src/certspotter/cmd/certspotter@latest - -2. Create a file called ~/.certspotter/watchlist listing the DNS names - you want to monitor, one per line. To monitor an entire domain tree - (including the domain itself and all sub-domains) prefix the domain - name with a dot (e.g. ".example.com"). To monitor a single DNS name - only, do not prefix the name with a dot. - -3. Create a cron job to periodically run `certspotter`. See below for - command line options. - -Every time you run Cert Spotter, it scans all browser-recognized -Certificate Transparency logs for certificates matching domains on -your watch list. When Cert Spotter detects a matching certificate, it -writes a report to standard out, which the Cron daemon emails to you. -Make sure you are able to receive emails sent by Cron. - -Cert Spotter also saves a copy of matching certificates in -~/.certspotter/certs (unless you specify the -no_save option). - -When Cert Spotter has previously monitored a log, it scans the log -from the previous position, to avoid downloading the same log entry -more than once. (To override this behavior and scan all logs from the -beginning, specify the -all_time option.) - -When Cert Spotter has not previously monitored a log, it can either start -monitoring the log from the beginning, or seek to the end of the log and -start monitoring from there. Monitoring from the beginning guarantees -detection of all certificates, but requires downloading hundreds of -millions of certificates, which takes days. The default behavior is to -monitor from the beginning. To start monitoring new logs from the end, -specify the -start_at_end option. - -You can add and remove domains on your watchlist at any time. However, -the certspotter command only notifies you of certificates that were -logged since adding a domain to the watchlist, unless you specify the --all_time option, which requires scanning the entirety of every log -and takes many days to complete with a fast Internet connection. -To examine preexisting certificates, it's better to use the Cert -Spotter service , the Cert Spotter -API , or a CT search engine such -as . - - -COMMAND LINE FLAGS - - -watchlist FILENAME - File containing identifiers to watch, one per line, as described - above (use - to read from stdin). Default: ~/.certspotter/watchlist - -no_save - Do not save a copy of matching certificates. - -start_at_end - Start monitoring logs from the end, rather than the beginning. - This significantly reduces the time to run Cert Spotter, but - you will miss certificates that were added to a log before Cert - Spotter started monitoring it. - -all_time - Scan for certificates from all time, not just those logged since - the previous run of Cert Spotter. - -logs FILENAME_OR_URL - Filename of HTTPS URL of a JSON file containing logs to monitor, in the format - documented at . - Default: https://loglist.certspotter.org/monitor.json which includes the union - of active logs recognized by Chrome and Apple. - -state_dir PATH - Directory for storing state. Default: ~/.certspotter - -verbose - Be verbose. - - -WHAT CERTIFICATES ARE DETECTED BY CERT SPOTTER? - -Any certificate that is logged to a Certificate Transparency log trusted -by Chromium will be detected by Cert Spotter. All certificates issued -after April 30, 2018 must be logged to such a log to be trusted by Chromium. - -Generally, certificate authorities will automatically submit certificates -to logs so that they will work in Chromium. In addition, certificates -that are discovered during Internet-wide scans are submitted to Certificate -Transparency logs. - - -SECURITY - -Cert Spotter assumes an adversarial model in which an attacker produces -a certificate that is accepted by at least some clients but goes -undetected because of an encoding error that prevents CT monitors from -understanding it. To defend against this attack, Cert Spotter uses a -special certificate parser that keeps the certificate unparsed except -for the identifiers. If one of the identifiers matches a domain on your -watchlist, you will be notified, even if other parts of the certificate -are unparsable. - -Cert Spotter takes special precautions to ensure identifiers are parsed -correctly, and implements defenses against identifier-based attacks. -For instance, if a DNS identifier contains a null byte, Cert Spotter -interprets it as two identifiers: the complete identifier, and the -identifier formed by truncating at the first null byte. For example, a -certificate for example.org\0.example.com will alert the owners of both -example.org and example.com. This defends against null prefix attacks -. - -SSLMate continuously monitors CT logs to make sure every certificate's -identifiers can be successfully parsed, and will release updates to -Cert Spotter as necessary to fix parsing failures. - -Cert Spotter understands wildcard and redacted DNS names, and will alert -you if a wildcard or redacted certificate might match an identifier on -your watchlist. For example, a watchlist entry for sub.example.com would -match certificates for *.example.com or ?.example.com. - -Cert Spotter is not just a log monitor, but also a log auditor which -checks that the log is obeying its append-only property. A future -release of Cert Spotter will support gossiping with other log monitors -to ensure the log is presenting a single view. - - -BygoneSSL - -Cert Spotter can also notify users of bygone SSL certificates, which are SSL -certificates that outlived their prior domain owner's registration into the -next owners registration. To detect these certificates add a valid_at -argument to each domain in the watchlist followed by the date the domain was -registered in the following format YYYY-MM-DD. For example: -example.com valid_at:2014-05-02 - diff -Nru certspotter-0.14.0/README.md certspotter-0.15.1/README.md --- certspotter-0.14.0/README.md 1970-01-01 00:00:00.000000000 +0000 +++ certspotter-0.15.1/README.md 2023-02-09 18:44:06.000000000 +0000 @@ -0,0 +1,111 @@ +# Cert Spotter - Certificate Transparency Monitor + +**Cert Spotter** is a Certificate Transparency log monitor from SSLMate that +alerts you when an SSL/TLS certificate is issued for one of your domains. +Cert Spotter is easier to use than other open source CT monitors, since it does not require +a database. It's also more robust, since it uses a special certificate parser +that ensures it won't miss certificates. + +Cert Spotter is also available as a hosted service by SSLMate that +requires zero setup and provides an easy web dashboard to centrally +manage your certificates. Visit +to sign up. + +You can use Cert Spotter to detect: + + * Certificates issued to attackers who have compromised your DNS and + are redirecting your visitors to their malicious site. + * Certificates issued to attackers who have taken over an abandoned + sub-domain in order to serve malware under your name. + * Certificates issued to attackers who have compromised a certificate + authority and want to impersonate your site. + * Certificates issued in violation of your corporate policy + or outside of your centralized certificate procurement process. + +## Quickstart + +Cert Spotter requires Go version 1.19 or higher. + +1. Install the certspotter command using the `go` command: + + ``` + go install software.sslmate.com/src/certspotter/cmd/certspotter@latest + ``` + +2. Create a watch list file containing the DNS names you want to monitor, + one per line. To monitor an entire domain tree (including the domain itself + and all sub-domains) prefix the domain name with a dot (e.g. `.example.com`). + To monitor a single DNS name only, do not prefix the name with a dot. + +3. Configure your system to run `certspotter` as a daemon. You should specify + the following command line options: + + * `-watchlist PATH` to specify the path to your watch list file. + + * `-email ADDRESS` to specify an email address which certspotter will contact + when it detects a domain on your watch list. (Your system must have a + working sendmail command.) + + * (Optional) `-start_at_end` to tell certspotter to start monitoring logs at the end + instead of the beginning. This saves significant bandwidth, but you won't be + notified about certificates which were logged before you started using certspotter. + + For example: + + ``` + certspotter -watchlist /etc/certspotter.watchlist -email pki@certspotteruser.example -start_at_end + ``` + +## Documentation + +* Command line options and operational details: [certspotter(8) man page](man/certspotter.md) +* The `-script` interface: [certspotter-script(8) man page](man/certspotter-script.md) +* [Change Log](CHANGELOG.md) + +## What certificates are detected by Cert Spotter? + +In the default configuration, any certificate that is logged to a Certificate +Transparency log recognized by Google Chrome or Apple will be detected by +Cert Spotter. By default, Google Chrome and Apple only accept certificates that +are logged, so any certificate that works in Chrome or Safari will be detected +by Cert Spotter. + +## Security + +Cert Spotter assumes an adversarial model in which an attacker produces +a certificate that is accepted by at least some clients but goes +undetected because of an encoding error that prevents CT monitors from +understanding it. To defend against this attack, Cert Spotter uses a +special certificate parser that keeps the certificate unparsed except +for the identifiers. If one of the identifiers matches a domain on your +watchlist, you will be notified, even if other parts of the certificate +are unparsable. + +Cert Spotter takes special precautions to ensure identifiers are parsed +correctly, and implements defenses against identifier-based attacks. +For instance, if a DNS identifier contains a null byte, Cert Spotter +interprets it as two identifiers: the complete identifier, and the +identifier formed by truncating at the first null byte. For example, a +certificate for `example.org\0.example.com` will alert the owners of both +`example.org` and `example.com`. This defends against [null prefix attacks] +(http://www.thoughtcrime.org/papers/null-prefix-attacks.pdf). + +SSLMate continuously monitors CT logs to make sure every certificate's +identifiers can be successfully parsed, and will release updates to +Cert Spotter as necessary to fix parsing failures. + +Cert Spotter understands wildcard DNS names, and will alert +you if a wildcard certificate might match an identifier on +your watchlist. For example, a watchlist entry for `sub.example.com` would +match certificates for `*.example.com`. + +Cert Spotter is not just a log monitor, but also a log auditor which +checks that the log is obeying its append-only property. A future +release of Cert Spotter will support gossiping with other log monitors +to ensure the log is presenting a single view. + +## Copyright + +Copyright © 2016-2023 Opsmate, Inc. + +Licensed under the [Mozilla Public License Version 2.0](LICENSE). diff -Nru certspotter-0.14.0/scanner.go certspotter-0.15.1/scanner.go --- certspotter-0.14.0/scanner.go 2022-06-13 15:23:35.000000000 +0000 +++ certspotter-0.15.1/scanner.go 1970-01-01 00:00:00.000000000 +0000 @@ -1,303 +0,0 @@ -// Copyright (C) 2016 Opsmate, Inc. -// -// This Source Code Form is subject to the terms of the Mozilla -// Public License, v. 2.0. If a copy of the MPL was not distributed -// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. -// -// This software is distributed WITHOUT A WARRANTY OF ANY KIND. -// See the Mozilla Public License for details. -// -// This file contains code from https://github.com/google/certificate-transparency/tree/master/go -// See ct/AUTHORS and ct/LICENSE for copyright and license information. - -package certspotter - -import ( - // "container/list" - "bytes" - "context" - "crypto" - "errors" - "fmt" - "log" - "strings" - "sync" - "sync/atomic" - "time" - - "software.sslmate.com/src/certspotter/ct" - "software.sslmate.com/src/certspotter/ct/client" -) - -type ProcessCallback func(*Scanner, *ct.LogEntry) - -// ScannerOptions holds configuration options for the Scanner -type ScannerOptions struct { - // Number of entries to request in one batch from the Log - BatchSize int - - // Number of concurrent proecssors to run - NumWorkers int - - // Don't print any status messages to stdout - Quiet bool -} - -// Creates a new ScannerOptions struct with sensible defaults -func DefaultScannerOptions() *ScannerOptions { - return &ScannerOptions{ - BatchSize: 1000, - NumWorkers: 1, - Quiet: false, - } -} - -// Scanner is a tool to scan all the entries in a CT Log. -type Scanner struct { - // Base URI of CT log - LogUri string - - // Public key of the log - publicKey crypto.PublicKey - LogId ct.SHA256Hash - - // Client used to talk to the CT log instance - logClient *client.LogClient - - // Configuration options for this Scanner instance - opts ScannerOptions -} - -// fetchRange represents a range of certs to fetch from a CT log -type fetchRange struct { - start int64 - end int64 -} - -// Worker function to process certs. -// Accepts ct.LogEntries over the |entries| channel, and invokes processCert on them. -// Returns true over the |done| channel when the |entries| channel is closed. -func (s *Scanner) processerJob(id int, certsProcessed *int64, entries <-chan ct.LogEntry, processCert ProcessCallback, wg *sync.WaitGroup) { - for entry := range entries { - atomic.AddInt64(certsProcessed, 1) - processCert(s, &entry) - } - wg.Done() -} - -func (s *Scanner) fetch(r fetchRange, entries chan<- ct.LogEntry, tree *CollapsedMerkleTree) error { - for r.start <= r.end { - s.Log(fmt.Sprintf("Fetching entries %d to %d", r.start, r.end)) - logEntries, err := s.logClient.GetEntries(context.Background(), r.start, r.end) - if err != nil { - return err - } - for _, logEntry := range logEntries { - if tree != nil { - tree.Add(hashLeaf(logEntry.LeafBytes)) - } - logEntry.Index = r.start - entries <- logEntry - r.start++ - } - } - return nil -} - -// Worker function for fetcher jobs. -// Accepts cert ranges to fetch over the |ranges| channel, and if the fetch is -// successful sends the individual LeafInputs out into the -// |entries| channel for the processors to chew on. -// Will retry failed attempts to retrieve ranges indefinitely. -// Sends true over the |done| channel when the |ranges| channel is closed. -/* disabled becuase error handling is broken -func (s *Scanner) fetcherJob(id int, ranges <-chan fetchRange, entries chan<- ct.LogEntry, wg *sync.WaitGroup) { - for r := range ranges { - s.fetch(r, entries, nil) - } - wg.Done() -} -*/ - -// Returns the smaller of |a| and |b| -func min(a int64, b int64) int64 { - if a < b { - return a - } else { - return b - } -} - -// Returns the larger of |a| and |b| -func max(a int64, b int64) int64 { - if a > b { - return a - } else { - return b - } -} - -// Pretty prints the passed in number of |seconds| into a more human readable -// string. -func humanTime(seconds int) string { - nanos := time.Duration(seconds) * time.Second - hours := int(nanos / (time.Hour)) - nanos %= time.Hour - minutes := int(nanos / time.Minute) - nanos %= time.Minute - seconds = int(nanos / time.Second) - s := "" - if hours > 0 { - s += fmt.Sprintf("%d hours ", hours) - } - if minutes > 0 { - s += fmt.Sprintf("%d minutes ", minutes) - } - if seconds > 0 { - s += fmt.Sprintf("%d seconds ", seconds) - } - return s -} - -func (s Scanner) Log(msg string) { - if !s.opts.Quiet { - log.Print(s.LogUri, ": ", msg) - } -} - -func (s Scanner) Warn(msg string) { - log.Print(s.LogUri, ": ", msg) -} - -func (s *Scanner) GetSTH() (*ct.SignedTreeHead, error) { - latestSth, err := s.logClient.GetSTH(context.Background()) - if err != nil { - return nil, err - } - if s.publicKey != nil { - verifier, err := ct.NewSignatureVerifier(s.publicKey) - if err != nil { - return nil, err - } - if err := verifier.VerifySTHSignature(*latestSth); err != nil { - return nil, errors.New("STH signature is invalid: " + err.Error()) - } - } - latestSth.LogID = s.LogId - return latestSth, nil -} - -func (s *Scanner) CheckConsistency(first *ct.SignedTreeHead, second *ct.SignedTreeHead) (bool, error) { - if first.TreeSize == 0 || second.TreeSize == 0 { - // RFC 6962 doesn't define how to generate a consistency proof in this case, - // and it doesn't matter anyways since the tree is empty. The DigiCert logs - // return a 400 error if we ask for such a proof. - return true, nil - } else if first.TreeSize < second.TreeSize { - proof, err := s.logClient.GetConsistencyProof(context.Background(), int64(first.TreeSize), int64(second.TreeSize)) - if err != nil { - return false, err - } - return VerifyConsistencyProof(proof, first, second), nil - } else if first.TreeSize > second.TreeSize { - proof, err := s.logClient.GetConsistencyProof(context.Background(), int64(second.TreeSize), int64(first.TreeSize)) - if err != nil { - return false, err - } - return VerifyConsistencyProof(proof, second, first), nil - } else { - // There is no need to ask the server for a consistency proof if the trees - // are the same size, and the DigiCert log returns a 400 error if we try. - return bytes.Equal(first.SHA256RootHash[:], second.SHA256RootHash[:]), nil - } -} - -func (s *Scanner) MakeCollapsedMerkleTree(sth *ct.SignedTreeHead) (*CollapsedMerkleTree, error) { - if sth.TreeSize == 0 { - return &CollapsedMerkleTree{}, nil - } - - entries, err := s.logClient.GetEntries(context.Background(), int64(sth.TreeSize-1), int64(sth.TreeSize-1)) - if err != nil { - return nil, err - } - if len(entries) == 0 { - return nil, fmt.Errorf("Log did not return entry %d", sth.TreeSize-1) - } - leafHash := hashLeaf(entries[0].LeafBytes) - - var tree *CollapsedMerkleTree - if sth.TreeSize > 1 { - auditPath, _, err := s.logClient.GetAuditProof(context.Background(), leafHash, sth.TreeSize) - if err != nil { - return nil, err - } - reverseHashes(auditPath) - tree, err = NewCollapsedMerkleTree(auditPath, sth.TreeSize-1) - if err != nil { - return nil, fmt.Errorf("Error returned bad audit proof for %x to %d", leafHash, sth.TreeSize) - } - } else { - tree = EmptyCollapsedMerkleTree() - } - - tree.Add(leafHash) - if !bytes.Equal(tree.CalculateRoot(), sth.SHA256RootHash[:]) { - return nil, fmt.Errorf("Calculated root hash does not match signed tree head at size %d", sth.TreeSize) - } - - return tree, nil -} - -func (s *Scanner) Scan(startIndex int64, endIndex int64, processCert ProcessCallback, tree *CollapsedMerkleTree) error { - s.Log("Starting scan...") - - certsProcessed := new(int64) - startTime := time.Now() - /* TODO: only launch ticker goroutine if in verbose mode; kill the goroutine when the scanner finishes - ticker := time.NewTicker(time.Second) - go func() { - for range ticker.C { - throughput := float64(s.certsProcessed) / time.Since(startTime).Seconds() - remainingCerts := int64(endIndex) - int64(startIndex) - s.certsProcessed - remainingSeconds := int(float64(remainingCerts) / throughput) - remainingString := humanTime(remainingSeconds) - s.Log(fmt.Sprintf("Processed: %d certs (to index %d). Throughput: %3.2f ETA: %s", s.certsProcessed, - startIndex+int64(s.certsProcessed), throughput, remainingString)) - } - }() - */ - - // Start processor workers - jobs := make(chan ct.LogEntry, 100) - var processorWG sync.WaitGroup - for w := 0; w < s.opts.NumWorkers; w++ { - processorWG.Add(1) - go s.processerJob(w, certsProcessed, jobs, processCert, &processorWG) - } - - for start := startIndex; start < int64(endIndex); { - end := min(start+int64(s.opts.BatchSize), int64(endIndex)) - 1 - if err := s.fetch(fetchRange{start, end}, jobs, tree); err != nil { - return err - } - start = end + 1 - } - close(jobs) - processorWG.Wait() - s.Log(fmt.Sprintf("Completed %d certs in %s", *certsProcessed, humanTime(int(time.Since(startTime).Seconds())))) - - return nil -} - -// Creates a new Scanner instance using |client| to talk to the log, and taking -// configuration options from |opts|. -func NewScanner(logUri string, logId ct.SHA256Hash, publicKey crypto.PublicKey, opts *ScannerOptions) *Scanner { - var scanner Scanner - scanner.LogUri = logUri - scanner.LogId = logId - scanner.publicKey = publicKey - scanner.logClient = client.New(strings.TrimRight(logUri, "/")) - scanner.opts = *opts - return &scanner -} diff -Nru certspotter-0.14.0/version.go certspotter-0.15.1/version.go --- certspotter-0.14.0/version.go 2022-06-13 15:23:35.000000000 +0000 +++ certspotter-0.15.1/version.go 1970-01-01 00:00:00.000000000 +0000 @@ -1,12 +0,0 @@ -// Copyright (C) 2022 Opsmate, Inc. -// -// This Source Code Form is subject to the terms of the Mozilla -// Public License, v. 2.0. If a copy of the MPL was not distributed -// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. -// -// This software is distributed WITHOUT A WARRANTY OF ANY KIND. -// See the Mozilla Public License for details. - -package certspotter - -const Version = "0.14.0"