diff -Nru settle-0.39.9/Cargo.lock settle-0.39.11/Cargo.lock --- settle-0.39.9/Cargo.lock 2023-06-10 16:47:05.000000000 +0000 +++ settle-0.39.11/Cargo.lock 2023-08-15 18:09:47.000000000 +0000 @@ -794,7 +794,7 @@ [[package]] name = "settle" -version = "0.39.9" +version = "0.39.11" dependencies = [ "chrono", "clap", diff -Nru settle-0.39.9/Cargo.toml settle-0.39.11/Cargo.toml --- settle-0.39.9/Cargo.toml 2023-06-10 16:47:05.000000000 +0000 +++ settle-0.39.11/Cargo.toml 2023-08-15 18:09:47.000000000 +0000 @@ -1,6 +1,6 @@ [package] name = "settle" -version = "0.39.9" +version = "0.39.11" edition = "2018" authors = [ "xylous " ] description = "CLI tool for managing a digital Zettelkasten" diff -Nru settle-0.39.9/Changelog.md settle-0.39.11/Changelog.md --- settle-0.39.9/Changelog.md 2023-06-10 16:47:05.000000000 +0000 +++ settle-0.39.11/Changelog.md 2023-08-15 18:09:47.000000000 +0000 @@ -2,6 +2,28 @@ NOTE: This Changelog is partially incomplete. +## v0.39.11 - 2023-08-15 + +- add: make configuration file location flexible by allowing using the + `SETTLE_CONFIG` environment option +- fix: check for environment variables at runtime, not at compile time +- fix: don't use environment variables if they are set but empty + +## v0.39.10 - 2023-08-15 + +- change: make titles be unique *globally*, i.e. *a single unique title per + Zettelkasten*, instead of having them be unique per-project basis. +- refactor: rework the database architecture entirely, but keep (roughly) the + same functionality +- fix: make finding backlinks just as fast as finding forward links +- fix: make `sync --create` return the proper error +- fix: prevent `compl` from creating configuration files +- fix: make `compl` recognise its input properly +- fix: prevent `query` from returning a capacity overflow error when using the + `--text` option. +- fix: don't insert duplicate links into the database +- fix: include more information in I/O panics, to make debugging easier + ## v0.39.9 - 2023-06-10 - fix: automatically rename Zettel with multiple consecutive whitespace, so that diff -Nru settle-0.39.9/debian/changelog settle-0.39.11/debian/changelog --- settle-0.39.9/debian/changelog 2023-08-14 15:47:43.000000000 +0000 +++ settle-0.39.11/debian/changelog 2023-08-15 18:46:28.000000000 +0000 @@ -1,3 +1,13 @@ +settle (0.39.11-1) unstable; urgency=low + + [ upstream ] + * new release(s) + + [ Jonas Smedegaard ] + * add patch 1001 to consistently resolve $HOME at runtime + + -- Jonas Smedegaard Tue, 15 Aug 2023 20:46:28 +0200 + settle (0.39.9-2) unstable; urgency=low * update patch 2001; diff -Nru settle-0.39.9/debian/copyright_hints settle-0.39.11/debian/copyright_hints --- settle-0.39.9/debian/copyright_hints 2023-08-14 15:47:43.000000000 +0000 +++ settle-0.39.11/debian/copyright_hints 2023-08-15 18:46:28.000000000 +0000 @@ -13,6 +13,7 @@ debian/copyright-check debian/dh-cargo/README.md debian/gbp.conf + debian/patches/1001_HOME.patch debian/patches/2001_deps.patch debian/patches/README debian/patches/series diff -Nru settle-0.39.9/debian/patches/1001_HOME.patch settle-0.39.11/debian/patches/1001_HOME.patch --- settle-0.39.9/debian/patches/1001_HOME.patch 1970-01-01 00:00:00.000000000 +0000 +++ settle-0.39.11/debian/patches/1001_HOME.patch 2023-08-15 18:45:15.000000000 +0000 @@ -0,0 +1,17 @@ +Description: consistently resolve $HOME at runtime +Author: Jonas Smedegaard +Bug: https://github.com/xylous/settle/issues/4 +Last-Update: 2023-07-15 +--- +This patch header follows DEP-3: http://dep.debian.net/deps/dep3/ +--- a/src/config.rs ++++ b/src/config.rs +@@ -17,7 +17,7 @@ + { + fn default() -> ConfigOptions + { +- ConfigOptions { zettelkasten: format!("{}/zettelkasten", env!("HOME")), ++ ConfigOptions { zettelkasten: format!("{}/zettelkasten", env::var("HOME").unwrap()), + template: String::from("") } + } + } diff -Nru settle-0.39.9/debian/patches/series settle-0.39.11/debian/patches/series --- settle-0.39.9/debian/patches/series 2023-06-22 10:16:13.000000000 +0000 +++ settle-0.39.11/debian/patches/series 2023-08-15 18:39:38.000000000 +0000 @@ -1 +1,2 @@ +1001_HOME.patch 2001_deps.patch diff -Nru settle-0.39.9/doc/configuration.md settle-0.39.11/doc/configuration.md --- settle-0.39.9/doc/configuration.md 2023-06-10 16:47:05.000000000 +0000 +++ settle-0.39.11/doc/configuration.md 2023-08-15 18:09:47.000000000 +0000 @@ -1,13 +1,19 @@ # Configuration -The configuration file is at either `$XDG_CONFIG_HOME/settle/settle.yaml`, if -`$XDG_CONFIG_HOME` is set, either `~/.config/settle/settle.yaml`, by default, -and is written in YAML Format. +The configuration options are passed in YAML format. The location of the +configuration file may be influenced by environment variables: -Note that paths specified in configuration may contain environment variables, or -a leading tilde. +1. if `$SETTLE_CONFIG` is set: `$SETTLE_CONFIG` +2. if `$XDG_CONFIG_HOME` is set: `$XDG_CONFIG_HOME/settle/settle.yaml` +3. default: `$HOME/.config/settle/settle.yaml` + +A generic configuration file is automatically created when `settle` is ran with +any command (except `compl`), if it doesn't already exist. -Here are the configuration options: +### Configuration option + +NOTE: the paths specified in configuration may contain environment variables, or +a leading tilde. - `zettelkasten` - path to the directory in which the notes are stored at @@ -20,9 +26,6 @@ ### Example configuration file -The configuration file is automatically created when `settle` is ran, if it -doesn't already exist. They can be as simple as: - ```YAML zettelkasten: ~/docs/zettelkasten template: ~/.config/settle/template.md diff -Nru settle-0.39.9/doc/creating-notes.md settle-0.39.11/doc/creating-notes.md --- settle-0.39.9/doc/creating-notes.md 2023-06-10 16:47:05.000000000 +0000 +++ settle-0.39.11/doc/creating-notes.md 2023-08-15 18:09:47.000000000 +0000 @@ -1,29 +1,18 @@ # Creating notes Most of your writing revolves around creating new notes. Granted, that's a -simple thing to do with `settle`. - -For example: +simple thing to do: - `settle sync --create 'This is an interesting note'` would create a note with 'This is an interesting note' as title, within the main Zettelkasten. - - `settle sync --create 'My second note' --project inbox` would create a note with 'My second note' as title, but in the 'inbox' project. -NOTE: if you specify a [project](./projects.md) that doesn't exist yet, then -it's automatically created. - -However, based on certain conditions, this operation may have three outcomes: +If you specify a [project](./projects.md) that doesn't exist yet, then it's +automatically created. -- if you try to create a new note but one with the same title in the same - project already exists, then nothing is changed and an error is returned - (duplicates are forbidden) -- if you try to create a new note but there is a file (on the filesystem) that - exists with that title and in the same project as the one specified, then - the file is not overwritten, and its metadata is added to the database -- if a corresponding file doesn't exist and a database entry for it doesn't - exist, then indeed, a new note is created +However, note that this operation may fail if a note with the same title in the +Zettelkasten already exists; duplicate titles are forbidden. ### Templates diff -Nru settle-0.39.9/doc/design-principles.md settle-0.39.11/doc/design-principles.md --- settle-0.39.9/doc/design-principles.md 2023-06-10 16:47:05.000000000 +0000 +++ settle-0.39.11/doc/design-principles.md 2023-08-15 18:09:47.000000000 +0000 @@ -18,3 +18,5 @@ isn't any hard boundary - any note may reference any other note - [links between Zettel](./links-and-backlinks.md) are wiki-style links, which are extremely straightforward. +- no two Zettel should have the same title, even if they are in different + projects. This is only to avoid potential ambiguity with links. diff -Nru settle-0.39.9/doc/history.md settle-0.39.11/doc/history.md --- settle-0.39.9/doc/history.md 2023-06-10 16:47:05.000000000 +0000 +++ settle-0.39.11/doc/history.md 2023-08-15 18:09:47.000000000 +0000 @@ -8,10 +8,11 @@ I had already written quite a few notes, and I didn't want to change them to make the links and tags work with other programs. -So, in early August, I had the idea of writing a CLI program that I could easily -use with vim, and at the same time use Obsidian-style links and tags. Thus, -`settle` was born: it combines maximum human readability with the potential to -be used with any text editor [that has plugin support.] +So, in early August, I had the idea of writing a standalone CLI program that I +could easily use with vim, and at the same time use Obsidian-style links and +tags. Thus, `settle` was born: it combines maximum human readability with the +potential to be used with any text editor, no plugins required (even if they may +improve usability). In the meantime, I've read Sonke Ahrens's *How to take smart notes* and have added many features to `settle`, such as [projects](./projects.md). diff -Nru settle-0.39.9/doc/links-and-backlinks.md settle-0.39.11/doc/links-and-backlinks.md --- settle-0.39.9/doc/links-and-backlinks.md 2023-06-10 16:47:05.000000000 +0000 +++ settle-0.39.11/doc/links-and-backlinks.md 2023-08-15 18:09:47.000000000 +0000 @@ -4,7 +4,7 @@ `settle` can only understand wiki-style links, such as `[[Neurons]]`, or `[[Language learning]]`, which would redirect to a note called `Neurons` or -`Lanuage leraning`, respectively. Case-in-point, a wiki-style link is any text +`Language leraning`, respectively. Case-in-point, a wiki-style link is any text between two matching `[[` and `]]`. Such links may be embedded anywhere inside a note. diff -Nru settle-0.39.9/doc/projects.md settle-0.39.11/doc/projects.md --- settle-0.39.9/doc/projects.md 2023-06-10 16:47:05.000000000 +0000 +++ settle-0.39.11/doc/projects.md 2023-08-15 18:09:47.000000000 +0000 @@ -1,7 +1,5 @@ # Projects -### What is a project? - `settle` understands your Zettelkasten in terms of *projects*. A project is any directory that contains at least one note, i.e. at least one Markdown file. @@ -10,14 +8,10 @@ the root of your Zettelkasten is at `~/docs/zettelkasten`, then `~/docs/zettelkasten/myproject` can be a project, but `~/docs/zettelkasten/myproject/mysubproject` cannot. Likewise, since `~/docs` -isn't a subdirectory of your Zettelkasten's root, it can't can't count as a -project. - -The reason for this design choice is that, with subprojects like this, it would -become a hierarchical nightmare extremely fast. +isn't a subdirectory of your Zettelkasten's root, it can't count as a project. -NOTE: the root of the Zettelkasten can be referenced by two names: `"main"`, or -an empty string (`""`). +Note, however, the root of the Zettelkasten can be referenced by two names: +`"main"`, or an empty string (`""`). ### The role of projects @@ -27,9 +21,8 @@ Your most basic projects are the root of the Zettelkasten and your inbox - the former should contain permanent notes, the other should contain temporary notes. -It's up to you how you use your ability to create and manage projects. My advice -would be to use as few as possible. Some projects, such as having, say, a -`literature` project for literature notes, would be useful. Aside from that, -use-cases like writing about game's lore, or writing a book's chapter, or using -projects for anything that isn't and can never be related to the rest of your -Zettelkasten, come to mind. +It's up to you how you use projects. General Zettelkasten guidelines indicate +using as few hierarchical structures as possible. They are most practical when +you want to separate notes that shouldn't mix together: you may create a +`writings` project to hold your publishings, or a `literature` project to hold +notes on what you read or plan to read. diff -Nru settle-0.39.9/doc/SETTLE_MANUAL.md settle-0.39.11/doc/SETTLE_MANUAL.md --- settle-0.39.9/doc/SETTLE_MANUAL.md 2023-06-10 16:47:05.000000000 +0000 +++ settle-0.39.11/doc/SETTLE_MANUAL.md 2023-08-15 18:09:47.000000000 +0000 @@ -65,6 +65,9 @@ - `sync` or `-S` (described below) +All warnings and errors are printed to `stderr`, so you can suppress them (e.g. +`2>/dev/null`). + ##### A short briefing on regular expressions (REGEX) Regex is a useful tool that `settle` has support for, because it provides @@ -205,7 +208,8 @@ - `-c | --create ` - create a new Zettel with the provided title. If the `--project` flag is provided, then make it part of that project; if not, - then add it to the main Zettelkasten project + then add it to the main Zettelkasten project. This operation may only fail + if there is already an entry in the database with the same title. - `-u | --update <PATH>` - update a note's metadata, given its path (relative or absolute) on the filesystem. If the file is not part of the Zettelkasten or @@ -320,7 +324,7 @@ ```zsh mkdir ~/.zsh_completion.d -settle compl zsh >~/zsh_completion.d/_settle +settle compl zsh >~/.zsh_completion.d/_settle ``` Then add this line in your zshrc: diff -Nru settle-0.39.9/README.md settle-0.39.11/README.md --- settle-0.39.9/README.md 2023-06-10 16:47:05.000000000 +0000 +++ settle-0.39.11/README.md 2023-08-15 18:09:47.000000000 +0000 @@ -6,12 +6,12 @@ ### Requirements -* cargo/rust toolchain -* SQLite +* cargo/rust toolchain (for building) +* SQLite (for running) ### Installation -There's a crate on crates.io, so you can simply run: +There's a Rust crate available, so you can simply run: ``` cargo install settle @@ -20,8 +20,8 @@ ### Overview - [full usage manual](./doc/SETTLE_MANUAL.md), contains more technical descriptions +- [project motivation](./doc/history.md) - [design principles](./doc/design-principles.md) -- [project history](./doc/history.md) - [configuration](./doc/configuration.md) - [tags and subtags](./doc/tags-and-subtags.md) - [links and backlinks](./doc/links-and-backlinks.md) @@ -35,8 +35,6 @@ ## Roadmap -#### Before 2023 - - [x] generate the database from existing files - [x] create Zettel - [x] list Zettel @@ -59,24 +57,19 @@ - [x] move notes from project to project - [x] rename notes - [x] update all links to the renamed note - -#### After/During 2023 - +- [x] simplify command structure - [x] query: filter notes based on various criteria (title, tags, etc.) - [x] support regex - [x] print according to a format - [x] put custom separator between links, both forward and backward -- [x] graph - - [x] DOT output - - [ ] ~~render DOT as image~~ + - [x] add option to print DOT graphs, which can be read with e.g. `graphviz` - [ ] writing experience (help deal with writer's block) - [ ] find related notes - - [ ] suggest random notes ## Contributing -Pull requests are welcome. For major changes, please open an issue first to -discuss what you would like to change. +Pull requests are welcome. For any minor or major changes, you can open an issue +to discuss what you would like to change. <!-- Please make sure to update tests as appropriate. diff -Nru settle-0.39.9/src/config.rs settle-0.39.11/src/config.rs --- settle-0.39.9/src/config.rs 2023-06-10 16:47:05.000000000 +0000 +++ settle-0.39.11/src/config.rs 2023-08-15 18:09:47.000000000 +0000 @@ -1,5 +1,6 @@ -use crate::io::{dir_exists, file_exists, file_to_string, mkdir, write_to_file}; +use crate::io::{dir_exists, dirname, file_exists, file_to_string, mkdir, write_to_file}; use serde::{Deserialize, Serialize}; +use std::env; /// The location of the database file. Unchangeable: the user doesn't need to know the location of /// this file @@ -48,30 +49,48 @@ { pub fn load() -> ConfigOptions { - let xdg_cfg_dir = option_env!("XDG_CONFIG_HOME"); - let config_path = if xdg_cfg_dir.is_none() { - // Use $HOME/.config/settle/settle.yaml if XDG_CONFIG_HOME isn't set - format!("{}/.config/settle", env!("HOME")) - } else { - // Use $XDG_CONFIG_HOME/settle/settle.yaml otherwise - format!("{}/settle", xdg_cfg_dir.unwrap()) - }; - let config_file = format!("{}/settle.yaml", config_path); - - // If the dir doesn't exist, create it - if !dir_exists(&config_path) { - mkdir(&config_path); + let config_file = Self::cfg_file(); + let config_dir = dirname(&config_file); + + // The configuration directory is necessary to creating the configuration file + if !dir_exists(&config_dir) { + mkdir(&config_dir); } - // If the file doesn't exist, create it + // Provide a default configuration file if it doesn't exist if !file_exists(&config_file) { let data = serde_yaml::to_string(&ConfigOptions::default()).unwrap(); write_to_file(&config_file, &data); } - // The paths inside the config file may not be absolute, and so we need to expand them + // The config file may have relative paths, but we only deal in absolutes let tmp: ConfigOptions = serde_yaml::from_str(&file_to_string(&config_file)).unwrap(); - ConfigOptions { zettelkasten: expand_path(&tmp.zettelkasten), - template: expand_path(&tmp.template) } + let cfg = ConfigOptions { zettelkasten: expand_path(&tmp.zettelkasten), + template: expand_path(&tmp.template) }; + + // Create the Zettelkasten directory and the 'inbox' project if it doesn't exist + if !dir_exists(&cfg.zettelkasten) { + mkdir(&cfg.zettelkasten); + mkdir(&format!("{}/inbox", &cfg.zettelkasten)); + } + + cfg + } + + // The configuration is determined by looking at the environment variables in this order: + // 1. If `$SETTLE_CONFIG` is set: `$SETTLE_CONFIG` + // 2. If `$XDG_CONFIG_HOME` is set: `$XDG_CONFIG_HOME/settle/settle.yaml` + // 3. default: `$HOME/.config/settle/settle.yaml` + pub fn cfg_file() -> String + { + let settle_cfg = env::var("SETTLE_CONFIG").unwrap_or_default(); + let xdg_cfg_home = env::var("XDG_CONFIG_HOME").unwrap_or_default(); + if !settle_cfg.is_empty() { + settle_cfg + } else if !xdg_cfg_home.is_empty() { + format!("{}/settle/settle.yaml", xdg_cfg_home) + } else { + format!("{}/.config/settle/settle.yaml", env::var("HOME").unwrap()) + } } } diff -Nru settle-0.39.9/src/database.rs settle-0.39.11/src/database.rs --- settle-0.39.9/src/database.rs 2023-06-10 16:47:05.000000000 +0000 +++ settle-0.39.11/src/database.rs 2023-08-15 18:09:47.000000000 +0000 @@ -1,48 +1,60 @@ -use rusqlite::{named_params, Connection, DatabaseName, Error, Result, Row}; - -use crate::{config::ConfigOptions, str_to_vec, zettel::Zettel}; +use crate::{config::ConfigOptions, zettel::Zettel}; use rayon::prelude::*; +use rusqlite::{ + named_params, Connection, DatabaseName, Error, Result, Row, Transaction, TransactionBehavior, +}; +use std::sync::{mpsc, Arc, Mutex, MutexGuard}; +use std::thread; impl Zettel { /// Construct a Zettel from an entry in the database metadata /// Return an Error if the `row` was invalid - fn from_db(row: &Row) -> Result<Zettel, rusqlite::Error> + fn from_db(conn_lock: &MutexGuard<Connection>, row: &Row) -> Result<Zettel, rusqlite::Error> { let title: String = row.get(0)?; let project: String = row.get(1)?; - let links: String = row.get(2)?; - let tags: String = row.get(3)?; let mut z = Zettel::new(&title, &project); - z.links = str_to_vec(&links); - z.tags = str_to_vec(&tags); + + let mut stmt = conn_lock.prepare("SELECT link_id FROM links WHERE zettel_id = ?1")?; + let mut links = stmt.query([&z.title])?; + while let Some(link_row) = links.next()? { + z.links.push(link_row.get(0)?); + } + let mut stmt = conn_lock.prepare("SELECT zettel_id FROM links WHERE link_id = ?1")?; + let mut backlinks = stmt.query([&z.title])?; + while let Some(backlink_row) = backlinks.next()? { + z.backlinks.push(backlink_row.get(0)?); + } + let mut stmt = conn_lock.prepare("SELECT tag FROM tags WHERE zettel_id = ?1")?; + let mut tags = stmt.query([&z.title])?; + while let Some(tag_row) = tags.next()? { + z.tags.push(tag_row.get(0)?); + } Ok(z) } } pub struct Database { - name: String, - conn: Connection, + conn: Arc<Mutex<Connection>>, } impl Database { /// Create a `Database` interface to an SQLite database /// Return an Error if the connection couldn't be made - pub fn new(name: &str) -> Result<Self, Error> + pub fn new(uri: &str) -> Result<Self, Error> { - Ok(Database { name: name.to_string(), - conn: Connection::open(name)? }) + Ok(Database { conn: Arc::new(Mutex::new(Connection::open(uri)?)) }) } /// Create a `Database` interface to a named SQLite database, opened in memory /// Return an Error if the connection couldn't be made - pub fn in_memory(name: &str) -> Result<Self, Error> + pub fn new_in_memory(filename: &str) -> Result<Self, Error> { - let uri = &format!("file:{}?mode=memory&cache=shared", name); - Ok(Database { name: name.to_string(), - conn: Connection::open(uri)? }) + let uri = &format!("file:{}?mode=memory&cache=shared", filename); + Database::new(uri) } /// Initialise the current Database with a `zettelkasten` table that holds the properties of @@ -50,16 +62,25 @@ /// Return an Error if this wasn't possible pub fn init(&self) -> Result<(), Error> { - self.conn.execute( - "CREATE TABLE IF NOT EXISTS zettelkasten ( - title TEXT NOT NULL, - project TEXT, - links TEXT, - tags TEXT, - UNIQUE(title, project) - )", - [], - )?; + let conn_lock = self.conn.lock().unwrap(); + conn_lock.execute("CREATE TABLE IF NOT EXISTS zettelkasten ( + title TEXT NOT NULL, + project TEXT, + UNIQUE(title) + )", + [])?; + conn_lock.execute("CREATE TABLE IF NOT EXISTS links ( + zettel_id TEXT, + link_id TEXT, + FOREIGN KEY (zettel_id) REFERENCES zettelkasten (title) + )", + [])?; + conn_lock.execute("CREATE TABLE IF NOT EXISTS tags ( + zettel_id TEXT, + tag TEXT, + FOREIGN KEY (zettel_id) REFERENCES zettelkasten (title) + )", + [])?; Ok(()) } @@ -67,19 +88,36 @@ /// Return an Error if this wasn't possible pub fn write_to(&self, path: &str) -> Result<(), Error> { - self.conn.backup(DatabaseName::Main, path, None)?; + self.conn + .lock() + .unwrap() + .backup(DatabaseName::Main, path, None)?; Ok(()) } /// Save a Zettel's metadata to the database pub fn save(&self, zettel: &Zettel) -> Result<(), Error> { - let links = crate::vec_to_str(&zettel.links); - let tags = crate::vec_to_str(&zettel.tags); - self.conn.execute( - "INSERT INTO zettelkasten (title, project, links, tags) values (?1, ?2, ?3, ?4)", - [&zettel.title, &zettel.project, &links, &tags], - )?; + let conn_lock = self.conn.lock().unwrap(); + let tsx = Transaction::new_unchecked(&conn_lock, TransactionBehavior::Immediate).unwrap(); + Self::save_tsx(&tsx, zettel)?; + tsx.commit()?; + Ok(()) + } + + /// Save a Zettel's metadata in the given transaction + pub fn save_tsx(tsx: &Transaction, zettel: &Zettel) -> Result<(), Error> + { + tsx.execute("INSERT INTO zettelkasten (title, project) values (?1, ?2)", + [&zettel.title, &zettel.project])?; + for link in &zettel.links { + tsx.execute("INSERT INTO links (zettel_id, link_id) values (?1, ?2)", + [&zettel.title, link])?; + } + for tag in &zettel.tags { + tsx.execute("INSERT INTO tags (zettel_id, tag) values (?1, ?2)", + [&zettel.title, tag])?; + } Ok(()) } @@ -87,8 +125,9 @@ pub fn delete(&self, zettel: &Zettel) -> Result<(), Error> { self.conn - .execute("DELETE FROM zettelkasten WHERE title=?1 AND project=?2", - [&zettel.title, &zettel.project])?; + .lock() + .unwrap() + .execute("DELETE FROM zettelkasten WHERE title=?1", [&zettel.title])?; Ok(()) } @@ -97,12 +136,13 @@ /// unreachable pub fn all(&self) -> Result<Vec<Zettel>, Error> { - let mut stmt = self.conn.prepare("SELECT * FROM zettelkasten")?; + let conn_lock = self.conn.lock().unwrap(); + let mut stmt = conn_lock.prepare("SELECT * FROM zettelkasten")?; let mut rows = stmt.query([])?; let mut results: Vec<Zettel> = Vec::new(); while let Some(row) = rows.next()? { - let zettel = Zettel::from_db(row)?; + let zettel = Zettel::from_db(&conn_lock, row)?; results.push(zettel); } @@ -116,13 +156,13 @@ /// `pattern` uses SQL pattern syntax, e.g. `%` to match zero or more characters. pub fn find_by_title(&self, pattern: &str) -> Result<Vec<Zettel>, Error> { - let mut stmt = self.conn - .prepare("SELECT * FROM zettelkasten WHERE title LIKE :pattern")?; + let conn_lock = self.conn.lock().unwrap(); + let mut stmt = conn_lock.prepare("SELECT * FROM zettelkasten WHERE title LIKE :pattern")?; let mut rows = stmt.query(named_params! {":pattern": pattern})?; let mut results: Vec<Zettel> = Vec::new(); while let Some(row) = rows.next()? { - let zettel = Zettel::from_db(row)?; + let zettel = Zettel::from_db(&conn_lock, row)?; results.push(zettel); } @@ -134,18 +174,14 @@ /// Return an Error if the database was unreachable pub fn list_tags(&self) -> Result<Vec<String>, Error> { - let mut stmt = self.conn.prepare("SELECT tags FROM zettelkasten")?; + let conn_lock = self.conn.lock().unwrap(); + let mut stmt = conn_lock.prepare("SELECT DISTINCT tag FROM tags")?; let mut rows = stmt.query([])?; let mut results: Vec<String> = Vec::new(); while let Some(row) = rows.next()? { - let tags: String = row.get(0)?; - for tag in str_to_vec(&tags) { - results.push(tag); - } + results.push(row.get(0)?); } - results.par_sort(); - results.dedup(); Ok(results) } @@ -154,7 +190,8 @@ /// Return an Error if the database was unreachable pub fn list_projects(&self) -> Result<Vec<String>, Error> { - let mut stmt = self.conn.prepare("SELECT project FROM zettelkasten")?; + let conn_lock = self.conn.lock().unwrap(); + let mut stmt = conn_lock.prepare("SELECT DISTINCT project FROM zettelkasten")?; let mut rows = stmt.query([])?; let mut results: Vec<String> = Vec::new(); @@ -164,63 +201,88 @@ results.push(project); } } - results.par_sort(); - results.dedup(); Ok(results) } /// Search in the database for Zettel that have been linked to, but don't yet exist + /// /// Return an Error if the database was unreachable or if the data in a Row couldn't have been /// accessed - pub fn zettel_not_yet_created(&self) -> Result<Vec<String>> + pub fn zettel_not_yet_created(&self) -> Result<Vec<String>, Error> { - let mut stmt = self.conn.prepare("SELECT links FROM zettelkasten")?; + let conn_lock = self.conn.lock().unwrap(); + let mut stmt = conn_lock.prepare("SELECT DISTINCT link_id FROM links WHERE link_id NOT IN (SELECT title FROM zettelkasten)")?; let mut rows = stmt.query([])?; - let mut unique_links: Vec<String> = Vec::new(); + let mut ghosts: Vec<String> = Vec::new(); while let Some(row) = rows.next()? { - let links_str: String = row.get(0)?; - let links = str_to_vec(&links_str); - unique_links.extend(links); + ghosts.push(row.get(0)?); } - unique_links.par_sort(); - unique_links.dedup(); - - Ok(unique_links.into_iter() - .filter(|link| { - // if the response was empty, then nothing has been found, meaning it doesn't exist - // in the database - self.find_by_title(link).unwrap().is_empty() - }) - .collect()) + ghosts.sort(); + ghosts.dedup(); + + Ok(ghosts) } /// Look for Markdown files in the Zettelkasten directory and populate the database with their /// metadata - pub fn generate(&self, cfg: &ConfigOptions) + pub fn generate(&self, cfg: &ConfigOptions) -> Result<(), Error> { - let db_name = &self.name; - let mut directories = crate::io::list_subdirectories(&cfg.zettelkasten); + + let (tx, rx) = mpsc::sync_channel::<Zettel>(1); + let conn = self.conn.clone(); + + // Add a separate thread to handle transactioning everything at once + thread::spawn(move || { + let conn_lock = conn.lock().unwrap(); + let tsx = + Transaction::new_unchecked(&conn_lock, TransactionBehavior::Immediate).unwrap(); + loop { + let data = rx.recv(); + match data { + Ok(zettel) => match Database::save_tsx(&tsx, &zettel) { + Ok(_) => continue, + Err(_) => { + eprintln!("Warning: couldn't add Zettel '{}' to the '{}' project; there is another note with that title, and titles must be unique", + &zettel.title, + if zettel.project.is_empty() { + "main" + } else { + &zettel.project + }, + ) + } + }, + // If we get a RecvError, then we know we've encountered the end + Err(mpsc::RecvError) => { + tsx.commit().unwrap(); + return; + } + } + } + }); + directories.push(cfg.zettelkasten.clone()); directories.par_iter().for_each(|dir| { - let notes: Vec<String> = - // don't add markdown file that starts with a dot (which - // includes the empty title file, the '.md') - crate::io::list_md_files(dir).into_iter() - .filter(|f| { - !crate::io::basename(f).starts_with('.') - }) - .collect(); - notes.par_iter().for_each(|note| { - let thread_db = - Self::in_memory(db_name).unwrap(); - let thread_zettel = - Zettel::from_file(cfg, note); - thread_db.save(&thread_zettel).unwrap(); - }); - }); + let paths: Vec<String> = + // don't add markdown file that starts with a dot (which + // includes the empty title file, the '.md') + crate::io::list_md_files(dir).into_iter() + .filter(|f| { + !crate::io::basename(f).starts_with('.') + }) + .collect(); + paths.par_iter().for_each(|path| { + let zettel = Zettel::from_file(cfg, path); + tx.send(zettel).unwrap(); + }); + }); + // Send RecvError to the thread + drop(tx); + + Ok(()) } /// Update the metadata for a given Zettel. The specified path *must* exist @@ -237,8 +299,10 @@ pub fn change_project(&self, zettel: &Zettel, new_project: &str) -> Result<(), Error> { self.conn - .execute("UPDATE zettelkasten SET project=?1 WHERE title=?2 AND project=?3", - [new_project, &zettel.title, &zettel.project])?; + .lock() + .unwrap() + .execute("UPDATE zettelkasten SET project=?1 WHERE title=?2", + [new_project, &zettel.title])?; Ok(()) } @@ -246,8 +310,10 @@ pub fn change_title(&self, zettel: &Zettel, new_title: &str) -> Result<(), Error> { self.conn - .execute("UPDATE zettelkasten SET title=?1 WHERE title=?2 AND project=?3", - [new_title, &zettel.title, &zettel.project])?; + .lock() + .unwrap() + .execute("UPDATE zettelkasten SET title=?1 WHERE title=?2", + [new_title, &zettel.title])?; Ok(()) } } diff -Nru settle-0.39.9/src/io.rs settle-0.39.11/src/io.rs --- settle-0.39.9/src/io.rs 2023-06-10 16:47:05.000000000 +0000 +++ settle-0.39.11/src/io.rs 2023-08-15 18:09:47.000000000 +0000 @@ -2,12 +2,6 @@ use std::fs::{canonicalize, read_to_string, write}; use std::path::{Path, PathBuf}; -/// Read `path` and return the contents -pub fn file_to_string(path: &str) -> String -{ - read_to_string(path).expect("failed to read file") -} - /// Return true if the path specified exists and is a file pub fn file_exists(path: &str) -> bool { @@ -34,44 +28,50 @@ pieces[0..pieces.len() - 1].join("/") } +/// Given a path, replace its extension with `new_ext` and return the resulting path +pub fn replace_extension(file: &str, new_ext: &str) -> String +{ + let mut path = PathBuf::from(file); + path.set_extension(new_ext); + path.to_string_lossy().to_string() +} + +/// Read `path` and return the contents +pub fn file_to_string(path: &str) -> String +{ + read_to_string(path).unwrap_or_else(|_| panic!("Can't read file '{}'", path)) +} + /// Write `data` to `path` pub fn write_to_file(path: &str, data: &str) { - write(path, data).expect("Unable to write file") + write(path, data).unwrap_or_else(|_| panic!("Can't write to file '{}'", path)) } /// Rename `from` to `to` pub fn rename(from: &str, to: &str) { - std::fs::rename(from, to).unwrap(); + std::fs::rename(from, to).unwrap_or_else(|_| panic!("Can't rename file '{}' to '{}'", from, to)) } -/// Given a filename, replace its extension with `new_ext` -pub fn replace_extension(file: &str, new_ext: &str) -> String +/// Create specified `path` as a directory +pub fn mkdir(path: &str) { - let mut path = PathBuf::from(file); - path.set_extension(new_ext); - path.to_string_lossy().to_string() + std::fs::create_dir_all(path).unwrap_or_else(|_| panic!("Can't create directory '{}'", path)) } /// List all markdown files in the specified directory pub fn list_md_files(dir: &str) -> Vec<String> { - glob(&format!("{}/*.md", dir)).expect("failed to read directory") + glob(&format!("{}/*.md", dir)).unwrap_or_else(|_| panic!("Can't read directory '{}'", dir)) .map(|f| f.unwrap().to_string_lossy().to_string()) .collect() } -/// Create specified `path` as a directory -pub fn mkdir(path: &str) -{ - std::fs::create_dir_all(path).expect("Wasn't able to create directory:") -} - /// List all subdirectories in the specified directory pub fn list_subdirectories(dir: &str) -> Vec<String> { - glob(&format!("{}/*/", dir)).expect("failed to read directory") + glob(&format!("{}/*/", dir)).unwrap_or_else(|_| panic!("Can't read directory '{}'", dir)) .map(|f| f.unwrap().to_string_lossy().to_string()) .collect() } diff -Nru settle-0.39.9/src/main.rs settle-0.39.11/src/main.rs --- settle-0.39.9/src/main.rs 2023-06-10 16:47:05.000000000 +0000 +++ settle-0.39.11/src/main.rs 2023-08-15 18:09:47.000000000 +0000 @@ -11,50 +11,19 @@ use crate::subcommands::*; use crate::zettel::Zettel; -const SQL_ARRAY_SEPARATOR: &str = "::"; - -/// Join a vector of `String`s, and return a string starting and ending with `SQL_ARRAY_SEPARATOR`, -/// and with the elements of the vector separated by `SQL_ARRAY_SEPARATOR` -fn vec_to_str(vec: &[String]) -> String -{ - format!("{}{}{}", - SQL_ARRAY_SEPARATOR, - vec.join(SQL_ARRAY_SEPARATOR), - SQL_ARRAY_SEPARATOR,) -} - -/// Split `str` on `SQL_ARRAY_SEPARATOR` and return non-empty results as a vector -fn str_to_vec(str: &str) -> Vec<String> -{ - str.split(SQL_ARRAY_SEPARATOR) - .filter(|s| s != &"") - .map(|s| s.to_string()) - .collect() -} - fn main() -> Result<(), rusqlite::Error> { let matches = cli::build().get_matches(); - let cfg = ConfigOptions::load(); - io::mkdir(&cfg.zettelkasten); - io::mkdir(&format!("{}/inbox", &cfg.zettelkasten)); - let cmd = matches.subcommand_name().unwrap_or_default(); // NOTE: this won't crash on unwrap, because if no subcommand was specified, clap-rs would // print the help message let cmd_matches = matches.subcommand_matches(cmd).unwrap(); - // If the database isn't initialised, the program would likely panic, complaining that it isn't - // able to find SQL tables. - // Yes, it's kind of lazy to put this here, but putting it in every function that accesses the - // database, just to make sure that the database exists in the first place, is worse. - Database::new(&cfg.db_file())?.init()?; - match cmd { - "sync" => sync(cmd_matches, &cfg)?, - "query" => query(cmd_matches, &cfg)?, - "ls" => ls(cmd_matches, &cfg)?, + "sync" => sync(cmd_matches, &ConfigOptions::load())?, + "query" => query(cmd_matches, &ConfigOptions::load())?, + "ls" => ls(cmd_matches, &ConfigOptions::load())?, "compl" => compl(cmd_matches)?, _ => (), }; diff -Nru settle-0.39.9/src/subcommands.rs settle-0.39.11/src/subcommands.rs --- settle-0.39.9/src/subcommands.rs 2023-06-10 16:47:05.000000000 +0000 +++ settle-0.39.11/src/subcommands.rs 2023-08-15 18:09:47.000000000 +0000 @@ -1,6 +1,5 @@ use clap::ArgMatches; use clap_complete::Shell::*; -use rayon::prelude::*; use regex::Regex; use rusqlite::Error; @@ -13,31 +12,6 @@ use crate::cli; use crate::io::{abs_path, file_exists}; -pub fn sync(matches: &ArgMatches, cfg: &ConfigOptions) -> Result<(), Error> -{ - let project = realproject(if let Some(p) = matches.get_one::<String>("PROJECT") { - p - } else { - "" - }); - if let Some(title) = matches.get_one::<String>("CREATE") { - create(cfg, title, project)?; - } else if let Some(path) = matches.get_one::<String>("UPDATE") { - update(cfg, path)?; - } else if let Some(title) = matches.get_one::<String>("MOVE") { - mv(cfg, title, project)?; - } else if matches.contains_id("RENAME") { - let args_arr = matches.get_many::<String>("RENAME") - .unwrap_or_default() - .map(|a| a.to_string()) - .collect::<Vec<_>>(); - rename(cfg, &args_arr)?; - } else if matches.get_flag("GENERATE") { - generate(cfg)?; - } - Ok(()) -} - /// A printer that prints. Because it's convenient. struct Printer { @@ -62,11 +36,6 @@ self.zettel = new_zettel; } - fn set_single_zettel(&mut self, new_zettel: Zettel) - { - self.zettel = vec![new_zettel]; - } - fn set_additional(&mut self, new_additional: Vec<String>) { self.additional = new_additional; @@ -82,6 +51,12 @@ self.link_separator = new_separator; } + fn print_one(&mut self, cfg: &ConfigOptions, zettel: Zettel) + { + self.zettel = vec![zettel]; + self.print(cfg); + } + /// Abracadabra, yadda yadda. Print everything properly. fn print(&mut self, cfg: &ConfigOptions) { @@ -100,18 +75,7 @@ result = result.replace("%P", &z.filename(cfg)); result = result.replace("%l", &z.links.join(&self.link_separator)); result = result.replace("%a", a); - // Based on the provided ConfigOptions, we may or may not get the backlinks for the given - // Zettel, so if we don't, we just consume the `%b` token and move on - if result.contains("%b") { - let maybe_get_backlinks = || -> Result<Vec<String>, Error> { - let all = Database::new(&cfg.db_file())?.all()?; - let bks = backlinks(&all, &z.title, true); - Ok(bks.iter().map(|z| z.title.clone()).collect()) - }; - if let Ok(bks) = maybe_get_backlinks() { - result = result.replace("%b", &bks.join(&self.link_separator)); - } - } + result = result.replace("%b", &z.backlinks.join(&self.link_separator)); println!("{}", result); } @@ -129,10 +93,38 @@ } } +pub fn sync(matches: &ArgMatches, cfg: &ConfigOptions) -> Result<(), Error> +{ + Database::new(&cfg.db_file())?.init()?; + + let project = realproject(if let Some(p) = matches.get_one::<String>("PROJECT") { + p + } else { + "" + }); + if let Some(title) = matches.get_one::<String>("CREATE") { + create(cfg, title, project)?; + } else if let Some(path) = matches.get_one::<String>("UPDATE") { + update(cfg, path)?; + } else if let Some(title) = matches.get_one::<String>("MOVE") { + mv(cfg, title, project)?; + } else if matches.contains_id("RENAME") { + let args_arr = matches.get_many::<String>("RENAME") + .unwrap_or_default() + .map(|a| a.to_string()) + .collect::<Vec<_>>(); + rename(cfg, &args_arr)?; + } else if matches.get_flag("GENERATE") { + generate(cfg)?; + } + Ok(()) +} + /// Query the database, applying various filters if proivded pub fn query(matches: &ArgMatches, cfg: &ConfigOptions) -> Result<(), Error> { let db = Database::new(&cfg.db_file())?; + db.init()?; let mut zs: Vec<Zettel> = db.all()?; let mut printer = Printer::default(); @@ -145,17 +137,6 @@ if let Some(project) = matches.get_one::<String>("PROJECT") { zs = filter_project(zs, realproject(project), exact); } - if let Some(text) = matches.get_one::<String>("TEXT_REGEX") { - let vs = filter_text(zs.clone(), text, cfg); - let mut texts: Vec<String> = vec![]; - zs = vec![]; // reset Zettel vector and append indiivdually - // maybe speed this up by using unzip? I digress - for (z, t) in vs { - zs.push(z); - texts.push(t); - } - printer.set_additional(texts); - } if let Some(tag) = matches.get_one::<String>("TAG") { zs = filter_tag(zs, tag, exact); } @@ -165,6 +146,19 @@ if let Some(links_to) = matches.get_one::<String>("BACKLINKS") { zs = intersect(&zs, &backlinks(&zs, links_to, exact)); } + if let Some(text) = matches.get_one::<String>("TEXT_REGEX") { + let vs = filter_text(zs.clone(), text, cfg); + let mut texts: Vec<String> = vec![]; + let mut found = vec![]; + // maybe speed this up by using unzip? I digress + for (z, t) in vs { + found.push(z); + texts.push(t); + } + printer.set_additional(texts); + zs = intersect(&zs, &found); + } + if matches.get_flag("LONERS") { zs = filter_isolated(zs); } @@ -196,6 +190,7 @@ pub fn ls(matches: &ArgMatches, cfg: &ConfigOptions) -> Result<(), Error> { let db = Database::new(&cfg.db_file())?; + db.init()?; let obj = if let Some(m) = matches.get_one::<String>("OBJECT") { m @@ -218,13 +213,9 @@ /// Generate completions for a shell pub fn compl(matches: &ArgMatches) -> Result<(), Error> { - let shell = if let Some(m) = matches.get_one::<String>("OBJECT") { - m - } else { - "" - }; + let shell = matches.get_one::<String>("SHELL").unwrap(); - let sh = match shell { + let sh = match shell.as_str() { "zsh" => Some(Zsh), "bash" => Some(Bash), "fish" => Some(Fish), @@ -305,9 +296,8 @@ /// Keep only those Zettel that neither link to other notes, nor have links pointing to them fn filter_isolated(zs: Vec<Zettel>) -> Vec<Zettel> { - zs.clone() - .into_iter() - .filter(|z| z.links.is_empty() && backlinks(&zs, &z.title, true).is_empty()) + zs.into_iter() + .filter(|z| z.links.is_empty() && z.backlinks.is_empty()) .collect() } @@ -376,22 +366,25 @@ } let exists_in_fs = file_exists(&zettel.filename(cfg)); - let exists_in_db = db.all()?.into_par_iter().any(|z| z == zettel); + let exists_in_db = db.find_by_title(&zettel.title)?.len() == 1; // If the corresponding file exists and there's an entry in the database, abort. + // If there's an entry in the database but no corresponding file, replace the database entry // If there's a file but there's no entry in the database, create an entry. // Otherwise, create a new file from template and add a database entry. if exists_in_fs && exists_in_db { eprintln!("error: couldn't create new Zettel: one with the same title already exists"); return Ok(()); - } else if exists_in_fs { - println!("file exists in the filesystem but not in the database; added entry"); + } else if !exists_in_fs && exists_in_db { + eprintln!("Zettel exists in the database but not on the filesystem; replaced entry"); + db.delete(&zettel)?; + // saved outside of the loop + } else if exists_in_fs && !exists_in_db { + eprintln!("Zettel exists on the filesystem but not in the database; added entry"); // saved outside of the loop } else { zettel.create(cfg); - let mut printer = Printer::default(); - printer.set_single_zettel(zettel.clone()); - printer.print(cfg); + Printer::default().print_one(cfg, zettel.clone()); } db.save(&zettel)?; @@ -443,8 +436,6 @@ db.change_title(old_zettel, &new_title).unwrap(); // It's not enough that we renamed the file. We need to update all references to it! let backlinks = backlinks(&db.all()?, &old_title, true); - // for some reason rustfmt has absolutely cursed formatting here. this is not my fault, I - // swear backlinks.iter().for_each(|bl| { let contents = crate::io::file_to_string(&bl.filename(cfg)); // The link might span over multiple lines. We must account for that @@ -491,6 +482,7 @@ let new_notes = zs.iter().map(|z| Zettel { title: z.title.clone(), project: project.to_string(), links: z.links.clone(), + backlinks: z.backlinks.clone(), tags: z.tags.clone() }); let pairs = zs.iter().zip(new_notes); pairs.for_each(|(old, new)| { @@ -528,9 +520,9 @@ { let start = std::time::Instant::now(); - let mem_db = Database::in_memory(&cfg.db_file())?; + let mem_db = Database::new_in_memory(&cfg.db_file())?; mem_db.init()?; - mem_db.generate(cfg); + mem_db.generate(cfg)?; mem_db.write_to(&cfg.db_file())?; println!("database generated successfully, took {}ms", diff -Nru settle-0.39.9/src/zettel.rs settle-0.39.11/src/zettel.rs --- settle-0.39.9/src/zettel.rs 2023-06-10 16:47:05.000000000 +0000 +++ settle-0.39.11/src/zettel.rs 2023-08-15 18:09:47.000000000 +0000 @@ -1,5 +1,5 @@ use chrono::prelude::*; -use rayon::iter::{ParallelBridge, ParallelIterator}; +use rayon::prelude::*; use regex::Regex; use crate::config::ConfigOptions; @@ -10,13 +10,17 @@ fn find_links(contents: &str) -> Vec<String> { let re = Regex::new(r#"\[\[((?s).*?)\]\]"#).unwrap(); - re.captures_iter(contents) - .par_bridge() - .map(|cap| { - cap.get(1) - .map_or("".to_string(), |m| strip_multiple_whitespace(m.as_str())) - }) - .collect() + let mut links: Vec<String> = re.captures_iter(contents) + .par_bridge() + .map(|cap| { + cap.get(1).map_or("".to_string(), |m| { + strip_multiple_whitespace(m.as_str()) + }) + }) + .collect(); + links.par_sort(); + links.dedup(); + links } /// Find tags inside of `contents` string and return them @@ -47,8 +51,9 @@ { pub title: String, pub project: String, - pub links: Vec<String>, pub tags: Vec<String>, + pub links: Vec<String>, + pub backlinks: Vec<String>, } impl Zettel @@ -58,8 +63,9 @@ { Zettel { title: title.to_string(), project: project.to_string(), + tags: vec![], links: vec![], - tags: vec![] } + backlinks: vec![] } } /// Create a Zettel from a file, provided the ABSOLUTE path to the Zettel @@ -95,13 +101,13 @@ pub fn create(&self, cfg: &ConfigOptions) { mkdir(&format!("{}/{}", &cfg.zettelkasten, &self.project)); - if file_exists(&cfg.template) { + let new_contents = if file_exists(&cfg.template) { let template_contents = file_to_string(&cfg.template); - let new_zettel_contents = self.replace_template_placeholders(&template_contents); - write_to_file(&self.filename(cfg), &new_zettel_contents); + self.replace_template_placeholders(&template_contents) } else { - write_to_file(&self.filename(cfg), ""); - } + String::from("") + }; + write_to_file(&self.filename(cfg), &new_contents); } /// Return the absolute path to the Zettel