highlandcows_isam/lib.rs
1//! # highlandcows-isam
2//!
3//! An ISAM (Indexed Sequential Access Method) library with ACID transactions
4//! and optional secondary indices.
5//!
6//! ## Quick start
7//!
8//! Use the [`write`](Isam::write) and [`read`](Isam::read) helpers for simple
9//! single-operation transactions:
10//!
11//! ```
12//! # use tempfile::TempDir;
13//! use highlandcows_isam::Isam;
14//!
15//! # let dir = TempDir::new().unwrap();
16//! # let path = dir.path().join("db");
17//! let db: Isam<String, String> = Isam::create(&path).unwrap();
18//! db.write(|txn| db.insert(txn, "hello".to_string(), &"world".to_string())).unwrap();
19//! let v = db.read(|txn| db.get(txn, &"hello".to_string())).unwrap();
20//! assert_eq!(v, Some("world".to_string()));
21//! ```
22//!
23//! For multi-operation transactions, use [`begin_transaction`](Isam::begin_transaction) directly:
24//!
25//! ```
26//! # use tempfile::TempDir;
27//! use highlandcows_isam::Isam;
28//!
29//! # let dir = TempDir::new().unwrap();
30//! # let path = dir.path().join("db");
31//! # let db: Isam<String, String> = Isam::create(&path).unwrap();
32//! let mut txn = db.begin_transaction().unwrap();
33//! db.insert(&mut txn, "a".to_string(), &"1".to_string()).unwrap();
34//! db.insert(&mut txn, "b".to_string(), &"2".to_string()).unwrap();
35//! txn.commit().unwrap();
36//! ```
37//!
38//! ## Secondary indices
39//!
40//! Secondary indices let you look up records by a field other than the primary
41//! key. Implement [`DeriveKey`] on a marker struct, then register it via
42//! [`Isam::builder`] when creating or opening the database.
43//!
44//! ```
45//! # use tempfile::TempDir;
46//! use serde::{Serialize, Deserialize};
47//! use highlandcows_isam::{Isam, DeriveKey};
48//!
49//! #[derive(Serialize, Deserialize, Clone)]
50//! struct User { name: String, city: String }
51//!
52//! struct CityIndex;
53//! impl DeriveKey<User> for CityIndex {
54//! type Key = String;
55//! fn derive(u: &User) -> String { u.city.clone() }
56//! }
57//!
58//! # let dir = TempDir::new().unwrap();
59//! # let path = dir.path().join("users");
60//! let db = Isam::<u64, User>::builder()
61//! .with_index("city", CityIndex)
62//! .create(&path)
63//! .unwrap();
64//! let city_idx = db.index::<CityIndex>("city");
65//!
66//! db.write(|txn| {
67//! db.insert(txn, 1, &User { name: "Alice".into(), city: "London".into() })?;
68//! db.insert(txn, 2, &User { name: "Bob".into(), city: "London".into() })?;
69//! db.insert(txn, 3, &User { name: "Carol".into(), city: "Paris".into() })
70//! }).unwrap();
71//!
72//! let londoners = db.read(|txn| city_idx.lookup(txn, &"London".to_string())).unwrap();
73//! assert_eq!(londoners.len(), 2);
74//! ```
75//!
76//! ### Inspecting registered indices, rebuilding, and migrating
77//!
78//! Use [`Isam::secondary_indices`] to list the indices registered on an open
79//! database. Each [`IndexInfo`] entry includes the index name, the
80//! fully-qualified extractor type name, and a `schema_version` that reflects
81//! the last [`migrate_index`](Isam::migrate_index) call.
82//!
83//! To rebuild a stale index from primary data without versioning — for example
84//! after the [`DeriveKey`] extractor logic has changed — drop the current handle
85//! and reopen with [`IsamBuilder::rebuild_index`]:
86//!
87//! ```
88//! # use tempfile::TempDir;
89//! # use serde::{Serialize, Deserialize};
90//! # use highlandcows_isam::{Isam, DeriveKey};
91//! # #[derive(Serialize, Deserialize, Clone)]
92//! # struct User { name: String, city: String }
93//! # struct CityIndex;
94//! # impl DeriveKey<User> for CityIndex {
95//! # type Key = String;
96//! # fn derive(u: &User) -> String { u.city.clone() }
97//! # }
98//! # let dir = TempDir::new().unwrap();
99//! # let path = dir.path().join("users");
100//! # Isam::<u64, User>::builder().with_index("city", CityIndex).create(&path).unwrap();
101//! // Inspect which indices are registered.
102//! let indices = {
103//! let db = Isam::<u64, User>::builder()
104//! .with_index("city", CityIndex)
105//! .open(&path)
106//! .unwrap();
107//! db.secondary_indices().unwrap()
108//! // db is fully dropped here — all file handles released.
109//! };
110//! assert_eq!(indices[0].name, "city");
111//!
112//! // Reopen, forcing a full rebuild of the "city" index.
113//! let db = Isam::<u64, User>::builder()
114//! .with_index("city", CityIndex)
115//! .rebuild_index("city")
116//! .open(&path)
117//! .unwrap();
118//! ```
119//!
120//! To migrate a secondary index with a version bump — for instance when the
121//! derivation logic changes and you want to record that the migration was
122//! applied — use [`Isam::migrate_index`] on a live database handle. Pass a closure that
123//! transforms each primary value before [`DeriveKey::derive`] runs; pass the
124//! identity closure (`|v| Ok(v)`) for a plain rebuild. Primary records are
125//! not modified.
126//!
127//! ```
128//! # use tempfile::TempDir;
129//! # use serde::{Serialize, Deserialize};
130//! # use highlandcows_isam::{Isam, DeriveKey, DEFAULT_SINGLE_USER_TIMEOUT};
131//! # #[derive(Serialize, Deserialize, Clone)]
132//! # struct User { name: String, city: String }
133//! # struct CityIndex;
134//! # impl DeriveKey<User> for CityIndex {
135//! # type Key = String;
136//! # // derive now normalizes to lowercase
137//! # fn derive(u: &User) -> String { u.city.to_lowercase() }
138//! # }
139//! # let dir = TempDir::new().unwrap();
140//! # let path = dir.path().join("users");
141//! # let db = Isam::<u64, User>::builder().with_index("city", CityIndex).create(&path).unwrap();
142//! # db.write(|txn| db.insert(txn, 1u64, &User { name: "Alice".into(), city: "London".into() })).unwrap();
143//! // Rebuild the city index, normalizing city names to lowercase so the
144//! // on-disk data matches the updated DeriveKey logic. Bumps schema_version to 1.
145//! let db = db.as_single_user(DEFAULT_SINGLE_USER_TIMEOUT, |token, db| {
146//! db.migrate_index("city", 1, |mut u: User| {
147//! u.city = u.city.to_lowercase();
148//! Ok(u)
149//! }, token)?;
150//! Ok(db)
151//! }).unwrap();
152//!
153//! let info = db.secondary_indices().unwrap();
154//! assert_eq!(info[0].schema_version, 1);
155//! ```
156//!
157//! ## Single-user mode
158//!
159//! [`Isam::as_single_user`] lets one thread take exclusive access to the
160//! database for administration operations such as compaction and index
161//! migration. While the closure is running, any other thread that calls any
162//! [`Isam`] operation on a clone of the same handle receives
163//! [`IsamError::SingleUserMode`] immediately — those threads are never
164//! blocked, they fail fast.
165//!
166//! ```
167//! # use tempfile::TempDir;
168//! use highlandcows_isam::{Isam, DEFAULT_SINGLE_USER_TIMEOUT};
169//!
170//! # let dir = TempDir::new().unwrap();
171//! # let path = dir.path().join("db");
172//! # let db: Isam<u32, String> = Isam::create(&path).unwrap();
173//! let db = db.as_single_user(DEFAULT_SINGLE_USER_TIMEOUT, |token, db| {
174//! db.compact(token)?;
175//! Ok(db)
176//! }).unwrap();
177//! ```
178//!
179//! When a migration changes the value type, return the *new* handle from the
180//! closure rather than the original `db`:
181//!
182//! ```
183//! # use tempfile::TempDir;
184//! use highlandcows_isam::{Isam, DEFAULT_SINGLE_USER_TIMEOUT};
185//!
186//! # let dir = TempDir::new().unwrap();
187//! # let path = dir.path().join("db");
188//! # let db: Isam<u32, String> = Isam::create(&path).unwrap();
189//! // db is Isam<u32, String>; migrate to Isam<u32, Vec<u8>>.
190//! let db: Isam<u32, Vec<u8>> =
191//! db.as_single_user(DEFAULT_SINGLE_USER_TIMEOUT, |token, db| {
192//! db.migrate_values(1, |s: String| Ok(s.into_bytes()), token)
193//! }).unwrap();
194//! ```
195//!
196//! [`DEFAULT_SINGLE_USER_TIMEOUT`] is 30 seconds. Pass a custom
197//! [`std::time::Duration`] if you need a shorter or longer window.
198//!
199//! ### How it works
200//!
201//! 1. The exclusive flag is set atomically. From this point on, other threads
202//! fail immediately with [`IsamError::SingleUserMode`].
203//! 2. The call waits (spinning with 1 ms sleeps) for any in-flight transaction
204//! on another thread to finish and release the storage lock.
205//! 3. Once the lock is confirmed free, the closure runs with exclusive access.
206//! 4. When the closure returns — normally or via panic — the exclusive flag is
207//! cleared and other threads can operate again.
208//!
209//! If step 2 does not complete within `timeout`, the flag is cleared and
210//! [`IsamError::Timeout`] is returned. The database is left fully operational.
211//!
212//! ### What to run inside the closure
213//!
214//! Single-user mode is intended for operations that must not run concurrently
215//! with reads or writes:
216//!
217//! ```
218//! # use tempfile::TempDir;
219//! # use serde::{Serialize, Deserialize};
220//! # use highlandcows_isam::{Isam, DeriveKey, DEFAULT_SINGLE_USER_TIMEOUT};
221//! # #[derive(Serialize, Deserialize, Clone)]
222//! # struct User { name: String, city: String }
223//! # struct CityIndex;
224//! # impl DeriveKey<User> for CityIndex {
225//! # type Key = String;
226//! # fn derive(u: &User) -> String { u.city.to_lowercase() }
227//! # }
228//! # let dir = TempDir::new().unwrap();
229//! # let path = dir.path().join("db");
230//! # let db = Isam::<u64, User>::builder().with_index("city", CityIndex).create(&path).unwrap();
231//! # db.write(|txn| db.insert(txn, 1u64, &User { name: "Alice".into(), city: "London".into() })).unwrap();
232//! let db = db.as_single_user(DEFAULT_SINGLE_USER_TIMEOUT, |token, db| {
233//! // Reclaim disk space from deleted/updated records.
234//! db.compact(token)?;
235//! // Rebuild a secondary index after updating the DeriveKey logic.
236//! db.migrate_index("city", 1, |mut u: User| {
237//! u.city = u.city.to_lowercase();
238//! Ok(u)
239//! }, token)?;
240//! Ok(db)
241//! }).unwrap();
242//! ```
243//!
244//! Inside the closure you can call [`Isam::write`], [`Isam::read`],
245//! [`Isam::begin_transaction`], and any of the offline administration methods
246//! ([`Isam::compact`], [`Isam::migrate_values`], [`Isam::migrate_keys`],
247//! [`Isam::migrate_index`]).
248//!
249//! ### Caveats
250//!
251//! - **Consumes `self`**: `as_single_user` takes ownership of the handle.
252//! Return `db` from the closure (as `Ok(db)`) if you need to keep using
253//! it afterward. If the call returns `Err`, the handle is dropped; clone
254//! before calling if you need to retry on failure.
255//! - **Deadlock if you hold a transaction**: `as_single_user` waits for the
256//! storage lock to be free. If the calling thread already holds an open
257//! [`Transaction`], the storage lock is already taken, so the spin will
258//! never succeed and the call will time out. Commit or roll back all open
259//! transactions on the calling thread before calling `as_single_user`.
260//! - **Not re-entrant**: calling `as_single_user` again from inside the
261//! closure returns [`IsamError::SingleUserMode`].
262//! - **In-process only**: the exclusive flag is an in-memory atomic; it does
263//! not prevent access from a separate process opening the same database
264//! files.
265//!
266//! ## Files on disk
267//!
268//! | File | Contents |
269//! |-----------------------|------------------------------------------------|
270//! | `*.idb` | Append-only data records (bincode) |
271//! | `*.idx` | On-disk B-tree index (page-based) |
272//! | `*_<name>.sidb` | Secondary index data store (one per index) |
273//! | `*_<name>.sidx` | Secondary index B-tree (one per index) |
274
275pub mod error;
276pub mod index;
277pub mod isam;
278pub mod manager;
279pub mod secondary_index;
280pub mod storage;
281pub mod store;
282pub mod transaction;
283
284// Re-export the main types at the crate root for convenience.
285pub use error::{IsamError, IsamResult};
286pub use isam::{IndexInfo, Isam, IsamBuilder, IsamIter, RangeIter, SecondaryIndexHandle, SingleUserToken, DEFAULT_SINGLE_USER_TIMEOUT};
287pub use secondary_index::DeriveKey;
288pub use transaction::Transaction;