Re-introduce fuzzy-matching on the new WorkTree implementation

Co-Authored-By: Nathan Sobo <nathan@zed.dev>
This commit is contained in:
Antonio Scandurra
2021-04-15 18:18:53 +02:00
co-authored by Nathan Sobo
parent 070069d65e
commit cefc753123
4 changed files with 698 additions and 49 deletions
+1 -1
View File
@@ -10,7 +10,7 @@ const TREE_BASE: usize = 2;
#[cfg(not(test))]
const TREE_BASE: usize = 6;
pub trait Item: Clone + Eq + fmt::Debug {
pub trait Item: Clone + fmt::Debug {
type Summary: for<'a> AddAssign<&'a Self::Summary> + Default + Clone + fmt::Debug;
fn summary(&self) -> Self::Summary;
+110 -48
View File
@@ -1,10 +1,16 @@
mod char_bag;
mod fuzzy;
use crate::sum_tree::{self, Edit, SumTree};
use anyhow::{anyhow, Result};
pub use fuzzy::match_paths;
use fuzzy::PathEntry;
use gpui::{scoped_pool, Entity, ModelContext};
use ignore::dir::{Ignore, IgnoreBuilder};
use parking_lot::Mutex;
use smol::{channel::Sender, Timer};
use std::{
ffi::{OsStr, OsString},
ffi::OsStr,
fmt, fs, io,
ops::AddAssign,
os::unix::fs::MetadataExt,
@@ -13,7 +19,7 @@ use std::{
atomic::{self, AtomicU64},
Arc,
},
time::{Duration, Instant},
time::Duration,
};
#[derive(Debug)]
@@ -24,24 +30,26 @@ enum ScanState {
}
pub struct Worktree {
id: usize,
path: Arc<Path>,
entries: SumTree<Entry>,
scanner: BackgroundScanner,
scan_state: ScanState,
will_poll_entries: bool,
poll_scheduled: bool,
}
impl Worktree {
fn new(path: impl Into<Arc<Path>>, ctx: &mut ModelContext<Self>) -> Self {
let path = path.into();
let scan_state = smol::channel::unbounded();
let scanner = BackgroundScanner::new(path.clone(), scan_state.0, ctx.thread_pool().clone());
let scanner = BackgroundScanner::new(path.clone(), scan_state.0);
let tree = Self {
id: ctx.model_id(),
path,
entries: Default::default(),
scanner,
scan_state: ScanState::Idle,
will_poll_entries: false,
poll_scheduled: false,
};
let scanner = tree.scanner.clone();
@@ -56,18 +64,19 @@ impl Worktree {
fn observe_scan_state(&mut self, scan_state: ScanState, ctx: &mut ModelContext<Self>) {
self.scan_state = scan_state;
self.poll_entries(ctx);
ctx.notify();
}
fn poll_entries(&mut self, ctx: &mut ModelContext<Self>) {
self.entries = self.scanner.snapshot();
if self.is_scanning() && !self.will_poll_entries {
self.will_poll_entries = true;
ctx.notify();
if self.is_scanning() && !self.poll_scheduled {
ctx.spawn(Timer::after(Duration::from_millis(100)), |this, _, ctx| {
this.will_poll_entries = false;
this.poll_scheduled = false;
this.poll_entries(ctx);
})
.detach();
self.poll_scheduled = true;
}
}
@@ -79,18 +88,46 @@ impl Worktree {
}
}
fn is_empty(&self) -> bool {
self.root_ino() == 0
fn root_ino(&self) -> Option<u64> {
let ino = self.scanner.root_ino.load(atomic::Ordering::SeqCst);
if ino == 0 {
None
} else {
Some(ino)
}
}
fn root_ino(&self) -> u64 {
self.scanner.root_ino.load(atomic::Ordering::SeqCst)
fn root_entry(&self) -> Option<&Entry> {
self.root_ino().and_then(|ino| self.entries.get(&ino))
}
fn file_count(&self) -> usize {
self.entries.summary().file_count
}
fn abs_entry_path(&self, ino: u64) -> Result<PathBuf> {
Ok(self.path.join(self.entry_path(ino)?))
}
fn entry_path(&self, ino: u64) -> Result<PathBuf> {
let mut components = Vec::new();
let mut entry = self
.entries
.get(&ino)
.ok_or_else(|| anyhow!("entry does not exist in worktree"))?;
components.push(entry.name());
while let Some(parent) = entry.parent() {
entry = self.entries.get(&parent).unwrap();
components.push(entry.name());
}
let mut path = PathBuf::new();
for component in components.into_iter().rev() {
path.push(component);
}
Ok(path)
}
fn fmt_entry(&self, f: &mut fmt::Formatter<'_>, ino: u64, indent: usize) -> fmt::Result {
match self.entries.get(&ino).unwrap() {
Entry::Dir { name, children, .. } => {
@@ -123,15 +160,15 @@ impl Entity for Worktree {
impl fmt::Debug for Worktree {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
if self.is_empty() {
write!(f, "Empty tree\n")
if let Some(root_ino) = self.root_ino() {
self.fmt_entry(f, root_ino, 0)
} else {
self.fmt_entry(f, self.root_ino(), 0)
write!(f, "Empty tree\n")
}
}
}
#[derive(Clone, Debug, Eq, PartialEq)]
#[derive(Clone, Debug)]
pub enum Entry {
Dir {
parent: Option<u64>,
@@ -145,6 +182,7 @@ pub enum Entry {
File {
parent: Option<u64>,
name: Arc<OsStr>,
path: PathEntry,
ino: u64,
is_symlink: bool,
is_ignored: bool,
@@ -158,6 +196,20 @@ impl Entry {
Entry::File { ino, .. } => *ino,
}
}
fn parent(&self) -> Option<u64> {
match self {
Entry::Dir { parent, .. } => *parent,
Entry::File { parent, .. } => *parent,
}
}
fn name(&self) -> &OsStr {
match self {
Entry::Dir { name, .. } => name,
Entry::File { name, .. } => name,
}
}
}
impl sum_tree::Item for Entry {
@@ -202,6 +254,15 @@ impl<'a> sum_tree::Dimension<'a, EntrySummary> for u64 {
}
}
#[derive(Copy, Clone, Default, Debug, Eq, PartialEq, Ord, PartialOrd)]
struct FileCount(usize);
impl<'a> sum_tree::Dimension<'a, EntrySummary> for FileCount {
fn add_summary(&mut self, summary: &'a EntrySummary) {
self.0 += summary.file_count;
}
}
#[derive(Clone)]
struct BackgroundScanner {
path: Arc<Path>,
@@ -212,13 +273,13 @@ struct BackgroundScanner {
}
impl BackgroundScanner {
fn new(path: Arc<Path>, notify: Sender<ScanState>, thread_pool: scoped_pool::Pool) -> Self {
fn new(path: Arc<Path>, notify: Sender<ScanState>) -> Self {
Self {
path,
root_ino: Arc::new(AtomicU64::new(0)),
entries: Default::default(),
notify,
thread_pool,
thread_pool: scoped_pool::Pool::new(16),
}
}
@@ -245,7 +306,6 @@ impl BackgroundScanner {
}
fn scan_dirs(&self) -> io::Result<()> {
println!("Scanning dirs ;)");
let metadata = fs::metadata(&self.path)?;
let ino = metadata.ino();
let is_symlink = fs::symlink_metadata(&self.path)?.file_type().is_symlink();
@@ -273,6 +333,7 @@ impl BackgroundScanner {
pending: true,
};
self.insert_entries(Some(dir_entry.clone()));
self.root_ino.store(ino, atomic::Ordering::SeqCst);
let (tx, rx) = crossbeam_channel::unbounded();
@@ -288,7 +349,7 @@ impl BackgroundScanner {
drop(tx);
let mut results = Vec::new();
results.resize_with(16, || Ok(()));
results.resize_with(self.thread_pool.workers(), || Ok(()));
self.thread_pool.scoped(|pool| {
for result in &mut results {
pool.execute(|| {
@@ -307,14 +368,14 @@ impl BackgroundScanner {
self.insert_entries(Some(Entry::File {
parent: None,
name,
path: PathEntry::new(ino, &relative_path, is_ignored),
ino,
is_symlink,
is_ignored,
}));
self.root_ino.store(ino, atomic::Ordering::SeqCst);
}
self.root_ino.store(ino, atomic::Ordering::SeqCst);
Ok(())
}
@@ -371,10 +432,11 @@ impl BackgroundScanner {
let is_ignored = job
.ignore
.as_ref()
.map_or(true, |i| i.matched(path, false).is_ignore());
.map_or(true, |i| i.matched(&path, false).is_ignore());
new_entries.push(Entry::File {
parent: Some(job.ino),
name,
path: PathEntry::new(ino, &relative_path, is_ignored),
ino,
is_symlink,
is_ignored,
@@ -395,7 +457,7 @@ impl BackgroundScanner {
self.insert_entries(new_entries);
for new_job in new_jobs {
let _ = scan_queue.send(Ok(new_job));
scan_queue.send(Ok(new_job)).unwrap();
}
Ok(())
@@ -463,29 +525,29 @@ mod tests {
let tree = app.add_model(|ctx| Worktree::new(root_link_path, ctx));
assert_condition(1, 300, || app.read(|ctx| tree.read(ctx).file_count() == 4)).await;
// app.read(|ctx| {
// let tree = tree.read(ctx);
// assert_eq!(tree.file_count(), 4);
// let results = match_paths(
// &[tree.clone()],
// "bna",
// false,
// false,
// 10,
// ctx.thread_pool().clone(),
// )
// .iter()
// .map(|result| tree.entry_path(result.entry_id))
// .collect::<Result<Vec<PathBuf>, _>>()
// .unwrap();
// assert_eq!(
// results,
// vec![
// PathBuf::from("root_link/banana/carrot/date"),
// PathBuf::from("root_link/banana/carrot/endive"),
// ]
// );
// })
app.read(|ctx| {
let tree = tree.read(ctx);
let results = match_paths(
Some(tree).into_iter(),
"bna",
false,
false,
false,
10,
ctx.thread_pool().clone(),
)
.iter()
.map(|result| tree.entry_path(result.entry_id))
.collect::<Result<Vec<PathBuf>, _>>()
.unwrap();
assert_eq!(
results,
vec![
PathBuf::from("root_link/banana/carrot/date"),
PathBuf::from("root_link/banana/carrot/endive"),
]
);
})
});
}
}
+44
View File
@@ -0,0 +1,44 @@
#[derive(Copy, Clone, Debug)]
pub struct CharBag(u64);
impl CharBag {
pub fn is_superset(self, other: CharBag) -> bool {
self.0 & other.0 == other.0
}
fn insert(&mut self, c: char) {
if c >= 'a' && c <= 'z' {
let mut count = self.0;
let idx = c as u8 - 'a' as u8;
count = count >> (idx * 2);
count = ((count << 1) | 1) & 3;
count = count << idx * 2;
self.0 |= count;
} else if c >= '0' && c <= '9' {
let idx = c as u8 - '0' as u8;
self.0 |= 1 << (idx + 52);
} else if c == '-' {
self.0 |= 1 << 62;
}
}
}
impl From<&str> for CharBag {
fn from(s: &str) -> Self {
let mut bag = Self(0);
for c in s.chars() {
bag.insert(c);
}
bag
}
}
impl From<&[char]> for CharBag {
fn from(chars: &[char]) -> Self {
let mut bag = Self(0);
for c in chars {
bag.insert(*c);
}
bag
}
}
+543
View File
@@ -0,0 +1,543 @@
use gpui::scoped_pool;
use crate::sum_tree::SeekBias;
use super::{char_bag::CharBag, Entry, FileCount, Worktree};
use std::{
cmp::{max, min, Ordering, Reverse},
collections::BinaryHeap,
path::Path,
sync::Arc,
};
const BASE_DISTANCE_PENALTY: f64 = 0.6;
const ADDITIONAL_DISTANCE_PENALTY: f64 = 0.05;
const MIN_DISTANCE_PENALTY: f64 = 0.2;
#[derive(Clone, Debug)]
pub struct PathEntry {
pub ino: u64,
pub path_chars: CharBag,
pub path: Arc<[char]>,
pub lowercase_path: Arc<[char]>,
pub is_ignored: bool,
}
impl PathEntry {
pub fn new(ino: u64, path: &Path, is_ignored: bool) -> Self {
let path = path.to_string_lossy();
let lowercase_path = path.to_lowercase().chars().collect::<Vec<_>>().into();
let path: Arc<[char]> = path.chars().collect::<Vec<_>>().into();
let path_chars = CharBag::from(path.as_ref());
Self {
ino,
path_chars,
path,
lowercase_path,
is_ignored,
}
}
}
#[derive(Clone, Debug)]
pub struct PathMatch {
pub score: f64,
pub positions: Vec<usize>,
pub tree_id: usize,
pub entry_id: u64,
pub skipped_prefix_len: usize,
}
impl PartialEq for PathMatch {
fn eq(&self, other: &Self) -> bool {
self.score.eq(&other.score)
}
}
impl Eq for PathMatch {}
impl PartialOrd for PathMatch {
fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
self.score.partial_cmp(&other.score)
}
}
impl Ord for PathMatch {
fn cmp(&self, other: &Self) -> Ordering {
self.partial_cmp(other).unwrap_or(Ordering::Equal)
}
}
pub fn match_paths<'a, T>(
trees: T,
query: &str,
include_root_name: bool,
include_ignored: bool,
smart_case: bool,
max_results: usize,
pool: scoped_pool::Pool,
) -> Vec<PathMatch>
where
T: Clone + Send + Iterator<Item = &'a Worktree>,
{
let lowercase_query = query.to_lowercase().chars().collect::<Vec<_>>();
let query = query.chars().collect::<Vec<_>>();
let lowercase_query = &lowercase_query;
let query = &query;
let query_chars = CharBag::from(&lowercase_query[..]);
let cpus = num_cpus::get();
let path_count: usize = trees.clone().map(Worktree::file_count).sum();
let segment_size = (path_count + cpus - 1) / cpus;
let mut segment_results = (0..cpus).map(|_| BinaryHeap::new()).collect::<Vec<_>>();
pool.scoped(|scope| {
for (segment_idx, results) in segment_results.iter_mut().enumerate() {
let trees = trees.clone();
scope.execute(move || {
let segment_start = segment_idx * segment_size;
let segment_end = segment_start + segment_size;
let mut min_score = 0.0;
let mut last_positions = Vec::new();
last_positions.resize(query.len(), 0);
let mut match_positions = Vec::new();
match_positions.resize(query.len(), 0);
let mut score_matrix = Vec::new();
let mut best_position_matrix = Vec::new();
let mut tree_start = 0;
for tree in trees {
let tree_end = tree_start + tree.file_count();
if tree_start < segment_end && segment_start < tree_end {
let start = max(tree_start, segment_start) - tree_start;
let end = min(tree_end, segment_end) - tree_start;
let mut cursor = tree.entries.cursor::<_, ()>();
cursor.seek(&FileCount(start), SeekBias::Right);
let path_entries = cursor
.filter_map(|e| {
if let Entry::File { path, .. } = e {
Some(path)
} else {
None
}
})
.take(end - start);
let skipped_prefix_len = if include_root_name {
0
} else if let Some(Entry::Dir { name, .. }) = tree.root_entry() {
let name = name.to_string_lossy();
if name == "/" {
1
} else {
name.chars().count() + 1
}
} else {
0
};
match_single_tree_paths(
tree.id,
skipped_prefix_len,
path_entries,
query,
lowercase_query,
query_chars.clone(),
include_ignored,
smart_case,
results,
max_results,
&mut min_score,
&mut match_positions,
&mut last_positions,
&mut score_matrix,
&mut best_position_matrix,
);
}
if tree_end >= segment_end {
break;
}
tree_start = tree_end;
}
})
}
});
let mut results = segment_results
.into_iter()
.flatten()
.map(|r| r.0)
.collect::<Vec<_>>();
results.sort_unstable_by(|a, b| b.score.partial_cmp(&a.score).unwrap());
results.truncate(max_results);
results
}
fn match_single_tree_paths<'a>(
tree_id: usize,
skipped_prefix_len: usize,
path_entries: impl Iterator<Item = &'a PathEntry>,
query: &[char],
lowercase_query: &[char],
query_chars: CharBag,
include_ignored: bool,
smart_case: bool,
results: &mut BinaryHeap<Reverse<PathMatch>>,
max_results: usize,
min_score: &mut f64,
match_positions: &mut Vec<usize>,
last_positions: &mut Vec<usize>,
score_matrix: &mut Vec<Option<f64>>,
best_position_matrix: &mut Vec<usize>,
) {
for path_entry in path_entries {
if !include_ignored && path_entry.is_ignored {
continue;
}
if !path_entry.path_chars.is_superset(query_chars.clone()) {
continue;
}
if !find_last_positions(
last_positions,
skipped_prefix_len,
&path_entry.lowercase_path,
&lowercase_query[..],
) {
continue;
}
let matrix_len = query.len() * (path_entry.path.len() - skipped_prefix_len);
score_matrix.clear();
score_matrix.resize(matrix_len, None);
best_position_matrix.clear();
best_position_matrix.resize(matrix_len, skipped_prefix_len);
let score = score_match(
&query[..],
&lowercase_query[..],
&path_entry.path,
&path_entry.lowercase_path,
skipped_prefix_len,
smart_case,
&last_positions,
score_matrix,
best_position_matrix,
match_positions,
*min_score,
);
if score > 0.0 {
results.push(Reverse(PathMatch {
tree_id,
entry_id: path_entry.ino,
score,
positions: match_positions.clone(),
skipped_prefix_len,
}));
if results.len() == max_results {
*min_score = results.peek().unwrap().0.score;
}
}
}
}
fn find_last_positions(
last_positions: &mut Vec<usize>,
skipped_prefix_len: usize,
path: &[char],
query: &[char],
) -> bool {
let mut path = path.iter();
for (i, char) in query.iter().enumerate().rev() {
if let Some(j) = path.rposition(|c| c == char) {
if j >= skipped_prefix_len {
last_positions[i] = j;
} else {
return false;
}
} else {
return false;
}
}
true
}
fn score_match(
query: &[char],
query_cased: &[char],
path: &[char],
path_cased: &[char],
skipped_prefix_len: usize,
smart_case: bool,
last_positions: &[usize],
score_matrix: &mut [Option<f64>],
best_position_matrix: &mut [usize],
match_positions: &mut [usize],
min_score: f64,
) -> f64 {
let score = recursive_score_match(
query,
query_cased,
path,
path_cased,
skipped_prefix_len,
smart_case,
last_positions,
score_matrix,
best_position_matrix,
min_score,
0,
skipped_prefix_len,
query.len() as f64,
) * query.len() as f64;
if score <= 0.0 {
return 0.0;
}
let path_len = path.len() - skipped_prefix_len;
let mut cur_start = 0;
for i in 0..query.len() {
match_positions[i] = best_position_matrix[i * path_len + cur_start] - skipped_prefix_len;
cur_start = match_positions[i] + 1;
}
score
}
fn recursive_score_match(
query: &[char],
query_cased: &[char],
path: &[char],
path_cased: &[char],
skipped_prefix_len: usize,
smart_case: bool,
last_positions: &[usize],
score_matrix: &mut [Option<f64>],
best_position_matrix: &mut [usize],
min_score: f64,
query_idx: usize,
path_idx: usize,
cur_score: f64,
) -> f64 {
if query_idx == query.len() {
return 1.0;
}
let path_len = path.len() - skipped_prefix_len;
if let Some(memoized) = score_matrix[query_idx * path_len + path_idx - skipped_prefix_len] {
return memoized;
}
let mut score = 0.0;
let mut best_position = 0;
let query_char = query_cased[query_idx];
let limit = last_positions[query_idx];
let mut last_slash = 0;
for j in path_idx..=limit {
let path_char = path_cased[j];
let is_path_sep = path_char == '/' || path_char == '\\';
if query_idx == 0 && is_path_sep {
last_slash = j;
}
if query_char == path_char || (is_path_sep && query_char == '_' || query_char == '\\') {
let mut char_score = 1.0;
if j > path_idx {
let last = path[j - 1];
let curr = path[j];
if last == '/' {
char_score = 0.9;
} else if last == '-' || last == '_' || last == ' ' || last.is_numeric() {
char_score = 0.8;
} else if last.is_lowercase() && curr.is_uppercase() {
char_score = 0.8;
} else if last == '.' {
char_score = 0.7;
} else if query_idx == 0 {
char_score = BASE_DISTANCE_PENALTY;
} else {
char_score = MIN_DISTANCE_PENALTY.max(
BASE_DISTANCE_PENALTY
- (j - path_idx - 1) as f64 * ADDITIONAL_DISTANCE_PENALTY,
);
}
}
// Apply a severe penalty if the case doesn't match.
// This will make the exact matches have higher score than the case-insensitive and the
// path insensitive matches.
if (smart_case || path[j] == '/') && query[query_idx] != path[j] {
char_score *= 0.001;
}
let mut multiplier = char_score;
// Scale the score based on how deep within the patch we found the match.
if query_idx == 0 {
multiplier /= (path.len() - last_slash) as f64;
}
let mut next_score = 1.0;
if min_score > 0.0 {
next_score = cur_score * multiplier;
// Scores only decrease. If we can't pass the previous best, bail
if next_score < min_score {
// Ensure that score is non-zero so we use it in the memo table.
if score == 0.0 {
score = 1e-18;
}
continue;
}
}
let new_score = recursive_score_match(
query,
query_cased,
path,
path_cased,
skipped_prefix_len,
smart_case,
last_positions,
score_matrix,
best_position_matrix,
min_score,
query_idx + 1,
j + 1,
next_score,
) * multiplier;
if new_score > score {
score = new_score;
best_position = j;
// Optimization: can't score better than 1.
if new_score == 1.0 {
break;
}
}
}
}
if best_position != 0 {
best_position_matrix[query_idx * path_len + path_idx - skipped_prefix_len] = best_position;
}
score_matrix[query_idx * path_len + path_idx - skipped_prefix_len] = Some(score);
score
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_match_path_entries() {
let paths = vec![
"",
"a",
"ab",
"abC",
"abcd",
"alphabravocharlie",
"AlphaBravoCharlie",
"thisisatestdir",
"/////ThisIsATestDir",
"/this/is/a/test/dir",
"/test/tiatd",
];
assert_eq!(
match_query("abc", false, &paths),
vec![
("abC", vec![0, 1, 2]),
("abcd", vec![0, 1, 2]),
("AlphaBravoCharlie", vec![0, 5, 10]),
("alphabravocharlie", vec![4, 5, 10]),
]
);
assert_eq!(
match_query("t/i/a/t/d", false, &paths),
vec![("/this/is/a/test/dir", vec![1, 5, 6, 8, 9, 10, 11, 15, 16]),]
);
assert_eq!(
match_query("tiatd", false, &paths),
vec![
("/test/tiatd", vec![6, 7, 8, 9, 10]),
("/this/is/a/test/dir", vec![1, 6, 9, 11, 16]),
("/////ThisIsATestDir", vec![5, 9, 11, 12, 16]),
("thisisatestdir", vec![0, 2, 6, 7, 11]),
]
);
}
fn match_query<'a>(
query: &str,
smart_case: bool,
paths: &Vec<&'a str>,
) -> Vec<(&'a str, Vec<usize>)> {
let lowercase_query = query.to_lowercase().chars().collect::<Vec<_>>();
let query = query.chars().collect::<Vec<_>>();
let query_chars = CharBag::from(&lowercase_query[..]);
let mut path_entries = Vec::new();
for (i, path) in paths.iter().enumerate() {
let lowercase_path: Arc<[char]> =
path.to_lowercase().chars().collect::<Vec<_>>().into();
let path_chars = CharBag::from(lowercase_path.as_ref());
let path = path.chars().collect();
path_entries.push(PathEntry {
ino: i as u64,
path_chars,
path,
lowercase_path,
is_ignored: false,
});
}
let mut match_positions = Vec::new();
let mut last_positions = Vec::new();
match_positions.resize(query.len(), 0);
last_positions.resize(query.len(), 0);
let mut results = BinaryHeap::new();
match_single_tree_paths(
0,
0,
path_entries.iter(),
&query[..],
&lowercase_query[..],
query_chars,
true,
smart_case,
&mut results,
100,
&mut 0.0,
&mut match_positions,
&mut last_positions,
&mut Vec::new(),
&mut Vec::new(),
);
results
.into_iter()
.rev()
.map(|result| {
(
paths[result.0.entry_id as usize].clone(),
result.0.positions,
)
})
.collect()
}
}