oak-team/oak-gpui
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

editor: Use unbounded shifts for chunk bitmaps (#40879)

Browse Source 
This simplifies some code and is also more correct in some others (I
believe some of these might've overflowed causing panics in sentry)

Release Notes:

- N/A *or* Added/Fixed/Improved ...
This commit is contained in:
Lukas Wirth
2025-10-22 11:05:32 +00:00
committed by GitHub
parent bd69124f2b
commit c81ffaffb6
12 changed files with 169 additions and 227 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/rope/src/chunk.rs
+58 -53
View File
@@ -5,29 +5,36 @@ use sum_tree::Bias;
use unicode_segmentation::GraphemeCursor;
use util::debug_panic;
pub(crate) const MIN_BASE: usize = if cfg!(test) { 6 } else { 64 };
pub(crate) const MAX_BASE: usize = MIN_BASE * 2;
#[cfg(not(all(test, not(rust_analyzer))))]
pub(crate) type Bitmap = u128;
#[cfg(all(test, not(rust_analyzer)))]
pub(crate) type Bitmap = u16;
pub(crate) const MIN_BASE: usize = MAX_BASE / 2;
pub(crate) const MAX_BASE: usize = Bitmap::BITS as usize;
#[derive(Clone, Debug, Default)]
pub struct Chunk {
/// If bit[i] is set, then the character at index i is the start of a UTF-8 character in the
/// text.
chars: u128,
chars: Bitmap,
/// The number of set bits is the number of UTF-16 code units it would take to represent the
/// text.
///
/// Bit[i] is set if text[i] is the start of a UTF-8 character. If the character would
/// take two UTF-16 code units, then bit[i+1] is also set. (Rust chars never take more
/// than two UTF-16 code units.)
chars_utf16: u128,
chars_utf16: Bitmap,
/// If bit[i] is set, then the character at index i is an ascii newline.
newlines: u128,
newlines: Bitmap,
/// If bit[i] is set, then the character at index i is an ascii tab.
pub tabs: u128,
tabs: Bitmap,
pub text: ArrayString<MAX_BASE>,
}
impl Chunk {
pub const MASK_BITS: usize = Bitmap::BITS as usize;
#[inline(always)]
pub fn new(text: &str) -> Self {
let mut this = Chunk::default();
@@ -41,9 +48,9 @@ impl Chunk {
let ix = self.text.len() + char_ix;
self.chars |= 1 << ix;
self.chars_utf16 |= 1 << ix;
self.chars_utf16 |= (c.len_utf16() as u128) << ix;
self.newlines |= ((c == '\n') as u128) << ix;
self.tabs |= ((c == '\t') as u128) << ix;
self.chars_utf16 |= (c.len_utf16() as Bitmap) << ix;
self.newlines |= ((c == '\n') as Bitmap) << ix;
self.tabs |= ((c == '\t') as Bitmap) << ix;
}
self.text.push_str(text);
}
@@ -79,17 +86,21 @@ impl Chunk {
}
#[inline(always)]
pub fn chars(&self) -> u128 {
pub fn chars(&self) -> Bitmap {
self.chars
}
pub fn tabs(&self) -> Bitmap {
self.tabs
}
}
#[derive(Clone, Copy, Debug)]
pub struct ChunkSlice<'a> {
chars: u128,
chars_utf16: u128,
newlines: u128,
tabs: u128,
chars: Bitmap,
chars_utf16: Bitmap,
newlines: Bitmap,
tabs: Bitmap,
text: &'a str,
}
@@ -129,7 +140,7 @@ impl<'a> ChunkSlice<'a> {
};
(left, right)
} else {
let mask = (1u128 << mid) - 1;
let mask = ((1 as Bitmap) << mid) - 1;
let (left_text, right_text) = self.text.split_at(mid);
let left = ChunkSlice {
chars: self.chars & mask,
@@ -151,17 +162,15 @@ impl<'a> ChunkSlice<'a> {
#[inline(always)]
pub fn slice(self, range: Range<usize>) -> Self {
let mask = if range.end == MAX_BASE {
u128::MAX
} else {
debug_assert!(
self.is_char_boundary(range.end),
"Invalid range end {} in {:?}",
range.end,
self
);
(1u128 << range.end) - 1
};
debug_assert!(
self.is_char_boundary(range.end),
"Invalid range end {} in {:?}",
range.end,
self
);
let mask = (1 as Bitmap)
.unbounded_shl(range.end as u32)
.wrapping_sub(1);
if range.start == MAX_BASE {
Self {
chars: 0,
@@ -220,7 +229,7 @@ impl<'a> ChunkSlice<'a> {
#[inline(always)]
pub fn lines(&self) -> Point {
let row = self.newlines.count_ones();
let column = self.newlines.leading_zeros() - (u128::BITS - self.text.len() as u32);
let column = self.newlines.leading_zeros() - (Bitmap::BITS - self.text.len() as u32);
Point::new(row, column)
}
@@ -230,7 +239,7 @@ impl<'a> ChunkSlice<'a> {
if self.newlines == 0 {
self.chars.count_ones()
} else {
let mask = (1u128 << self.newlines.trailing_zeros()) - 1;
let mask = ((1 as Bitmap) << self.newlines.trailing_zeros()) - 1;
(self.chars & mask).count_ones()
}
}
@@ -241,7 +250,7 @@ impl<'a> ChunkSlice<'a> {
if self.newlines == 0 {
self.chars.count_ones()
} else {
let mask = !(u128::MAX >> self.newlines.leading_zeros());
let mask = !(Bitmap::MAX >> self.newlines.leading_zeros());
(self.chars & mask).count_ones()
}
}
@@ -252,7 +261,7 @@ impl<'a> ChunkSlice<'a> {
if self.newlines == 0 {
self.chars_utf16.count_ones()
} else {
let mask = !(u128::MAX >> self.newlines.leading_zeros());
let mask = !(Bitmap::MAX >> self.newlines.leading_zeros());
(self.chars_utf16 & mask).count_ones()
}
}
@@ -295,13 +304,9 @@ impl<'a> ChunkSlice<'a> {
#[inline(always)]
pub fn offset_to_point(&self, offset: usize) -> Point {
let mask = if offset == MAX_BASE {
u128::MAX
} else {
(1u128 << offset) - 1
};
let mask = (1 as Bitmap).unbounded_shl(offset as u32).wrapping_sub(1);
let row = (self.newlines & mask).count_ones();
let newline_ix = u128::BITS - (self.newlines & mask).leading_zeros();
let newline_ix = Bitmap::BITS - (self.newlines & mask).leading_zeros();
let column = (offset - newline_ix as usize) as u32;
Point::new(row, column)
}
@@ -332,11 +337,7 @@ impl<'a> ChunkSlice<'a> {
#[inline(always)]
pub fn offset_to_offset_utf16(&self, offset: usize) -> OffsetUtf16 {
let mask = if offset == MAX_BASE {
u128::MAX
} else {
(1u128 << offset) - 1
};
let mask = (1 as Bitmap).unbounded_shl(offset as u32).wrapping_sub(1);
OffsetUtf16((self.chars_utf16 & mask).count_ones() as usize)
}
@@ -345,7 +346,11 @@ impl<'a> ChunkSlice<'a> {
if target.0 == 0 {
0
} else {
let ix = nth_set_bit(self.chars_utf16, target.0) + 1;
#[cfg(not(test))]
let chars_utf16 = self.chars_utf16;
#[cfg(test)]
let chars_utf16 = self.chars_utf16 as u128;
let ix = nth_set_bit(chars_utf16, target.0) + 1;
if ix == MAX_BASE {
MAX_BASE
} else {
@@ -360,13 +365,9 @@ impl<'a> ChunkSlice<'a> {
#[inline(always)]
pub fn offset_to_point_utf16(&self, offset: usize) -> PointUtf16 {
let mask = if offset == MAX_BASE {
u128::MAX
} else {
(1u128 << offset) - 1
};
let mask = (1 as Bitmap).unbounded_shl(offset as u32).wrapping_sub(1);
let row = (self.newlines & mask).count_ones();
let newline_ix = u128::BITS - (self.newlines & mask).leading_zeros();
let newline_ix = Bitmap::BITS - (self.newlines & mask).leading_zeros();
let column = if newline_ix as usize == MAX_BASE {
0
} else {
@@ -520,7 +521,11 @@ impl<'a> ChunkSlice<'a> {
#[inline(always)]
fn offset_range_for_row(&self, row: u32) -> Range<usize> {
let row_start = if row > 0 {
nth_set_bit(self.newlines, row as usize) + 1
#[cfg(not(test))]
let newlines = self.newlines;
#[cfg(test)]
let newlines = self.newlines as u128;
nth_set_bit(newlines, row as usize) + 1
} else {
0
};
@@ -545,8 +550,8 @@ impl<'a> ChunkSlice<'a> {
}
pub struct Tabs {
tabs: u128,
chars: u128,
tabs: Bitmap,
chars: Bitmap,
}
#[derive(Debug, PartialEq, Eq)]
@@ -647,8 +652,8 @@ mod tests {
// Verify Chunk::chars() bitmap
let expected_chars = char_offsets(&text)
.into_iter()
.inspect(|i| assert!(*i < 128))
.fold(0u128, |acc, i| acc | (1 << i));
.inspect(|i| assert!(*i < MAX_BASE))
.fold(0 as Bitmap, |acc, i| acc | (1 << i));
assert_eq!(chunk.chars(), expected_chars);
for _ in 0..10 {
crates/rope/src/rope.rs
+8 -39
View File
@@ -4,7 +4,6 @@ mod point;
mod point_utf16;
mod unclipped;
use chunk::Chunk;
use rayon::iter::{IntoParallelIterator, ParallelIterator as _};
use smallvec::SmallVec;
use std::{
@@ -14,12 +13,14 @@ use std::{
};
use sum_tree::{Bias, Dimension, Dimensions, SumTree};
pub use chunk::ChunkSlice;
pub use chunk::{Chunk, ChunkSlice};
pub use offset_utf16::OffsetUtf16;
pub use point::Point;
pub use point_utf16::PointUtf16;
pub use unclipped::Unclipped;
use crate::chunk::Bitmap;
#[derive(Clone, Default)]
pub struct Rope {
chunks: SumTree<Chunk>,
@@ -676,9 +677,9 @@ pub struct ChunkBitmaps<'a> {
/// A slice of text up to 128 bytes in size
pub text: &'a str,
/// Bitmap of character locations in text. LSB ordered
pub chars: u128,
pub chars: Bitmap,
/// Bitmap of tab locations in text. LSB ordered
pub tabs: u128,
pub tabs: Bitmap,
}
#[derive(Clone)]
@@ -850,39 +851,6 @@ impl<'a> Chunks<'a> {
self.offset < initial_offset && self.offset == 0
}
/// Returns bitmaps that represent character positions and tab positions
pub fn peek_with_bitmaps(&self) -> Option<ChunkBitmaps<'a>> {
if !self.offset_is_valid() {
return None;
}
let chunk = self.chunks.item()?;
let chunk_start = *self.chunks.start();
let slice_range = if self.reversed {
let slice_start = cmp::max(chunk_start, self.range.start) - chunk_start;
let slice_end = self.offset - chunk_start;
slice_start..slice_end
} else {
let slice_start = self.offset - chunk_start;
let slice_end = cmp::min(self.chunks.end(), self.range.end) - chunk_start;
slice_start..slice_end
};
// slice range has a bounds between 0 and 128 in non test builds
// We use a non wrapping sub because we want to overflow in the case where slice_range.end == 128
// because that represents a full chunk and the bitmask shouldn't remove anything
let bitmask = (1u128.unbounded_shl(slice_range.end as u32)).wrapping_sub(1);
let chars = (chunk.chars() & bitmask) >> slice_range.start;
let tabs = (chunk.tabs & bitmask) >> slice_range.start;
Some(ChunkBitmaps {
text: &chunk.text[slice_range],
chars,
tabs,
})
}
pub fn peek(&self) -> Option<&'a str> {
if !self.offset_is_valid() {
return None;
@@ -903,7 +871,8 @@ impl<'a> Chunks<'a> {
Some(&chunk.text[slice_range])
}
pub fn peek_tabs(&self) -> Option<ChunkBitmaps<'a>> {
/// Returns bitmaps that represent character positions and tab positions
pub fn peek_with_bitmaps(&self) -> Option<ChunkBitmaps<'a>> {
if !self.offset_is_valid() {
return None;
}
@@ -923,7 +892,7 @@ impl<'a> Chunks<'a> {
let slice_text = &chunk.text[slice_range];
// Shift the tabs to align with our slice window
let shifted_tabs = chunk.tabs >> chunk_start_offset;
let shifted_tabs = chunk.tabs() >> chunk_start_offset;
let shifted_chars = chunk.chars() >> chunk_start_offset;
Some(ChunkBitmaps {