Merge pull request #1276 from zed-industries/handle-carriage-returns

Handle files with CRLF (windows-style) line endings

crates/collab/src/integration_tests.rs

@@ -22,7 +22,7 @@ use gpui::{
 };
 use language::{
     range_to_lsp, tree_sitter_rust, Diagnostic, DiagnosticEntry, FakeLspAdapter, Language,
-    LanguageConfig, LanguageRegistry, OffsetRangeExt, Point, Rope,
+    LanguageConfig, LanguageRegistry, LineEnding, OffsetRangeExt, Point, Rope,
 };
 use lsp::{self, FakeLanguageServer};
 use parking_lot::Mutex;
@@ -1263,7 +1263,11 @@ async fn test_buffer_reloading(cx_a: &mut TestAppContext, cx_b: &mut TestAppCont
 
     client_a
         .fs
-        .save("/dir/a.txt".as_ref(), &"new contents".into())
+        .save(
+            "/dir/a.txt".as_ref(),
+            &"new contents".into(),
+            LineEnding::Unix,
+        )
         .await
         .unwrap();
     buffer_b
@@ -1857,7 +1861,11 @@ async fn test_reloading_buffer_manually(cx_a: &mut TestAppContext, cx_b: &mut Te
 
     client_a
         .fs
-        .save("/a/a.rs".as_ref(), &Rope::from("let seven = 7;"))
+        .save(
+            "/a/a.rs".as_ref(),
+            &Rope::from("let seven = 7;"),
+            LineEnding::Unix,
+        )
         .await
         .unwrap();
     buffer_a

crates/language/src/buffer.rs

@@ -53,6 +53,7 @@ pub struct Buffer {
     saved_version: clock::Global,
     saved_version_fingerprint: String,
     saved_mtime: SystemTime,
+    line_ending: LineEnding,
     transaction_depth: usize,
     was_dirty_before_starting_transaction: Option<bool>,
     language: Option<Arc<Language>>,
@@ -97,6 +98,12 @@ pub enum IndentKind {
     Tab,
 }
 
+#[derive(Copy, Debug, Clone, PartialEq, Eq)]
+pub enum LineEnding {
+    Unix,
+    Windows,
+}
+
 #[derive(Clone, Debug)]
 struct SelectionSet {
     line_mode: bool,
@@ -194,6 +201,7 @@ pub trait File: Send + Sync {
         buffer_id: u64,
         text: Rope,
         version: clock::Global,
+        line_ending: LineEnding,
         cx: &mut MutableAppContext,
     ) -> Task<Result<(clock::Global, String, SystemTime)>>;
 
@@ -275,6 +283,7 @@ pub(crate) struct Diff {
     base_version: clock::Global,
     new_text: Arc<str>,
     changes: Vec<(ChangeTag, usize)>,
+    line_ending: LineEnding,
     start_offset: usize,
 }
 
@@ -309,13 +318,12 @@ impl Buffer {
         base_text: T,
         cx: &mut ModelContext<Self>,
     ) -> Self {
+        let history = History::new(base_text.into());
+        let line_ending = LineEnding::detect(&history.base_text);
         Self::build(
-            TextBuffer::new(
-                replica_id,
-                cx.model_id() as u64,
-                History::new(base_text.into()),
-            ),
+            TextBuffer::new(replica_id, cx.model_id() as u64, history),
             None,
+            line_ending,
         )
     }
 
@@ -325,13 +333,12 @@ impl Buffer {
         file: Arc<dyn File>,
         cx: &mut ModelContext<Self>,
     ) -> Self {
+        let history = History::new(base_text.into());
+        let line_ending = LineEnding::detect(&history.base_text);
         Self::build(
-            TextBuffer::new(
-                replica_id,
-                cx.model_id() as u64,
-                History::new(base_text.into()),
-            ),
+            TextBuffer::new(replica_id, cx.model_id() as u64, history),
             Some(file),
+            line_ending,
         )
     }
 
@@ -346,7 +353,9 @@ impl Buffer {
             message.id,
             History::new(Arc::from(message.base_text)),
         );
-        let mut this = Self::build(buffer, file);
+        let line_ending = proto::LineEnding::from_i32(message.line_ending)
+            .ok_or_else(|| anyhow!("missing line_ending"))?;
+        let mut this = Self::build(buffer, file, LineEnding::from_proto(line_ending));
         let ops = message
             .operations
             .into_iter()
@@ -411,6 +420,7 @@ impl Buffer {
             diagnostics: proto::serialize_diagnostics(self.diagnostics.iter()),
             diagnostics_timestamp: self.diagnostics_timestamp.value,
             completion_triggers: self.completion_triggers.clone(),
+            line_ending: self.line_ending.to_proto() as i32,
         }
     }
 
@@ -419,7 +429,7 @@ impl Buffer {
         self
     }
 
-    fn build(buffer: TextBuffer, file: Option<Arc<dyn File>>) -> Self {
+    fn build(buffer: TextBuffer, file: Option<Arc<dyn File>>, line_ending: LineEnding) -> Self {
         let saved_mtime;
         if let Some(file) = file.as_ref() {
             saved_mtime = file.mtime();
@@ -435,6 +445,7 @@ impl Buffer {
             was_dirty_before_starting_transaction: None,
             text: buffer,
             file,
+            line_ending,
             syntax_tree: Mutex::new(None),
             parsing_in_background: false,
             parse_count: 0,
@@ -491,7 +502,13 @@ impl Buffer {
         };
         let text = self.as_rope().clone();
         let version = self.version();
-        let save = file.save(self.remote_id(), text, version, cx.as_mut());
+        let save = file.save(
+            self.remote_id(),
+            text,
+            version,
+            self.line_ending,
+            cx.as_mut(),
+        );
         cx.spawn(|this, mut cx| async move {
             let (version, fingerprint, mtime) = save.await?;
             this.update(&mut cx, |this, cx| {
@@ -538,7 +555,7 @@ impl Buffer {
             }) {
                 let new_text = new_text.await?;
                 let diff = this
-                    .read_with(&cx, |this, cx| this.diff(new_text.into(), cx))
+                    .read_with(&cx, |this, cx| this.diff(new_text, cx))
                     .await;
                 this.update(&mut cx, |this, cx| {
                     if let Some(transaction) = this.apply_diff(diff, cx).cloned() {
@@ -952,19 +969,22 @@ impl Buffer {
         }
     }
 
-    pub(crate) fn diff(&self, new_text: Arc<str>, cx: &AppContext) -> Task<Diff> {
-        // TODO: it would be nice to not allocate here.
-        let old_text = self.text();
+    pub(crate) fn diff(&self, new_text: String, cx: &AppContext) -> Task<Diff> {
+        let old_text = self.as_rope().clone();
         let base_version = self.version();
         cx.background().spawn(async move {
-            let changes = TextDiff::from_lines(old_text.as_str(), new_text.as_ref())
+            let old_text = old_text.to_string();
+            let line_ending = LineEnding::detect(&new_text);
+            let new_text = new_text.replace("\r\n", "\n").replace('\r', "\n");
+            let changes = TextDiff::from_lines(old_text.as_str(), new_text.as_str())
                 .iter_all_changes()
                 .map(|c| (c.tag(), c.value().len()))
                 .collect::<Vec<_>>();
             Diff {
                 base_version,
-                new_text,
+                new_text: new_text.into(),
                 changes,
+                line_ending,
                 start_offset: 0,
             }
         })
@@ -978,6 +998,7 @@ impl Buffer {
         if self.version == diff.base_version {
             self.finalize_last_transaction();
             self.start_transaction();
+            self.line_ending = diff.line_ending;
             let mut offset = diff.start_offset;
             for (tag, len) in diff.changes {
                 let range = offset..(offset + len);
@@ -1492,6 +1513,10 @@ impl Buffer {
     pub fn completion_triggers(&self) -> &[String] {
         &self.completion_triggers
     }
+
+    pub fn line_ending(&self) -> LineEnding {
+        self.line_ending
+    }
 }
 
 #[cfg(any(test, feature = "test-support"))]
@@ -2512,6 +2537,52 @@ impl std::ops::SubAssign for IndentSize {
     }
 }
 
+impl LineEnding {
+    fn from_proto(style: proto::LineEnding) -> Self {
+        match style {
+            proto::LineEnding::Unix => Self::Unix,
+            proto::LineEnding::Windows => Self::Windows,
+        }
+    }
+
+    fn detect(text: &str) -> Self {
+        let text = &text[..cmp::min(text.len(), 1000)];
+        if let Some(ix) = text.find('\n') {
+            if ix == 0 || text.as_bytes()[ix - 1] != b'\r' {
+                Self::Unix
+            } else {
+                Self::Windows
+            }
+        } else {
+            Default::default()
+        }
+    }
+
+    pub fn as_str(self) -> &'static str {
+        match self {
+            LineEnding::Unix => "\n",
+            LineEnding::Windows => "\r\n",
+        }
+    }
+
+    fn to_proto(self) -> proto::LineEnding {
+        match self {
+            LineEnding::Unix => proto::LineEnding::Unix,
+            LineEnding::Windows => proto::LineEnding::Windows,
+        }
+    }
+}
+
+impl Default for LineEnding {
+    fn default() -> Self {
+        #[cfg(unix)]
+        return Self::Unix;
+
+        #[cfg(not(unix))]
+        return Self::Windows;
+    }
+}
+
 impl Completion {
     pub fn sort_key(&self) -> (usize, &str) {
         let kind_key = match self.lsp_completion.kind {

crates/language/src/proto.rs

@@ -9,7 +9,7 @@ use rpc::proto;
 use std::{ops::Range, sync::Arc};
 use text::*;
 
-pub use proto::{Buffer, BufferState, SelectionSet};
+pub use proto::{Buffer, BufferState, LineEnding, SelectionSet};
 
 pub fn serialize_operation(operation: &Operation) -> proto::Operation {
     proto::Operation {

crates/project/src/fs.rs

@@ -1,6 +1,7 @@
 use anyhow::{anyhow, Result};
 use fsevent::EventStream;
 use futures::{Stream, StreamExt};
+use language::LineEnding;
 use smol::io::{AsyncReadExt, AsyncWriteExt};
 use std::{
     io,
@@ -21,7 +22,7 @@ pub trait Fs: Send + Sync {
     async fn remove_file(&self, path: &Path, options: RemoveOptions) -> Result<()>;
     async fn open_sync(&self, path: &Path) -> Result<Box<dyn io::Read>>;
     async fn load(&self, path: &Path) -> Result<String>;
-    async fn save(&self, path: &Path, text: &Rope) -> Result<()>;
+    async fn save(&self, path: &Path, text: &Rope, line_ending: LineEnding) -> Result<()>;
     async fn canonicalize(&self, path: &Path) -> Result<PathBuf>;
     async fn is_file(&self, path: &Path) -> bool;
     async fn metadata(&self, path: &Path) -> Result<Option<Metadata>>;
@@ -169,11 +170,11 @@ impl Fs for RealFs {
         Ok(text)
     }
 
-    async fn save(&self, path: &Path, text: &Rope) -> Result<()> {
+    async fn save(&self, path: &Path, text: &Rope, line_ending: LineEnding) -> Result<()> {
         let buffer_size = text.summary().bytes.min(10 * 1024);
         let file = smol::fs::File::create(path).await?;
         let mut writer = smol::io::BufWriter::with_capacity(buffer_size, file);
-        for chunk in text.chunks() {
+        for chunk in chunks(text, line_ending) {
             writer.write_all(chunk.as_bytes()).await?;
         }
         writer.flush().await?;
@@ -646,16 +647,17 @@ impl Fs for FakeFs {
         Ok(text.clone())
     }
 
-    async fn save(&self, path: &Path, text: &Rope) -> Result<()> {
+    async fn save(&self, path: &Path, text: &Rope, line_ending: LineEnding) -> Result<()> {
         self.simulate_random_delay().await;
         let mut state = self.state.lock().await;
         let path = normalize_path(path);
         state.validate_path(&path)?;
+        let content = chunks(text, line_ending).collect();
         if let Some(entry) = state.entries.get_mut(&path) {
             if entry.metadata.is_dir {
                 Err(anyhow!("cannot overwrite a directory with a file"))
             } else {
-                entry.content = Some(text.chunks().collect());
+                entry.content = Some(content);
                 entry.metadata.mtime = SystemTime::now();
                 state.emit_event(&[path]).await;
                 Ok(())
@@ -670,7 +672,7 @@ impl Fs for FakeFs {
                     is_dir: false,
                     is_symlink: false,
                 },
-                content: Some(text.chunks().collect()),
+                content: Some(content),
             };
             state.entries.insert(path.to_path_buf(), entry);
             state.emit_event(&[path]).await;
@@ -752,6 +754,21 @@ impl Fs for FakeFs {
     }
 }
 
+fn chunks(rope: &Rope, line_ending: LineEnding) -> impl Iterator<Item = &str> {
+    rope.chunks().flat_map(move |chunk| {
+        let mut newline = false;
+        chunk.split('\n').flat_map(move |line| {
+            let ending = if newline {
+                Some(line_ending.as_str())
+            } else {
+                None
+            };
+            newline = true;
+            ending.into_iter().chain([line])
+        })
+    })
+}
+
 pub fn normalize_path(path: &Path) -> PathBuf {
     let mut components = path.components().peekable();
     let mut ret = if let Some(c @ Component::Prefix(..)) = components.peek().cloned() {

crates/project/src/worktree.rs

@@ -24,7 +24,7 @@ use gpui::{
 };
 use language::{
     proto::{deserialize_version, serialize_version},
-    Buffer, DiagnosticEntry, PointUtf16, Rope,
+    Buffer, DiagnosticEntry, LineEnding, PointUtf16, Rope,
 };
 use lazy_static::lazy_static;
 use parking_lot::Mutex;
@@ -595,7 +595,7 @@ impl LocalWorktree {
         let text = buffer.as_rope().clone();
         let fingerprint = text.fingerprint();
         let version = buffer.version();
-        let save = self.write_file(path, text, cx);
+        let save = self.write_file(path, text, buffer.line_ending(), cx);
         let handle = cx.handle();
         cx.as_mut().spawn(|mut cx| async move {
             let entry = save.await?;
@@ -636,9 +636,10 @@ impl LocalWorktree {
         &self,
         path: impl Into<Arc<Path>>,
         text: Rope,
+        line_ending: LineEnding,
         cx: &mut ModelContext<Worktree>,
     ) -> Task<Result<Entry>> {
-        self.write_entry_internal(path, Some(text), cx)
+        self.write_entry_internal(path, Some((text, line_ending)), cx)
     }
 
     pub fn delete_entry(
@@ -754,7 +755,7 @@ impl LocalWorktree {
     fn write_entry_internal(
         &self,
         path: impl Into<Arc<Path>>,
-        text_if_file: Option<Rope>,
+        text_if_file: Option<(Rope, LineEnding)>,
         cx: &mut ModelContext<Worktree>,
     ) -> Task<Result<Entry>> {
         let path = path.into();
@@ -763,8 +764,8 @@ impl LocalWorktree {
             let fs = self.fs.clone();
             let abs_path = abs_path.clone();
             async move {
-                if let Some(text) = text_if_file {
-                    fs.save(&abs_path, &text).await
+                if let Some((text, line_ending)) = text_if_file {
+                    fs.save(&abs_path, &text, line_ending).await
                 } else {
                     fs.create_dir(&abs_path).await
                 }
@@ -1653,6 +1654,7 @@ impl language::File for File {
         buffer_id: u64,
         text: Rope,
         version: clock::Global,
+        line_ending: LineEnding,
         cx: &mut MutableAppContext,
     ) -> Task<Result<(clock::Global, String, SystemTime)>> {
         self.worktree.update(cx, |worktree, cx| match worktree {
@@ -1660,7 +1662,7 @@ impl language::File for File {
                 let rpc = worktree.client.clone();
                 let project_id = worktree.share.as_ref().map(|share| share.project_id);
                 let fingerprint = text.fingerprint();
-                let save = worktree.write_file(self.path.clone(), text, cx);
+                let save = worktree.write_file(self.path.clone(), text, line_ending, cx);
                 cx.background().spawn(async move {
                     let entry = save.await?;
                     if let Some(project_id) = project_id {
@@ -2841,6 +2843,7 @@ mod tests {
             tree.as_local().unwrap().write_file(
                 Path::new("tracked-dir/file.txt"),
                 "hello".into(),
+                Default::default(),
                 cx,
             )
         })
@@ -2850,6 +2853,7 @@ mod tests {
             tree.as_local().unwrap().write_file(
                 Path::new("ignored-dir/file.txt"),
                 "world".into(),
+                Default::default(),
                 cx,
             )
         })

crates/rpc/proto/zed.proto

@@ -810,6 +810,12 @@ message BufferState {
     repeated Diagnostic diagnostics = 6;
     uint32 diagnostics_timestamp = 7;
     repeated string completion_triggers = 8;
+    LineEnding line_ending = 9;
+}
+
+enum LineEnding {
+    Unix = 0;
+    Windows = 1;
 }
 
 message SelectionSet {

crates/text/src/rope.rs

@@ -58,11 +58,19 @@ impl Rope {
     pub fn push(&mut self, text: &str) {
         let mut new_chunks = SmallVec::<[_; 16]>::new();
         let mut new_chunk = ArrayString::new();
-        for ch in text.chars() {
+        let mut chars = text.chars().peekable();
+        while let Some(mut ch) = chars.next() {
             if new_chunk.len() + ch.len_utf8() > 2 * CHUNK_BASE {
                 new_chunks.push(Chunk(new_chunk));
                 new_chunk = ArrayString::new();
             }
+
+            if ch == '\r' {
+                ch = '\n';
+                if chars.peek().copied() == Some('\n') {
+                    chars.next();
+                }
+            }
             new_chunk.push(ch);
         }
         if !new_chunk.is_empty() {

crates/zed/src/settings_file.rs

@@ -117,6 +117,7 @@ mod tests {
             }
             "#
             .into(),
+            Default::default(),
         )
         .await
         .unwrap();
@@ -174,6 +175,7 @@ mod tests {
             }
             "#
             .into(),
+            Default::default(),
         )
         .await
         .unwrap();