Add gpt-4-1106-preview model (#3319)

This PR adds the `gpt-4-1106-preview` model. It required forking two
repos from different organizations, as one is a submodule of another.

Update 1:

~Currently, I'm just temporarily pointing towards a personal fork of the
repo, but once [this
PR](https://github.com/zurawiki/tiktoken-rs/pull/49) lands, we can
switch back to using the official crate.~

Update 2:

~PR was merged, we are now pointing to a specific commit on the original
official repo - may want to bump it to an official version once a new
one has been released.~

Update 3:

Author published a [new
version](https://crates.io/crates/tiktoken-rs/0.5.7). Now, we are back
to relying on the official crate.

<img width="1118" alt="SCR-20231114-ibgs"
src="https://github.com/zed-industries/zed/assets/19867440/cc2657b6-98f9-413f-9a0c-f337636f367c">

Release Notes:

- Added support for the
[`gpt-4-1106-preview`](https://openai.com/blog/new-models-and-developer-products-announced-at-devday)
model in the assistant panel.
- Updated the `assistant.default_open_ai_model` setting to default to
`gpt-4-1106-preview`. To override the default, use the following snippet
to your `settings.json` file:

```rs
"assistant": {
    // 1. "gpt-3.5-turbo-0613""
    // 2. "gpt-4-0613""
    // 3. "gpt-4-1106-preview"
    "default_open_ai_model": "<Your choice of model here>"
  },
```

Cargo.lock

@@ -9256,9 +9256,9 @@ dependencies = [
 
 [[package]]
 name = "tiktoken-rs"
-version = "0.5.4"
+version = "0.5.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f9ae5a3c24361e5f038af22517ba7f8e3af4099e30e78a3d56f86b48238fce9d"
+checksum = "a4427b6b1c6b38215b92dd47a83a0ecc6735573d0a5a4c14acc0ac5b33b28adb"
 dependencies = [
  "anyhow",
  "base64 0.21.4",

Cargo.toml

@@ -154,6 +154,7 @@ tempdir = { version = "0.3.7" }
 thiserror = { version = "1.0.29" }
 time = { version = "0.3", features = ["serde", "serde-well-known"] }
 toml = { version = "0.5" }
+tiktoken-rs = "0.5.7"
 tree-sitter = "0.20"
 unindent = { version = "0.1.7" }
 pretty_assertions = "1.3.0"

assets/settings/default.json

@@ -174,7 +174,8 @@
     //
     // 1. "gpt-3.5-turbo-0613""
     // 2. "gpt-4-0613""
-    "default_open_ai_model": "gpt-4-0613"
+    // 3. "gpt-4-1106-preview"
+    "default_open_ai_model": "gpt-4-1106-preview"
   },
   // Whether the screen sharing icon is shown in the os status bar.
   "show_call_status_icon": true,
@@ -270,9 +271,7 @@
   "copilot": {
     // The set of glob patterns for which copilot should be disabled
     // in any matching file.
-    "disabled_globs": [
-      ".env"
-    ]
+    "disabled_globs": [".env"]
   },
   // Settings specific to journaling
   "journal": {
@@ -381,12 +380,7 @@
         // Default directories to search for virtual environments, relative
         // to the current working directory. We recommend overriding this
         // in your project's settings, rather than globally.
-        "directories": [
-          ".env",
-          "env",
-          ".venv",
-          "venv"
-        ],
+        "directories": [".env", "env", ".venv", "venv"],
         // Can also be 'csh', 'fish', and `nushell`
         "activate_script": "default"
       }

crates/Cargo.toml

@@ -29,7 +29,7 @@ postage.workspace = true
 rand.workspace = true
 log.workspace = true
 parse_duration = "2.1.1"
-tiktoken-rs = "0.5.0"
+tiktoken-rs.workspace = true
 matrixmultiply = "0.3.7"
 rusqlite = { version = "0.29.0", features = ["blob", "array", "modern_sqlite"] }
 bincode = "1.3.3"

crates/ai/Cargo.toml

@@ -29,7 +29,7 @@ postage.workspace = true
 rand.workspace = true
 log.workspace = true
 parse_duration = "2.1.1"
-tiktoken-rs = "0.5.0"
+tiktoken-rs.workspace = true
 matrixmultiply = "0.3.7"
 rusqlite = { version = "0.29.0", features = ["blob", "array", "modern_sqlite"] }
 bincode = "1.3.3"

crates/ai2/Cargo.toml

@@ -29,7 +29,7 @@ postage.workspace = true
 rand.workspace = true
 log.workspace = true
 parse_duration = "2.1.1"
-tiktoken-rs = "0.5.0"
+tiktoken-rs.workspace = true
 matrixmultiply = "0.3.7"
 rusqlite = { version = "0.29.0", features = ["blob", "array", "modern_sqlite"] }
 bincode = "1.3.3"

crates/assistant/Cargo.toml

@@ -40,7 +40,7 @@ schemars.workspace = true
 serde.workspace = true
 serde_json.workspace = true
 smol.workspace = true
-tiktoken-rs = "0.5"
+tiktoken-rs.workspace = true
 
 [dev-dependencies]
 editor = { path = "../editor", features = ["test-support"] }

crates/assistant/src/assistant_settings.rs

@@ -9,6 +9,8 @@ pub enum OpenAIModel {
     ThreePointFiveTurbo,
     #[serde(rename = "gpt-4-0613")]
     Four,
+    #[serde(rename = "gpt-4-1106-preview")]
+    FourTurbo,
 }
 
 impl OpenAIModel {
@@ -16,6 +18,7 @@ impl OpenAIModel {
         match self {
             OpenAIModel::ThreePointFiveTurbo => "gpt-3.5-turbo-0613",
             OpenAIModel::Four => "gpt-4-0613",
+            OpenAIModel::FourTurbo => "gpt-4-1106-preview",
         }
     }
 
@@ -23,13 +26,15 @@ impl OpenAIModel {
         match self {
             OpenAIModel::ThreePointFiveTurbo => "gpt-3.5-turbo",
             OpenAIModel::Four => "gpt-4",
+            OpenAIModel::FourTurbo => "gpt-4-turbo",
         }
     }
 
     pub fn cycle(&self) -> Self {
         match self {
             OpenAIModel::ThreePointFiveTurbo => OpenAIModel::Four,
-            OpenAIModel::Four => OpenAIModel::ThreePointFiveTurbo,
+            OpenAIModel::Four => OpenAIModel::FourTurbo,
+            OpenAIModel::FourTurbo => OpenAIModel::ThreePointFiveTurbo,
         }
     }
 }

crates/semantic_index/Cargo.toml

@@ -33,7 +33,7 @@ lazy_static.workspace = true
 serde.workspace = true
 serde_json.workspace = true
 async-trait.workspace = true
-tiktoken-rs = "0.5.0"
+tiktoken-rs.workspace = true
 parking_lot.workspace = true
 rand.workspace = true
 schemars.workspace = true