oak-team/oak-gpui
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Start to send data to Snowflake too (#20698)

Browse Source 
This PR adds support for sending telemetry events to AWS Kinesis.

In our AWS account we now have three new things:
* The [Kinesis data
stream](https://us-east-1.console.aws.amazon.com/kinesis/home?region=us-east-1#/streams/details/zed-telemetry/monitoring)
that we will actually write to.
* A [Firehose for
Axiom](https://us-east-1.console.aws.amazon.com/firehose/home?region=us-east-1#/details/telemetry-to-axiom/monitoring)
that sends events from that stream to Axiom for ad-hoc queries over
recent data.
* A [Firehose for
Snowflake](https://us-east-1.console.aws.amazon.com/firehose/home?region=us-east-1#/details/telemetry-to-snowflake/monitoring)
that sends events from that stream to Snowflake for long-term retention.
This Firehose also backs up data into an S3 bucket in case we want to
change how the system works in the future.

In a follow-up PR, we'll add support for ad-hoc telemetry events; and
slowly move away from the current Clickhouse defined schemas; though we
won't move off click house until we have what we need in Snowflake.

Co-Authored-By: Nathan <nathan@zed.dev>

Release Notes:

- N/A
This commit is contained in:
Conrad Irwin
2024-11-15 12:58:00 -07:00
committed by GitHub
co-authored by Nathan
parent f449e8d3d3
commit 6ff69faf37
7 changed files with 305 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Cargo.lock
Generated
+41 -18
View File
@@ -1099,9 +1099,9 @@ dependencies = [
[[package]]
name = "aws-runtime"
version = "1.4.2"
version = "1.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2424565416eef55906f9f8cece2072b6b6a76075e3ff81483ebe938a89a4c05f"
checksum = "a10d5c055aa540164d9561a0e2e74ad30f0dcf7393c3a92f6733ddf9c5762468"
dependencies = [
"aws-credential-types",
"aws-sigv4",
@@ -1123,6 +1123,28 @@ dependencies = [
"uuid",
]
[[package]]
name = "aws-sdk-kinesis"
version = "1.51.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ad48026d3d53881146469b36358d633f1b8c9ad6eb3033f348600f981f2f449b"
dependencies = [
"aws-credential-types",
"aws-runtime",
"aws-smithy-async",
"aws-smithy-http",
"aws-smithy-json",
"aws-smithy-runtime",
"aws-smithy-runtime-api",
"aws-smithy-types",
"aws-types",
"bytes 1.7.2",
"http 0.2.12",
"once_cell",
"regex-lite",
"tracing",
]
[[package]]
name = "aws-sdk-s3"
version = "1.47.0"
@@ -1227,9 +1249,9 @@ dependencies = [
[[package]]
name = "aws-sigv4"
version = "1.2.3"
version = "1.2.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5df1b0fa6be58efe9d4ccc257df0a53b89cd8909e86591a13ca54817c87517be"
checksum = "5619742a0d8f253be760bfbb8e8e8368c69e3587e4637af5754e488a611499b1"
dependencies = [
"aws-credential-types",
"aws-smithy-eventstream",
@@ -1288,9 +1310,9 @@ dependencies = [
[[package]]
name = "aws-smithy-eventstream"
version = "0.60.4"
version = "0.60.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e6363078f927f612b970edf9d1903ef5cef9a64d1e8423525ebb1f0a1633c858"
checksum = "cef7d0a272725f87e51ba2bf89f8c21e4df61b9e49ae1ac367a6d69916ef7c90"
dependencies = [
"aws-smithy-types",
"bytes 1.7.2",
@@ -1299,9 +1321,9 @@ dependencies = [
[[package]]
name = "aws-smithy-http"
version = "0.60.10"
version = "0.60.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "01dbcb6e2588fd64cfb6d7529661b06466419e4c54ed1c62d6510d2d0350a728"
checksum = "5c8bc3e8fdc6b8d07d976e301c02fe553f72a39b7a9fea820e023268467d7ab6"
dependencies = [
"aws-smithy-eventstream",
"aws-smithy-runtime-api",
@@ -1339,9 +1361,9 @@ dependencies = [
[[package]]
name = "aws-smithy-runtime"
version = "1.7.1"
version = "1.7.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d1ce695746394772e7000b39fe073095db6d45a862d0767dd5ad0ac0d7f8eb87"
checksum = "be28bd063fa91fd871d131fc8b68d7cd4c5fa0869bea68daca50dcb1cbd76be2"
dependencies = [
"aws-smithy-async",
"aws-smithy-http",
@@ -1366,9 +1388,9 @@ dependencies = [
[[package]]
name = "aws-smithy-runtime-api"
version = "1.7.2"
version = "1.7.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e086682a53d3aa241192aa110fa8dfce98f2f5ac2ead0de84d41582c7e8fdb96"
checksum = "92165296a47a812b267b4f41032ff8069ab7ff783696d217f0994a0d7ab585cd"
dependencies = [
"aws-smithy-async",
"aws-smithy-types",
@@ -1383,9 +1405,9 @@ dependencies = [
[[package]]
name = "aws-smithy-types"
version = "1.2.4"
version = "1.2.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "273dcdfd762fae3e1650b8024624e7cd50e484e37abdab73a7a706188ad34543"
checksum = "4fbd94a32b3a7d55d3806fe27d98d3ad393050439dd05eb53ece36ec5e3d3510"
dependencies = [
"base64-simd",
"bytes 1.7.2",
@@ -1591,7 +1613,7 @@ dependencies = [
"bitflags 2.6.0",
"cexpr",
"clang-sys",
"itertools 0.10.5",
"itertools 0.12.1",
"lazy_static",
"lazycell",
"proc-macro2",
@@ -2560,6 +2582,7 @@ dependencies = [
"async-tungstenite",
"audio",
"aws-config",
"aws-sdk-kinesis",
"aws-sdk-s3",
"axum",
"axum-extra",
@@ -5644,7 +5667,7 @@ dependencies = [
"httpdate",
"itoa",
"pin-project-lite",
"socket2 0.4.10",
"socket2 0.5.7",
"tokio",
"tower-service",
"tracing",
@@ -6553,7 +6576,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4979f22fdb869068da03c9f7528f8297c6fd2606bc3a4affe42e6a823fdb8da4"
dependencies = [
"cfg-if",
"windows-targets 0.48.5",
"windows-targets 0.52.6",
]
[[package]]
@@ -14228,7 +14251,7 @@ version = "0.1.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb"
dependencies = [
"windows-sys 0.48.0",
"windows-sys 0.59.0",
]
[[package]]