Skip to content

Seed DynamoDB Local #197

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
jonhoo merged 10 commits into from
Jan 5, 2025
Merged

Seed DynamoDB Local #197

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
78797fd
Add seeding to dynamodb local arm in main
rustworthy Dec 28, 2024
cfb3e4b
Re-use votes autogeneration logic
rustworthy Dec 28, 2024
377a8a8
Update test event url in docs
rustworthy Dec 28, 2024
2d10509
Use let-else in main
rustworthy Dec 29, 2024
9d15443
Put seeding logic to dedicated fn
rustworthy Dec 29, 2024
529d83d
Update server/src/main.rs
rustworthy Dec 30, 2024
d8b0284
Restore local seed arm: ask + update
rustworthy Jan 2, 2025
1501e77
Add not on tracing subscriber
rustworthy Jan 2, 2025
ea6582a
Use Backend::ask in dynamodb arm in main
rustworthy Jan 2, 2025
7651394
Re-use Backend::event, Backend::ask, Backend::list in seed
rustworthy Jan 5, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion CONTRIBUTING.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ auto-update. If you modify files under `server/`, you'll have to re-run
Note that when run this way, to aid in development, the server will
auto-populate an event with a set of questions from a past live Q&A
session I ran at
<http://localhost:5173/#/event/00000000-0000-0000-0000-000000000000/secret>.
<http://localhost:5173/event/00000000000000000000000000/secret>.
Comment thread
jonhoo marked this conversation as resolved.
It will also auto-generate user votes over time for the questions there.

If you're curious about the technologies used in the server and client,
Expand Down
14 changes: 3 additions & 11 deletions server/src/ask.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
use super::{Backend, Local};
use crate::to_dynamo_timestamp;
use crate::{to_dynamo_timestamp, QUESTIONS_TTL};
use aws_sdk_dynamodb::{
error::SdkError,
operation::put_item::{PutItemError, PutItemOutput},
Expand All @@ -9,17 +9,12 @@ use axum::extract::{Path, State};
use axum::response::Json;
use http::StatusCode;
use serde::Deserialize;
use std::{
collections::HashMap,
time::{Duration, SystemTime},
};
use std::{collections::HashMap, time::SystemTime};
use ulid::Ulid;

#[allow(unused_imports)]
use tracing::{debug, error, info, trace, warn};

const QUESTIONS_EXPIRE_AFTER_DAYS: u64 = 30;

impl Backend {
pub(super) async fn ask(
&self,
Expand All @@ -35,10 +30,7 @@ impl Backend {
("when", to_dynamo_timestamp(SystemTime::now())),
(
"expire",
to_dynamo_timestamp(
SystemTime::now()
+ Duration::from_secs(QUESTIONS_EXPIRE_AFTER_DAYS * 24 * 60 * 60),
),
to_dynamo_timestamp(SystemTime::now() + QUESTIONS_TTL),
),
("hidden", AttributeValue::Bool(false)),
];
Expand Down
191 changes: 168 additions & 23 deletions server/src/main.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ use axum::Router;
use http::StatusCode;
use http_body_util::BodyExt;
use lambda_http::Error;
use std::time::SystemTime;
use std::time::{Duration, SystemTime};
use std::{
collections::HashMap,
future::Future,
Expand All @@ -19,12 +19,32 @@ use tower_service::Service;
use tracing_subscriber::EnvFilter;
use ulid::Ulid;

const QUESTIONS_EXPIRE_AFTER_DAYS: u64 = 30;
const QUESTIONS_TTL: Duration = Duration::from_secs(QUESTIONS_EXPIRE_AFTER_DAYS * 24 * 60 * 60);

const EVENTS_EXPIRE_AFTER_DAYS: u64 = 60;
const EVENTS_TTL: Duration = Duration::from_secs(EVENTS_EXPIRE_AFTER_DAYS * 24 * 60 * 60);

#[allow(unused_imports)]
use tracing::{debug, error, info, trace, warn};

#[cfg(debug_assertions)]
const SEED: &str = include_str!("test.json");

#[cfg(debug_assertions)]
use serde::Deserialize;

#[cfg(debug_assertions)]
#[derive(Deserialize)]
struct LiveAskQuestion {
likes: usize,
text: String,
hidden: bool,
answered: bool,
#[serde(rename = "createTimeUnix")]
created: usize,
}

#[derive(Clone, Debug)]
#[allow(dead_code)]
enum Backend {
Expand Down Expand Up @@ -170,24 +190,141 @@ async fn main() -> Result<(), Error> {
#[cfg(not(debug_assertions))]
let backend = Backend::dynamo().await;
#[cfg(debug_assertions)]
let backend = if std::env::var_os("USE_DYNAMODB").is_some() {
Backend::dynamo().await
} else {
use rand::prelude::SliceRandom;
use serde::Deserialize;
use std::time::Duration;

#[cfg(debug_assertions)]
#[derive(Deserialize)]
struct LiveAskQuestion {
likes: usize,
text: String,
hidden: bool,
answered: bool,
#[serde(rename = "createTimeUnix")]
created: usize,
}
let (backend, qids) = if std::env::var_os("USE_DYNAMODB").is_some() {
use aws_sdk_dynamodb::types::{PutRequest, WriteRequest};
Comment thread
jonhoo marked this conversation as resolved.
Outdated

let mut backend = Backend::dynamo().await;
let qids: Vec<Ulid> = if let Backend::Dynamo(ref mut client) = backend {
info!("going to seed test event");
let seed: Vec<LiveAskQuestion> = serde_json::from_str(SEED).unwrap();
let seed_e = Ulid::from_string("00000000000000000000000000").unwrap();
match client
.put_item()
.table_name("events")
.condition_expression("attribute_not_exists(id)")
.item("id", AttributeValue::S(seed_e.to_string()))
.item("secret", AttributeValue::S("secret".into()))
.item("when", to_dynamo_timestamp(SystemTime::now()))
.item(
"expire",
to_dynamo_timestamp(SystemTime::now() + EVENTS_TTL),
)
.send()
.await
{
Err(ref error @ SdkError::ServiceError(ref e)) => {
if e.err().is_conditional_check_failed_exception() {
warn!("test event is already there, skipping seeding questions");
} else {
panic!("failed to seed test event {:?}", error)
}
}
Err(e) => panic!("failed to seed test event {:?}", e),
Ok(_) => {
info!("successfully registered test event, going to seed questions now");
// DynamoDB supports batch write operations with `25` as max batch size
// https://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_BatchWriteItem.html
for chunk in seed.chunks(25) {
client
.batch_write_item()
.request_items(
"questions",
chunk
.iter()
.map(
|LiveAskQuestion {
likes,
text,
hidden,
answered,
created,
}| {
let mut item = HashMap::from([
(
"id".to_string(),
AttributeValue::S(
ulid::Ulid::new().to_string(),
),
),
(
"eid".to_string(),
AttributeValue::S(seed_e.to_string()),
),
(
"votes".to_string(),
AttributeValue::N(likes.to_string()),
),
(
"text".to_string(),
AttributeValue::S(text.clone()),
),
(
"when".to_string(),
AttributeValue::N(created.to_string()),
),
(
"expire".to_string(),
to_dynamo_timestamp(
SystemTime::now() + QUESTIONS_TTL,
),
),
(
"hidden".to_string(),
AttributeValue::Bool(*hidden),
),
]);
if *answered {
item.insert(
"answered".to_string(),
to_dynamo_timestamp(SystemTime::now()),
);
}
WriteRequest::builder()
.put_request(
PutRequest::builder()
.set_item(Some(item))
.build()
.expect("request to have been built ok"),
)
.build()
},
)
.collect::<Vec<_>>(),
)
.send()
.await
.expect("batch to have been written ok");
}
info!("successfully registered questions");
}
}
// let's collect ids of the questions related to the test event,
// we can then use them to auto-generate user votes over time
client
.query()
.table_name("questions")
.index_name("top")
.key_condition_expression("eid = :eid")
.expression_attribute_values(":eid", AttributeValue::S(seed_e.to_string()))
.send()
.await
.expect("scanned index ok")
.items()
.iter()
.filter_map(|item| {
let id = item
.get("id")
.expect("id is in projection")
.as_s()
.expect("id is of type string");
ulid::Ulid::from_string(id).ok()
})
.collect()
} else {
unreachable!()
Comment thread
jonhoo marked this conversation as resolved.
Outdated
};
(backend, qids)
} else {
let mut state = Local::default();
let seed: Vec<LiveAskQuestion> = serde_json::from_str(SEED).unwrap();
let seed_e = "00000000000000000000000000";
Expand Down Expand Up @@ -229,16 +366,24 @@ async fn main() -> Result<(), Error> {
qids.push(qid);
}
}
let cheat = state.clone();
(state, qids)
};

// to aid in development, auto-generate user votes over time
#[cfg(debug_assertions)]
{
use rand::prelude::SliceRandom;
let backend = backend.clone();
tokio::spawn(async move {
let mut interval = tokio::time::interval(Duration::from_secs(1));
interval.tick().await;
loop {
tokio::time::sleep(Duration::from_secs(1)).await;
interval.tick().await;
let qid = qids.choose(&mut rand::thread_rng()).unwrap();
let _ = cheat.vote(qid, vote::UpDown::Up).await;
let _ = backend.vote(qid, vote::UpDown::Up).await;
}
});
state
};
}

let app = Router::new()
.route("/api/event", post(new::new))
Expand Down
11 changes: 3 additions & 8 deletions server/src/new.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
use crate::to_dynamo_timestamp;
use crate::{to_dynamo_timestamp, EVENTS_TTL};

use super::{Backend, Local};
use aws_sdk_dynamodb::{
Expand All @@ -11,14 +11,12 @@ use axum::response::Json;
use http::StatusCode;
use rand::distributions::Alphanumeric;
use rand::{thread_rng, Rng};
use std::time::{Duration, SystemTime};
use std::time::SystemTime;
use ulid::Ulid;

#[allow(unused_imports)]
use tracing::{debug, error, info, trace, warn};

const EVENTS_EXPIRE_AFTER_DAYS: u64 = 60;

impl Backend {
#[allow(clippy::wrong_self_convention)]
#[allow(clippy::new_ret_no_self)]
Expand All @@ -37,10 +35,7 @@ impl Backend {
.item("when", to_dynamo_timestamp(SystemTime::now()))
.item(
"expire",
to_dynamo_timestamp(
SystemTime::now()
+ Duration::from_secs(EVENTS_EXPIRE_AFTER_DAYS * 24 * 60 * 60),
),
to_dynamo_timestamp(SystemTime::now() + EVENTS_TTL),
)
.send()
.await
Expand Down