feat: initial make-space full-stack release with weighted multi-provider LLM and Dockerfile

This commit is contained in:
dahyeon committed 2026-11-04 20:49:24 +09:00
commit 37b323c9ac
55 files changed
+17087

No files matched your search

+250
View File
@@ -0,0 +1,250 @@
//! Step 4 & 5 of the pipeline:
//! - VLM batch photo classification (`PhotoTag` per photo)
//! - Multi-evidence floor voting (photo `floor_level` + text `floor_mentions`)
use std::collections::BTreeMap;
use domain::{
floor_label, FloorEvidence, FurnitureCount, FurnitureKind, Photo, PhotoKind, PhotoTag,
PhotoZone,
};
use llm::{LlmClient, Part};
use providers::text::floor_mentions;
use serde::Deserialize;
use serde_json::{json, Value};
use storage::TextDoc;
const SYSTEM_PROMPT: &str = "\
당신은 한국 식당·카페의 사진을 분류하고 공간 단서를 추출하는 비전 분석가입니다.
주어진 각 사진(index 0..N-1)과 캡션 단서를 보고 다음을 판별하세요:
- kind: interior(실내 공간/좌석/카운터/계단), exterior(건물 외관/간판), food(음식/음료 클로즈업), menu(메뉴판/가격표), person(인물 위주), other
- zone: entrance, counter, hall, window_side, terrace, restroom, stairs, kitchen, unknown
- floor_level: 사진이나 캡션에서 층수가 확인되면 정수(1=1층, 2=2층, -1=지하1층, 99=루프탑/옥상), 알 수 없으면 null
- floor_evidence: 층수를 판단한 근거(계단, 표지판, 캡션 문구 등, 없으면 빈 문자열)
- furniture: 사진에 보이는 가구 종류(table2, table4, table6, communal_table, bar_seat, sofa, counter_bar, plant)와 대략적 개수
- quality: 공간 배치 파악에 도움이 되는 정도 (0.0~1.0)
- description: 공간 구조/좌석/창문/계단 중심 한국어 1문장 요약";
pub async fn classify_batch(
llm: &LlmClient,
batch: &[(&Photo, Vec<u8>)],
) -> anyhow::Result<Vec<(String, PhotoTag)>> {
if batch.is_empty() {
return Ok(Vec::new());
}
let mut parts = Vec::with_capacity(batch.len() * 2 + 1);
parts.push(Part::text(format!(
"총 {}장의 사진입니다. 각 사진의 index(0..{})에 맞춰 분류 결과를 반환하세요.",
batch.len(),
batch.len() - 1
)));
for (idx, (photo, bytes)) in batch.iter().enumerate() {
parts.push(Part::text(format!(
"Photo index={idx}, id={}, source={:?}, caption={:?}",
photo.id, photo.source, photo.caption
)));
parts.push(Part::Image {
bytes: bytes.clone(),
mime: "image/jpeg".into(),
});
}
let resp: BatchResp = llm
.chat_json(SYSTEM_PROMPT, &parts, "photo_classification", &schema())
.await?;
let mut out = Vec::new();
for item in resp.items {
if let Some((photo, _)) = batch.get(item.index) {
out.push((
photo.id.clone(),
PhotoTag {
kind: item.kind,
zone: item.zone,
floor_level: item.floor_level.filter(|&l| l == 99 || (-5..=20).contains(&l) && l != 0),
floor_evidence: item.floor_evidence,
furniture: item.furniture,
quality: item.quality.clamp(0.0, 1.0),
description: item.description,
},
));
}
}
Ok(out)
}
#[derive(Deserialize)]
struct BatchResp {
items: Vec<BatchItem>,
}
#[derive(Deserialize)]
struct BatchItem {
index: usize,
kind: PhotoKind,
zone: PhotoZone,
floor_level: Option<i32>,
floor_evidence: String,
furniture: Vec<FurnitureCount>,
quality: f32,
description: String,
}
fn schema() -> Value {
json!({
"type": "object",
"additionalProperties": false,
"required": ["items"],
"properties": {
"items": {
"type": "array",
"items": {
"type": "object",
"additionalProperties": false,
"required": ["index", "kind", "zone", "floor_level", "floor_evidence", "furniture", "quality", "description"],
"properties": {
"index": { "type": "integer" },
"kind": { "type": "string", "enum": ["interior", "exterior", "food", "menu", "person", "other"] },
"zone": { "type": "string", "enum": ["entrance", "counter", "hall", "window_side", "terrace", "restroom", "stairs", "kitchen", "unknown"] },
"floor_level": { "type": ["integer", "null"] },
"floor_evidence": { "type": "string" },
"furniture": {
"type": "array",
"items": {
"type": "object",
"additionalProperties": false,
"required": ["kind", "count"],
"properties": {
"kind": { "type": "string", "enum": ["table2", "table4", "table6", "communal_table", "bar_seat", "sofa", "counter_bar", "plant"] },
"count": { "type": "integer" }
}
}
},
"quality": { "type": "number" },
"description": { "type": "string" }
}
}
}
}
})
}
/// Combine photo `floor_level` hints, photo captions, and text snippets into a ranked list of floors.
/// Always includes at least `1F` so single-floor venues have a ground floor.
pub fn detect_floors(photos: &[Photo], texts: &[TextDoc]) -> Vec<FloorEvidence> {
let mut map: BTreeMap<i32, FloorEvidence> = BTreeMap::new();
let ensure = |map: &mut BTreeMap<i32, FloorEvidence>, level: i32| {
map.entry(level).or_insert_with(|| FloorEvidence {
level,
label: floor_label(level),
score: 0.0,
photo_ids: Vec::new(),
text_snippets: Vec::new(),
});
};
for p in photos {
if let Some(tag) = &p.tag {
if let Some(lvl) = tag.floor_level {
ensure(&mut map, lvl);
let entry = map.get_mut(&lvl).unwrap();
entry.score += if tag.kind == PhotoKind::Interior { 1.5 } else { 0.8 };
if !entry.photo_ids.contains(&p.id) {
entry.photo_ids.push(p.id.clone());
}
if !tag.floor_evidence.is_empty() && entry.text_snippets.len() < 4 {
entry.text_snippets.push(tag.floor_evidence.clone());
}
}
}
for (lvl, snip) in floor_mentions(&p.caption) {
ensure(&mut map, lvl);
let entry = map.get_mut(&lvl).unwrap();
entry.score += 0.7;
if !entry.photo_ids.contains(&p.id) {
entry.photo_ids.push(p.id.clone());
}
if entry.text_snippets.len() < 4 && !entry.text_snippets.contains(&snip) {
entry.text_snippets.push(snip);
}
}
}
for t in texts {
for (lvl, snip) in floor_mentions(&t.text) {
ensure(&mut map, lvl);
let entry = map.get_mut(&lvl).unwrap();
entry.score += 0.5;
if entry.text_snippets.len() < 4 && !entry.text_snippets.contains(&snip) {
entry.text_snippets.push(snip);
}
}
}
// Filter out weak stray mentions (e.g., a single "3층" referring to another tenant in the building)
// unless it's 1F.
let mut floors: Vec<FloorEvidence> = map
.into_values()
.filter(|f| f.level == 1 || f.score >= 1.0)
.collect();
if floors.is_empty() {
floors.push(FloorEvidence {
level: 1,
label: "1F".into(),
score: 1.0,
photo_ids: photos
.iter()
.filter(|p| p.tag.as_ref().map(|t| t.kind == PhotoKind::Interior).unwrap_or(false))
.take(4)
.map(|p| p.id.clone())
.collect(),
text_snippets: Vec::new(),
});
}
floors.sort_by_key(|f| f.level);
let _ = FurnitureKind::Table2;
floors
}
#[cfg(test)]
mod tests {
use super::*;
use domain::PhotoSourceId;
#[test]
fn floor_voting_combines_photo_and_text() {
let p = Photo {
id: "p1".into(),
place_id: "kakao:1".into(),
source: PhotoSourceId::NaverBlog,
image_url: String::new(),
page_url: String::new(),
caption: "2층 창가석".into(),
object_key: String::new(),
dhash: String::new(),
width: 800,
height: 600,
tag: Some(PhotoTag {
kind: PhotoKind::Interior,
zone: PhotoZone::WindowSide,
floor_level: Some(2),
floor_evidence: "계단 위 2층 홀".into(),
furniture: vec![],
quality: 0.9,
description: "2층 창가".into(),
}),
};
let t = TextDoc {
place_id: "kakao:1".into(),
source: PhotoSourceId::NaverBlog,
url: String::new(),
text: "1층에서 주문하고 2층으로 올라가면 루프탑도 있어요. 루프탑 뷰가 최고".into(),
};
let floors = detect_floors(&[p], &[t]);
let levels: Vec<i32> = floors.iter().map(|f| f.level).collect();
assert!(levels.contains(&2));
assert!(levels.contains(&domain::ROOFTOP_LEVEL));
}
}
+79
View File
@@ -0,0 +1,79 @@
//! Image decoding, resizing, and 64-bit perceptual difference hash (dHash).
use image::imageops::FilterType;
use image::{DynamicImage, GenericImageView};
use std::io::Cursor;
pub struct ProcessedImage {
pub jpeg_bytes: Vec<u8>,
pub width: u32,
pub height: u32,
pub dhash: u64,
}
/// Decode raw image bytes, reject tiny icons (< 160px), downscale to at most `max_dim`
/// on the longest side, re-encode as JPEG, and compute a 9x8 dHash.
pub fn process_image(bytes: &[u8], max_dim: u32) -> anyhow::Result<ProcessedImage> {
let img = image::load_from_memory(bytes)?;
let (w, h) = img.dimensions();
if w < 160 || h < 160 {
anyhow::bail!("image too small ({w}x{h})");
}
let dhash = compute_dhash(&img);
let resized = if w > max_dim || h > max_dim {
img.resize(max_dim, max_dim, FilterType::Triangle)
} else {
img
};
let (rw, rh) = resized.dimensions();
let mut out = Vec::new();
resized.to_rgb8().write_to(&mut Cursor::new(&mut out), image::ImageFormat::Jpeg)?;
Ok(ProcessedImage {
jpeg_bytes: out,
width: rw,
height: rh,
dhash,
})
}
pub fn compute_dhash(img: &DynamicImage) -> u64 {
let gray = img.resize_exact(9, 8, FilterType::Triangle).to_luma8();
let mut hash = 0u64;
let mut bit = 0;
for y in 0..8 {
for x in 0..8 {
if gray.get_pixel(x, y)[0] > gray.get_pixel(x + 1, y)[0] {
hash |= 1u64 << bit;
}
bit += 1;
}
}
hash
}
pub fn hamming(a: u64, b: u64) -> u32 {
(a ^ b).count_ones()
}
pub fn is_near_duplicate(existing: &[u64], candidate: u64) -> bool {
existing.iter().any(|&h| hamming(h, candidate) <= 6)
}
#[cfg(test)]
mod tests {
use super::*;
use image::{Rgb, RgbImage};
#[test]
fn identical_images_match() {
let mut buf = RgbImage::new(200, 200);
for (x, y, p) in buf.enumerate_pixels_mut() {
*p = Rgb([(x % 255) as u8, (y % 255) as u8, 120]);
}
let dyn_img = DynamicImage::ImageRgb8(buf);
let h1 = compute_dhash(&dyn_img);
let h2 = compute_dhash(&dyn_img);
assert_eq!(hamming(h1, h2), 0);
assert!(is_near_duplicate(&[h1], h2));
}
}
+430
View File
@@ -0,0 +1,430 @@
//! Step 6 & 7 of the pipeline:
//! - VLM spatial layout inference -> `domain::Layout`
//! - Validation & clamping (bounds, non-zero dimensions, valid `evidence_photos` IDs)
//! - Single retry with validation feedback if needed
use std::collections::HashSet;
use domain::{
floor_label, Accessibility, Arrangement, Floor, FloorEvidence, FurnitureGroup, FurnitureKind,
Layout, Opening, OpeningKind, Photo, PhotoKind, Place, Summary, Tri, Vertical, VerticalKind,
Wall, Zone, ZoneKind,
};
use llm::{LlmClient, Part};
use serde::Deserialize;
use serde_json::{json, Value};
use storage::TextDoc;
const SYSTEM_PROMPT: &str = "\
당신은 식당·카페의 실내 사진들과 텍스트 리뷰를 종합해 three.js 미니어처 3D 뷰어용 공간 배치(Layout JSON)를 추론하는 공간 건축 AI입니다.
좌표 규칙 (층별 미터 단위):
- 각 층은 직사각형 바닥(width_m: 동서 폭, depth_m: 남북 깊이, 보통 6~22m 범위)입니다.
- 원점 (0, 0)은 남서쪽(South-West) 모서리이며, x는 동쪽(0..width_m), y는 북쪽(0..depth_m)으로 증가합니다.
- 주 출입구(entrance)는 기본적으로 south 벽에 두되, 사진 단서에 맞게 조정하세요.
- 각 zone(구역)의 직사각형 (x, y, w, d)는 바닥 범위(0 <= x, x+w <= width_m, 0 <= y, y+d <= depth_m) 안에 있어야 하며 서로 과도하게 겹치지 않게 배치하세요.
- evidence_photos 배열에는 반드시 입력으로 주어진 사진 id만 넣으세요 (절대 없는 id를 만들지 마세요).
- 다층 건물인 경우 층간 이동 수단(stairs 또는 elevator)을 verticals에 포함하세요.
- summary에는 예상 평수 범위, 예상 좌석 수, 좌석 종류, 방문 전 핵심 체크 포인트(highlights), 주의사항(caveats)을 한국어로 작성하세요.";
pub async fn generate_layout(
llm: &LlmClient,
place: &Place,
photos_with_bytes: &[(&Photo, Vec<u8>)],
floors: &[FloorEvidence],
texts: &[TextDoc],
mut on_attempt: impl FnMut(u32),
) -> anyhow::Result<Layout> {
let valid_ids: HashSet<String> = photos_with_bytes.iter().map(|(p, _)| p.id.clone()).collect();
let mut feedback: Option<String> = None;
for attempt in 1..=2 {
on_attempt(attempt);
let parts = build_prompt_parts(place, photos_with_bytes, floors, texts, feedback.as_deref());
match llm
.chat_json_with_meta::<DraftLayout>(SYSTEM_PROMPT, &parts, "space_layout", &schema())
.await
{
Ok(out) => {
let model_tag = out.model_tag();
let mut layout = into_layout(place, &model_tag, out.data, &valid_ids);
match validate_and_clamp(&mut layout) {
Ok(()) => return Ok(layout),
Err(err) if attempt == 1 => {
tracing::warn!(attempt, %err, "layout validation failed, retrying once");
feedback = Some(err);
}
Err(err) => {
tracing::warn!(attempt, %err, "layout validation failed on second attempt, using clamped layout");
return Ok(layout);
}
}
}
Err(err) if attempt == 1 => {
tracing::warn!(attempt, %err, "layout LLM call failed, retrying once");
feedback = Some(format!("이전 응답 파싱 실패: {err}"));
}
Err(err) => {
tracing::warn!(%err, "layout LLM call failed twice, using fallback layout");
return Ok(fallback_layout(place, &llm.model, photos_with_bytes, floors));
}
}
}
Ok(fallback_layout(place, &llm.model, photos_with_bytes, floors))
}
fn build_prompt_parts(
place: &Place,
photos_with_bytes: &[(&Photo, Vec<u8>)],
floors: &[FloorEvidence],
texts: &[TextDoc],
feedback: Option<&str>,
) -> Vec<Part> {
let mut header = format!(
"장소명: {}\n카테고리: {}\n주소: {} ({})\n\n[감지된 층 단서]\n",
place.name, place.category, place.road_address, place.address
);
for f in floors {
header.push_str(&format!(
"- {} (level={}, 점수={:.1}, 관련 사진={:?}, 텍스트 단서={:?})\n",
f.label, f.level, f.score, f.photo_ids, f.text_snippets
));
}
header.push_str("\n[수집된 텍스트/리뷰 발췌]\n");
for t in texts.iter().take(6) {
let excerpt: String = t.text.chars().take(450).collect();
header.push_str(&format!("({:?}) {}\n---\n", t.source, excerpt));
}
if let Some(fb) = feedback {
header.push_str(&format!(
"\n[이전 생성 결과 오류 - 반드시 수정하세요]\n{fb}\n"
));
}
header.push_str("\n[실내/외관 핵심 사진 목록 — evidence_photos에는 아래 photo_id만 사용하세요]\n");
let mut parts = vec![Part::text(header)];
for (p, bytes) in photos_with_bytes.iter().take(12) {
let tag_desc = p
.tag
.as_ref()
.map(|t| {
format!(
"kind={:?}, zone={:?}, floor={:?}, desc={}, furniture={:?}",
t.kind, t.zone, t.floor_level, t.description, t.furniture
)
})
.unwrap_or_default();
parts.push(Part::text(format!(
"photo_id=\"{}\" | caption={:?} | {tag_desc}",
p.id, p.caption
)));
parts.push(Part::Image {
bytes: bytes.clone(),
mime: "image/jpeg".into(),
});
}
parts
}
fn into_layout(
place: &Place,
model: &str,
draft: DraftLayout,
valid_ids: &HashSet<String>,
) -> Layout {
let floors = draft
.floors
.into_iter()
.map(|mut f| {
if f.label.trim().is_empty() {
f.label = floor_label(f.level);
}
for z in &mut f.zones {
z.evidence_photos.retain(|id| valid_ids.contains(id));
}
f
})
.collect();
Layout {
place_id: place.id.clone(),
generated_at: chrono::Utc::now().to_rfc3339(),
model: model.to_string(),
confidence: draft.confidence.clamp(0.1, 0.98),
floors,
summary: draft.summary,
}
}
/// Clamp coordinates into valid ranges and return `Err` if the layout had structural issues worth retrying.
pub fn validate_and_clamp(layout: &mut Layout) -> Result<(), String> {
if layout.floors.is_empty() {
return Err("floors 배열이 비어 있습니다. 최소 1개 층(1F)을 포함해야 합니다.".into());
}
let mut issues = Vec::new();
for f in &mut layout.floors {
if f.width_m < 4.0 || f.width_m > 40.0 {
issues.push(format!("층 {} width_m({})가 비정상 범위", f.label, f.width_m));
f.width_m = f.width_m.clamp(5.0, 28.0);
}
if f.depth_m < 4.0 || f.depth_m > 40.0 {
issues.push(format!("층 {} depth_m({})가 비정상 범위", f.label, f.depth_m));
f.depth_m = f.depth_m.clamp(5.0, 28.0);
}
f.confidence = f.confidence.clamp(0.1, 0.99);
if f.zones.is_empty() {
issues.push(format!("층 {}에 zones가 비어 있습니다.", f.label));
}
for z in &mut f.zones {
z.w = z.w.clamp(1.2, f.width_m);
z.d = z.d.clamp(1.2, f.depth_m);
z.x = z.x.clamp(0.0, (f.width_m - z.w).max(0.0));
z.y = z.y.clamp(0.0, (f.depth_m - z.d).max(0.0));
for fg in &mut z.furniture {
fg.count = fg.count.clamp(1, 30);
}
}
for o in &mut f.openings {
let wall_len = match o.wall {
Wall::North | Wall::South => f.width_m,
Wall::East | Wall::West => f.depth_m,
};
o.length_m = o.length_m.clamp(0.8, (wall_len - 0.4).max(0.8));
o.offset_m = o.offset_m.clamp(0.2, (wall_len - o.length_m - 0.2).max(0.2));
}
for v in &mut f.verticals {
v.w = v.w.clamp(1.0, 3.5);
v.d = v.d.clamp(1.2, 4.0);
v.x = v.x.clamp(0.0, (f.width_m - v.w).max(0.0));
v.y = v.y.clamp(0.0, (f.depth_m - v.d).max(0.0));
}
}
layout.floors.sort_by_key(|f| f.level);
if issues.is_empty() {
Ok(())
} else {
Err(issues.join("; "))
}
}
pub fn fallback_layout(
place: &Place,
model: &str,
photos_with_bytes: &[(&Photo, Vec<u8>)],
floors: &[FloorEvidence],
) -> Layout {
let evidence: Vec<String> = photos_with_bytes.iter().take(4).map(|(p, _)| p.id.clone()).collect();
let levels: Vec<i32> = if floors.is_empty() {
vec![1]
} else {
floors.iter().map(|f| f.level).collect()
};
let built_floors = levels
.iter()
.map(|&lvl| Floor {
level: lvl,
label: floor_label(lvl),
confidence: 0.45,
width_m: 10.0,
depth_m: 12.0,
openings: if lvl == 1 {
vec![
Opening { kind: OpeningKind::Entrance, wall: Wall::South, offset_m: 1.5, length_m: 1.6 },
Opening { kind: OpeningKind::Window, wall: Wall::South, offset_m: 4.0, length_m: 4.5 },
]
} else {
vec![Opening { kind: OpeningKind::Window, wall: Wall::South, offset_m: 2.0, length_m: 6.0 }]
},
zones: vec![
Zone {
kind: if lvl == 1 { ZoneKind::Counter } else { ZoneKind::Seating },
label: if lvl == 1 { "주문 카운터".into() } else { "창가 좌석".into() },
x: 0.6,
y: if lvl == 1 { 8.2 } else { 0.6 },
w: 4.2,
d: 3.0,
furniture: vec![FurnitureGroup {
kind: if lvl == 1 { FurnitureKind::CounterBar } else { FurnitureKind::BarSeat },
count: if lvl == 1 { 1 } else { 5 },
arrangement: Arrangement::AlongWall,
along_wall: Some(if lvl == 1 { Wall::North } else { Wall::South }),
}],
evidence_photos: evidence.clone(),
},
Zone {
kind: ZoneKind::Seating,
label: "메인 홀 좌석".into(),
x: 0.8,
y: if lvl == 1 { 1.0 } else { 4.2 },
w: 8.4,
d: 6.5,
furniture: vec![
FurnitureGroup {
kind: FurnitureKind::Table2,
count: 5,
arrangement: Arrangement::Grid,
along_wall: None,
},
FurnitureGroup {
kind: FurnitureKind::Table4,
count: 3,
arrangement: Arrangement::Grid,
along_wall: None,
},
],
evidence_photos: evidence.clone(),
},
],
verticals: if levels.len() > 1 {
vec![Vertical {
kind: VerticalKind::Stairs,
x: 7.8,
y: 9.2,
w: 1.6,
d: 2.2,
to_level: if lvl == 1 { levels.get(1).copied().unwrap_or(2) } else { 1 },
}]
} else {
vec![]
},
notes: "수집된 사진을 기반으로 생성한 기본 추정 배치입니다.".into(),
})
.collect();
let _ = PhotoKind::Interior;
Layout {
place_id: place.id.clone(),
generated_at: chrono::Utc::now().to_rfc3339(),
model: model.to_string(),
confidence: 0.45,
floors: built_floors,
summary: Summary {
area_pyeong_min: 18.0,
area_pyeong_max: 28.0,
seat_estimate: 26,
seat_types: vec!["2인 테이블".into(), "4인 테이블".into(), "바 좌석".into()],
highlights: vec![format!("{} 공간 배치 추정 완료", place.name)],
accessibility: Accessibility {
elevator: Tri::Unknown,
step_free_entrance: Tri::Likely,
},
caveats: vec!["실내 사진 수량이 제한적이어서 일부 구역은 표준 배치로 추정되었습니다.".into()],
},
}
}
#[derive(Deserialize)]
struct DraftLayout {
confidence: f32,
floors: Vec<Floor>,
summary: Summary,
}
fn schema() -> Value {
json!({
"type": "object",
"additionalProperties": false,
"required": ["confidence", "floors", "summary"],
"properties": {
"confidence": { "type": "number" },
"floors": {
"type": "array",
"items": {
"type": "object",
"additionalProperties": false,
"required": ["level", "label", "confidence", "width_m", "depth_m", "openings", "zones", "verticals", "notes"],
"properties": {
"level": { "type": "integer" },
"label": { "type": "string" },
"confidence": { "type": "number" },
"width_m": { "type": "number" },
"depth_m": { "type": "number" },
"openings": {
"type": "array",
"items": {
"type": "object",
"additionalProperties": false,
"required": ["kind", "wall", "offset_m", "length_m"],
"properties": {
"kind": { "type": "string", "enum": ["entrance", "window", "door"] },
"wall": { "type": "string", "enum": ["north", "south", "east", "west"] },
"offset_m": { "type": "number" },
"length_m": { "type": "number" }
}
}
},
"zones": {
"type": "array",
"items": {
"type": "object",
"additionalProperties": false,
"required": ["kind", "label", "x", "y", "w", "d", "furniture", "evidence_photos"],
"properties": {
"kind": { "type": "string", "enum": ["seating", "counter", "bar", "kitchen", "restroom", "terrace", "entrance", "other"] },
"label": { "type": "string" },
"x": { "type": "number" },
"y": { "type": "number" },
"w": { "type": "number" },
"d": { "type": "number" },
"furniture": {
"type": "array",
"items": {
"type": "object",
"additionalProperties": false,
"required": ["kind", "count", "arrangement", "along_wall"],
"properties": {
"kind": { "type": "string", "enum": ["table2", "table4", "table6", "communal_table", "bar_seat", "sofa", "counter_bar", "plant"] },
"count": { "type": "integer" },
"arrangement": { "type": "string", "enum": ["grid", "along_wall", "row", "cluster"] },
"along_wall": { "type": ["string", "null"], "enum": ["north", "south", "east", "west", null] }
}
}
},
"evidence_photos": { "type": "array", "items": { "type": "string" } }
}
}
},
"verticals": {
"type": "array",
"items": {
"type": "object",
"additionalProperties": false,
"required": ["kind", "x", "y", "w", "d", "to_level"],
"properties": {
"kind": { "type": "string", "enum": ["stairs", "elevator"] },
"x": { "type": "number" },
"y": { "type": "number" },
"w": { "type": "number" },
"d": { "type": "number" },
"to_level": { "type": "integer" }
}
}
},
"notes": { "type": "string" }
}
}
},
"summary": {
"type": "object",
"additionalProperties": false,
"required": ["area_pyeong_min", "area_pyeong_max", "seat_estimate", "seat_types", "highlights", "accessibility", "caveats"],
"properties": {
"area_pyeong_min": { "type": "number" },
"area_pyeong_max": { "type": "number" },
"seat_estimate": { "type": "integer" },
"seat_types": { "type": "array", "items": { "type": "string" } },
"highlights": { "type": "array", "items": { "type": "string" } },
"accessibility": {
"type": "object",
"additionalProperties": false,
"required": ["elevator", "step_free_entrance"],
"properties": {
"elevator": { "type": "string", "enum": ["yes", "likely", "no", "unknown"] },
"step_free_entrance": { "type": "string", "enum": ["yes", "likely", "no", "unknown"] }
}
},
"caveats": { "type": "array", "items": { "type": "string" } }
}
}
}
})
}
+482
View File
@@ -0,0 +1,482 @@
#![recursion_limit = "256"]
//! End-to-end place analysis pipeline:
//! 1. Normalize place (Kakao candidate + NCP Geocode address verification + Naver place ID resolution)
//! 2. Collect photos & texts in parallel across all `PhotoSource`s (per-source timeout)
//! 3. Download images, deduplicate with 64-bit dHash, store JPEG blobs in MinIO & metadata in MongoDB
//! 4. Classify photos in batches with the VLM (`PhotoTag`)
//! 5. Vote on floors (`FloorEvidence`) from photo tags, captions, and text snippets
//! 6. Generate 3D `Layout` JSON with the VLM
//! 7. Validate & clamp layout (retry once on structural issues) and save to MongoDB
pub mod classify;
pub mod dhash;
pub mod layout_gen;
use std::sync::Arc;
use std::time::Duration;
use bytes::Bytes;
use dashmap::DashMap;
use domain::{
JobEvent, Photo, PhotoKind, PhotoSummary, Place, PlaceCandidate, PlaceView, RawPhoto, RecentPlace,
};
use futures::stream::{self, StreamExt};
use llm::LlmClient;
use providers::browser::{browser_source, BrowserSource};
use providers::kakao_local::KakaoLocal;
use providers::kakao_place::KakaoPlacePhotos;
use providers::naver_blog::NaverBlog;
use providers::naver_map::NaverMapSearch;
use providers::naver_place::NaverPlacePhotos;
use providers::ncp_geocode::NcpGeocode;
use providers::{PhotoSource, PlaceSearch};
use storage::{Store, TextDoc};
use tokio::sync::broadcast;
#[derive(Clone)]
pub struct PipelineConfig {
pub kakao_api_key: String,
pub ncp_client_id: String,
pub ncp_client_secret: String,
pub browser_scraper_url: Option<String>,
pub max_photos_per_source: usize,
pub max_photos_total: usize,
}
#[derive(Clone)]
pub struct Pipeline {
pub store: Store,
pub llm: LlmClient,
kakao: Arc<KakaoLocal>,
naver_map: Arc<NaverMapSearch>,
ncp: Option<Arc<NcpGeocode>>,
sources: Arc<Vec<Box<dyn PhotoSource>>>,
http: reqwest::Client,
cfg: PipelineConfig,
/// Active job event broadcasters keyed by `job_id`.
jobs: Arc<DashMap<String, broadcast::Sender<JobEvent>>>,
/// Map `place_id` -> currently running `job_id` so concurrent visitors share one job.
running_by_place: Arc<DashMap<String, String>>,
}
impl Pipeline {
pub fn new(store: Store, llm: LlmClient, cfg: PipelineConfig) -> Self {
let kakao = Arc::new(KakaoLocal::new(&cfg.kakao_api_key));
let naver_map = Arc::new(NaverMapSearch::new());
let ncp = if !cfg.ncp_client_id.trim().is_empty() && !cfg.ncp_client_secret.trim().is_empty() {
Some(Arc::new(NcpGeocode::new(&cfg.ncp_client_id, &cfg.ncp_client_secret)))
} else {
None
};
let sources: Vec<Box<dyn PhotoSource>> = vec![
Box::new(NaverPlacePhotos::new()),
Box::new(KakaoPlacePhotos::new()),
Box::new(NaverBlog::new()),
browser_source(cfg.browser_scraper_url.as_deref(), BrowserSource::GoogleMaps),
browser_source(cfg.browser_scraper_url.as_deref(), BrowserSource::Instagram),
];
Self {
store,
llm,
kakao,
naver_map,
ncp,
sources: Arc::new(sources),
http: providers::http::scraping_client(),
cfg,
jobs: Arc::new(DashMap::new()),
running_by_place: Arc::new(DashMap::new()),
}
}
/// Search place candidates by free text (Kakao primary, Naver Map fallback).
pub async fn search_places(&self, query: &str) -> anyhow::Result<Vec<PlaceCandidate>> {
let q = query.trim();
if q.is_empty() {
return Ok(Vec::new());
}
let mut results = self.kakao.search(q).await.unwrap_or_default();
if results.is_empty() {
results = self.naver_map.search(q).await.unwrap_or_default();
}
Ok(results)
}
/// Saved analyses ordered by their most recent view (or analysis time for older records).
pub async fn recent_places(&self) -> anyhow::Result<Vec<RecentPlace>> {
self.store.recent_places().await
}
/// Return current `PlaceView` (place + latest cached layout + stored photos + running job id).
pub async fn get_place_view(&self, place_id: &str) -> anyhow::Result<Option<PlaceView>> {
let Some(place) = self.store.get_place(place_id).await? else {
return Ok(None);
};
let layout = self.store.latest_layout(place_id).await?.map(|d| d.layout);
let photos = self
.store
.photos_for_place(place_id)
.await?
.iter()
.map(PhotoSummary::from)
.collect();
let running_job = self.running_by_place.get(place_id).map(|v| v.value().clone());
if layout.is_some() {
if let Err(error) = self.store.record_place_view(place_id).await {
tracing::warn!(%place_id, %error, "could not record place view");
}
}
Ok(Some(PlaceView {
place,
layout,
photos,
running_job,
}))
}
/// Subscribe to live events for `job_id`, plus any events already recorded in MongoDB.
pub async fn subscribe_job(
&self,
job_id: &str,
) -> anyhow::Result<(Vec<JobEvent>, Option<broadcast::Receiver<JobEvent>>)> {
let rx = self.jobs.get(job_id).map(|s| s.value().subscribe());
let past = self
.store
.get_job(job_id)
.await?
.map(|j| j.events)
.unwrap_or_default();
Ok((past, rx))
}
/// Ensure the candidate's `Place` exists in MongoDB; if no cached layout exists (or `force_reanalyze` is true),
/// spawn the analysis job in the background and return `(place_id, Option<job_id>)`.
pub async fn select_or_analyze(
&self,
candidate: PlaceCandidate,
force_reanalyze: bool,
) -> anyhow::Result<(String, Option<String>)> {
let place_id = candidate.place_id();
let mut place = match self.store.get_place(&place_id).await? {
Some(existing) => existing,
None => {
let p = Place::from_candidate(&candidate);
self.store.upsert_place(&p).await?;
p
}
};
if !force_reanalyze {
if let Some( job_id ) = self.running_by_place.get(&place_id).map(|r| r.value().clone()) {
return Ok((place_id, Some(job_id)));
}
if self.store.latest_layout(&place_id).await?.is_some() {
return Ok((place_id, None));
}
}
let job_id = self.spawn_job(place_id.clone(), place.clone(), force_reanalyze).await?;
let _ = &mut place;
Ok((place_id, Some(job_id)))
}
/// Trigger reanalysis for an already-stored `place_id`.
pub async fn reanalyze(&self, place_id: &str) -> anyhow::Result<String> {
if let Some(existing) = self.running_by_place.get(place_id) {
return Ok(existing.value().clone());
}
let place = self
.store
.get_place(place_id)
.await?
.ok_or_else(|| anyhow::anyhow!("장소를 찾을 수 없습니다: {place_id}"))?;
self.spawn_job(place_id.to_string(), place, true).await
}
async fn spawn_job(
&self,
place_id: String,
place: Place,
clear_existing_photos: bool,
) -> anyhow::Result<String> {
let job_id = uuid::Uuid::new_v4().to_string();
self.store.create_job(&job_id, &place_id).await?;
let (tx, _) = broadcast::channel(64);
self.jobs.insert(job_id.clone(), tx.clone());
self.running_by_place.insert(place_id.clone(), job_id.clone());
let this = self.clone();
let jid = job_id.clone();
let pid = place_id.clone();
tokio::spawn(async move {
if let Err(err) = this.run_job(&jid, place, clear_existing_photos, &tx).await {
tracing::error!(job_id = %jid, place_id = %pid, "pipeline job failed: {err:#}");
this.emit(&jid, &tx, JobEvent::Error { message: err.to_string() }).await;
}
this.running_by_place.remove(&pid);
// Keep channel briefly so SSE clients receive the terminal event, then drop.
tokio::time::sleep(Duration::from_secs(5)).await;
this.jobs.remove(&jid);
});
Ok(job_id)
}
/// Run a place analysis synchronously (used by `precompute` CLI and `spawn_job`).
pub async fn run_sync(&self, place: Place) -> anyhow::Result<domain::Layout> {
let job_id = uuid::Uuid::new_v4().to_string();
self.store.upsert_place(&place).await?;
self.store.create_job(&job_id, &place.id).await?;
let (tx, _) = broadcast::channel(64);
self.run_job(&job_id, place.clone(), false, &tx).await?;
let doc = self
.store
.latest_layout(&place.id)
.await?
.ok_or_else(|| anyhow::anyhow!("layout not saved"))?;
Ok(doc.layout)
}
async fn emit(&self, job_id: &str, tx: &broadcast::Sender<JobEvent>, ev: JobEvent) {
let _ = self.store.push_job_event(job_id, &ev).await;
let _ = tx.send(ev);
}
async fn run_job(
&self,
job_id: &str,
mut place: Place,
clear_existing_photos: bool,
tx: &broadcast::Sender<JobEvent>,
) -> anyhow::Result<()> {
self.emit(job_id, tx, JobEvent::Started { place_id: place.id.clone() }).await;
// 1. Address verification (NCP Geocode) + Naver Place ID matching
if let Some(ncp) = &self.ncp {
let addr_query = if !place.road_address.is_empty() {
place.road_address.clone()
} else {
place.address.clone()
};
if !addr_query.is_empty() {
if let Ok(Some(geo)) = ncp.geocode(&addr_query).await {
if place.road_address.is_empty() && !geo.road_address.is_empty() {
place.road_address = geo.road_address;
}
if place.address.is_empty() && !geo.jibun_address.is_empty() {
place.address = geo.jibun_address;
}
}
}
}
if place.naver_place_id.is_none() {
if let Ok(Some(nid)) = self.naver_map.resolve_place_id(&place).await {
place.naver_place_id = Some(nid);
}
}
self.store.upsert_place(&place).await?;
self.emit(
job_id,
tx,
JobEvent::PlaceResolved {
name: place.name.clone(),
naver_place_id: place.naver_place_id.clone(),
},
)
.await;
// 2. Collect photos & text snippets from all sources in parallel
let per_source = self.cfg.max_photos_per_source;
let source_futs = self.sources.iter().map(|src| {
let place_ref = &place;
async move {
let id = src.id();
let res = tokio::time::timeout(Duration::from_secs(30), src.collect(place_ref, per_source)).await;
match res {
Ok(Ok(out)) => (id, out, None),
Ok(Err(e)) => (id, Default::default(), Some(e.to_string())),
Err(_) => (id, Default::default(), Some("시간 초과 (30s)".into())),
}
}
});
let collected = futures::future::join_all(source_futs).await;
let mut raw_photos: Vec<RawPhoto> = Vec::new();
let mut text_docs: Vec<TextDoc> = Vec::new();
for (source, out, error) in collected {
self.emit(
job_id,
tx,
JobEvent::SourceDone {
source,
photos: out.photos.len(),
texts: out.texts.len(),
error,
},
)
.await;
raw_photos.extend(out.photos);
for t in out.texts {
text_docs.push(TextDoc {
place_id: place.id.clone(),
source: t.source,
url: t.url,
text: t.text,
});
}
}
let _ = self.store.replace_texts(&place.id, &text_docs).await;
raw_photos.truncate(self.cfg.max_photos_total);
// 3. Download, deduplicate (dHash), and store in MinIO + MongoDB
if clear_existing_photos {
let _ = self.store.delete_photos_for_place(&place.id).await;
}
let downloaded: Vec<Option<(RawPhoto, dhash::ProcessedImage)>> = stream::iter(raw_photos)
.map(|raw| {
let client = self.http.clone();
async move {
let mut req = client.get(&raw.image_url);
if let Some(r) = &raw.referer {
req = req.header(reqwest::header::REFERER, r);
}
let bytes = tokio::time::timeout(Duration::from_secs(15), async {
Ok::<_, anyhow::Error>(req.send().await?.error_for_status()?.bytes().await?)
})
.await
.ok()?
.ok()?;
let processed = dhash::process_image(&bytes, 1024).ok()?;
Some((raw, processed))
}
})
.buffer_unordered(8)
.collect()
.await;
let mut seen_hashes: Vec<u64> = Vec::new();
let mut stored_photos: Vec<(Photo, Vec<u8>)> = Vec::new();
let mut duplicates = 0usize;
let mut failed = 0usize;
for item in downloaded {
let Some((raw, proc_img)) = item else {
failed += 1;
continue;
};
if dhash::is_near_duplicate(&seen_hashes, proc_img.dhash) {
duplicates += 1;
continue;
}
seen_hashes.push(proc_img.dhash);
let photo_id = format!("p_{}", &uuid::Uuid::new_v4().simple().to_string()[..10]);
let object_key = format!("places/{}/{photo_id}.jpg", place.id.replace(':', "_"));
if let Err(e) = self.store.put_blob(&object_key, Bytes::from(proc_img.jpeg_bytes.clone())).await {
tracing::warn!(%e, "minio put_blob failed");
failed += 1;
continue;
}
let photo = Photo {
id: photo_id,
place_id: place.id.clone(),
source: raw.source,
image_url: raw.image_url,
page_url: raw.page_url,
caption: raw.caption,
object_key,
dhash: format!("{:016x}", proc_img.dhash),
width: proc_img.width,
height: proc_img.height,
tag: None,
};
self.store.insert_photo(&photo).await?;
stored_photos.push((photo, proc_img.jpeg_bytes));
}
self.emit(
job_id,
tx,
JobEvent::PhotosStored {
stored: stored_photos.len(),
duplicates,
failed,
},
)
.await;
// 4. Classify photos in batches of 6 via VLM
let to_classify_len = stored_photos.len().min(36);
for chunk in stored_photos[..to_classify_len].chunks_mut(6) {
let refs: Vec<(&Photo, Vec<u8>)> = chunk.iter().map(|(p, b)| (p, b.clone())).collect();
match classify::classify_batch(&self.llm, &refs).await {
Ok(tags) => {
for (pid, tag) in tags {
let _ = self.store.set_photo_tag(&pid, &tag).await;
if let Some((p, _)) = chunk.iter_mut().find(|(p, _)| p.id == pid) {
p.tag = Some(tag);
}
}
}
Err(e) => {
tracing::warn!(%e, "photo batch classification failed");
}
}
}
let mut interior = 0usize;
let mut exterior = 0usize;
let mut food = 0usize;
let mut other = 0usize;
for (p, _) in &stored_photos {
match p.tag.as_ref().map(|t| t.kind) {
Some(PhotoKind::Interior) => interior += 1,
Some(PhotoKind::Exterior) => exterior += 1,
Some(PhotoKind::Food) => food += 1,
_ => other += 1,
}
}
self.emit(job_id, tx, JobEvent::Classified { interior, exterior, food, other }).await;
// 5. Floor detection
let all_photos: Vec<Photo> = stored_photos.iter().map(|(p, _)| p.clone()).collect();
let floors = classify::detect_floors(&all_photos, &text_docs);
self.emit(job_id, tx, JobEvent::FloorsDetected { floors: floors.clone() }).await;
// 6 & 7. Select top interior/exterior photos and infer 3D layout
let mut ranked: Vec<(&Photo, Vec<u8>)> = stored_photos
.iter()
.filter(|(p, _)| {
matches!(
p.tag.as_ref().map(|t| t.kind),
Some(PhotoKind::Interior | PhotoKind::Exterior) | None
)
})
.map(|(p, b)| (p, b.clone()))
.collect();
ranked.sort_by(|(a, _), (b, _)| {
let score = |p: &Photo| match &p.tag {
Some(t) if t.kind == PhotoKind::Interior => 2.0 + t.quality,
Some(t) if t.kind == PhotoKind::Exterior => 1.0 + t.quality,
Some(t) => t.quality,
None => 0.5,
};
score(b).total_cmp(&score(a))
});
ranked.truncate(12);
self.emit(job_id, tx, JobEvent::LayoutDraft { attempt: 1 }).await;
let layout = layout_gen::generate_layout(
&self.llm,
&place,
&ranked,
&floors,
&text_docs,
|_attempt| {},
)
.await?;
self.store.save_layout(&layout).await?;
self.emit(job_id, tx, JobEvent::Done { place_id: place.id.clone() }).await;
Ok(())
}
}