feat: initial make-space full-stack release with weighted multi-provider LLM and Dockerfile
This commit is contained in:
commit
37b323c9ac
55 files changed
+17087
No files matched your search
@@ -0,0 +1,250 @@
|
||||
//! Step 4 & 5 of the pipeline:
|
||||
//! - VLM batch photo classification (`PhotoTag` per photo)
|
||||
//! - Multi-evidence floor voting (photo `floor_level` + text `floor_mentions`)
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
use domain::{
|
||||
floor_label, FloorEvidence, FurnitureCount, FurnitureKind, Photo, PhotoKind, PhotoTag,
|
||||
PhotoZone,
|
||||
};
|
||||
use llm::{LlmClient, Part};
|
||||
use providers::text::floor_mentions;
|
||||
use serde::Deserialize;
|
||||
use serde_json::{json, Value};
|
||||
use storage::TextDoc;
|
||||
|
||||
const SYSTEM_PROMPT: &str = "\
|
||||
당신은 한국 식당·카페의 사진을 분류하고 공간 단서를 추출하는 비전 분석가입니다.
|
||||
주어진 각 사진(index 0..N-1)과 캡션 단서를 보고 다음을 판별하세요:
|
||||
- kind: interior(실내 공간/좌석/카운터/계단), exterior(건물 외관/간판), food(음식/음료 클로즈업), menu(메뉴판/가격표), person(인물 위주), other
|
||||
- zone: entrance, counter, hall, window_side, terrace, restroom, stairs, kitchen, unknown
|
||||
- floor_level: 사진이나 캡션에서 층수가 확인되면 정수(1=1층, 2=2층, -1=지하1층, 99=루프탑/옥상), 알 수 없으면 null
|
||||
- floor_evidence: 층수를 판단한 근거(계단, 표지판, 캡션 문구 등, 없으면 빈 문자열)
|
||||
- furniture: 사진에 보이는 가구 종류(table2, table4, table6, communal_table, bar_seat, sofa, counter_bar, plant)와 대략적 개수
|
||||
- quality: 공간 배치 파악에 도움이 되는 정도 (0.0~1.0)
|
||||
- description: 공간 구조/좌석/창문/계단 중심 한국어 1문장 요약";
|
||||
|
||||
pub async fn classify_batch(
|
||||
llm: &LlmClient,
|
||||
batch: &[(&Photo, Vec<u8>)],
|
||||
) -> anyhow::Result<Vec<(String, PhotoTag)>> {
|
||||
if batch.is_empty() {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
let mut parts = Vec::with_capacity(batch.len() * 2 + 1);
|
||||
parts.push(Part::text(format!(
|
||||
"총 {}장의 사진입니다. 각 사진의 index(0..{})에 맞춰 분류 결과를 반환하세요.",
|
||||
batch.len(),
|
||||
batch.len() - 1
|
||||
)));
|
||||
for (idx, (photo, bytes)) in batch.iter().enumerate() {
|
||||
parts.push(Part::text(format!(
|
||||
"Photo index={idx}, id={}, source={:?}, caption={:?}",
|
||||
photo.id, photo.source, photo.caption
|
||||
)));
|
||||
parts.push(Part::Image {
|
||||
bytes: bytes.clone(),
|
||||
mime: "image/jpeg".into(),
|
||||
});
|
||||
}
|
||||
|
||||
let resp: BatchResp = llm
|
||||
.chat_json(SYSTEM_PROMPT, &parts, "photo_classification", &schema())
|
||||
.await?;
|
||||
|
||||
let mut out = Vec::new();
|
||||
for item in resp.items {
|
||||
if let Some((photo, _)) = batch.get(item.index) {
|
||||
out.push((
|
||||
photo.id.clone(),
|
||||
PhotoTag {
|
||||
kind: item.kind,
|
||||
zone: item.zone,
|
||||
floor_level: item.floor_level.filter(|&l| l == 99 || (-5..=20).contains(&l) && l != 0),
|
||||
floor_evidence: item.floor_evidence,
|
||||
furniture: item.furniture,
|
||||
quality: item.quality.clamp(0.0, 1.0),
|
||||
description: item.description,
|
||||
},
|
||||
));
|
||||
}
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct BatchResp {
|
||||
items: Vec<BatchItem>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct BatchItem {
|
||||
index: usize,
|
||||
kind: PhotoKind,
|
||||
zone: PhotoZone,
|
||||
floor_level: Option<i32>,
|
||||
floor_evidence: String,
|
||||
furniture: Vec<FurnitureCount>,
|
||||
quality: f32,
|
||||
description: String,
|
||||
}
|
||||
|
||||
fn schema() -> Value {
|
||||
json!({
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["items"],
|
||||
"properties": {
|
||||
"items": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["index", "kind", "zone", "floor_level", "floor_evidence", "furniture", "quality", "description"],
|
||||
"properties": {
|
||||
"index": { "type": "integer" },
|
||||
"kind": { "type": "string", "enum": ["interior", "exterior", "food", "menu", "person", "other"] },
|
||||
"zone": { "type": "string", "enum": ["entrance", "counter", "hall", "window_side", "terrace", "restroom", "stairs", "kitchen", "unknown"] },
|
||||
"floor_level": { "type": ["integer", "null"] },
|
||||
"floor_evidence": { "type": "string" },
|
||||
"furniture": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["kind", "count"],
|
||||
"properties": {
|
||||
"kind": { "type": "string", "enum": ["table2", "table4", "table6", "communal_table", "bar_seat", "sofa", "counter_bar", "plant"] },
|
||||
"count": { "type": "integer" }
|
||||
}
|
||||
}
|
||||
},
|
||||
"quality": { "type": "number" },
|
||||
"description": { "type": "string" }
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
/// Combine photo `floor_level` hints, photo captions, and text snippets into a ranked list of floors.
|
||||
/// Always includes at least `1F` so single-floor venues have a ground floor.
|
||||
pub fn detect_floors(photos: &[Photo], texts: &[TextDoc]) -> Vec<FloorEvidence> {
|
||||
let mut map: BTreeMap<i32, FloorEvidence> = BTreeMap::new();
|
||||
let ensure = |map: &mut BTreeMap<i32, FloorEvidence>, level: i32| {
|
||||
map.entry(level).or_insert_with(|| FloorEvidence {
|
||||
level,
|
||||
label: floor_label(level),
|
||||
score: 0.0,
|
||||
photo_ids: Vec::new(),
|
||||
text_snippets: Vec::new(),
|
||||
});
|
||||
};
|
||||
|
||||
for p in photos {
|
||||
if let Some(tag) = &p.tag {
|
||||
if let Some(lvl) = tag.floor_level {
|
||||
ensure(&mut map, lvl);
|
||||
let entry = map.get_mut(&lvl).unwrap();
|
||||
entry.score += if tag.kind == PhotoKind::Interior { 1.5 } else { 0.8 };
|
||||
if !entry.photo_ids.contains(&p.id) {
|
||||
entry.photo_ids.push(p.id.clone());
|
||||
}
|
||||
if !tag.floor_evidence.is_empty() && entry.text_snippets.len() < 4 {
|
||||
entry.text_snippets.push(tag.floor_evidence.clone());
|
||||
}
|
||||
}
|
||||
}
|
||||
for (lvl, snip) in floor_mentions(&p.caption) {
|
||||
ensure(&mut map, lvl);
|
||||
let entry = map.get_mut(&lvl).unwrap();
|
||||
entry.score += 0.7;
|
||||
if !entry.photo_ids.contains(&p.id) {
|
||||
entry.photo_ids.push(p.id.clone());
|
||||
}
|
||||
if entry.text_snippets.len() < 4 && !entry.text_snippets.contains(&snip) {
|
||||
entry.text_snippets.push(snip);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for t in texts {
|
||||
for (lvl, snip) in floor_mentions(&t.text) {
|
||||
ensure(&mut map, lvl);
|
||||
let entry = map.get_mut(&lvl).unwrap();
|
||||
entry.score += 0.5;
|
||||
if entry.text_snippets.len() < 4 && !entry.text_snippets.contains(&snip) {
|
||||
entry.text_snippets.push(snip);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Filter out weak stray mentions (e.g., a single "3층" referring to another tenant in the building)
|
||||
// unless it's 1F.
|
||||
let mut floors: Vec<FloorEvidence> = map
|
||||
.into_values()
|
||||
.filter(|f| f.level == 1 || f.score >= 1.0)
|
||||
.collect();
|
||||
|
||||
if floors.is_empty() {
|
||||
floors.push(FloorEvidence {
|
||||
level: 1,
|
||||
label: "1F".into(),
|
||||
score: 1.0,
|
||||
photo_ids: photos
|
||||
.iter()
|
||||
.filter(|p| p.tag.as_ref().map(|t| t.kind == PhotoKind::Interior).unwrap_or(false))
|
||||
.take(4)
|
||||
.map(|p| p.id.clone())
|
||||
.collect(),
|
||||
text_snippets: Vec::new(),
|
||||
});
|
||||
}
|
||||
|
||||
floors.sort_by_key(|f| f.level);
|
||||
let _ = FurnitureKind::Table2;
|
||||
floors
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use domain::PhotoSourceId;
|
||||
|
||||
#[test]
|
||||
fn floor_voting_combines_photo_and_text() {
|
||||
let p = Photo {
|
||||
id: "p1".into(),
|
||||
place_id: "kakao:1".into(),
|
||||
source: PhotoSourceId::NaverBlog,
|
||||
image_url: String::new(),
|
||||
page_url: String::new(),
|
||||
caption: "2층 창가석".into(),
|
||||
object_key: String::new(),
|
||||
dhash: String::new(),
|
||||
width: 800,
|
||||
height: 600,
|
||||
tag: Some(PhotoTag {
|
||||
kind: PhotoKind::Interior,
|
||||
zone: PhotoZone::WindowSide,
|
||||
floor_level: Some(2),
|
||||
floor_evidence: "계단 위 2층 홀".into(),
|
||||
furniture: vec![],
|
||||
quality: 0.9,
|
||||
description: "2층 창가".into(),
|
||||
}),
|
||||
};
|
||||
let t = TextDoc {
|
||||
place_id: "kakao:1".into(),
|
||||
source: PhotoSourceId::NaverBlog,
|
||||
url: String::new(),
|
||||
text: "1층에서 주문하고 2층으로 올라가면 루프탑도 있어요. 루프탑 뷰가 최고".into(),
|
||||
};
|
||||
let floors = detect_floors(&[p], &[t]);
|
||||
let levels: Vec<i32> = floors.iter().map(|f| f.level).collect();
|
||||
assert!(levels.contains(&2));
|
||||
assert!(levels.contains(&domain::ROOFTOP_LEVEL));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,79 @@
|
||||
//! Image decoding, resizing, and 64-bit perceptual difference hash (dHash).
|
||||
|
||||
use image::imageops::FilterType;
|
||||
use image::{DynamicImage, GenericImageView};
|
||||
use std::io::Cursor;
|
||||
|
||||
pub struct ProcessedImage {
|
||||
pub jpeg_bytes: Vec<u8>,
|
||||
pub width: u32,
|
||||
pub height: u32,
|
||||
pub dhash: u64,
|
||||
}
|
||||
|
||||
/// Decode raw image bytes, reject tiny icons (< 160px), downscale to at most `max_dim`
|
||||
/// on the longest side, re-encode as JPEG, and compute a 9x8 dHash.
|
||||
pub fn process_image(bytes: &[u8], max_dim: u32) -> anyhow::Result<ProcessedImage> {
|
||||
let img = image::load_from_memory(bytes)?;
|
||||
let (w, h) = img.dimensions();
|
||||
if w < 160 || h < 160 {
|
||||
anyhow::bail!("image too small ({w}x{h})");
|
||||
}
|
||||
let dhash = compute_dhash(&img);
|
||||
let resized = if w > max_dim || h > max_dim {
|
||||
img.resize(max_dim, max_dim, FilterType::Triangle)
|
||||
} else {
|
||||
img
|
||||
};
|
||||
let (rw, rh) = resized.dimensions();
|
||||
let mut out = Vec::new();
|
||||
resized.to_rgb8().write_to(&mut Cursor::new(&mut out), image::ImageFormat::Jpeg)?;
|
||||
Ok(ProcessedImage {
|
||||
jpeg_bytes: out,
|
||||
width: rw,
|
||||
height: rh,
|
||||
dhash,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn compute_dhash(img: &DynamicImage) -> u64 {
|
||||
let gray = img.resize_exact(9, 8, FilterType::Triangle).to_luma8();
|
||||
let mut hash = 0u64;
|
||||
let mut bit = 0;
|
||||
for y in 0..8 {
|
||||
for x in 0..8 {
|
||||
if gray.get_pixel(x, y)[0] > gray.get_pixel(x + 1, y)[0] {
|
||||
hash |= 1u64 << bit;
|
||||
}
|
||||
bit += 1;
|
||||
}
|
||||
}
|
||||
hash
|
||||
}
|
||||
|
||||
pub fn hamming(a: u64, b: u64) -> u32 {
|
||||
(a ^ b).count_ones()
|
||||
}
|
||||
|
||||
pub fn is_near_duplicate(existing: &[u64], candidate: u64) -> bool {
|
||||
existing.iter().any(|&h| hamming(h, candidate) <= 6)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use image::{Rgb, RgbImage};
|
||||
|
||||
#[test]
|
||||
fn identical_images_match() {
|
||||
let mut buf = RgbImage::new(200, 200);
|
||||
for (x, y, p) in buf.enumerate_pixels_mut() {
|
||||
*p = Rgb([(x % 255) as u8, (y % 255) as u8, 120]);
|
||||
}
|
||||
let dyn_img = DynamicImage::ImageRgb8(buf);
|
||||
let h1 = compute_dhash(&dyn_img);
|
||||
let h2 = compute_dhash(&dyn_img);
|
||||
assert_eq!(hamming(h1, h2), 0);
|
||||
assert!(is_near_duplicate(&[h1], h2));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,430 @@
|
||||
//! Step 6 & 7 of the pipeline:
|
||||
//! - VLM spatial layout inference -> `domain::Layout`
|
||||
//! - Validation & clamping (bounds, non-zero dimensions, valid `evidence_photos` IDs)
|
||||
//! - Single retry with validation feedback if needed
|
||||
|
||||
use std::collections::HashSet;
|
||||
|
||||
use domain::{
|
||||
floor_label, Accessibility, Arrangement, Floor, FloorEvidence, FurnitureGroup, FurnitureKind,
|
||||
Layout, Opening, OpeningKind, Photo, PhotoKind, Place, Summary, Tri, Vertical, VerticalKind,
|
||||
Wall, Zone, ZoneKind,
|
||||
};
|
||||
use llm::{LlmClient, Part};
|
||||
use serde::Deserialize;
|
||||
use serde_json::{json, Value};
|
||||
use storage::TextDoc;
|
||||
|
||||
const SYSTEM_PROMPT: &str = "\
|
||||
당신은 식당·카페의 실내 사진들과 텍스트 리뷰를 종합해 three.js 미니어처 3D 뷰어용 공간 배치(Layout JSON)를 추론하는 공간 건축 AI입니다.
|
||||
|
||||
좌표 규칙 (층별 미터 단위):
|
||||
- 각 층은 직사각형 바닥(width_m: 동서 폭, depth_m: 남북 깊이, 보통 6~22m 범위)입니다.
|
||||
- 원점 (0, 0)은 남서쪽(South-West) 모서리이며, x는 동쪽(0..width_m), y는 북쪽(0..depth_m)으로 증가합니다.
|
||||
- 주 출입구(entrance)는 기본적으로 south 벽에 두되, 사진 단서에 맞게 조정하세요.
|
||||
- 각 zone(구역)의 직사각형 (x, y, w, d)는 바닥 범위(0 <= x, x+w <= width_m, 0 <= y, y+d <= depth_m) 안에 있어야 하며 서로 과도하게 겹치지 않게 배치하세요.
|
||||
- evidence_photos 배열에는 반드시 입력으로 주어진 사진 id만 넣으세요 (절대 없는 id를 만들지 마세요).
|
||||
- 다층 건물인 경우 층간 이동 수단(stairs 또는 elevator)을 verticals에 포함하세요.
|
||||
- summary에는 예상 평수 범위, 예상 좌석 수, 좌석 종류, 방문 전 핵심 체크 포인트(highlights), 주의사항(caveats)을 한국어로 작성하세요.";
|
||||
|
||||
pub async fn generate_layout(
|
||||
llm: &LlmClient,
|
||||
place: &Place,
|
||||
photos_with_bytes: &[(&Photo, Vec<u8>)],
|
||||
floors: &[FloorEvidence],
|
||||
texts: &[TextDoc],
|
||||
mut on_attempt: impl FnMut(u32),
|
||||
) -> anyhow::Result<Layout> {
|
||||
let valid_ids: HashSet<String> = photos_with_bytes.iter().map(|(p, _)| p.id.clone()).collect();
|
||||
let mut feedback: Option<String> = None;
|
||||
|
||||
for attempt in 1..=2 {
|
||||
on_attempt(attempt);
|
||||
let parts = build_prompt_parts(place, photos_with_bytes, floors, texts, feedback.as_deref());
|
||||
match llm
|
||||
.chat_json_with_meta::<DraftLayout>(SYSTEM_PROMPT, &parts, "space_layout", &schema())
|
||||
.await
|
||||
{
|
||||
Ok(out) => {
|
||||
let model_tag = out.model_tag();
|
||||
let mut layout = into_layout(place, &model_tag, out.data, &valid_ids);
|
||||
match validate_and_clamp(&mut layout) {
|
||||
Ok(()) => return Ok(layout),
|
||||
Err(err) if attempt == 1 => {
|
||||
tracing::warn!(attempt, %err, "layout validation failed, retrying once");
|
||||
feedback = Some(err);
|
||||
}
|
||||
Err(err) => {
|
||||
tracing::warn!(attempt, %err, "layout validation failed on second attempt, using clamped layout");
|
||||
return Ok(layout);
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(err) if attempt == 1 => {
|
||||
tracing::warn!(attempt, %err, "layout LLM call failed, retrying once");
|
||||
feedback = Some(format!("이전 응답 파싱 실패: {err}"));
|
||||
}
|
||||
Err(err) => {
|
||||
tracing::warn!(%err, "layout LLM call failed twice, using fallback layout");
|
||||
return Ok(fallback_layout(place, &llm.model, photos_with_bytes, floors));
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(fallback_layout(place, &llm.model, photos_with_bytes, floors))
|
||||
}
|
||||
|
||||
fn build_prompt_parts(
|
||||
place: &Place,
|
||||
photos_with_bytes: &[(&Photo, Vec<u8>)],
|
||||
floors: &[FloorEvidence],
|
||||
texts: &[TextDoc],
|
||||
feedback: Option<&str>,
|
||||
) -> Vec<Part> {
|
||||
let mut header = format!(
|
||||
"장소명: {}\n카테고리: {}\n주소: {} ({})\n\n[감지된 층 단서]\n",
|
||||
place.name, place.category, place.road_address, place.address
|
||||
);
|
||||
for f in floors {
|
||||
header.push_str(&format!(
|
||||
"- {} (level={}, 점수={:.1}, 관련 사진={:?}, 텍스트 단서={:?})\n",
|
||||
f.label, f.level, f.score, f.photo_ids, f.text_snippets
|
||||
));
|
||||
}
|
||||
header.push_str("\n[수집된 텍스트/리뷰 발췌]\n");
|
||||
for t in texts.iter().take(6) {
|
||||
let excerpt: String = t.text.chars().take(450).collect();
|
||||
header.push_str(&format!("({:?}) {}\n---\n", t.source, excerpt));
|
||||
}
|
||||
if let Some(fb) = feedback {
|
||||
header.push_str(&format!(
|
||||
"\n[이전 생성 결과 오류 - 반드시 수정하세요]\n{fb}\n"
|
||||
));
|
||||
}
|
||||
header.push_str("\n[실내/외관 핵심 사진 목록 — evidence_photos에는 아래 photo_id만 사용하세요]\n");
|
||||
|
||||
let mut parts = vec![Part::text(header)];
|
||||
for (p, bytes) in photos_with_bytes.iter().take(12) {
|
||||
let tag_desc = p
|
||||
.tag
|
||||
.as_ref()
|
||||
.map(|t| {
|
||||
format!(
|
||||
"kind={:?}, zone={:?}, floor={:?}, desc={}, furniture={:?}",
|
||||
t.kind, t.zone, t.floor_level, t.description, t.furniture
|
||||
)
|
||||
})
|
||||
.unwrap_or_default();
|
||||
parts.push(Part::text(format!(
|
||||
"photo_id=\"{}\" | caption={:?} | {tag_desc}",
|
||||
p.id, p.caption
|
||||
)));
|
||||
parts.push(Part::Image {
|
||||
bytes: bytes.clone(),
|
||||
mime: "image/jpeg".into(),
|
||||
});
|
||||
}
|
||||
parts
|
||||
}
|
||||
|
||||
fn into_layout(
|
||||
place: &Place,
|
||||
model: &str,
|
||||
draft: DraftLayout,
|
||||
valid_ids: &HashSet<String>,
|
||||
) -> Layout {
|
||||
let floors = draft
|
||||
.floors
|
||||
.into_iter()
|
||||
.map(|mut f| {
|
||||
if f.label.trim().is_empty() {
|
||||
f.label = floor_label(f.level);
|
||||
}
|
||||
for z in &mut f.zones {
|
||||
z.evidence_photos.retain(|id| valid_ids.contains(id));
|
||||
}
|
||||
f
|
||||
})
|
||||
.collect();
|
||||
Layout {
|
||||
place_id: place.id.clone(),
|
||||
generated_at: chrono::Utc::now().to_rfc3339(),
|
||||
model: model.to_string(),
|
||||
confidence: draft.confidence.clamp(0.1, 0.98),
|
||||
floors,
|
||||
summary: draft.summary,
|
||||
}
|
||||
}
|
||||
|
||||
/// Clamp coordinates into valid ranges and return `Err` if the layout had structural issues worth retrying.
|
||||
pub fn validate_and_clamp(layout: &mut Layout) -> Result<(), String> {
|
||||
if layout.floors.is_empty() {
|
||||
return Err("floors 배열이 비어 있습니다. 최소 1개 층(1F)을 포함해야 합니다.".into());
|
||||
}
|
||||
let mut issues = Vec::new();
|
||||
for f in &mut layout.floors {
|
||||
if f.width_m < 4.0 || f.width_m > 40.0 {
|
||||
issues.push(format!("층 {} width_m({})가 비정상 범위", f.label, f.width_m));
|
||||
f.width_m = f.width_m.clamp(5.0, 28.0);
|
||||
}
|
||||
if f.depth_m < 4.0 || f.depth_m > 40.0 {
|
||||
issues.push(format!("층 {} depth_m({})가 비정상 범위", f.label, f.depth_m));
|
||||
f.depth_m = f.depth_m.clamp(5.0, 28.0);
|
||||
}
|
||||
f.confidence = f.confidence.clamp(0.1, 0.99);
|
||||
if f.zones.is_empty() {
|
||||
issues.push(format!("층 {}에 zones가 비어 있습니다.", f.label));
|
||||
}
|
||||
for z in &mut f.zones {
|
||||
z.w = z.w.clamp(1.2, f.width_m);
|
||||
z.d = z.d.clamp(1.2, f.depth_m);
|
||||
z.x = z.x.clamp(0.0, (f.width_m - z.w).max(0.0));
|
||||
z.y = z.y.clamp(0.0, (f.depth_m - z.d).max(0.0));
|
||||
for fg in &mut z.furniture {
|
||||
fg.count = fg.count.clamp(1, 30);
|
||||
}
|
||||
}
|
||||
for o in &mut f.openings {
|
||||
let wall_len = match o.wall {
|
||||
Wall::North | Wall::South => f.width_m,
|
||||
Wall::East | Wall::West => f.depth_m,
|
||||
};
|
||||
o.length_m = o.length_m.clamp(0.8, (wall_len - 0.4).max(0.8));
|
||||
o.offset_m = o.offset_m.clamp(0.2, (wall_len - o.length_m - 0.2).max(0.2));
|
||||
}
|
||||
for v in &mut f.verticals {
|
||||
v.w = v.w.clamp(1.0, 3.5);
|
||||
v.d = v.d.clamp(1.2, 4.0);
|
||||
v.x = v.x.clamp(0.0, (f.width_m - v.w).max(0.0));
|
||||
v.y = v.y.clamp(0.0, (f.depth_m - v.d).max(0.0));
|
||||
}
|
||||
}
|
||||
layout.floors.sort_by_key(|f| f.level);
|
||||
if issues.is_empty() {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(issues.join("; "))
|
||||
}
|
||||
}
|
||||
|
||||
pub fn fallback_layout(
|
||||
place: &Place,
|
||||
model: &str,
|
||||
photos_with_bytes: &[(&Photo, Vec<u8>)],
|
||||
floors: &[FloorEvidence],
|
||||
) -> Layout {
|
||||
let evidence: Vec<String> = photos_with_bytes.iter().take(4).map(|(p, _)| p.id.clone()).collect();
|
||||
let levels: Vec<i32> = if floors.is_empty() {
|
||||
vec![1]
|
||||
} else {
|
||||
floors.iter().map(|f| f.level).collect()
|
||||
};
|
||||
let built_floors = levels
|
||||
.iter()
|
||||
.map(|&lvl| Floor {
|
||||
level: lvl,
|
||||
label: floor_label(lvl),
|
||||
confidence: 0.45,
|
||||
width_m: 10.0,
|
||||
depth_m: 12.0,
|
||||
openings: if lvl == 1 {
|
||||
vec![
|
||||
Opening { kind: OpeningKind::Entrance, wall: Wall::South, offset_m: 1.5, length_m: 1.6 },
|
||||
Opening { kind: OpeningKind::Window, wall: Wall::South, offset_m: 4.0, length_m: 4.5 },
|
||||
]
|
||||
} else {
|
||||
vec![Opening { kind: OpeningKind::Window, wall: Wall::South, offset_m: 2.0, length_m: 6.0 }]
|
||||
},
|
||||
zones: vec![
|
||||
Zone {
|
||||
kind: if lvl == 1 { ZoneKind::Counter } else { ZoneKind::Seating },
|
||||
label: if lvl == 1 { "주문 카운터".into() } else { "창가 좌석".into() },
|
||||
x: 0.6,
|
||||
y: if lvl == 1 { 8.2 } else { 0.6 },
|
||||
w: 4.2,
|
||||
d: 3.0,
|
||||
furniture: vec![FurnitureGroup {
|
||||
kind: if lvl == 1 { FurnitureKind::CounterBar } else { FurnitureKind::BarSeat },
|
||||
count: if lvl == 1 { 1 } else { 5 },
|
||||
arrangement: Arrangement::AlongWall,
|
||||
along_wall: Some(if lvl == 1 { Wall::North } else { Wall::South }),
|
||||
}],
|
||||
evidence_photos: evidence.clone(),
|
||||
},
|
||||
Zone {
|
||||
kind: ZoneKind::Seating,
|
||||
label: "메인 홀 좌석".into(),
|
||||
x: 0.8,
|
||||
y: if lvl == 1 { 1.0 } else { 4.2 },
|
||||
w: 8.4,
|
||||
d: 6.5,
|
||||
furniture: vec![
|
||||
FurnitureGroup {
|
||||
kind: FurnitureKind::Table2,
|
||||
count: 5,
|
||||
arrangement: Arrangement::Grid,
|
||||
along_wall: None,
|
||||
},
|
||||
FurnitureGroup {
|
||||
kind: FurnitureKind::Table4,
|
||||
count: 3,
|
||||
arrangement: Arrangement::Grid,
|
||||
along_wall: None,
|
||||
},
|
||||
],
|
||||
evidence_photos: evidence.clone(),
|
||||
},
|
||||
],
|
||||
verticals: if levels.len() > 1 {
|
||||
vec![Vertical {
|
||||
kind: VerticalKind::Stairs,
|
||||
x: 7.8,
|
||||
y: 9.2,
|
||||
w: 1.6,
|
||||
d: 2.2,
|
||||
to_level: if lvl == 1 { levels.get(1).copied().unwrap_or(2) } else { 1 },
|
||||
}]
|
||||
} else {
|
||||
vec![]
|
||||
},
|
||||
notes: "수집된 사진을 기반으로 생성한 기본 추정 배치입니다.".into(),
|
||||
})
|
||||
.collect();
|
||||
|
||||
let _ = PhotoKind::Interior;
|
||||
Layout {
|
||||
place_id: place.id.clone(),
|
||||
generated_at: chrono::Utc::now().to_rfc3339(),
|
||||
model: model.to_string(),
|
||||
confidence: 0.45,
|
||||
floors: built_floors,
|
||||
summary: Summary {
|
||||
area_pyeong_min: 18.0,
|
||||
area_pyeong_max: 28.0,
|
||||
seat_estimate: 26,
|
||||
seat_types: vec!["2인 테이블".into(), "4인 테이블".into(), "바 좌석".into()],
|
||||
highlights: vec![format!("{} 공간 배치 추정 완료", place.name)],
|
||||
accessibility: Accessibility {
|
||||
elevator: Tri::Unknown,
|
||||
step_free_entrance: Tri::Likely,
|
||||
},
|
||||
caveats: vec!["실내 사진 수량이 제한적이어서 일부 구역은 표준 배치로 추정되었습니다.".into()],
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct DraftLayout {
|
||||
confidence: f32,
|
||||
floors: Vec<Floor>,
|
||||
summary: Summary,
|
||||
}
|
||||
|
||||
fn schema() -> Value {
|
||||
json!({
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["confidence", "floors", "summary"],
|
||||
"properties": {
|
||||
"confidence": { "type": "number" },
|
||||
"floors": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["level", "label", "confidence", "width_m", "depth_m", "openings", "zones", "verticals", "notes"],
|
||||
"properties": {
|
||||
"level": { "type": "integer" },
|
||||
"label": { "type": "string" },
|
||||
"confidence": { "type": "number" },
|
||||
"width_m": { "type": "number" },
|
||||
"depth_m": { "type": "number" },
|
||||
"openings": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["kind", "wall", "offset_m", "length_m"],
|
||||
"properties": {
|
||||
"kind": { "type": "string", "enum": ["entrance", "window", "door"] },
|
||||
"wall": { "type": "string", "enum": ["north", "south", "east", "west"] },
|
||||
"offset_m": { "type": "number" },
|
||||
"length_m": { "type": "number" }
|
||||
}
|
||||
}
|
||||
},
|
||||
"zones": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["kind", "label", "x", "y", "w", "d", "furniture", "evidence_photos"],
|
||||
"properties": {
|
||||
"kind": { "type": "string", "enum": ["seating", "counter", "bar", "kitchen", "restroom", "terrace", "entrance", "other"] },
|
||||
"label": { "type": "string" },
|
||||
"x": { "type": "number" },
|
||||
"y": { "type": "number" },
|
||||
"w": { "type": "number" },
|
||||
"d": { "type": "number" },
|
||||
"furniture": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["kind", "count", "arrangement", "along_wall"],
|
||||
"properties": {
|
||||
"kind": { "type": "string", "enum": ["table2", "table4", "table6", "communal_table", "bar_seat", "sofa", "counter_bar", "plant"] },
|
||||
"count": { "type": "integer" },
|
||||
"arrangement": { "type": "string", "enum": ["grid", "along_wall", "row", "cluster"] },
|
||||
"along_wall": { "type": ["string", "null"], "enum": ["north", "south", "east", "west", null] }
|
||||
}
|
||||
}
|
||||
},
|
||||
"evidence_photos": { "type": "array", "items": { "type": "string" } }
|
||||
}
|
||||
}
|
||||
},
|
||||
"verticals": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["kind", "x", "y", "w", "d", "to_level"],
|
||||
"properties": {
|
||||
"kind": { "type": "string", "enum": ["stairs", "elevator"] },
|
||||
"x": { "type": "number" },
|
||||
"y": { "type": "number" },
|
||||
"w": { "type": "number" },
|
||||
"d": { "type": "number" },
|
||||
"to_level": { "type": "integer" }
|
||||
}
|
||||
}
|
||||
},
|
||||
"notes": { "type": "string" }
|
||||
}
|
||||
}
|
||||
},
|
||||
"summary": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["area_pyeong_min", "area_pyeong_max", "seat_estimate", "seat_types", "highlights", "accessibility", "caveats"],
|
||||
"properties": {
|
||||
"area_pyeong_min": { "type": "number" },
|
||||
"area_pyeong_max": { "type": "number" },
|
||||
"seat_estimate": { "type": "integer" },
|
||||
"seat_types": { "type": "array", "items": { "type": "string" } },
|
||||
"highlights": { "type": "array", "items": { "type": "string" } },
|
||||
"accessibility": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["elevator", "step_free_entrance"],
|
||||
"properties": {
|
||||
"elevator": { "type": "string", "enum": ["yes", "likely", "no", "unknown"] },
|
||||
"step_free_entrance": { "type": "string", "enum": ["yes", "likely", "no", "unknown"] }
|
||||
}
|
||||
},
|
||||
"caveats": { "type": "array", "items": { "type": "string" } }
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,482 @@
|
||||
#![recursion_limit = "256"]
|
||||
//! End-to-end place analysis pipeline:
|
||||
//! 1. Normalize place (Kakao candidate + NCP Geocode address verification + Naver place ID resolution)
|
||||
//! 2. Collect photos & texts in parallel across all `PhotoSource`s (per-source timeout)
|
||||
//! 3. Download images, deduplicate with 64-bit dHash, store JPEG blobs in MinIO & metadata in MongoDB
|
||||
//! 4. Classify photos in batches with the VLM (`PhotoTag`)
|
||||
//! 5. Vote on floors (`FloorEvidence`) from photo tags, captions, and text snippets
|
||||
//! 6. Generate 3D `Layout` JSON with the VLM
|
||||
//! 7. Validate & clamp layout (retry once on structural issues) and save to MongoDB
|
||||
|
||||
pub mod classify;
|
||||
pub mod dhash;
|
||||
pub mod layout_gen;
|
||||
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
use bytes::Bytes;
|
||||
use dashmap::DashMap;
|
||||
use domain::{
|
||||
JobEvent, Photo, PhotoKind, PhotoSummary, Place, PlaceCandidate, PlaceView, RawPhoto, RecentPlace,
|
||||
};
|
||||
use futures::stream::{self, StreamExt};
|
||||
use llm::LlmClient;
|
||||
use providers::browser::{browser_source, BrowserSource};
|
||||
use providers::kakao_local::KakaoLocal;
|
||||
use providers::kakao_place::KakaoPlacePhotos;
|
||||
use providers::naver_blog::NaverBlog;
|
||||
use providers::naver_map::NaverMapSearch;
|
||||
use providers::naver_place::NaverPlacePhotos;
|
||||
use providers::ncp_geocode::NcpGeocode;
|
||||
use providers::{PhotoSource, PlaceSearch};
|
||||
use storage::{Store, TextDoc};
|
||||
use tokio::sync::broadcast;
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct PipelineConfig {
|
||||
pub kakao_api_key: String,
|
||||
pub ncp_client_id: String,
|
||||
pub ncp_client_secret: String,
|
||||
pub browser_scraper_url: Option<String>,
|
||||
pub max_photos_per_source: usize,
|
||||
pub max_photos_total: usize,
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct Pipeline {
|
||||
pub store: Store,
|
||||
pub llm: LlmClient,
|
||||
kakao: Arc<KakaoLocal>,
|
||||
naver_map: Arc<NaverMapSearch>,
|
||||
ncp: Option<Arc<NcpGeocode>>,
|
||||
sources: Arc<Vec<Box<dyn PhotoSource>>>,
|
||||
http: reqwest::Client,
|
||||
cfg: PipelineConfig,
|
||||
/// Active job event broadcasters keyed by `job_id`.
|
||||
jobs: Arc<DashMap<String, broadcast::Sender<JobEvent>>>,
|
||||
/// Map `place_id` -> currently running `job_id` so concurrent visitors share one job.
|
||||
running_by_place: Arc<DashMap<String, String>>,
|
||||
}
|
||||
|
||||
impl Pipeline {
|
||||
pub fn new(store: Store, llm: LlmClient, cfg: PipelineConfig) -> Self {
|
||||
let kakao = Arc::new(KakaoLocal::new(&cfg.kakao_api_key));
|
||||
let naver_map = Arc::new(NaverMapSearch::new());
|
||||
let ncp = if !cfg.ncp_client_id.trim().is_empty() && !cfg.ncp_client_secret.trim().is_empty() {
|
||||
Some(Arc::new(NcpGeocode::new(&cfg.ncp_client_id, &cfg.ncp_client_secret)))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let sources: Vec<Box<dyn PhotoSource>> = vec![
|
||||
Box::new(NaverPlacePhotos::new()),
|
||||
Box::new(KakaoPlacePhotos::new()),
|
||||
Box::new(NaverBlog::new()),
|
||||
browser_source(cfg.browser_scraper_url.as_deref(), BrowserSource::GoogleMaps),
|
||||
browser_source(cfg.browser_scraper_url.as_deref(), BrowserSource::Instagram),
|
||||
];
|
||||
Self {
|
||||
store,
|
||||
llm,
|
||||
kakao,
|
||||
naver_map,
|
||||
ncp,
|
||||
sources: Arc::new(sources),
|
||||
http: providers::http::scraping_client(),
|
||||
cfg,
|
||||
jobs: Arc::new(DashMap::new()),
|
||||
running_by_place: Arc::new(DashMap::new()),
|
||||
}
|
||||
}
|
||||
|
||||
/// Search place candidates by free text (Kakao primary, Naver Map fallback).
|
||||
pub async fn search_places(&self, query: &str) -> anyhow::Result<Vec<PlaceCandidate>> {
|
||||
let q = query.trim();
|
||||
if q.is_empty() {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
let mut results = self.kakao.search(q).await.unwrap_or_default();
|
||||
if results.is_empty() {
|
||||
results = self.naver_map.search(q).await.unwrap_or_default();
|
||||
}
|
||||
Ok(results)
|
||||
}
|
||||
|
||||
/// Saved analyses ordered by their most recent view (or analysis time for older records).
|
||||
pub async fn recent_places(&self) -> anyhow::Result<Vec<RecentPlace>> {
|
||||
self.store.recent_places().await
|
||||
}
|
||||
|
||||
/// Return current `PlaceView` (place + latest cached layout + stored photos + running job id).
|
||||
pub async fn get_place_view(&self, place_id: &str) -> anyhow::Result<Option<PlaceView>> {
|
||||
let Some(place) = self.store.get_place(place_id).await? else {
|
||||
return Ok(None);
|
||||
};
|
||||
let layout = self.store.latest_layout(place_id).await?.map(|d| d.layout);
|
||||
let photos = self
|
||||
.store
|
||||
.photos_for_place(place_id)
|
||||
.await?
|
||||
.iter()
|
||||
.map(PhotoSummary::from)
|
||||
.collect();
|
||||
let running_job = self.running_by_place.get(place_id).map(|v| v.value().clone());
|
||||
if layout.is_some() {
|
||||
if let Err(error) = self.store.record_place_view(place_id).await {
|
||||
tracing::warn!(%place_id, %error, "could not record place view");
|
||||
}
|
||||
}
|
||||
Ok(Some(PlaceView {
|
||||
place,
|
||||
layout,
|
||||
photos,
|
||||
running_job,
|
||||
}))
|
||||
}
|
||||
|
||||
/// Subscribe to live events for `job_id`, plus any events already recorded in MongoDB.
|
||||
pub async fn subscribe_job(
|
||||
&self,
|
||||
job_id: &str,
|
||||
) -> anyhow::Result<(Vec<JobEvent>, Option<broadcast::Receiver<JobEvent>>)> {
|
||||
let rx = self.jobs.get(job_id).map(|s| s.value().subscribe());
|
||||
let past = self
|
||||
.store
|
||||
.get_job(job_id)
|
||||
.await?
|
||||
.map(|j| j.events)
|
||||
.unwrap_or_default();
|
||||
Ok((past, rx))
|
||||
}
|
||||
|
||||
/// Ensure the candidate's `Place` exists in MongoDB; if no cached layout exists (or `force_reanalyze` is true),
|
||||
/// spawn the analysis job in the background and return `(place_id, Option<job_id>)`.
|
||||
pub async fn select_or_analyze(
|
||||
&self,
|
||||
candidate: PlaceCandidate,
|
||||
force_reanalyze: bool,
|
||||
) -> anyhow::Result<(String, Option<String>)> {
|
||||
let place_id = candidate.place_id();
|
||||
let mut place = match self.store.get_place(&place_id).await? {
|
||||
Some(existing) => existing,
|
||||
None => {
|
||||
let p = Place::from_candidate(&candidate);
|
||||
self.store.upsert_place(&p).await?;
|
||||
p
|
||||
}
|
||||
};
|
||||
|
||||
if !force_reanalyze {
|
||||
if let Some( job_id ) = self.running_by_place.get(&place_id).map(|r| r.value().clone()) {
|
||||
return Ok((place_id, Some(job_id)));
|
||||
}
|
||||
if self.store.latest_layout(&place_id).await?.is_some() {
|
||||
return Ok((place_id, None));
|
||||
}
|
||||
}
|
||||
|
||||
let job_id = self.spawn_job(place_id.clone(), place.clone(), force_reanalyze).await?;
|
||||
let _ = &mut place;
|
||||
Ok((place_id, Some(job_id)))
|
||||
}
|
||||
|
||||
/// Trigger reanalysis for an already-stored `place_id`.
|
||||
pub async fn reanalyze(&self, place_id: &str) -> anyhow::Result<String> {
|
||||
if let Some(existing) = self.running_by_place.get(place_id) {
|
||||
return Ok(existing.value().clone());
|
||||
}
|
||||
let place = self
|
||||
.store
|
||||
.get_place(place_id)
|
||||
.await?
|
||||
.ok_or_else(|| anyhow::anyhow!("장소를 찾을 수 없습니다: {place_id}"))?;
|
||||
self.spawn_job(place_id.to_string(), place, true).await
|
||||
}
|
||||
|
||||
async fn spawn_job(
|
||||
&self,
|
||||
place_id: String,
|
||||
place: Place,
|
||||
clear_existing_photos: bool,
|
||||
) -> anyhow::Result<String> {
|
||||
let job_id = uuid::Uuid::new_v4().to_string();
|
||||
self.store.create_job(&job_id, &place_id).await?;
|
||||
let (tx, _) = broadcast::channel(64);
|
||||
self.jobs.insert(job_id.clone(), tx.clone());
|
||||
self.running_by_place.insert(place_id.clone(), job_id.clone());
|
||||
|
||||
let this = self.clone();
|
||||
let jid = job_id.clone();
|
||||
let pid = place_id.clone();
|
||||
tokio::spawn(async move {
|
||||
if let Err(err) = this.run_job(&jid, place, clear_existing_photos, &tx).await {
|
||||
tracing::error!(job_id = %jid, place_id = %pid, "pipeline job failed: {err:#}");
|
||||
this.emit(&jid, &tx, JobEvent::Error { message: err.to_string() }).await;
|
||||
}
|
||||
this.running_by_place.remove(&pid);
|
||||
// Keep channel briefly so SSE clients receive the terminal event, then drop.
|
||||
tokio::time::sleep(Duration::from_secs(5)).await;
|
||||
this.jobs.remove(&jid);
|
||||
});
|
||||
|
||||
Ok(job_id)
|
||||
}
|
||||
|
||||
/// Run a place analysis synchronously (used by `precompute` CLI and `spawn_job`).
|
||||
pub async fn run_sync(&self, place: Place) -> anyhow::Result<domain::Layout> {
|
||||
let job_id = uuid::Uuid::new_v4().to_string();
|
||||
self.store.upsert_place(&place).await?;
|
||||
self.store.create_job(&job_id, &place.id).await?;
|
||||
let (tx, _) = broadcast::channel(64);
|
||||
self.run_job(&job_id, place.clone(), false, &tx).await?;
|
||||
let doc = self
|
||||
.store
|
||||
.latest_layout(&place.id)
|
||||
.await?
|
||||
.ok_or_else(|| anyhow::anyhow!("layout not saved"))?;
|
||||
Ok(doc.layout)
|
||||
}
|
||||
|
||||
async fn emit(&self, job_id: &str, tx: &broadcast::Sender<JobEvent>, ev: JobEvent) {
|
||||
let _ = self.store.push_job_event(job_id, &ev).await;
|
||||
let _ = tx.send(ev);
|
||||
}
|
||||
|
||||
async fn run_job(
|
||||
&self,
|
||||
job_id: &str,
|
||||
mut place: Place,
|
||||
clear_existing_photos: bool,
|
||||
tx: &broadcast::Sender<JobEvent>,
|
||||
) -> anyhow::Result<()> {
|
||||
self.emit(job_id, tx, JobEvent::Started { place_id: place.id.clone() }).await;
|
||||
|
||||
// 1. Address verification (NCP Geocode) + Naver Place ID matching
|
||||
if let Some(ncp) = &self.ncp {
|
||||
let addr_query = if !place.road_address.is_empty() {
|
||||
place.road_address.clone()
|
||||
} else {
|
||||
place.address.clone()
|
||||
};
|
||||
if !addr_query.is_empty() {
|
||||
if let Ok(Some(geo)) = ncp.geocode(&addr_query).await {
|
||||
if place.road_address.is_empty() && !geo.road_address.is_empty() {
|
||||
place.road_address = geo.road_address;
|
||||
}
|
||||
if place.address.is_empty() && !geo.jibun_address.is_empty() {
|
||||
place.address = geo.jibun_address;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if place.naver_place_id.is_none() {
|
||||
if let Ok(Some(nid)) = self.naver_map.resolve_place_id(&place).await {
|
||||
place.naver_place_id = Some(nid);
|
||||
}
|
||||
}
|
||||
self.store.upsert_place(&place).await?;
|
||||
self.emit(
|
||||
job_id,
|
||||
tx,
|
||||
JobEvent::PlaceResolved {
|
||||
name: place.name.clone(),
|
||||
naver_place_id: place.naver_place_id.clone(),
|
||||
},
|
||||
)
|
||||
.await;
|
||||
|
||||
// 2. Collect photos & text snippets from all sources in parallel
|
||||
let per_source = self.cfg.max_photos_per_source;
|
||||
let source_futs = self.sources.iter().map(|src| {
|
||||
let place_ref = &place;
|
||||
async move {
|
||||
let id = src.id();
|
||||
let res = tokio::time::timeout(Duration::from_secs(30), src.collect(place_ref, per_source)).await;
|
||||
match res {
|
||||
Ok(Ok(out)) => (id, out, None),
|
||||
Ok(Err(e)) => (id, Default::default(), Some(e.to_string())),
|
||||
Err(_) => (id, Default::default(), Some("시간 초과 (30s)".into())),
|
||||
}
|
||||
}
|
||||
});
|
||||
let collected = futures::future::join_all(source_futs).await;
|
||||
|
||||
let mut raw_photos: Vec<RawPhoto> = Vec::new();
|
||||
let mut text_docs: Vec<TextDoc> = Vec::new();
|
||||
for (source, out, error) in collected {
|
||||
self.emit(
|
||||
job_id,
|
||||
tx,
|
||||
JobEvent::SourceDone {
|
||||
source,
|
||||
photos: out.photos.len(),
|
||||
texts: out.texts.len(),
|
||||
error,
|
||||
},
|
||||
)
|
||||
.await;
|
||||
raw_photos.extend(out.photos);
|
||||
for t in out.texts {
|
||||
text_docs.push(TextDoc {
|
||||
place_id: place.id.clone(),
|
||||
source: t.source,
|
||||
url: t.url,
|
||||
text: t.text,
|
||||
});
|
||||
}
|
||||
}
|
||||
let _ = self.store.replace_texts(&place.id, &text_docs).await;
|
||||
raw_photos.truncate(self.cfg.max_photos_total);
|
||||
|
||||
// 3. Download, deduplicate (dHash), and store in MinIO + MongoDB
|
||||
if clear_existing_photos {
|
||||
let _ = self.store.delete_photos_for_place(&place.id).await;
|
||||
}
|
||||
let downloaded: Vec<Option<(RawPhoto, dhash::ProcessedImage)>> = stream::iter(raw_photos)
|
||||
.map(|raw| {
|
||||
let client = self.http.clone();
|
||||
async move {
|
||||
let mut req = client.get(&raw.image_url);
|
||||
if let Some(r) = &raw.referer {
|
||||
req = req.header(reqwest::header::REFERER, r);
|
||||
}
|
||||
let bytes = tokio::time::timeout(Duration::from_secs(15), async {
|
||||
Ok::<_, anyhow::Error>(req.send().await?.error_for_status()?.bytes().await?)
|
||||
})
|
||||
.await
|
||||
.ok()?
|
||||
.ok()?;
|
||||
let processed = dhash::process_image(&bytes, 1024).ok()?;
|
||||
Some((raw, processed))
|
||||
}
|
||||
})
|
||||
.buffer_unordered(8)
|
||||
.collect()
|
||||
.await;
|
||||
|
||||
let mut seen_hashes: Vec<u64> = Vec::new();
|
||||
let mut stored_photos: Vec<(Photo, Vec<u8>)> = Vec::new();
|
||||
let mut duplicates = 0usize;
|
||||
let mut failed = 0usize;
|
||||
|
||||
for item in downloaded {
|
||||
let Some((raw, proc_img)) = item else {
|
||||
failed += 1;
|
||||
continue;
|
||||
};
|
||||
if dhash::is_near_duplicate(&seen_hashes, proc_img.dhash) {
|
||||
duplicates += 1;
|
||||
continue;
|
||||
}
|
||||
seen_hashes.push(proc_img.dhash);
|
||||
let photo_id = format!("p_{}", &uuid::Uuid::new_v4().simple().to_string()[..10]);
|
||||
let object_key = format!("places/{}/{photo_id}.jpg", place.id.replace(':', "_"));
|
||||
if let Err(e) = self.store.put_blob(&object_key, Bytes::from(proc_img.jpeg_bytes.clone())).await {
|
||||
tracing::warn!(%e, "minio put_blob failed");
|
||||
failed += 1;
|
||||
continue;
|
||||
}
|
||||
let photo = Photo {
|
||||
id: photo_id,
|
||||
place_id: place.id.clone(),
|
||||
source: raw.source,
|
||||
image_url: raw.image_url,
|
||||
page_url: raw.page_url,
|
||||
caption: raw.caption,
|
||||
object_key,
|
||||
dhash: format!("{:016x}", proc_img.dhash),
|
||||
width: proc_img.width,
|
||||
height: proc_img.height,
|
||||
tag: None,
|
||||
};
|
||||
self.store.insert_photo(&photo).await?;
|
||||
stored_photos.push((photo, proc_img.jpeg_bytes));
|
||||
}
|
||||
|
||||
self.emit(
|
||||
job_id,
|
||||
tx,
|
||||
JobEvent::PhotosStored {
|
||||
stored: stored_photos.len(),
|
||||
duplicates,
|
||||
failed,
|
||||
},
|
||||
)
|
||||
.await;
|
||||
|
||||
// 4. Classify photos in batches of 6 via VLM
|
||||
let to_classify_len = stored_photos.len().min(36);
|
||||
for chunk in stored_photos[..to_classify_len].chunks_mut(6) {
|
||||
let refs: Vec<(&Photo, Vec<u8>)> = chunk.iter().map(|(p, b)| (p, b.clone())).collect();
|
||||
match classify::classify_batch(&self.llm, &refs).await {
|
||||
Ok(tags) => {
|
||||
for (pid, tag) in tags {
|
||||
let _ = self.store.set_photo_tag(&pid, &tag).await;
|
||||
if let Some((p, _)) = chunk.iter_mut().find(|(p, _)| p.id == pid) {
|
||||
p.tag = Some(tag);
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::warn!(%e, "photo batch classification failed");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let mut interior = 0usize;
|
||||
let mut exterior = 0usize;
|
||||
let mut food = 0usize;
|
||||
let mut other = 0usize;
|
||||
for (p, _) in &stored_photos {
|
||||
match p.tag.as_ref().map(|t| t.kind) {
|
||||
Some(PhotoKind::Interior) => interior += 1,
|
||||
Some(PhotoKind::Exterior) => exterior += 1,
|
||||
Some(PhotoKind::Food) => food += 1,
|
||||
_ => other += 1,
|
||||
}
|
||||
}
|
||||
self.emit(job_id, tx, JobEvent::Classified { interior, exterior, food, other }).await;
|
||||
|
||||
// 5. Floor detection
|
||||
let all_photos: Vec<Photo> = stored_photos.iter().map(|(p, _)| p.clone()).collect();
|
||||
let floors = classify::detect_floors(&all_photos, &text_docs);
|
||||
self.emit(job_id, tx, JobEvent::FloorsDetected { floors: floors.clone() }).await;
|
||||
|
||||
// 6 & 7. Select top interior/exterior photos and infer 3D layout
|
||||
let mut ranked: Vec<(&Photo, Vec<u8>)> = stored_photos
|
||||
.iter()
|
||||
.filter(|(p, _)| {
|
||||
matches!(
|
||||
p.tag.as_ref().map(|t| t.kind),
|
||||
Some(PhotoKind::Interior | PhotoKind::Exterior) | None
|
||||
)
|
||||
})
|
||||
.map(|(p, b)| (p, b.clone()))
|
||||
.collect();
|
||||
ranked.sort_by(|(a, _), (b, _)| {
|
||||
let score = |p: &Photo| match &p.tag {
|
||||
Some(t) if t.kind == PhotoKind::Interior => 2.0 + t.quality,
|
||||
Some(t) if t.kind == PhotoKind::Exterior => 1.0 + t.quality,
|
||||
Some(t) => t.quality,
|
||||
None => 0.5,
|
||||
};
|
||||
score(b).total_cmp(&score(a))
|
||||
});
|
||||
ranked.truncate(12);
|
||||
|
||||
self.emit(job_id, tx, JobEvent::LayoutDraft { attempt: 1 }).await;
|
||||
let layout = layout_gen::generate_layout(
|
||||
&self.llm,
|
||||
&place,
|
||||
&ranked,
|
||||
&floors,
|
||||
&text_docs,
|
||||
|_attempt| {},
|
||||
)
|
||||
.await?;
|
||||
|
||||
self.store.save_layout(&layout).await?;
|
||||
self.emit(job_id, tx, JobEvent::Done { place_id: place.id.clone() }).await;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user