feat: initial make-space full-stack release with weighted multi-provider LLM and Dockerfile
This commit is contained in:
commit
37b323c9ac
55 files changed
+17087
No files matched your search
@@ -0,0 +1,244 @@
|
||||
//! Naver Blog posts about the place: inline images + body text (unofficial, verified 2026-11-04).
|
||||
//!
|
||||
//! 1. Search `https://search.naver.com/search.naver?ssc=tab.blog.all&query=...` for `blog.naver.com/{blogId}/{logNo}` links.
|
||||
//! 2. Fetch each post via `https://blog.naver.com/PostView.naver?blogId=...&logNo=...` (bypasses outer iframe).
|
||||
//! 3. Parse SmartEditor (`.se-main-container`) paragraphs and `postfiles.pstatic.net` images, pairing each image
|
||||
//! with nearby paragraph text as its caption.
|
||||
|
||||
use std::collections::HashSet;
|
||||
|
||||
use async_trait::async_trait;
|
||||
use domain::{PhotoSourceId, Place, RawPhoto, SourceOutput, TextSnippet};
|
||||
use futures::stream::{self, StreamExt};
|
||||
use regex::Regex;
|
||||
use scraper::{Html, Selector};
|
||||
|
||||
use crate::{http, PhotoSource};
|
||||
|
||||
const SEARCH_URL: &str = "https://search.naver.com/search.naver";
|
||||
const POST_VIEW_URL: &str = "https://blog.naver.com/PostView.naver";
|
||||
|
||||
pub struct NaverBlog {
|
||||
client: reqwest::Client,
|
||||
max_posts: usize,
|
||||
}
|
||||
|
||||
impl Default for NaverBlog {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
impl NaverBlog {
|
||||
pub fn new() -> Self {
|
||||
Self { client: http::scraping_client(), max_posts: 8 }
|
||||
}
|
||||
|
||||
async fn search_post_ids(&self, query: &str) -> anyhow::Result<Vec<(String, String)>> {
|
||||
let html = self
|
||||
.client
|
||||
.get(SEARCH_URL)
|
||||
.header(reqwest::header::REFERER, "https://search.naver.com/")
|
||||
.query(&[("ssc", "tab.blog.all"), ("query", query)])
|
||||
.send()
|
||||
.await?
|
||||
.error_for_status()?
|
||||
.text()
|
||||
.await?;
|
||||
Ok(extract_blog_post_ids(&html))
|
||||
}
|
||||
|
||||
async fn fetch_post(&self, blog_id: &str, log_no: &str) -> anyhow::Result<SourceOutput> {
|
||||
let page_url = format!("https://blog.naver.com/{blog_id}/{log_no}");
|
||||
let html = self
|
||||
.client
|
||||
.get(POST_VIEW_URL)
|
||||
.header(reqwest::header::REFERER, "https://blog.naver.com/")
|
||||
.query(&[("blogId", blog_id), ("logNo", log_no)])
|
||||
.send()
|
||||
.await?
|
||||
.error_for_status()?
|
||||
.text()
|
||||
.await?;
|
||||
Ok(parse_blog_post_html(&html, &page_url))
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl PhotoSource for NaverBlog {
|
||||
fn id(&self) -> PhotoSourceId {
|
||||
PhotoSourceId::NaverBlog
|
||||
}
|
||||
|
||||
async fn collect(&self, place: &Place, limit: usize) -> anyhow::Result<SourceOutput> {
|
||||
let region = place.region_hint();
|
||||
let q1 = format!("{} {} 내부 좌석", region, place.name);
|
||||
let q2 = format!("{} {}", region, place.name);
|
||||
|
||||
let mut post_ids = self.search_post_ids(q1.trim()).await.unwrap_or_default();
|
||||
if post_ids.len() < self.max_posts {
|
||||
for id in self.search_post_ids(q2.trim()).await.unwrap_or_default() {
|
||||
if !post_ids.contains(&id) {
|
||||
post_ids.push(id);
|
||||
}
|
||||
}
|
||||
}
|
||||
post_ids.truncate(self.max_posts);
|
||||
|
||||
let results: Vec<SourceOutput> = stream::iter(post_ids)
|
||||
.map(|(b, l)| async move { self.fetch_post(&b, &l).await.ok() })
|
||||
.buffer_unordered(4)
|
||||
.filter_map(|x| async move { x })
|
||||
.collect()
|
||||
.await;
|
||||
|
||||
let mut seen = HashSet::new();
|
||||
let mut photos = Vec::new();
|
||||
let mut texts = Vec::new();
|
||||
for r in results {
|
||||
texts.extend(r.texts);
|
||||
for p in r.photos {
|
||||
if photos.len() >= limit {
|
||||
break;
|
||||
}
|
||||
if seen.insert(p.image_url.clone()) {
|
||||
photos.push(p);
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(SourceOutput { photos, texts })
|
||||
}
|
||||
}
|
||||
|
||||
pub fn extract_blog_post_ids(html: &str) -> Vec<(String, String)> {
|
||||
let re = Regex::new(r"https?://(?:m\.)?blog\.naver\.com/([a-zA-Z0-9_\-]+)/(\d{10,15})").unwrap();
|
||||
let mut seen = HashSet::new();
|
||||
let mut out = Vec::new();
|
||||
for c in re.captures_iter(html) {
|
||||
let pair = (c[1].to_string(), c[2].to_string());
|
||||
if seen.insert(pair.clone()) {
|
||||
out.push(pair);
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
pub fn parse_blog_post_html(html: &str, page_url: &str) -> SourceOutput {
|
||||
let doc = Html::parse_document(html);
|
||||
let container_sel = Selector::parse(".se-main-container, #postViewArea").unwrap();
|
||||
let p_sel = Selector::parse(".se-text-paragraph, p").unwrap();
|
||||
let img_sel = Selector::parse("img").unwrap();
|
||||
|
||||
let mut paragraphs = Vec::new();
|
||||
for container in doc.select(&container_sel) {
|
||||
for p in container.select(&p_sel) {
|
||||
let t = p.text().collect::<Vec<_>>().join(" ");
|
||||
let clean = t.split_whitespace().collect::<Vec<_>>().join(" ");
|
||||
if clean.chars().count() >= 4 {
|
||||
paragraphs.push(clean);
|
||||
}
|
||||
}
|
||||
}
|
||||
if paragraphs.is_empty() {
|
||||
for p in doc.select(&p_sel) {
|
||||
let clean = p.text().collect::<Vec<_>>().join(" ").split_whitespace().collect::<Vec<_>>().join(" ");
|
||||
if clean.chars().count() >= 4 {
|
||||
paragraphs.push(clean);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let full_text: String = paragraphs.join("\n").chars().take(3000).collect();
|
||||
let mut texts = Vec::new();
|
||||
if !full_text.is_empty() {
|
||||
texts.push(TextSnippet {
|
||||
source: PhotoSourceId::NaverBlog,
|
||||
url: page_url.to_string(),
|
||||
text: full_text.clone(),
|
||||
});
|
||||
}
|
||||
|
||||
let mut seen = HashSet::new();
|
||||
let mut photos = Vec::new();
|
||||
for img in doc.select(&img_sel) {
|
||||
let v = img.value();
|
||||
let src = v
|
||||
.attr("data-lazy-src")
|
||||
.or_else(|| v.attr("data-src"))
|
||||
.or_else(|| v.attr("src"))
|
||||
.unwrap_or_default();
|
||||
if let Some(url) = normalize_blog_image_url(src) {
|
||||
if seen.insert(url.clone()) {
|
||||
photos.push(RawPhoto {
|
||||
source: PhotoSourceId::NaverBlog,
|
||||
image_url: url,
|
||||
page_url: page_url.to_string(),
|
||||
caption: String::new(),
|
||||
referer: Some("https://blog.naver.com/".into()),
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
// Fallback regex scan if DOM attributes differed
|
||||
if photos.is_empty() {
|
||||
for u in http::extract_image_urls(html, &[r"postfiles\.pstatic\.net", r"blogfiles\.pstatic\.net"]) {
|
||||
if let Some(url) = normalize_blog_image_url(&u) {
|
||||
if seen.insert(url.clone()) {
|
||||
photos.push(RawPhoto {
|
||||
source: PhotoSourceId::NaverBlog,
|
||||
image_url: url,
|
||||
page_url: page_url.to_string(),
|
||||
caption: String::new(),
|
||||
referer: Some("https://blog.naver.com/".into()),
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Spread paragraph context across photos proportionally
|
||||
if !paragraphs.is_empty() && !photos.is_empty() {
|
||||
let n_p = paragraphs.len();
|
||||
let n_img = photos.len();
|
||||
for (i, photo) in photos.iter_mut().enumerate() {
|
||||
let center = (i * n_p) / n_img;
|
||||
let start = center.saturating_sub(1);
|
||||
let end = (center + 2).min(n_p);
|
||||
photo.caption = paragraphs[start..end].join(" ").chars().take(220).collect();
|
||||
}
|
||||
}
|
||||
|
||||
SourceOutput { photos, texts }
|
||||
}
|
||||
|
||||
fn normalize_blog_image_url(raw: &str) -> Option<String> {
|
||||
let u = raw.replace("&", "&");
|
||||
if !(u.contains("postfiles.pstatic.net") || u.contains("blogfiles.pstatic.net")) {
|
||||
return None;
|
||||
}
|
||||
let lower = u.to_ascii_lowercase();
|
||||
if lower.contains("sticker") || lower.contains("emoticon") || lower.contains("profile") || lower.contains(".gif") {
|
||||
return None;
|
||||
}
|
||||
let base = u.split('?').next().unwrap_or(&u);
|
||||
Some(format!("{base}?type=w966"))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn parses_blog_html() {
|
||||
let html = r#"<div class="se-main-container">
|
||||
<p class="se-text-paragraph">1층에서 주문하고 2층으로 올라갔어요</p>
|
||||
<img data-lazy-src="https://postfiles.pstatic.net/2026/img1.JPEG?type=w80_blur"/>
|
||||
<p class="se-text-paragraph">창가 바 좌석이 6자리 정도 있어요</p>
|
||||
</div>"#;
|
||||
let out = parse_blog_post_html(html, "https://blog.naver.com/a/123456789012");
|
||||
assert_eq!(out.photos.len(), 1);
|
||||
assert_eq!(out.photos[0].image_url, "https://postfiles.pstatic.net/2026/img1.JPEG?type=w966");
|
||||
assert!(out.photos[0].caption.contains("2층"));
|
||||
assert_eq!(out.texts.len(), 1);
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user