Files
make-space/crates/providers/src/naver_blog.rs
T

245 lines
8.5 KiB
Rust

//! Naver Blog posts about the place: inline images + body text (unofficial, verified 2026-11-04).
//!
//! 1. Search `https://search.naver.com/search.naver?ssc=tab.blog.all&query=...` for `blog.naver.com/{blogId}/{logNo}` links.
//! 2. Fetch each post via `https://blog.naver.com/PostView.naver?blogId=...&logNo=...` (bypasses outer iframe).
//! 3. Parse SmartEditor (`.se-main-container`) paragraphs and `postfiles.pstatic.net` images, pairing each image
//! with nearby paragraph text as its caption.
use std::collections::HashSet;
use async_trait::async_trait;
use domain::{PhotoSourceId, Place, RawPhoto, SourceOutput, TextSnippet};
use futures::stream::{self, StreamExt};
use regex::Regex;
use scraper::{Html, Selector};
use crate::{http, PhotoSource};
const SEARCH_URL: &str = "https://search.naver.com/search.naver";
const POST_VIEW_URL: &str = "https://blog.naver.com/PostView.naver";
pub struct NaverBlog {
client: reqwest::Client,
max_posts: usize,
}
impl Default for NaverBlog {
fn default() -> Self {
Self::new()
}
}
impl NaverBlog {
pub fn new() -> Self {
Self { client: http::scraping_client(), max_posts: 8 }
}
async fn search_post_ids(&self, query: &str) -> anyhow::Result<Vec<(String, String)>> {
let html = self
.client
.get(SEARCH_URL)
.header(reqwest::header::REFERER, "https://search.naver.com/")
.query(&[("ssc", "tab.blog.all"), ("query", query)])
.send()
.await?
.error_for_status()?
.text()
.await?;
Ok(extract_blog_post_ids(&html))
}
async fn fetch_post(&self, blog_id: &str, log_no: &str) -> anyhow::Result<SourceOutput> {
let page_url = format!("https://blog.naver.com/{blog_id}/{log_no}");
let html = self
.client
.get(POST_VIEW_URL)
.header(reqwest::header::REFERER, "https://blog.naver.com/")
.query(&[("blogId", blog_id), ("logNo", log_no)])
.send()
.await?
.error_for_status()?
.text()
.await?;
Ok(parse_blog_post_html(&html, &page_url))
}
}
#[async_trait]
impl PhotoSource for NaverBlog {
fn id(&self) -> PhotoSourceId {
PhotoSourceId::NaverBlog
}
async fn collect(&self, place: &Place, limit: usize) -> anyhow::Result<SourceOutput> {
let region = place.region_hint();
let q1 = format!("{} {} 내부 좌석", region, place.name);
let q2 = format!("{} {}", region, place.name);
let mut post_ids = self.search_post_ids(q1.trim()).await.unwrap_or_default();
if post_ids.len() < self.max_posts {
for id in self.search_post_ids(q2.trim()).await.unwrap_or_default() {
if !post_ids.contains(&id) {
post_ids.push(id);
}
}
}
post_ids.truncate(self.max_posts);
let results: Vec<SourceOutput> = stream::iter(post_ids)
.map(|(b, l)| async move { self.fetch_post(&b, &l).await.ok() })
.buffer_unordered(4)
.filter_map(|x| async move { x })
.collect()
.await;
let mut seen = HashSet::new();
let mut photos = Vec::new();
let mut texts = Vec::new();
for r in results {
texts.extend(r.texts);
for p in r.photos {
if photos.len() >= limit {
break;
}
if seen.insert(p.image_url.clone()) {
photos.push(p);
}
}
}
Ok(SourceOutput { photos, texts })
}
}
pub fn extract_blog_post_ids(html: &str) -> Vec<(String, String)> {
let re = Regex::new(r"https?://(?:m\.)?blog\.naver\.com/([a-zA-Z0-9_\-]+)/(\d{10,15})").unwrap();
let mut seen = HashSet::new();
let mut out = Vec::new();
for c in re.captures_iter(html) {
let pair = (c[1].to_string(), c[2].to_string());
if seen.insert(pair.clone()) {
out.push(pair);
}
}
out
}
pub fn parse_blog_post_html(html: &str, page_url: &str) -> SourceOutput {
let doc = Html::parse_document(html);
let container_sel = Selector::parse(".se-main-container, #postViewArea").unwrap();
let p_sel = Selector::parse(".se-text-paragraph, p").unwrap();
let img_sel = Selector::parse("img").unwrap();
let mut paragraphs = Vec::new();
for container in doc.select(&container_sel) {
for p in container.select(&p_sel) {
let t = p.text().collect::<Vec<_>>().join(" ");
let clean = t.split_whitespace().collect::<Vec<_>>().join(" ");
if clean.chars().count() >= 4 {
paragraphs.push(clean);
}
}
}
if paragraphs.is_empty() {
for p in doc.select(&p_sel) {
let clean = p.text().collect::<Vec<_>>().join(" ").split_whitespace().collect::<Vec<_>>().join(" ");
if clean.chars().count() >= 4 {
paragraphs.push(clean);
}
}
}
let full_text: String = paragraphs.join("\n").chars().take(3000).collect();
let mut texts = Vec::new();
if !full_text.is_empty() {
texts.push(TextSnippet {
source: PhotoSourceId::NaverBlog,
url: page_url.to_string(),
text: full_text.clone(),
});
}
let mut seen = HashSet::new();
let mut photos = Vec::new();
for img in doc.select(&img_sel) {
let v = img.value();
let src = v
.attr("data-lazy-src")
.or_else(|| v.attr("data-src"))
.or_else(|| v.attr("src"))
.unwrap_or_default();
if let Some(url) = normalize_blog_image_url(src) {
if seen.insert(url.clone()) {
photos.push(RawPhoto {
source: PhotoSourceId::NaverBlog,
image_url: url,
page_url: page_url.to_string(),
caption: String::new(),
referer: Some("https://blog.naver.com/".into()),
});
}
}
}
// Fallback regex scan if DOM attributes differed
if photos.is_empty() {
for u in http::extract_image_urls(html, &[r"postfiles\.pstatic\.net", r"blogfiles\.pstatic\.net"]) {
if let Some(url) = normalize_blog_image_url(&u) {
if seen.insert(url.clone()) {
photos.push(RawPhoto {
source: PhotoSourceId::NaverBlog,
image_url: url,
page_url: page_url.to_string(),
caption: String::new(),
referer: Some("https://blog.naver.com/".into()),
});
}
}
}
}
// Spread paragraph context across photos proportionally
if !paragraphs.is_empty() && !photos.is_empty() {
let n_p = paragraphs.len();
let n_img = photos.len();
for (i, photo) in photos.iter_mut().enumerate() {
let center = (i * n_p) / n_img;
let start = center.saturating_sub(1);
let end = (center + 2).min(n_p);
photo.caption = paragraphs[start..end].join(" ").chars().take(220).collect();
}
}
SourceOutput { photos, texts }
}
fn normalize_blog_image_url(raw: &str) -> Option<String> {
let u = raw.replace("&amp;", "&");
if !(u.contains("postfiles.pstatic.net") || u.contains("blogfiles.pstatic.net")) {
return None;
}
let lower = u.to_ascii_lowercase();
if lower.contains("sticker") || lower.contains("emoticon") || lower.contains("profile") || lower.contains(".gif") {
return None;
}
let base = u.split('?').next().unwrap_or(&u);
Some(format!("{base}?type=w966"))
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn parses_blog_html() {
let html = r#"<div class="se-main-container">
<p class="se-text-paragraph">1층에서 주문하고 2층으로 올라갔어요</p>
<img data-lazy-src="https://postfiles.pstatic.net/2026/img1.JPEG?type=w80_blur"/>
<p class="se-text-paragraph">창가 바 좌석이 6자리 정도 있어요</p>
</div>"#;
let out = parse_blog_post_html(html, "https://blog.naver.com/a/123456789012");
assert_eq!(out.photos.len(), 1);
assert_eq!(out.photos[0].image_url, "https://postfiles.pstatic.net/2026/img1.JPEG?type=w966");
assert!(out.photos[0].caption.contains("2층"));
assert_eq!(out.texts.len(), 1);
}
}