245 lines
8.5 KiB
Rust
245 lines
8.5 KiB
Rust
//! Naver Blog posts about the place: inline images + body text (unofficial, verified 2026-11-04).
|
|
//!
|
|
//! 1. Search `https://search.naver.com/search.naver?ssc=tab.blog.all&query=...` for `blog.naver.com/{blogId}/{logNo}` links.
|
|
//! 2. Fetch each post via `https://blog.naver.com/PostView.naver?blogId=...&logNo=...` (bypasses outer iframe).
|
|
//! 3. Parse SmartEditor (`.se-main-container`) paragraphs and `postfiles.pstatic.net` images, pairing each image
|
|
//! with nearby paragraph text as its caption.
|
|
|
|
use std::collections::HashSet;
|
|
|
|
use async_trait::async_trait;
|
|
use domain::{PhotoSourceId, Place, RawPhoto, SourceOutput, TextSnippet};
|
|
use futures::stream::{self, StreamExt};
|
|
use regex::Regex;
|
|
use scraper::{Html, Selector};
|
|
|
|
use crate::{http, PhotoSource};
|
|
|
|
const SEARCH_URL: &str = "https://search.naver.com/search.naver";
|
|
const POST_VIEW_URL: &str = "https://blog.naver.com/PostView.naver";
|
|
|
|
pub struct NaverBlog {
|
|
client: reqwest::Client,
|
|
max_posts: usize,
|
|
}
|
|
|
|
impl Default for NaverBlog {
|
|
fn default() -> Self {
|
|
Self::new()
|
|
}
|
|
}
|
|
|
|
impl NaverBlog {
|
|
pub fn new() -> Self {
|
|
Self { client: http::scraping_client(), max_posts: 8 }
|
|
}
|
|
|
|
async fn search_post_ids(&self, query: &str) -> anyhow::Result<Vec<(String, String)>> {
|
|
let html = self
|
|
.client
|
|
.get(SEARCH_URL)
|
|
.header(reqwest::header::REFERER, "https://search.naver.com/")
|
|
.query(&[("ssc", "tab.blog.all"), ("query", query)])
|
|
.send()
|
|
.await?
|
|
.error_for_status()?
|
|
.text()
|
|
.await?;
|
|
Ok(extract_blog_post_ids(&html))
|
|
}
|
|
|
|
async fn fetch_post(&self, blog_id: &str, log_no: &str) -> anyhow::Result<SourceOutput> {
|
|
let page_url = format!("https://blog.naver.com/{blog_id}/{log_no}");
|
|
let html = self
|
|
.client
|
|
.get(POST_VIEW_URL)
|
|
.header(reqwest::header::REFERER, "https://blog.naver.com/")
|
|
.query(&[("blogId", blog_id), ("logNo", log_no)])
|
|
.send()
|
|
.await?
|
|
.error_for_status()?
|
|
.text()
|
|
.await?;
|
|
Ok(parse_blog_post_html(&html, &page_url))
|
|
}
|
|
}
|
|
|
|
#[async_trait]
|
|
impl PhotoSource for NaverBlog {
|
|
fn id(&self) -> PhotoSourceId {
|
|
PhotoSourceId::NaverBlog
|
|
}
|
|
|
|
async fn collect(&self, place: &Place, limit: usize) -> anyhow::Result<SourceOutput> {
|
|
let region = place.region_hint();
|
|
let q1 = format!("{} {} 내부 좌석", region, place.name);
|
|
let q2 = format!("{} {}", region, place.name);
|
|
|
|
let mut post_ids = self.search_post_ids(q1.trim()).await.unwrap_or_default();
|
|
if post_ids.len() < self.max_posts {
|
|
for id in self.search_post_ids(q2.trim()).await.unwrap_or_default() {
|
|
if !post_ids.contains(&id) {
|
|
post_ids.push(id);
|
|
}
|
|
}
|
|
}
|
|
post_ids.truncate(self.max_posts);
|
|
|
|
let results: Vec<SourceOutput> = stream::iter(post_ids)
|
|
.map(|(b, l)| async move { self.fetch_post(&b, &l).await.ok() })
|
|
.buffer_unordered(4)
|
|
.filter_map(|x| async move { x })
|
|
.collect()
|
|
.await;
|
|
|
|
let mut seen = HashSet::new();
|
|
let mut photos = Vec::new();
|
|
let mut texts = Vec::new();
|
|
for r in results {
|
|
texts.extend(r.texts);
|
|
for p in r.photos {
|
|
if photos.len() >= limit {
|
|
break;
|
|
}
|
|
if seen.insert(p.image_url.clone()) {
|
|
photos.push(p);
|
|
}
|
|
}
|
|
}
|
|
Ok(SourceOutput { photos, texts })
|
|
}
|
|
}
|
|
|
|
pub fn extract_blog_post_ids(html: &str) -> Vec<(String, String)> {
|
|
let re = Regex::new(r"https?://(?:m\.)?blog\.naver\.com/([a-zA-Z0-9_\-]+)/(\d{10,15})").unwrap();
|
|
let mut seen = HashSet::new();
|
|
let mut out = Vec::new();
|
|
for c in re.captures_iter(html) {
|
|
let pair = (c[1].to_string(), c[2].to_string());
|
|
if seen.insert(pair.clone()) {
|
|
out.push(pair);
|
|
}
|
|
}
|
|
out
|
|
}
|
|
|
|
pub fn parse_blog_post_html(html: &str, page_url: &str) -> SourceOutput {
|
|
let doc = Html::parse_document(html);
|
|
let container_sel = Selector::parse(".se-main-container, #postViewArea").unwrap();
|
|
let p_sel = Selector::parse(".se-text-paragraph, p").unwrap();
|
|
let img_sel = Selector::parse("img").unwrap();
|
|
|
|
let mut paragraphs = Vec::new();
|
|
for container in doc.select(&container_sel) {
|
|
for p in container.select(&p_sel) {
|
|
let t = p.text().collect::<Vec<_>>().join(" ");
|
|
let clean = t.split_whitespace().collect::<Vec<_>>().join(" ");
|
|
if clean.chars().count() >= 4 {
|
|
paragraphs.push(clean);
|
|
}
|
|
}
|
|
}
|
|
if paragraphs.is_empty() {
|
|
for p in doc.select(&p_sel) {
|
|
let clean = p.text().collect::<Vec<_>>().join(" ").split_whitespace().collect::<Vec<_>>().join(" ");
|
|
if clean.chars().count() >= 4 {
|
|
paragraphs.push(clean);
|
|
}
|
|
}
|
|
}
|
|
|
|
let full_text: String = paragraphs.join("\n").chars().take(3000).collect();
|
|
let mut texts = Vec::new();
|
|
if !full_text.is_empty() {
|
|
texts.push(TextSnippet {
|
|
source: PhotoSourceId::NaverBlog,
|
|
url: page_url.to_string(),
|
|
text: full_text.clone(),
|
|
});
|
|
}
|
|
|
|
let mut seen = HashSet::new();
|
|
let mut photos = Vec::new();
|
|
for img in doc.select(&img_sel) {
|
|
let v = img.value();
|
|
let src = v
|
|
.attr("data-lazy-src")
|
|
.or_else(|| v.attr("data-src"))
|
|
.or_else(|| v.attr("src"))
|
|
.unwrap_or_default();
|
|
if let Some(url) = normalize_blog_image_url(src) {
|
|
if seen.insert(url.clone()) {
|
|
photos.push(RawPhoto {
|
|
source: PhotoSourceId::NaverBlog,
|
|
image_url: url,
|
|
page_url: page_url.to_string(),
|
|
caption: String::new(),
|
|
referer: Some("https://blog.naver.com/".into()),
|
|
});
|
|
}
|
|
}
|
|
}
|
|
// Fallback regex scan if DOM attributes differed
|
|
if photos.is_empty() {
|
|
for u in http::extract_image_urls(html, &[r"postfiles\.pstatic\.net", r"blogfiles\.pstatic\.net"]) {
|
|
if let Some(url) = normalize_blog_image_url(&u) {
|
|
if seen.insert(url.clone()) {
|
|
photos.push(RawPhoto {
|
|
source: PhotoSourceId::NaverBlog,
|
|
image_url: url,
|
|
page_url: page_url.to_string(),
|
|
caption: String::new(),
|
|
referer: Some("https://blog.naver.com/".into()),
|
|
});
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// Spread paragraph context across photos proportionally
|
|
if !paragraphs.is_empty() && !photos.is_empty() {
|
|
let n_p = paragraphs.len();
|
|
let n_img = photos.len();
|
|
for (i, photo) in photos.iter_mut().enumerate() {
|
|
let center = (i * n_p) / n_img;
|
|
let start = center.saturating_sub(1);
|
|
let end = (center + 2).min(n_p);
|
|
photo.caption = paragraphs[start..end].join(" ").chars().take(220).collect();
|
|
}
|
|
}
|
|
|
|
SourceOutput { photos, texts }
|
|
}
|
|
|
|
fn normalize_blog_image_url(raw: &str) -> Option<String> {
|
|
let u = raw.replace("&", "&");
|
|
if !(u.contains("postfiles.pstatic.net") || u.contains("blogfiles.pstatic.net")) {
|
|
return None;
|
|
}
|
|
let lower = u.to_ascii_lowercase();
|
|
if lower.contains("sticker") || lower.contains("emoticon") || lower.contains("profile") || lower.contains(".gif") {
|
|
return None;
|
|
}
|
|
let base = u.split('?').next().unwrap_or(&u);
|
|
Some(format!("{base}?type=w966"))
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
|
|
#[test]
|
|
fn parses_blog_html() {
|
|
let html = r#"<div class="se-main-container">
|
|
<p class="se-text-paragraph">1층에서 주문하고 2층으로 올라갔어요</p>
|
|
<img data-lazy-src="https://postfiles.pstatic.net/2026/img1.JPEG?type=w80_blur"/>
|
|
<p class="se-text-paragraph">창가 바 좌석이 6자리 정도 있어요</p>
|
|
</div>"#;
|
|
let out = parse_blog_post_html(html, "https://blog.naver.com/a/123456789012");
|
|
assert_eq!(out.photos.len(), 1);
|
|
assert_eq!(out.photos[0].image_url, "https://postfiles.pstatic.net/2026/img1.JPEG?type=w966");
|
|
assert!(out.photos[0].caption.contains("2층"));
|
|
assert_eq!(out.texts.len(), 1);
|
|
}
|
|
}
|