From a0d218a169747dbe7c522d8029070f7c939b156b Mon Sep 17 00:00:00 2001 From: Park Dahyeon Date: Sun, 21 Dec 2025 14:29:25 +0000 Subject: [PATCH] Implements a comprehensive web crawler microservice Introduces a complete NestJS-based web crawler microservice. This release establishes a robust system for web data extraction through a flexible Pipeline/Stage architecture. Key features include: - Supports both static (axios, cheerio) and dynamic (Playwright) page crawling. - Implements diverse triggering options: cron-based scheduling, NATS event-driven, and manual API calls. - Enhances system resilience with domain-specific rate limiting and configurable exponential backoff retry mechanisms. - Provides dedicated API endpoints and database schemas for managing pipelines, stages, and execution history. - Ensures data persistence by saving crawl results to MinIO. The `README.md` is entirely rewritten to provide comprehensive documentation on features, architecture, setup, API usage, and troubleshooting. Core dependencies for crawling, scheduling, and API validation are added, and database models are refactored to align with the new crawler system's data structures. --- README.md | 812 ++++++++++- env-vars/dev.env | 2 +- env-vars/example.env | 2 +- env-vars/prod.env | 2 +- env-vars/test.env | 2 +- package-lock.json | 467 +++++++ package.json | 10 + src/app.module.ts | 5 + src/crawler/crawler.controller.ts | 338 +++++ src/crawler/crawler.module.ts | 29 + src/crawler/crawler.service.ts | 0 src/crawler/dto/common.dto.ts | 103 ++ src/crawler/dto/create-pipelinedef.dto.ts | 39 + src/crawler/dto/create-stagedef.dto.ts | 48 + src/crawler/dto/index.ts | 5 + src/crawler/dto/trigger-pipeline.dto.ts | 21 + src/crawler/dto/update-pipelinedef.dto.ts | 4 + .../interfaces/crawler-result.interface.ts | 35 + src/crawler/services/crawler.service.ts | 291 ++++ .../services/pipeline-engine.service.ts | 289 ++++ src/crawler/services/rate-limiter.service.ts | 138 ++ src/crawler/services/retry.service.ts | 197 +++ .../services/stage-executor.service.ts | 255 ++++ src/crawler/services/trigger.service.ts | 327 +++++ src/global/database/connection.service.ts | 16 +- src/global/database/schema.ts | 1211 ++++++++++++++--- 26 files changed, 4367 insertions(+), 281 deletions(-) create mode 100644 src/crawler/crawler.controller.ts create mode 100644 src/crawler/crawler.module.ts create mode 100644 src/crawler/crawler.service.ts create mode 100644 src/crawler/dto/common.dto.ts create mode 100644 src/crawler/dto/create-pipelinedef.dto.ts create mode 100644 src/crawler/dto/create-stagedef.dto.ts create mode 100644 src/crawler/dto/index.ts create mode 100644 src/crawler/dto/trigger-pipeline.dto.ts create mode 100644 src/crawler/dto/update-pipelinedef.dto.ts create mode 100644 src/crawler/interfaces/crawler-result.interface.ts create mode 100644 src/crawler/services/crawler.service.ts create mode 100644 src/crawler/services/pipeline-engine.service.ts create mode 100644 src/crawler/services/rate-limiter.service.ts create mode 100644 src/crawler/services/retry.service.ts create mode 100644 src/crawler/services/stage-executor.service.ts create mode 100644 src/crawler/services/trigger.service.ts diff --git a/README.md b/README.md index 8f0f65f..ae6d607 100644 --- a/README.md +++ b/README.md @@ -1,98 +1,774 @@ -

- Nest Logo -

+# MSA 기반 웹 크롤러 서비스 -[circleci-image]: https://img.shields.io/circleci/build/github/nestjs/nest/master?token=abc123def456 -[circleci-url]: https://circleci.com/gh/nestjs/nest +NestJS 기반의 마이크로서비스 아키텍처 웹 크롤러 시스템입니다. 정적/동적 웹페이지 크롤링, Pipeline/Stage 기반 병렬 실행, 다중 트리거 지원, Rate Limiting 등의 기능을 제공합니다. -

A progressive Node.js framework for building efficient and scalable server-side applications.

-

-NPM Version -Package License -NPM Downloads -CircleCI -Discord -Backers on Open Collective -Sponsors on Open Collective - Donate us - Support us - Follow us on Twitter -

- +## 주요 기능 -## Description +### 크롤링 엔진 +- **정적 페이지 크롤링**: axios + cheerio 기반 HTML 파싱 +- **동적 SPA 크롤링**: Playwright 기반 JavaScript 렌더링 지원 +- **셀렉터 기반 데이터 추출**: CSS 셀렉터로 원하는 데이터만 추출 +- **속성 추출**: 이미지 URL 등 HTML 속성 추출 지원 -[Nest](https://github.com/nestjs/nest) framework TypeScript starter repository. +### Pipeline/Stage 아키텍처 +- **병렬 실행**: 여러 Stage를 동시에 실행하여 성능 최적화 +- **실행 기록**: 각 Stage의 실행 이력 및 결과 저장 +- **메타데이터 기반**: 유연한 설정 및 확장 가능한 구조 -## Project setup +### 트리거 시스템 +- **스케줄 트리거**: Cron 표현식 기반 정기 실행 +- **이벤트 트리거**: NATS 메시지 기반 이벤트 드리븐 실행 +- **API 트리거**: REST API 호출로 수동 실행 -```bash -$ npm install +### 안정성 및 성능 +- **Rate Limiting**: 도메인별 토큰 버킷 알고리즘으로 요청 제한 +- **재시도 로직**: Exponential backoff 기반 자동 재시도 +- **에러 알림**: 실패 시 NATS를 통한 알림 발송 +- **결과 저장**: MinIO에 크롤링 결과 자동 저장 + +## 기술 스택 + +### 프레임워크 및 런타임 +- **NestJS 11**: TypeScript 기반 서버 프레임워크 +- **Node.js**: JavaScript 런타임 + +### 크롤링 및 파싱 +- **axios**: HTTP 클라이언트 +- **cheerio**: 서버사이드 jQuery (HTML 파싱) +- **playwright**: 브라우저 자동화 (동적 페이지 크롤링) + +### 데이터베이스 및 스토리지 +- **MongoDB + Mongoose**: NoSQL 데이터베이스 +- **MinIO**: S3 호환 객체 스토리지 + +### 메시징 및 이벤트 +- **NATS**: 메시지 브로커 (이벤트 트리거 및 알림) + +### 스케줄링 및 인증 +- **@nestjs/schedule + cron**: 작업 스케줄링 +- **JWT (jose)**: 토큰 기반 인증 (Microsoft Entra ID 지원) +- **Infisical**: 중앙화된 비밀값 관리 + +## 아키텍처 + +``` +┌─────────────────────────────────────────────────────────────┐ +│ REST API │ +│ (JWT Authentication Required) │ +└────────────────────────┬────────────────────────────────────┘ + │ + ┌────────────────┼────────────────┐ + │ │ │ + ┌────▼─────┐ ┌──────▼──────┐ ┌────▼─────┐ + │ Cron │ │ NATS Event │ │ API │ + │ Trigger │ │ Trigger │ │ Trigger │ + └────┬─────┘ └──────┬──────┘ └────┬─────┘ + │ │ │ + └────────────────┼────────────────┘ + │ + ┌──────────▼──────────┐ + │ Pipeline Engine │ + │ (병렬 실행 관리) │ + └──────────┬──────────┘ + │ + ┌────────────────┼────────────────┐ + │ │ │ + ┌────▼─────┐ ┌──────▼──────┐ ┌────▼─────┐ + │ Stage 1 │ │ Stage 2 │ │ Stage 3 │ + │(EventSrc)│ │ (EventSrc) │ │(EventSrc)│ + └────┬─────┘ └──────┬──────┘ └────┬─────┘ + │ │ │ + ┌────▼─────┐ ┌──────▼──────┐ ┌────▼─────┐ + │ Crawler │ │ Crawler │ │ Crawler │ + │ Service │ │ Service │ │ Service │ + └────┬─────┘ └──────┬──────┘ └────┬─────┘ + │ │ │ + └────────────────┼────────────────┘ + │ + ┌──────────▼──────────┐ + │ MinIO Storage │ + │ (크롤링 결과 저장) │ + └─────────────────────┘ ``` -## Compile and run the project +## 설치 + +### 필수 요구사항 + +- **Node.js**: 18.x 이상 +- **MongoDB**: 4.x 이상 +- **NATS**: 2.x 이상 +- **MinIO**: S3 호환 객체 스토리지 +- **Infisical**: 비밀값 관리 (선택) + +### 1. 저장소 클론 및 패키지 설치 ```bash -# development -$ npm run start - -# watch mode -$ npm run start:dev - -# production mode -$ npm run start:prod +git clone https://git.dahyeon.us/dahyeon/crawler.git +cd crawler +npm install ``` -## Run tests +### 2. Playwright 브라우저 설치 ```bash -# unit tests -$ npm run test - -# e2e tests -$ npm run test:e2e - -# test coverage -$ npm run test:cov +npx playwright install chromium ``` -## Deployment +### 3. 환경 변수 설정 -When you're ready to deploy your NestJS application to production, there are some key steps you can take to ensure it runs as efficiently as possible. Check out the [deployment documentation](https://docs.nestjs.com/deployment) for more information. - -If you are looking for a cloud-based platform to deploy your NestJS application, check out [Mau](https://mau.nestjs.com), our official platform for deploying NestJS applications on AWS. Mau makes deployment straightforward and fast, requiring just a few simple steps: +환경별 `.env` 파일을 생성합니다: ```bash -$ npm install -g @nestjs/mau -$ mau deploy +# 개발 환경 +cp env-vars/example.env .env.dev + +# 테스트 환경 +cp env-vars/example.env .env.test + +# 프로덕션 환경 +cp env-vars/example.env .env.prod ``` -With Mau, you can deploy your application in just a few clicks, allowing you to focus on building features rather than managing infrastructure. +#### 필수 환경 변수 (Infisical 사용 시) -## Resources +```env +# Infisical 설정 +INFISICAL_SITE_URL=https://app.infisical.com +INFISICAL_CLIENT_ID=your-client-id +INFISICAL_CLIENT_SECRET=your-client-secret +INFISICAL_PROJECT_ID=a33f45fb-7564-4e4e-9931-96201395d110 +``` -Check out a few resources that may come in handy when working with NestJS: +Infisical을 통해 다음 값들을 관리: +- `MONGODB_CONNECTION_STRING`: MongoDB 연결 문자열 +- `NATS_ENDPOINT`: NATS 서버 주소 +- `NATS_USER`: NATS 사용자명 +- `NATS_PASS`: NATS 비밀번호 +- `JWT_CLAIM_AUDIENCE`: JWT 검증 audience +- `MICROSOFT_ENTRAID_TENANT_ID`: Azure AD 테넌트 ID +- `MINIO_ENDPOINT`: MinIO 서버 주소 +- `MINIO_ACCESS_KEY`: MinIO 액세스 키 +- `MINIO_SECRET_KEY`: MinIO 시크릿 키 -- Visit the [NestJS Documentation](https://docs.nestjs.com) to learn more about the framework. -- For questions and support, please visit our [Discord channel](https://discord.gg/G7Qnnhy). -- To dive deeper and get more hands-on experience, check out our official video [courses](https://courses.nestjs.com/). -- Deploy your application to AWS with the help of [NestJS Mau](https://mau.nestjs.com) in just a few clicks. -- Visualize your application graph and interact with the NestJS application in real-time using [NestJS Devtools](https://devtools.nestjs.com). -- Need help with your project (part-time to full-time)? Check out our official [enterprise support](https://enterprise.nestjs.com). -- To stay in the loop and get updates, follow us on [X](https://x.com/nestframework) and [LinkedIn](https://linkedin.com/company/nestjs). -- Looking for a job, or have a job to offer? Check out our official [Jobs board](https://jobs.nestjs.com). +### 4. TLS 인증서 (선택) -## Support +NATS TLS 연결을 위해 `internal-service-ca.pem` 파일을 프로젝트 루트에 배치합니다. -Nest is an MIT-licensed open source project. It can grow thanks to the sponsors and support by the amazing backers. If you'd like to join them, please [read more here](https://docs.nestjs.com/support). +## 실행 -## Stay in touch +### 개발 모드 -- Author - [Kamil Myśliwiec](https://twitter.com/kammysliwiec) -- Website - [https://nestjs.com](https://nestjs.com/) -- Twitter - [@nestframework](https://twitter.com/nestframework) +```bash +npm run start:dev +``` -## License +### 프로덕션 빌드 -Nest is [MIT licensed](https://github.com/nestjs/nest/blob/master/LICENSE). +```bash +npm run build +npm run start:prod +``` + +### 환경 전환 + +```bash +# 테스트 환경 +NODE_ENV=test npm run start:dev + +# 프로덕션 환경 +NODE_ENV=prod npm run start:prod +``` + +## REST API + +서버 실행 후 Swagger 문서: `http://localhost:3000/api` + +### 인증 + +모든 API는 JWT 인증이 필요합니다 (일부 Public 엔드포인트 제외). + +```http +Authorization: Bearer {JWT_TOKEN} +``` + +### Pipelinedef 관리 + +#### 1. Pipelinedef 생성 + +```http +POST /api/v1/crawler/pipelinedefs +Content-Type: application/json + +{ + "apiVersion": "v1", + "trigger": { + "type": "always" + }, + "stageIds": ["stage-id-1", "stage-id-2"], + "schedule": { + "cron": "0 0 * * *", + "timezone": "Asia/Seoul" + } +} +``` + +#### 2. Pipelinedef 목록 조회 + +```http +GET /api/v1/crawler/pipelinedefs?page=1&limit=20 +``` + +#### 3. Pipelinedef 상세 조회 + +```http +GET /api/v1/crawler/pipelinedefs/{id} +``` + +#### 4. Pipelinedef 수정 + +```http +PUT /api/v1/crawler/pipelinedefs/{id} +Content-Type: application/json + +{ + "schedule": { + "cron": "0 12 * * *" + } +} +``` + +#### 5. Pipelinedef 삭제 + +```http +DELETE /api/v1/crawler/pipelinedefs/{id} +``` + +### Stagedef 관리 + +#### 1. Stagedef 생성 + +```http +POST /api/v1/crawler/stagedefs +Content-Type: application/json + +{ + "apiVersion": "v1", + "name": "Example Crawler", + "topic": "crawler.results.example", + "eventSource": [ + { + "type": "web", + "name": "example-source", + "metadatas": [ + { + "key": "url", + "value": "https://example.com", + "dataType": "string" + }, + { + "key": "selectors", + "value": "{\"title\":\"h1\",\"content\":\".main\"}", + "dataType": "json" + }, + { + "key": "dynamic", + "value": "false", + "dataType": "string" + } + ] + } + ] +} +``` + +#### 2. Stagedef 목록 조회 + +```http +GET /api/v1/crawler/stagedefs?page=1&limit=20 +``` + +### Pipeline 실행 + +#### 1. Pipeline 수동 실행 + +```http +POST /api/v1/crawler/pipelines/trigger +Content-Type: application/json + +{ + "pipelineDefId": "pipeline-def-id", + "variables": { + "customVar": "value" + } +} +``` + +#### 2. 실행된 Pipeline 목록 + +```http +GET /api/v1/crawler/pipelines?page=1&limit=20&status=completed +``` + +**status 옵션**: `completed`, `failed`, `partial` + +#### 3. Pipeline 실행 상세 + +```http +GET /api/v1/crawler/pipelines/{pipelineId} +``` + +#### 4. Pipeline의 Stage 목록 + +```http +GET /api/v1/crawler/pipelines/{pipelineId}/stages +``` + +### 모니터링 + +#### 1. 등록된 스케줄 목록 + +```http +GET /api/v1/crawler/schedules +``` + +#### 2. Rate Limit 상태 + +```http +GET /api/v1/crawler/rate-limits +``` + +#### 3. 트리거 재로드 + +```http +POST /api/v1/crawler/triggers/reload +``` + +#### 4. Health Check (Public) + +```http +GET /api/v1/crawler/health +``` + +## 사용 예시 + +### 1. 정적 웹페이지 크롤링 + +```json +{ + "type": "web", + "name": "news-crawler", + "metadatas": [ + { + "key": "url", + "value": "https://news.example.com/latest", + "dataType": "string" + }, + { + "key": "selectors", + "value": "{\"title\":\"h2.article-title\",\"summary\":\".article-summary\",\"author\":\".author-name\"}", + "dataType": "json" + }, + { + "key": "dynamic", + "value": "false", + "dataType": "string" + } + ] +} +``` + +### 2. 동적 SPA 페이지 크롤링 + +```json +{ + "type": "web", + "name": "spa-crawler", + "metadatas": [ + { + "key": "url", + "value": "https://spa-app.example.com/products", + "dataType": "string" + }, + { + "key": "selectors", + "value": "{\"products\":\".product-item\",\"price\":\".price\"}", + "dataType": "json" + }, + { + "key": "dynamic", + "value": "true", + "dataType": "string" + }, + { + "key": "waitSelector", + "value": ".products-loaded", + "dataType": "string" + } + ] +} +``` + +### 3. 이미지 URL 추출 + +```json +{ + "type": "web", + "name": "image-crawler", + "metadatas": [ + { + "key": "url", + "value": "https://gallery.example.com", + "dataType": "string" + }, + { + "key": "selectors", + "value": "{\"title\":\"h1.gallery-title\"}", + "dataType": "json" + }, + { + "key": "extractAttributes", + "value": "{\"imageUrl\":\"img.gallery-image@src\",\"thumbnail\":\"img.thumb@data-src\"}", + "dataType": "json" + }, + { + "key": "dynamic", + "value": "false", + "dataType": "string" + } + ] +} +``` + +### 4. 매일 자정 실행하는 스케줄 설정 + +```json +{ + "apiVersion": "v1", + "trigger": { + "type": "always" + }, + "stageIds": ["stage-id-1"], + "schedule": { + "cron": "0 0 * * *", + "timezone": "Asia/Seoul" + } +} +``` + +### 5. NATS 이벤트로 트리거 + +```json +{ + "apiVersion": "v1", + "trigger": { + "type": "kafka-event", + "kafkaEvent": { + "topic": "crawler.trigger.news" + } + }, + "stageIds": ["stage-id-1"] +} +``` + +## 프로젝트 구조 + +``` +src/ +├── main.ts # 애플리케이션 진입점 +├── app.module.ts # 루트 모듈 +│ +├── global/ # 글로벌 공유 모듈 +│ ├── auth/ # JWT 인증 +│ │ ├── jwt.service.ts +│ │ ├── jwt.guard.ts +│ │ └── decorators... +│ ├── secret/ # Infisical 비밀값 관리 +│ │ └── secret.service.ts +│ ├── database/ # MongoDB 연결 +│ │ ├── connection.service.ts +│ │ ├── schema.ts # Mongoose 스키마 +│ │ └── types.d.ts +│ ├── nats/ # NATS 메시징 +│ │ └── nats.service.ts +│ └── minio/ # MinIO 파일 저장소 +│ └── minio.service.ts +│ +└── crawler/ # 크롤러 비즈니스 로직 + ├── crawler.module.ts + ├── crawler.controller.ts # REST API 컨트롤러 + │ + ├── services/ + │ ├── crawler.service.ts # HTTP + Playwright 크롤링 + │ ├── rate-limiter.service.ts # Rate Limiting + │ ├── retry.service.ts # 재시도 로직 + │ ├── stage-executor.service.ts # Stage 실행 + │ ├── pipeline-engine.service.ts # Pipeline 관리 + │ └── trigger.service.ts # 트리거 시스템 + │ + ├── dto/ + │ ├── common.dto.ts + │ ├── create-pipelinedef.dto.ts + │ ├── create-stagedef.dto.ts + │ └── trigger-pipeline.dto.ts + │ + └── interfaces/ + └── crawler-result.interface.ts +``` + +## 데이터베이스 스키마 + +### Pipelinedef +파이프라인 정의 (재사용 가능한 템플릿) + +```typescript +{ + apiVersion: string; // "v1" + trigger: PipelineTrigger; // always | kafka-event + stage: ObjectId[]; // Stagedef ID 배열 + metadatas: Metadata[]; // 커스텀 메타데이터 +} +``` + +### Pipeline +실행된 파이프라인 인스턴스 + +```typescript +{ + apiVersion: string; + name: string; // "pipeline-1234567890" + stage: Stage[]; // 실행 중인 Stage 배열 + metadatas: Metadata[]; // 실행 정보 (triggeredBy, status 등) + createdAt: Date; + updatedAt: Date; +} +``` + +### Stagedef +스테이지 정의 + +```typescript +{ + apiVersion: string; + name: string; + taxonomy?: string; + eventSource: EventSource[]; // 크롤링 대상 설정 + topic: string; // NATS 발행 토픽 + metadatas: Metadata[]; +} +``` + +### Stage +실행된 스테이지 + +```typescript +{ + apiVersion: string; + name: string; + executions: Execution[]; // 실행 기록 + status: "pending" | "in-progress" | "completed" | "failed" | "cancelled" | "skipped"; + metadatas: Metadata[]; +} +``` + +### Execution +개별 실행 기록 + +```typescript +{ + status: "pending" | "in-progress" | "completed" | "failed" | "cancelled" | "skipped"; + metadatas: Metadata[]; // crawlResult, storageKey 등 + createdAt: Date; + updatedAt: Date; +} +``` + +## Rate Limiting + +### 도메인별 제한 설정 + +```typescript +// 기본값: 10 req, 초당 2개 보충 +rateLimiter.setLimit('example.com', { + maxTokens: 20, // 최대 20개 토큰 + refillRate: 5 // 초당 5개 보충 +}); +``` + +### 현재 상태 조회 + +```http +GET /api/v1/crawler/rate-limits +``` + +응답: +```json +{ + "example.com": { + "config": { + "maxTokens": 20, + "refillRate": 5 + }, + "available": 15 + } +} +``` + +## 재시도 전략 + +### RetryStrategy 설정 + +```typescript +{ + limit: 3, // 최대 3회 재시도 + delay: 1000, // 초기 대기 시간 1초 + backoff: 2, // 2배씩 증가 (1초 → 2초 → 4초) + when: [ + "failed", // 모든 실패 + "5xx", // 5xx 에러 + "network" // 네트워크 에러 + ], + notify: [ + "admin-channel" // 실패 시 알림 채널 + ] +} +``` + +## NATS 이벤트 + +### 발행되는 이벤트 + +#### Pipeline 시작 +``` +Topic: crawler.pipeline.started +Payload: { + pipelineId: string; + pipelineDefId: string; + triggeredBy: "api" | "schedule" | "nats"; + timestamp: string; +} +``` + +#### Pipeline 완료 +``` +Topic: crawler.pipeline.completed +Payload: { + pipelineId: string; + status: "completed" | "failed" | "partial"; + triggeredBy: "api" | "schedule" | "nats"; + stageResults: Array<{ + stageName: string; + status: string; + error?: string; + }>; + completedAt: string; +} +``` + +#### Stage 시작 +``` +Topic: crawler.stage.started +Payload: { + pipelineId: string; + stageIndex: number; + stageName: string; + timestamp: string; +} +``` + +#### Stage 완료 (커스텀 토픽) +``` +Topic: {stagedef.topic} +Payload: { + pipelineId: string; + stageIndex: number; + stageName: string; + status: "completed" | "failed"; + results: Array<{ + url: string; + success: boolean; + parsedData: object; + }>; + timestamp: string; +} +``` + +## 개발 + +### 테스트 + +```bash +# 단위 테스트 +npm run test + +# E2E 테스트 +npm run test:e2e + +# 커버리지 +npm run test:cov +``` + +### 린트 + +```bash +npm run lint +npm run format +``` + +### 빌드 + +```bash +npm run build +``` + +## 트러블슈팅 + +### Playwright 브라우저 설치 실패 + +```bash +# 수동 설치 +npx playwright install-deps +npx playwright install chromium +``` + +### MongoDB 연결 실패 + +- `MONGODB_CONNECTION_STRING` 환경 변수 확인 +- MongoDB 서버 상태 확인 +- 네트워크 연결 및 방화벽 설정 확인 + +### NATS 연결 실패 + +- `NATS_ENDPOINT`, `NATS_USER`, `NATS_PASS` 확인 +- TLS 인증서 경로 확인 (`internal-service-ca.pem`) +- NATS 서버 상태 확인 + +### Rate Limit 초과 + +```typescript +// 도메인별 제한 조정 +await fetch('http://localhost:3000/api/v1/crawler/rate-limits'); +// 또는 코드에서 직접 설정 +rateLimiter.setLimit('slow-domain.com', { maxTokens: 5, refillRate: 1 }); +``` + +## 보안 고려사항 + +1. **JWT 인증**: 모든 API 엔드포인트는 JWT 인증 필요 +2. **비밀값 관리**: Infisical을 통한 중앙화된 비밀값 관리 +3. **TLS 통신**: NATS 연결 시 TLS 인증서 사용 +4. **Rate Limiting**: 크롤링 대상 서버 보호 +5. **Input Validation**: DTO 기반 요청 검증 + +## 라이선스 + +MIT + +## 작성자 + +- Dahyeon +- Repository: https://git.dahyeon.us/dahyeon/crawler + +## 기술 지원 + +이슈 및 문의: Repository Issues 페이지 diff --git a/env-vars/dev.env b/env-vars/dev.env index 6f0a153..bf37204 100644 --- a/env-vars/dev.env +++ b/env-vars/dev.env @@ -1 +1 @@ -INFISICAL_PROJECT_ID=1ebbd0b9-7f76-40fb-91c3-9cf0f12a1510 +INFISICAL_PROJECT_ID=a33f45fb-7564-4e4e-9931-96201395d110 \ No newline at end of file diff --git a/env-vars/example.env b/env-vars/example.env index 36bce0a..fbebf71 100644 --- a/env-vars/example.env +++ b/env-vars/example.env @@ -1,3 +1,3 @@ INFISICAL_CLIENT_SECRET= INFISICAL_CLIENT_ID= -INFISICAL_PROJECT_ID=1ebbd0b9-7f76-40fb-91c3-9cf0f12a1510 \ No newline at end of file +INFISICAL_PROJECT_ID=a33f45fb-7564-4e4e-9931-96201395d110 \ No newline at end of file diff --git a/env-vars/prod.env b/env-vars/prod.env index 0543dd8..bf37204 100644 --- a/env-vars/prod.env +++ b/env-vars/prod.env @@ -1 +1 @@ -INFISICAL_PROJECT_ID=1ebbd0b9-7f76-40fb-91c3-9cf0f12a1510 \ No newline at end of file +INFISICAL_PROJECT_ID=a33f45fb-7564-4e4e-9931-96201395d110 \ No newline at end of file diff --git a/env-vars/test.env b/env-vars/test.env index 0543dd8..bf37204 100644 --- a/env-vars/test.env +++ b/env-vars/test.env @@ -1 +1 @@ -INFISICAL_PROJECT_ID=1ebbd0b9-7f76-40fb-91c3-9cf0f12a1510 \ No newline at end of file +INFISICAL_PROJECT_ID=a33f45fb-7564-4e4e-9931-96201395d110 \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index 2046176..e8d18b5 100644 --- a/package-lock.json +++ b/package-lock.json @@ -14,10 +14,18 @@ "@nestjs/config": "^4.0.2", "@nestjs/core": "^11.0.1", "@nestjs/platform-express": "^11.0.1", + "@nestjs/schedule": "^6.1.0", "@nestjs/swagger": "^11.2.3", + "axios": "^1.13.2", + "cheerio": "^1.1.2", + "class-transformer": "^0.5.1", + "class-validator": "^0.14.3", + "cron": "^4.3.5", + "jose": "^6.1.3", "minio": "^8.0.6", "mongoose": "^9.0.2", "nats": "^2.29.3", + "playwright": "^1.57.0", "reflect-metadata": "^0.2.2", "rxjs": "^7.8.1", "temporal-polyfill": "^0.3.0" @@ -28,8 +36,10 @@ "@nestjs/cli": "^11.0.0", "@nestjs/schematics": "^11.0.0", "@nestjs/testing": "^11.0.1", + "@types/cheerio": "^0.22.35", "@types/express": "^5.0.0", "@types/jest": "^30.0.0", + "@types/multer": "^2.0.0", "@types/node": "^24.10.4", "@types/supertest": "^6.0.2", "eslint": "^9.18.0", @@ -5818,6 +5828,19 @@ "@nestjs/core": "^11.0.0" } }, + "node_modules/@nestjs/schedule": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/@nestjs/schedule/-/schedule-6.1.0.tgz", + "integrity": "sha512-W25Ydc933Gzb1/oo7+bWzzDiOissE+h/dhIAPugA39b9MuIzBbLybuXpc1AjoQLczO3v0ldmxaffVl87W0uqoQ==", + "license": "MIT", + "dependencies": { + "cron": "4.3.5" + }, + "peerDependencies": { + "@nestjs/common": "^10.0.0 || ^11.0.0", + "@nestjs/core": "^10.0.0 || ^11.0.0" + } + }, "node_modules/@nestjs/schematics": { "version": "11.0.9", "resolved": "https://registry.npmjs.org/@nestjs/schematics/-/schematics-11.0.9.tgz", @@ -6243,6 +6266,16 @@ "@types/node": "*" } }, + "node_modules/@types/cheerio": { + "version": "0.22.35", + "resolved": "https://registry.npmjs.org/@types/cheerio/-/cheerio-0.22.35.tgz", + "integrity": "sha512-yD57BchKRvTV+JD53UZ6PD8KWY5g5rvvMLRnZR3EQBCZXiDT/HR+pKpMzFGlWNhFrXlo7VPZXtKvIEwZkAWOIA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@types/connect": { "version": "3.4.38", "resolved": "https://registry.npmjs.org/@types/connect/-/connect-3.4.38.tgz", @@ -6367,6 +6400,12 @@ "dev": true, "license": "MIT" }, + "node_modules/@types/luxon": { + "version": "3.7.1", + "resolved": "https://registry.npmjs.org/@types/luxon/-/luxon-3.7.1.tgz", + "integrity": "sha512-H3iskjFIAn5SlJU7OuxUmTEpebK6TKB8rxZShDslBMZJ5u9S//KM1sbdAisiSrqwLQncVjnpi2OK2J51h+4lsg==", + "license": "MIT" + }, "node_modules/@types/methods": { "version": "1.1.4", "resolved": "https://registry.npmjs.org/@types/methods/-/methods-1.1.4.tgz", @@ -6374,6 +6413,16 @@ "dev": true, "license": "MIT" }, + "node_modules/@types/multer": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/@types/multer/-/multer-2.0.0.tgz", + "integrity": "sha512-C3Z9v9Evij2yST3RSBktxP9STm6OdMc5uR1xF1SGr98uv8dUlAL2hqwrZ3GVB3uyMyiegnscEK6PGtYvNrjTjw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/express": "*" + } + }, "node_modules/@types/node": { "version": "24.10.4", "resolved": "https://registry.npmjs.org/@types/node/-/node-24.10.4.tgz", @@ -6451,6 +6500,12 @@ "@types/superagent": "^8.1.0" } }, + "node_modules/@types/validator": { + "version": "13.15.10", + "resolved": "https://registry.npmjs.org/@types/validator/-/validator-13.15.10.tgz", + "integrity": "sha512-T8L6i7wCuyoK8A/ZeLYt1+q0ty3Zb9+qbSSvrIVitzT3YjZqkTZ40IbRsPanlB4h1QB3JVL1SYCdR6ngtFYcuA==", + "license": "MIT" + }, "node_modules/@types/webidl-conversions": { "version": "7.0.3", "resolved": "https://registry.npmjs.org/@types/webidl-conversions/-/webidl-conversions-7.0.3.tgz", @@ -7673,6 +7728,12 @@ "url": "https://opencollective.com/express" } }, + "node_modules/boolbase": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz", + "integrity": "sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww==", + "license": "ISC" + }, "node_modules/bowser": { "version": "2.13.1", "resolved": "https://registry.npmjs.org/bowser/-/bowser-2.13.1.tgz", @@ -7958,6 +8019,48 @@ "dev": true, "license": "MIT" }, + "node_modules/cheerio": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/cheerio/-/cheerio-1.1.2.tgz", + "integrity": "sha512-IkxPpb5rS/d1IiLbHMgfPuS0FgiWTtFIm/Nj+2woXDLTZ7fOT2eqzgYbdMlLweqlHbsZjxEChoVK+7iph7jyQg==", + "license": "MIT", + "dependencies": { + "cheerio-select": "^2.1.0", + "dom-serializer": "^2.0.0", + "domhandler": "^5.0.3", + "domutils": "^3.2.2", + "encoding-sniffer": "^0.2.1", + "htmlparser2": "^10.0.0", + "parse5": "^7.3.0", + "parse5-htmlparser2-tree-adapter": "^7.1.0", + "parse5-parser-stream": "^7.1.2", + "undici": "^7.12.0", + "whatwg-mimetype": "^4.0.0" + }, + "engines": { + "node": ">=20.18.1" + }, + "funding": { + "url": "https://github.com/cheeriojs/cheerio?sponsor=1" + } + }, + "node_modules/cheerio-select": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/cheerio-select/-/cheerio-select-2.1.0.tgz", + "integrity": "sha512-9v9kG0LvzrlcungtnJtpGNxY+fzECQKhK4EGJX2vByejiMX84MFNQw4UxPJl3bFbTMw+Dfs37XaIkCwTZfLh4g==", + "license": "BSD-2-Clause", + "dependencies": { + "boolbase": "^1.0.0", + "css-select": "^5.1.0", + "css-what": "^6.1.0", + "domelementtype": "^2.3.0", + "domhandler": "^5.0.3", + "domutils": "^3.0.1" + }, + "funding": { + "url": "https://github.com/sponsors/fb55" + } + }, "node_modules/chokidar": { "version": "4.0.3", "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-4.0.3.tgz", @@ -8008,6 +8111,25 @@ "dev": true, "license": "MIT" }, + "node_modules/class-transformer": { + "version": "0.5.1", + "resolved": "https://registry.npmjs.org/class-transformer/-/class-transformer-0.5.1.tgz", + "integrity": "sha512-SQa1Ws6hUbfC98vKGxZH3KFY0Y1lm5Zm0SY8XX9zbK7FJCyVEac3ATW0RIpwzW+oOfmHE5PMPufDG9hCfoEOMw==", + "license": "MIT", + "peer": true + }, + "node_modules/class-validator": { + "version": "0.14.3", + "resolved": "https://registry.npmjs.org/class-validator/-/class-validator-0.14.3.tgz", + "integrity": "sha512-rXXekcjofVN1LTOSw+u4u9WXVEUvNBVjORW154q/IdmYWy1nMbOU9aNtZB0t8m+FJQ9q91jlr2f9CwwUFdFMRA==", + "license": "MIT", + "peer": true, + "dependencies": { + "@types/validator": "^13.15.3", + "libphonenumber-js": "^1.11.1", + "validator": "^13.15.20" + } + }, "node_modules/cli-cursor": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/cli-cursor/-/cli-cursor-3.1.0.tgz", @@ -8327,6 +8449,19 @@ "dev": true, "license": "MIT" }, + "node_modules/cron": { + "version": "4.3.5", + "resolved": "https://registry.npmjs.org/cron/-/cron-4.3.5.tgz", + "integrity": "sha512-hKPP7fq1+OfyCqoePkKfVq7tNAdFwiQORr4lZUHwrf0tebC65fYEeWgOrXOL6prn1/fegGOdTfrM6e34PJfksg==", + "license": "MIT", + "dependencies": { + "@types/luxon": "~3.7.0", + "luxon": "~3.7.0" + }, + "engines": { + "node": ">=18.x" + } + }, "node_modules/cross-spawn": { "version": "7.0.6", "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", @@ -8342,6 +8477,34 @@ "node": ">= 8" } }, + "node_modules/css-select": { + "version": "5.2.2", + "resolved": "https://registry.npmjs.org/css-select/-/css-select-5.2.2.tgz", + "integrity": "sha512-TizTzUddG/xYLA3NXodFM0fSbNizXjOKhqiQQwvhlspadZokn1KDy0NZFS0wuEubIYAV5/c1/lAr0TaaFXEXzw==", + "license": "BSD-2-Clause", + "dependencies": { + "boolbase": "^1.0.0", + "css-what": "^6.1.0", + "domhandler": "^5.0.2", + "domutils": "^3.0.1", + "nth-check": "^2.0.1" + }, + "funding": { + "url": "https://github.com/sponsors/fb55" + } + }, + "node_modules/css-what": { + "version": "6.2.2", + "resolved": "https://registry.npmjs.org/css-what/-/css-what-6.2.2.tgz", + "integrity": "sha512-u/O3vwbptzhMs3L1fQE82ZSLHQQfto5gyZzwteVIEyeaY5Fc7R4dapF/BvRoSYFeqfBk4m0V1Vafq5Pjv25wvA==", + "license": "BSD-2-Clause", + "engines": { + "node": ">= 6" + }, + "funding": { + "url": "https://github.com/sponsors/fb55" + } + }, "node_modules/debug": { "version": "4.4.3", "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", @@ -8479,6 +8642,61 @@ "node": ">=0.3.1" } }, + "node_modules/dom-serializer": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-2.0.0.tgz", + "integrity": "sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg==", + "license": "MIT", + "dependencies": { + "domelementtype": "^2.3.0", + "domhandler": "^5.0.2", + "entities": "^4.2.0" + }, + "funding": { + "url": "https://github.com/cheeriojs/dom-serializer?sponsor=1" + } + }, + "node_modules/domelementtype": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-2.3.0.tgz", + "integrity": "sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fb55" + } + ], + "license": "BSD-2-Clause" + }, + "node_modules/domhandler": { + "version": "5.0.3", + "resolved": "https://registry.npmjs.org/domhandler/-/domhandler-5.0.3.tgz", + "integrity": "sha512-cgwlv/1iFQiFnU96XXgROh8xTeetsnJiDsTc7TYCLFd9+/WNkIqPTxiM/8pSd8VIrhXGTf1Ny1q1hquVqDJB5w==", + "license": "BSD-2-Clause", + "dependencies": { + "domelementtype": "^2.3.0" + }, + "engines": { + "node": ">= 4" + }, + "funding": { + "url": "https://github.com/fb55/domhandler?sponsor=1" + } + }, + "node_modules/domutils": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/domutils/-/domutils-3.2.2.tgz", + "integrity": "sha512-6kZKyUajlDuqlHKVX1w7gyslj9MPIXzIFiz/rGu35uC1wMi+kMhQwGhl4lt9unC9Vb9INnY9Z3/ZA3+FhASLaw==", + "license": "BSD-2-Clause", + "dependencies": { + "dom-serializer": "^2.0.0", + "domelementtype": "^2.3.0", + "domhandler": "^5.0.3" + }, + "funding": { + "url": "https://github.com/fb55/domutils?sponsor=1" + } + }, "node_modules/dotenv": { "version": "16.4.7", "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.4.7.tgz", @@ -8569,6 +8787,31 @@ "node": ">= 0.8" } }, + "node_modules/encoding-sniffer": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/encoding-sniffer/-/encoding-sniffer-0.2.1.tgz", + "integrity": "sha512-5gvq20T6vfpekVtqrYQsSCFZ1wEg5+wW0/QaZMWkFr6BqD3NfKs0rLCx4rrVlSWJeZb5NBJgVLswK/w2MWU+Gw==", + "license": "MIT", + "dependencies": { + "iconv-lite": "^0.6.3", + "whatwg-encoding": "^3.1.1" + }, + "funding": { + "url": "https://github.com/fb55/encoding-sniffer?sponsor=1" + } + }, + "node_modules/encoding-sniffer/node_modules/iconv-lite": { + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz", + "integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==", + "license": "MIT", + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/enhanced-resolve": { "version": "5.18.4", "resolved": "https://registry.npmjs.org/enhanced-resolve/-/enhanced-resolve-5.18.4.tgz", @@ -8583,6 +8826,18 @@ "node": ">=10.13.0" } }, + "node_modules/entities": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/entities/-/entities-4.5.0.tgz", + "integrity": "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw==", + "license": "BSD-2-Clause", + "engines": { + "node": ">=0.12" + }, + "funding": { + "url": "https://github.com/fb55/entities?sponsor=1" + } + }, "node_modules/error-ex": { "version": "1.3.4", "resolved": "https://registry.npmjs.org/error-ex/-/error-ex-1.3.4.tgz", @@ -9713,6 +9968,37 @@ "dev": true, "license": "MIT" }, + "node_modules/htmlparser2": { + "version": "10.0.0", + "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-10.0.0.tgz", + "integrity": "sha512-TwAZM+zE5Tq3lrEHvOlvwgj1XLWQCtaaibSN11Q+gGBAS7Y1uZSWwXXRe4iF6OXnaq1riyQAPFOBtYc77Mxq0g==", + "funding": [ + "https://github.com/fb55/htmlparser2?sponsor=1", + { + "type": "github", + "url": "https://github.com/sponsors/fb55" + } + ], + "license": "MIT", + "dependencies": { + "domelementtype": "^2.3.0", + "domhandler": "^5.0.3", + "domutils": "^3.2.1", + "entities": "^6.0.0" + } + }, + "node_modules/htmlparser2/node_modules/entities": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/entities/-/entities-6.0.1.tgz", + "integrity": "sha512-aN97NXWF6AWBTahfVOIrB/NShkzi5H7F9r1s9mD3cDj4Ko5f2qhhVoYMibXF7GlLveb/D2ioWay8lxI97Ven3g==", + "license": "BSD-2-Clause", + "engines": { + "node": ">=0.12" + }, + "funding": { + "url": "https://github.com/fb55/entities?sponsor=1" + } + }, "node_modules/http-errors": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.1.tgz", @@ -10892,6 +11178,15 @@ "url": "https://github.com/chalk/supports-color?sponsor=1" } }, + "node_modules/jose": { + "version": "6.1.3", + "resolved": "https://registry.npmjs.org/jose/-/jose-6.1.3.tgz", + "integrity": "sha512-0TpaTfihd4QMNwrz/ob2Bp7X04yuxJkjRGi4aKmOqwhov54i6u79oCv7T+C7lo70MKH6BesI3vscD1yb/yzKXQ==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/panva" + } + }, "node_modules/js-tokens": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", @@ -11028,6 +11323,12 @@ "node": ">= 0.8.0" } }, + "node_modules/libphonenumber-js": { + "version": "1.12.33", + "resolved": "https://registry.npmjs.org/libphonenumber-js/-/libphonenumber-js-1.12.33.tgz", + "integrity": "sha512-r9kw4OA6oDO4dPXkOrXTkArQAafIKAU71hChInV4FxZ69dxCfbwQGDPzqR5/vea94wU705/3AZroEbSoeVWrQw==", + "license": "MIT" + }, "node_modules/lines-and-columns": { "version": "1.2.4", "resolved": "https://registry.npmjs.org/lines-and-columns/-/lines-and-columns-1.2.4.tgz", @@ -11131,6 +11432,15 @@ "yallist": "^3.0.2" } }, + "node_modules/luxon": { + "version": "3.7.2", + "resolved": "https://registry.npmjs.org/luxon/-/luxon-3.7.2.tgz", + "integrity": "sha512-vtEhXh/gNjI9Yg1u4jX/0YVPMvxzHuGgCm6tC5kZyb08yjGWGnqAjGJvcXbqQR2P3MyMEFnRbpcdFS6PBcLqew==", + "license": "MIT", + "engines": { + "node": ">=12" + } + }, "node_modules/magic-string": { "version": "0.30.17", "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.30.17.tgz", @@ -11706,6 +12016,18 @@ "node": ">=8" } }, + "node_modules/nth-check": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/nth-check/-/nth-check-2.1.1.tgz", + "integrity": "sha512-lqjrjmaOoAnWfMmBPL+XNnynZh2+swxiX3WUE0s4yEHI6m+AwrK2UZOimIRl3X/4QctVqS8AiZjFqyOGrMXb/w==", + "license": "BSD-2-Clause", + "dependencies": { + "boolbase": "^1.0.0" + }, + "funding": { + "url": "https://github.com/fb55/nth-check?sponsor=1" + } + }, "node_modules/object-assign": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", @@ -11887,6 +12209,55 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/parse5": { + "version": "7.3.0", + "resolved": "https://registry.npmjs.org/parse5/-/parse5-7.3.0.tgz", + "integrity": "sha512-IInvU7fabl34qmi9gY8XOVxhYyMyuH2xUNpb2q8/Y+7552KlejkRvqvD19nMoUW/uQGGbqNpA6Tufu5FL5BZgw==", + "license": "MIT", + "dependencies": { + "entities": "^6.0.0" + }, + "funding": { + "url": "https://github.com/inikulin/parse5?sponsor=1" + } + }, + "node_modules/parse5-htmlparser2-tree-adapter": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/parse5-htmlparser2-tree-adapter/-/parse5-htmlparser2-tree-adapter-7.1.0.tgz", + "integrity": "sha512-ruw5xyKs6lrpo9x9rCZqZZnIUntICjQAd0Wsmp396Ul9lN/h+ifgVV1x1gZHi8euej6wTfpqX8j+BFQxF0NS/g==", + "license": "MIT", + "dependencies": { + "domhandler": "^5.0.3", + "parse5": "^7.0.0" + }, + "funding": { + "url": "https://github.com/inikulin/parse5?sponsor=1" + } + }, + "node_modules/parse5-parser-stream": { + "version": "7.1.2", + "resolved": "https://registry.npmjs.org/parse5-parser-stream/-/parse5-parser-stream-7.1.2.tgz", + "integrity": "sha512-JyeQc9iwFLn5TbvvqACIF/VXG6abODeB3Fwmv/TGdLk2LfbWkaySGY72at4+Ty7EkPZj854u4CrICqNk2qIbow==", + "license": "MIT", + "dependencies": { + "parse5": "^7.0.0" + }, + "funding": { + "url": "https://github.com/inikulin/parse5?sponsor=1" + } + }, + "node_modules/parse5/node_modules/entities": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/entities/-/entities-6.0.1.tgz", + "integrity": "sha512-aN97NXWF6AWBTahfVOIrB/NShkzi5H7F9r1s9mD3cDj4Ko5f2qhhVoYMibXF7GlLveb/D2ioWay8lxI97Ven3g==", + "license": "BSD-2-Clause", + "engines": { + "node": ">=0.12" + }, + "funding": { + "url": "https://github.com/fb55/entities?sponsor=1" + } + }, "node_modules/parseurl": { "version": "1.3.3", "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", @@ -12072,6 +12443,50 @@ "node": ">=8" } }, + "node_modules/playwright": { + "version": "1.57.0", + "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.57.0.tgz", + "integrity": "sha512-ilYQj1s8sr2ppEJ2YVadYBN0Mb3mdo9J0wQ+UuDhzYqURwSoW4n1Xs5vs7ORwgDGmyEh33tRMeS8KhdkMoLXQw==", + "license": "Apache-2.0", + "dependencies": { + "playwright-core": "1.57.0" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "fsevents": "2.3.2" + } + }, + "node_modules/playwright-core": { + "version": "1.57.0", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.57.0.tgz", + "integrity": "sha512-agTcKlMw/mjBWOnD6kFZttAAGHgi/Nw0CZ2o6JqWSbMlI219lAFLZZCyqByTsvVAJq5XA5H8cA6PrvBRpBWEuQ==", + "license": "Apache-2.0", + "bin": { + "playwright-core": "cli.js" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/playwright/node_modules/fsevents": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", + "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, "node_modules/pluralize": { "version": "8.0.0", "resolved": "https://registry.npmjs.org/pluralize/-/pluralize-8.0.0.tgz", @@ -12408,6 +12823,7 @@ "resolved": "https://registry.npmjs.org/rxjs/-/rxjs-7.8.2.tgz", "integrity": "sha512-dhKf903U/PQZY6boNNtAGdWbG85WAbjT/1xYoZIC7FAY0yWapOBQVsVrDl58W86//e1VpMNBtRV4MaXfdMySFA==", "license": "Apache-2.0", + "peer": true, "dependencies": { "tslib": "^2.1.0" } @@ -13702,6 +14118,15 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/undici": { + "version": "7.16.0", + "resolved": "https://registry.npmjs.org/undici/-/undici-7.16.0.tgz", + "integrity": "sha512-QEg3HPMll0o3t2ourKwOeUAZ159Kn9mx5pnzHRQO8+Wixmh88YdZRiIwat0iNzNNXn0yoEtXJqFpyW7eM8BV7g==", + "license": "MIT", + "engines": { + "node": ">=20.18.1" + } + }, "node_modules/undici-types": { "version": "7.16.0", "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.16.0.tgz", @@ -13858,6 +14283,15 @@ "node": ">=10.12.0" } }, + "node_modules/validator": { + "version": "13.15.26", + "resolved": "https://registry.npmjs.org/validator/-/validator-13.15.26.tgz", + "integrity": "sha512-spH26xU080ydGggxRyR1Yhcbgx+j3y5jbNXk/8L+iRvdIEQ4uTRH2Sgf2dokud6Q4oAtsbNvJ1Ft+9xmm6IZcA==", + "license": "MIT", + "engines": { + "node": ">= 0.10" + } + }, "node_modules/vary": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", @@ -14115,6 +14549,39 @@ "url": "https://opencollective.com/webpack" } }, + "node_modules/whatwg-encoding": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/whatwg-encoding/-/whatwg-encoding-3.1.1.tgz", + "integrity": "sha512-6qN4hJdMwfYBtE3YBTTHhoeuUrDBPZmbQaxWAqSALV/MeEnR5z1xd8UKud2RAkFoPkmB+hli1TZSnyi84xz1vQ==", + "license": "MIT", + "dependencies": { + "iconv-lite": "0.6.3" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/whatwg-encoding/node_modules/iconv-lite": { + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz", + "integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==", + "license": "MIT", + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/whatwg-mimetype": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/whatwg-mimetype/-/whatwg-mimetype-4.0.0.tgz", + "integrity": "sha512-QaKxh0eNIi2mE9p2vEdzfagOKHCcj1pJ56EEHGQOVxp8r9/iszLUUV7v89x9O1p/T+NlTM5W7jW6+cz4Fq1YVg==", + "license": "MIT", + "engines": { + "node": ">=18" + } + }, "node_modules/whatwg-url": { "version": "14.2.0", "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-14.2.0.tgz", diff --git a/package.json b/package.json index e6d60f1..b7620ac 100644 --- a/package.json +++ b/package.json @@ -29,10 +29,18 @@ "@nestjs/config": "^4.0.2", "@nestjs/core": "^11.0.1", "@nestjs/platform-express": "^11.0.1", + "@nestjs/schedule": "^6.1.0", "@nestjs/swagger": "^11.2.3", + "axios": "^1.13.2", + "cheerio": "^1.1.2", + "class-transformer": "^0.5.1", + "class-validator": "^0.14.3", + "cron": "^4.3.5", + "jose": "^6.1.3", "minio": "^8.0.6", "mongoose": "^9.0.2", "nats": "^2.29.3", + "playwright": "^1.57.0", "reflect-metadata": "^0.2.2", "rxjs": "^7.8.1", "temporal-polyfill": "^0.3.0" @@ -43,8 +51,10 @@ "@nestjs/cli": "^11.0.0", "@nestjs/schematics": "^11.0.0", "@nestjs/testing": "^11.0.1", + "@types/cheerio": "^0.22.35", "@types/express": "^5.0.0", "@types/jest": "^30.0.0", + "@types/multer": "^2.0.0", "@types/node": "^24.10.4", "@types/supertest": "^6.0.2", "eslint": "^9.18.0", diff --git a/src/app.module.ts b/src/app.module.ts index 435870f..0f0f7bd 100644 --- a/src/app.module.ts +++ b/src/app.module.ts @@ -1,6 +1,9 @@ import { Module } from '@nestjs/common'; import { AppController } from './app.controller'; import { AppService } from './app.service'; +import { ConfigModule } from '@nestjs/config'; +import { GlobalModule } from './global/global.module'; +import { CrawlerModule } from './crawler/crawler.module'; @Module({ imports: [ @@ -13,6 +16,8 @@ import { AppService } from './app.service'; ? '.env.test' : '.env.dev', }), + GlobalModule, + CrawlerModule, ], controllers: [AppController], providers: [AppService], diff --git a/src/crawler/crawler.controller.ts b/src/crawler/crawler.controller.ts new file mode 100644 index 0000000..9c7f7a3 --- /dev/null +++ b/src/crawler/crawler.controller.ts @@ -0,0 +1,338 @@ +import { + Controller, + Get, + Post, + Put, + Delete, + Body, + Param, + Query, + HttpStatus, + HttpCode, + UseGuards, + NotFoundException, +} from '@nestjs/common'; +import mongoose from 'mongoose'; +import { + ApiTags, + ApiOperation, + ApiResponse, + ApiBearerAuth, + ApiQuery, +} from '@nestjs/swagger'; +import { JwtAuthGuard } from '../global/auth/jwt.guard'; +import { AllowPublicAccess } from '../global/auth/allow-public-access.decorator'; +import { PipelineEngineService } from './services/pipeline-engine.service'; +import { TriggerService, ScheduledJob } from './services/trigger.service'; +import { RateLimiterService } from './services/rate-limiter.service'; +import { ConnectionService } from '../global/database/connection.service'; +import { + CreatePipelinedefDto, + UpdatePipelinedefDto, + CreateStagedefDto, + TriggerPipelineDto, +} from './dto'; + +@ApiTags('Crawler') +@Controller('api/v1/crawler') +@UseGuards(JwtAuthGuard) +@ApiBearerAuth() +export class CrawlerController { + constructor( + private readonly pipelineEngine: PipelineEngineService, + private readonly triggerService: TriggerService, + private readonly rateLimiter: RateLimiterService, + private readonly connectionService: ConnectionService, + ) {} + + // ============ Pipelinedef CRUD ============ + + @Post('pipelinedefs') + @ApiOperation({ summary: 'Create a new pipeline definition' }) + @ApiResponse({ status: 201, description: 'Pipeline definition created' }) + async createPipelinedef(@Body() dto: CreatePipelinedefDto) { + const model = await this.connectionService.getMoeumModels('pipelinedefs'); + + const metadatas = dto.metadatas || []; + + // 스케줄 정보가 있으면 메타데이터에 추가 + if (dto.schedule) { + metadatas.push({ + key: 'schedule', + value: JSON.stringify(dto.schedule), + dataType: 'json', + isReserved: true, + }); + } + + const pipelinedef = await model.create({ + apiVersion: dto.apiVersion || 'v1', + trigger: dto.trigger, + stage: dto.stageIds.map((id) => new mongoose.Types.ObjectId(id)), + metadatas, + }); + + // 트리거 등록 + if (dto.schedule) { + await this.triggerService.registerScheduleTrigger( + pipelinedef._id.toString(), + dto.schedule, + ); + } + + if (dto.trigger.type === 'kafka-event' && dto.trigger.kafkaEvent) { + await this.triggerService.registerNatsTrigger( + pipelinedef._id.toString(), + dto.trigger.kafkaEvent.topic, + ); + } + + return { + id: pipelinedef._id, + message: 'Pipeline definition created', + }; + } + + @Get('pipelinedefs') + @ApiOperation({ summary: 'List all pipeline definitions' }) + @ApiQuery({ name: 'page', required: false, type: Number }) + @ApiQuery({ name: 'limit', required: false, type: Number }) + async listPipelinedefs( + @Query('page') page: number = 1, + @Query('limit') limit: number = 20, + ) { + const { items, total } = await this.pipelineEngine.listPipelinedefs( + page, + limit, + ); + + return { + items, + pagination: { + page: Number(page), + limit: Number(limit), + total, + totalPages: Math.ceil(total / limit), + }, + }; + } + + @Get('pipelinedefs/:id') + @ApiOperation({ summary: 'Get a pipeline definition by ID' }) + async getPipelinedef(@Param('id') id: string) { + const model = await this.connectionService.getMoeumModels('pipelinedefs'); + const pipelinedef = await model.findById(id).exec(); + + if (!pipelinedef) { + throw new NotFoundException('Pipeline definition not found'); + } + + return pipelinedef; + } + + @Put('pipelinedefs/:id') + @ApiOperation({ summary: 'Update a pipeline definition' }) + async updatePipelinedef( + @Param('id') id: string, + @Body() dto: UpdatePipelinedefDto, + ) { + const model = await this.connectionService.getMoeumModels('pipelinedefs'); + + const updateData: any = {}; + if (dto.apiVersion) updateData.apiVersion = dto.apiVersion; + if (dto.trigger) updateData.trigger = dto.trigger; + if (dto.stageIds) updateData.stage = dto.stageIds; + if (dto.metadatas) updateData.metadatas = dto.metadatas; + + const updated = await model + .findByIdAndUpdate(id, updateData, { new: true }) + .exec(); + + if (!updated) { + throw new NotFoundException('Pipeline definition not found'); + } + + // 스케줄 업데이트 + if (dto.schedule) { + await this.triggerService.registerScheduleTrigger(id, dto.schedule); + } + + return updated; + } + + @Delete('pipelinedefs/:id') + @HttpCode(HttpStatus.NO_CONTENT) + @ApiOperation({ summary: 'Delete a pipeline definition' }) + async deletePipelinedef(@Param('id') id: string) { + const model = await this.connectionService.getMoeumModels('pipelinedefs'); + await model.findByIdAndDelete(id).exec(); + await this.triggerService.unregisterScheduleTrigger(id); + } + + // ============ Stagedef CRUD ============ + + @Post('stagedefs') + @ApiOperation({ summary: 'Create a new stage definition' }) + async createStagedef(@Body() dto: CreateStagedefDto) { + const model = await this.connectionService.getMoeumModels('stagedefs'); + + const stagedef = await model.create({ + apiVersion: dto.apiVersion || 'v1', + name: dto.name, + taxonomy: dto.taxonomy, + eventSource: dto.eventSource, + schema: dto.schema, + topic: dto.topic, + metadatas: dto.metadatas || [], + }); + + return { + id: stagedef._id, + message: 'Stage definition created', + }; + } + + @Get('stagedefs') + @ApiOperation({ summary: 'List all stage definitions' }) + @ApiQuery({ name: 'page', required: false, type: Number }) + @ApiQuery({ name: 'limit', required: false, type: Number }) + async listStagedefs( + @Query('page') page: number = 1, + @Query('limit') limit: number = 20, + ) { + const model = await this.connectionService.getMoeumModels('stagedefs'); + const skip = (Number(page) - 1) * Number(limit); + + const [items, total] = await Promise.all([ + model.find().skip(skip).limit(Number(limit)).exec(), + model.countDocuments().exec(), + ]); + + return { + items, + pagination: { + page: Number(page), + limit: Number(limit), + total, + totalPages: Math.ceil(total / Number(limit)), + }, + }; + } + + @Get('stagedefs/:id') + @ApiOperation({ summary: 'Get a stage definition by ID' }) + async getStagedef(@Param('id') id: string) { + const model = await this.connectionService.getMoeumModels('stagedefs'); + const stagedef = await model.findById(id).exec(); + + if (!stagedef) { + throw new NotFoundException('Stage definition not found'); + } + + return stagedef; + } + + @Delete('stagedefs/:id') + @HttpCode(HttpStatus.NO_CONTENT) + @ApiOperation({ summary: 'Delete a stage definition' }) + async deleteStagedef(@Param('id') id: string) { + const model = await this.connectionService.getMoeumModels('stagedefs'); + await model.findByIdAndDelete(id).exec(); + } + + // ============ Pipeline 실행 ============ + + @Post('pipelines/trigger') + @ApiOperation({ summary: 'Trigger a pipeline execution' }) + async triggerPipeline(@Body() dto: TriggerPipelineDto) { + const pipelineId = await this.triggerService.triggerPipelineManually( + dto.pipelineDefId, + ); + return { + pipelineId, + message: 'Pipeline triggered successfully', + }; + } + + @Get('pipelines') + @ApiOperation({ summary: 'List executed pipelines' }) + @ApiQuery({ name: 'page', required: false, type: Number }) + @ApiQuery({ name: 'limit', required: false, type: Number }) + @ApiQuery({ name: 'status', required: false, type: String }) + async listPipelines( + @Query('page') page: number = 1, + @Query('limit') limit: number = 20, + @Query('status') status?: string, + ) { + const { items, total } = await this.pipelineEngine.listPipelines( + Number(page), + Number(limit), + status, + ); + + return { + items, + pagination: { + page: Number(page), + limit: Number(limit), + total, + totalPages: Math.ceil(total / Number(limit)), + }, + }; + } + + @Get('pipelines/:id') + @ApiOperation({ summary: 'Get pipeline execution details' }) + async getPipeline(@Param('id') id: string) { + const pipeline = await this.pipelineEngine.getPipelineStatus(id); + + if (!pipeline) { + throw new NotFoundException('Pipeline not found'); + } + + return pipeline; + } + + @Get('pipelines/:id/stages') + @ApiOperation({ summary: 'Get stages of a pipeline' }) + async getPipelineStages(@Param('id') id: string) { + const pipeline = await this.pipelineEngine.getPipelineStatus(id); + + if (!pipeline) { + throw new NotFoundException('Pipeline not found'); + } + + return pipeline.stage; + } + + // ============ 모니터링 ============ + + @Get('schedules') + @ApiOperation({ summary: 'List registered schedules' }) + async listSchedules(): Promise { + return this.triggerService.getRegisteredSchedules(); + } + + @Post('triggers/reload') + @ApiOperation({ summary: 'Reload all triggers' }) + async reloadTriggers() { + await this.triggerService.reloadTriggers(); + return { message: 'Triggers reloaded successfully' }; + } + + @Get('rate-limits') + @ApiOperation({ summary: 'Get rate limit status for all domains' }) + async getRateLimits() { + return this.rateLimiter.getAllLimits(); + } + + @Get('health') + @AllowPublicAccess() + @ApiOperation({ summary: 'Health check' }) + async healthCheck() { + return { + status: 'ok', + timestamp: new Date().toISOString(), + }; + } +} diff --git a/src/crawler/crawler.module.ts b/src/crawler/crawler.module.ts new file mode 100644 index 0000000..24a412c --- /dev/null +++ b/src/crawler/crawler.module.ts @@ -0,0 +1,29 @@ +import { Module } from '@nestjs/common'; +import { ScheduleModule } from '@nestjs/schedule'; +import { CrawlerController } from './crawler.controller'; +import { CrawlerService } from './services/crawler.service'; +import { RateLimiterService } from './services/rate-limiter.service'; +import { RetryService } from './services/retry.service'; +import { StageExecutorService } from './services/stage-executor.service'; +import { PipelineEngineService } from './services/pipeline-engine.service'; +import { TriggerService } from './services/trigger.service'; + +@Module({ + imports: [ScheduleModule.forRoot()], + controllers: [CrawlerController], + providers: [ + RateLimiterService, + CrawlerService, + RetryService, + StageExecutorService, + PipelineEngineService, + TriggerService, + ], + exports: [ + CrawlerService, + PipelineEngineService, + TriggerService, + RateLimiterService, + ], +}) +export class CrawlerModule {} diff --git a/src/crawler/crawler.service.ts b/src/crawler/crawler.service.ts new file mode 100644 index 0000000..e69de29 diff --git a/src/crawler/dto/common.dto.ts b/src/crawler/dto/common.dto.ts new file mode 100644 index 0000000..656a5f4 --- /dev/null +++ b/src/crawler/dto/common.dto.ts @@ -0,0 +1,103 @@ +import { + IsString, + IsOptional, + IsEnum, + IsBoolean, + ValidateNested, + IsArray, +} from 'class-validator'; +import { Type } from 'class-transformer'; +import { ApiProperty, ApiPropertyOptional } from '@nestjs/swagger'; + +export class MetadataDto { + @ApiProperty({ description: '메타데이터 키' }) + @IsString() + key: string; + + @ApiProperty({ description: '메타데이터 값' }) + value: string | any[]; + + @ApiPropertyOptional({ + enum: ['string', 'number', 'boolean', 'array', 'json'], + default: 'string', + }) + @IsOptional() + @IsEnum(['string', 'number', 'boolean', 'array', 'json']) + dataType?: 'string' | 'number' | 'boolean' | 'array' | 'json'; + + @ApiPropertyOptional({ default: false }) + @IsOptional() + @IsBoolean() + isReserved?: boolean; +} + +export class KafkaEventDto { + @ApiProperty({ description: 'Kafka/NATS 토픽명' }) + @IsString() + topic: string; + + @ApiPropertyOptional({ description: '메시지 키' }) + @IsOptional() + @IsString() + key?: string; +} + +export class PipelineTriggerDto { + @ApiProperty({ enum: ['always', 'kafka-event'] }) + @IsEnum(['always', 'kafka-event']) + type: 'always' | 'kafka-event'; + + @ApiPropertyOptional({ type: KafkaEventDto }) + @IsOptional() + @ValidateNested() + @Type(() => KafkaEventDto) + kafkaEvent?: KafkaEventDto; +} + +export class ScheduleDto { + @ApiProperty({ description: 'Cron 표현식', example: '0 0 * * *' }) + @IsString() + cron: string; + + @ApiPropertyOptional({ description: '타임존', default: 'Asia/Seoul' }) + @IsOptional() + @IsString() + timezone?: string; +} + +export class EventSourceDto { + @ApiProperty({ description: '이벤트 소스 타입', example: 'web' }) + @IsString() + type: string; + + @ApiProperty({ description: '이벤트 소스 이름' }) + @IsString() + name: string; + + @ApiPropertyOptional({ type: [MetadataDto], description: '이벤트 소스 메타데이터' }) + @IsOptional() + @IsArray() + @ValidateNested({ each: true }) + @Type(() => MetadataDto) + metadatas?: MetadataDto[]; +} + +export class PaginationQueryDto { + @ApiPropertyOptional({ default: 1, description: '페이지 번호' }) + @IsOptional() + page?: number = 1; + + @ApiPropertyOptional({ default: 20, description: '페이지당 항목 수' }) + @IsOptional() + limit?: number = 20; +} + +export class PaginationResultDto { + items: T[]; + pagination: { + page: number; + limit: number; + total: number; + totalPages: number; + }; +} diff --git a/src/crawler/dto/create-pipelinedef.dto.ts b/src/crawler/dto/create-pipelinedef.dto.ts new file mode 100644 index 0000000..5879c91 --- /dev/null +++ b/src/crawler/dto/create-pipelinedef.dto.ts @@ -0,0 +1,39 @@ +import { + IsString, + IsOptional, + IsArray, + ValidateNested, +} from 'class-validator'; +import { Type } from 'class-transformer'; +import { ApiProperty, ApiPropertyOptional } from '@nestjs/swagger'; +import { MetadataDto, PipelineTriggerDto, ScheduleDto } from './common.dto'; + +export class CreatePipelinedefDto { + @ApiPropertyOptional({ default: 'v1' }) + @IsOptional() + @IsString() + apiVersion?: string; + + @ApiProperty({ type: PipelineTriggerDto }) + @ValidateNested() + @Type(() => PipelineTriggerDto) + trigger: PipelineTriggerDto; + + @ApiProperty({ type: [String], description: 'Stagedef ID 목록' }) + @IsArray() + @IsString({ each: true }) + stageIds: string[]; + + @ApiPropertyOptional({ type: ScheduleDto, description: '스케줄 설정 (선택)' }) + @IsOptional() + @ValidateNested() + @Type(() => ScheduleDto) + schedule?: ScheduleDto; + + @ApiPropertyOptional({ type: [MetadataDto] }) + @IsOptional() + @IsArray() + @ValidateNested({ each: true }) + @Type(() => MetadataDto) + metadatas?: MetadataDto[]; +} diff --git a/src/crawler/dto/create-stagedef.dto.ts b/src/crawler/dto/create-stagedef.dto.ts new file mode 100644 index 0000000..59ca603 --- /dev/null +++ b/src/crawler/dto/create-stagedef.dto.ts @@ -0,0 +1,48 @@ +import { + IsString, + IsOptional, + IsArray, + ValidateNested, + IsObject, +} from 'class-validator'; +import { Type } from 'class-transformer'; +import { ApiProperty, ApiPropertyOptional } from '@nestjs/swagger'; +import { MetadataDto, EventSourceDto } from './common.dto'; + +export class CreateStagedefDto { + @ApiPropertyOptional({ default: 'v1' }) + @IsOptional() + @IsString() + apiVersion?: string; + + @ApiProperty({ description: 'Stage 이름' }) + @IsString() + name: string; + + @ApiPropertyOptional({ description: '분류 체계' }) + @IsOptional() + @IsString() + taxonomy?: string; + + @ApiProperty({ type: [EventSourceDto], description: '이벤트 소스 목록' }) + @IsArray() + @ValidateNested({ each: true }) + @Type(() => EventSourceDto) + eventSource: EventSourceDto[]; + + @ApiPropertyOptional({ description: '데이터 스키마 (JSON Schema)' }) + @IsOptional() + @IsObject() + schema?: any; + + @ApiProperty({ description: '결과 발행할 NATS 토픽' }) + @IsString() + topic: string; + + @ApiPropertyOptional({ type: [MetadataDto] }) + @IsOptional() + @IsArray() + @ValidateNested({ each: true }) + @Type(() => MetadataDto) + metadatas?: MetadataDto[]; +} diff --git a/src/crawler/dto/index.ts b/src/crawler/dto/index.ts new file mode 100644 index 0000000..38ab08c --- /dev/null +++ b/src/crawler/dto/index.ts @@ -0,0 +1,5 @@ +export * from './create-pipelinedef.dto'; +export * from './update-pipelinedef.dto'; +export * from './create-stagedef.dto'; +export * from './trigger-pipeline.dto'; +export * from './common.dto'; diff --git a/src/crawler/dto/trigger-pipeline.dto.ts b/src/crawler/dto/trigger-pipeline.dto.ts new file mode 100644 index 0000000..9a7714a --- /dev/null +++ b/src/crawler/dto/trigger-pipeline.dto.ts @@ -0,0 +1,21 @@ +import { IsString, IsOptional, IsObject } from 'class-validator'; +import { ApiProperty, ApiPropertyOptional } from '@nestjs/swagger'; + +export class TriggerPipelineDto { + @ApiProperty({ description: 'Pipelinedef ID' }) + @IsString() + pipelineDefId: string; + + @ApiPropertyOptional({ description: '추가 변수 (선택)' }) + @IsOptional() + @IsObject() + variables?: Record; +} + +export class TriggerPipelineResponseDto { + @ApiProperty({ description: '생성된 Pipeline ID' }) + pipelineId: string; + + @ApiProperty() + message: string; +} diff --git a/src/crawler/dto/update-pipelinedef.dto.ts b/src/crawler/dto/update-pipelinedef.dto.ts new file mode 100644 index 0000000..3522a37 --- /dev/null +++ b/src/crawler/dto/update-pipelinedef.dto.ts @@ -0,0 +1,4 @@ +import { PartialType } from '@nestjs/swagger'; +import { CreatePipelinedefDto } from './create-pipelinedef.dto'; + +export class UpdatePipelinedefDto extends PartialType(CreatePipelinedefDto) {} diff --git a/src/crawler/interfaces/crawler-result.interface.ts b/src/crawler/interfaces/crawler-result.interface.ts new file mode 100644 index 0000000..0284c75 --- /dev/null +++ b/src/crawler/interfaces/crawler-result.interface.ts @@ -0,0 +1,35 @@ +export interface CrawlResult { + success: boolean; + url: string; + statusCode?: number; + html?: string; + parsedData?: Record; + error?: string; + timestamp: Date; +} + +export interface ParserConfig { + selectors: Record; // { title: 'h1.title', content: '.article-body' } + extractAttributes?: Record; // { imageUrl: 'img.main@src' } +} + +export interface RateLimitConfig { + maxTokens: number; // 최대 토큰 수 + refillRate: number; // 초당 토큰 보충 속도 +} + +export interface ExecutionContext { + pipelineId: string; + pipelineDefId: string; + triggeredBy: 'api' | 'schedule' | 'nats'; + triggeredAt: Date; + variables: Map; +} + +export interface StageExecutionResult { + stageIndex: number; + stageName: string; + status: 'completed' | 'failed'; + crawlResults?: CrawlResult[]; + error?: string; +} diff --git a/src/crawler/services/crawler.service.ts b/src/crawler/services/crawler.service.ts new file mode 100644 index 0000000..3bb868e --- /dev/null +++ b/src/crawler/services/crawler.service.ts @@ -0,0 +1,291 @@ +import { Injectable, Logger, OnModuleDestroy } from '@nestjs/common'; +import axios, { AxiosInstance, AxiosRequestConfig } from 'axios'; +import * as cheerio from 'cheerio'; +import { chromium, Browser, BrowserContext } from 'playwright'; +import { MinioService } from '../../global/minio/minio.service'; +import { RESTRequest, EventSource } from '../../global/database/schema'; +import { CrawlResult, ParserConfig } from '../interfaces/crawler-result.interface'; +import { RateLimiterService } from './rate-limiter.service'; + +@Injectable() +export class CrawlerService implements OnModuleDestroy { + private readonly logger = new Logger(CrawlerService.name); + private readonly httpClient: AxiosInstance; + private browser: Browser | null = null; + + constructor( + private readonly minioService: MinioService, + private readonly rateLimiter: RateLimiterService, + ) { + this.httpClient = axios.create({ + timeout: 30000, + headers: { + 'User-Agent': + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36', + }, + }); + } + + async onModuleDestroy() { + if (this.browser) { + await this.browser.close(); + this.logger.log('Playwright browser closed'); + } + } + + /** + * Playwright 브라우저 인스턴스 초기화 (Lazy) + */ + private async getBrowser(): Promise { + if (!this.browser) { + this.browser = await chromium.launch({ + headless: true, + args: ['--no-sandbox', '--disable-setuid-sandbox'], + }); + this.logger.log('Playwright browser launched'); + } + return this.browser; + } + + /** + * RESTRequest 스키마 기반 HTTP 요청 실행 (정적 페이지용) + */ + async executeRequest(request: RESTRequest): Promise { + const config = this.buildAxiosConfig(request); + const domain = new URL(request.url).hostname; + + try { + // Rate limiting 적용 + await this.rateLimiter.acquire(domain); + + const response = await this.httpClient.request(config); + return { + success: true, + url: request.url, + statusCode: response.status, + html: + typeof response.data === 'string' + ? response.data + : JSON.stringify(response.data), + timestamp: new Date(), + }; + } catch (error) { + this.logger.error(`HTTP request failed: ${request.url}`, error); + return { + success: false, + url: request.url, + error: error instanceof Error ? error.message : 'Unknown error', + timestamp: new Date(), + }; + } + } + + /** + * Playwright로 동적 페이지 크롤링 (SPA용) + */ + async crawlDynamicPage( + url: string, + options?: { + waitSelector?: string; + waitTimeout?: number; + javascript?: boolean; + }, + ): Promise { + const domain = new URL(url).hostname; + + try { + // Rate limiting 적용 + await this.rateLimiter.acquire(domain); + + const browser = await this.getBrowser(); + const context: BrowserContext = await browser.newContext({ + userAgent: + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36', + }); + const page = await context.newPage(); + + try { + await page.goto(url, { + waitUntil: 'networkidle', + timeout: options?.waitTimeout ?? 30000, + }); + + // 특정 셀렉터가 나타날 때까지 대기 + if (options?.waitSelector) { + await page.waitForSelector(options.waitSelector, { + timeout: options?.waitTimeout ?? 10000, + }); + } + + const html = await page.content(); + + return { + success: true, + url, + statusCode: 200, + html, + timestamp: new Date(), + }; + } finally { + await context.close(); + } + } catch (error) { + this.logger.error(`Dynamic page crawl failed: ${url}`, error); + return { + success: false, + url, + error: error instanceof Error ? error.message : 'Unknown error', + timestamp: new Date(), + }; + } + } + + /** + * HTML 파싱 및 데이터 추출 + */ + parseHtml(html: string, config: ParserConfig): Record { + const $ = cheerio.load(html); + const result: Record = {}; + + // 셀렉터 기반 텍스트 추출 + for (const [key, selector] of Object.entries(config.selectors)) { + const elements = $(selector); + result[key] = + elements.length > 1 + ? elements + .map((_, el) => $(el).text().trim()) + .get() + : elements.first().text().trim(); + } + + // 속성 추출 (예: img@src) + if (config.extractAttributes) { + for (const [key, selectorAttr] of Object.entries( + config.extractAttributes, + )) { + const [selector, attr] = selectorAttr.split('@'); + const elements = $(selector); + result[key] = + elements.length > 1 + ? elements.map((_, el) => $(el).attr(attr)).get() + : elements.first().attr(attr); + } + } + + return result; + } + + /** + * EventSource 기반 크롤링 실행 + */ + async crawlFromEventSource(eventSource: EventSource): Promise { + const urlMeta = eventSource.metadatas.find((m) => m.key === 'url'); + const selectorsMeta = eventSource.metadatas.find( + (m) => m.key === 'selectors', + ); + const dynamicMeta = eventSource.metadatas.find((m) => m.key === 'dynamic'); + const waitSelectorMeta = eventSource.metadatas.find( + (m) => m.key === 'waitSelector', + ); + + if (!urlMeta) { + return { + success: false, + url: '', + error: 'URL not found in EventSource metadatas', + timestamp: new Date(), + }; + } + + const url = urlMeta.value as string; + const isDynamic = dynamicMeta?.value === 'true' || String(dynamicMeta?.value) === 'true'; + + let crawlResult: CrawlResult; + + if (isDynamic) { + // 동적 페이지 크롤링 (Playwright) + crawlResult = await this.crawlDynamicPage(url, { + waitSelector: waitSelectorMeta?.value as string | undefined, + }); + } else { + // 정적 페이지 크롤링 (axios) + const request: RESTRequest = { + url, + method: 'GET', + }; + crawlResult = await this.executeRequest(request); + } + + // HTML 파싱 + if (crawlResult.success && crawlResult.html && selectorsMeta) { + try { + const parserConfig: ParserConfig = + typeof selectorsMeta.value === 'string' + ? JSON.parse(selectorsMeta.value) + : selectorsMeta.value; + crawlResult.parsedData = this.parseHtml(crawlResult.html, parserConfig); + } catch (error) { + this.logger.warn(`Failed to parse selectors config: ${error}`); + } + } + + return crawlResult; + } + + /** + * 크롤링 결과를 MinIO에 저장 + */ + async saveResultToStorage( + result: CrawlResult, + stageId: string, + ): Promise { + const buffer = Buffer.from(JSON.stringify(result, null, 2)); + const file = { + buffer, + originalname: `crawl-result-${stageId}-${Date.now()}.json`, + mimetype: 'application/json', + size: buffer.length, + } as Express.Multer.File; + + return this.minioService.uploadFile(file); + } + + /** + * axios 설정 빌드 + */ + private buildAxiosConfig(request: RESTRequest): AxiosRequestConfig { + const config: AxiosRequestConfig = { + url: request.url, + method: request.method.toLowerCase() as + | 'get' + | 'post' + | 'put' + | 'delete' + | 'patch', + headers: request.headers || {}, + params: request.queryParams, + data: request.body, + timeout: request.timeout || 30000, + }; + + // 인증 처리 + if (request.auth) { + switch (request.auth.type) { + case 'BearerToken': + config.headers!['Authorization'] = `Bearer ${request.auth.token}`; + break; + case 'BasicAuth': + config.auth = { + username: request.auth.username!, + password: request.auth.password!, + }; + break; + case 'APIKey': + config.headers!['X-API-Key'] = request.auth.token; + break; + } + } + + return config; + } +} diff --git a/src/crawler/services/pipeline-engine.service.ts b/src/crawler/services/pipeline-engine.service.ts new file mode 100644 index 0000000..80a7e7e --- /dev/null +++ b/src/crawler/services/pipeline-engine.service.ts @@ -0,0 +1,289 @@ +import { Injectable, Logger } from '@nestjs/common'; +import { ConnectionService } from '../../global/database/connection.service'; +import { NatsService } from '../../global/nats/nats.service'; +import { StageExecutorService } from './stage-executor.service'; +import { + Pipelinedef, + Pipeline, + Stagedef, + Stage, + PipelineDocument, +} from '../../global/database/schema'; + +export type TriggerType = 'api' | 'schedule' | 'nats'; + +export interface PipelineExecutionSummary { + pipelineId: string; + pipelineDefId: string; + status: 'completed' | 'failed' | 'partial'; + triggeredBy: TriggerType; + stageResults: { + stageName: string; + status: string; + error?: string; + }[]; + startedAt: Date; + completedAt?: Date; +} + +@Injectable() +export class PipelineEngineService { + private readonly logger = new Logger(PipelineEngineService.name); + + constructor( + private readonly connectionService: ConnectionService, + private readonly natsService: NatsService, + private readonly stageExecutor: StageExecutorService, + ) {} + + /** + * Pipelinedef -> Pipeline 인스턴스 생성 및 실행 + */ + async createAndExecutePipeline( + pipelineDefId: string, + triggeredBy: TriggerType, + ): Promise { + const startedAt = new Date(); + + // 1. Pipelinedef 조회 + const pipelinedefModel = + await this.connectionService.getMoeumModels('pipelinedefs'); + const pipelinedef = await pipelinedefModel.findById(pipelineDefId).exec(); + + if (!pipelinedef) { + throw new Error(`Pipelinedef not found: ${pipelineDefId}`); + } + + // 2. Stagedef들 조회 + const stagedefModel = + await this.connectionService.getMoeumModels('stagedefs'); + const stagedefs = await stagedefModel + .find({ _id: { $in: pipelinedef.stage } }) + .exec(); + + if (stagedefs.length === 0) { + throw new Error(`No stagedefs found for pipelinedef: ${pipelineDefId}`); + } + + // 3. Stage 인스턴스들 생성 + const stages: Stage[] = stagedefs.map((stagedef) => ({ + apiVersion: stagedef.apiVersion, + name: stagedef.name, + schema: stagedef.schema, + taxonomy: stagedef.taxonomy, + executions: [], + status: 'pending' as const, + metadatas: [...stagedef.metadatas], + retryStrategy: { + limit: 3, + delay: 1000, + backoff: 2, + when: ['failed'], + notify: [], + }, + })); + + // 4. Pipeline 인스턴스 생성 + const pipelineModel = + await this.connectionService.getMoeumModels('pipelines'); + const pipeline = await pipelineModel.create({ + apiVersion: pipelinedef.apiVersion, + name: `pipeline-${Date.now()}`, + stage: stages, + metadatas: [ + ...pipelinedef.metadatas, + { + key: 'triggeredBy', + value: triggeredBy, + dataType: 'string', + isReserved: true, + }, + { + key: 'triggeredAt', + value: startedAt.toISOString(), + dataType: 'string', + isReserved: true, + }, + { + key: 'pipelineDefId', + value: pipelineDefId, + dataType: 'string', + isReserved: true, + }, + ], + }); + + this.logger.log(`Pipeline created: ${pipeline._id}`); + + // 5. 시작 이벤트 발행 + await this.natsService.publish('crawler.pipeline.started', { + pipelineId: pipeline._id.toString(), + pipelineDefId, + triggeredBy, + timestamp: startedAt.toISOString(), + }); + + // 6. Stage들 병렬 실행 (비동기) + this.executeStagesInParallel( + pipeline._id.toString(), + stagedefs as Stagedef[], + triggeredBy, + ); + + return pipeline; + } + + /** + * Stage들을 병렬로 실행 + */ + private async executeStagesInParallel( + pipelineId: string, + stagedefs: Stagedef[], + triggeredBy: TriggerType, + ): Promise { + const pipelineModel = + await this.connectionService.getMoeumModels('pipelines'); + + try { + // Promise.allSettled로 모든 Stage 병렬 실행 + const results = await Promise.allSettled( + stagedefs.map((stagedef, index) => + this.stageExecutor.executeStage(pipelineId, index, stagedef), + ), + ); + + // 결과 집계 + const stageResults = results.map((r, i) => ({ + stageName: stagedefs[i].name, + status: r.status === 'fulfilled' ? r.value.status : 'failed', + error: + r.status === 'rejected' + ? r.reason?.message || 'Unknown error' + : undefined, + })); + + const allSucceeded = results.every( + (r) => r.status === 'fulfilled' && r.value.status === 'completed', + ); + const allFailed = results.every( + (r) => + r.status === 'rejected' || + (r.status === 'fulfilled' && r.value.status === 'failed'), + ); + + let pipelineStatus: 'completed' | 'failed' | 'partial'; + if (allSucceeded) { + pipelineStatus = 'completed'; + } else if (allFailed) { + pipelineStatus = 'failed'; + } else { + pipelineStatus = 'partial'; + } + + const completedAt = new Date(); + + // Pipeline 상태 업데이트 + await pipelineModel.findByIdAndUpdate(pipelineId, { + $push: { + metadatas: { + $each: [ + { + key: 'completedAt', + value: completedAt.toISOString(), + dataType: 'string', + isReserved: true, + }, + { + key: 'pipelineStatus', + value: pipelineStatus, + dataType: 'string', + isReserved: true, + }, + ], + }, + }, + }); + + // 완료 이벤트 발행 + await this.natsService.publish('crawler.pipeline.completed', { + pipelineId, + status: pipelineStatus, + triggeredBy, + stageResults, + completedAt: completedAt.toISOString(), + }); + + this.logger.log( + `Pipeline ${pipelineId} completed with status: ${pipelineStatus}`, + ); + } catch (error) { + this.logger.error(`Pipeline execution failed: ${pipelineId}`, error); + + await this.natsService.publish('crawler.pipeline.failed', { + pipelineId, + error: error instanceof Error ? error.message : 'Unknown error', + timestamp: new Date().toISOString(), + }); + } + } + + /** + * Pipeline 실행 상태 조회 + */ + async getPipelineStatus(pipelineId: string): Promise { + const pipelineModel = + await this.connectionService.getMoeumModels('pipelines'); + return pipelineModel.findById(pipelineId).exec(); + } + + /** + * Pipelinedef 목록 조회 + */ + async listPipelinedefs( + page: number = 1, + limit: number = 20, + ): Promise<{ items: Pipelinedef[]; total: number }> { + const pipelinedefModel = + await this.connectionService.getMoeumModels('pipelinedefs'); + const skip = (page - 1) * limit; + + const [items, total] = await Promise.all([ + pipelinedefModel.find().skip(skip).limit(limit).exec(), + pipelinedefModel.countDocuments().exec(), + ]); + + return { items: items as Pipelinedef[], total }; + } + + /** + * 실행된 Pipeline 목록 조회 + */ + async listPipelines( + page: number = 1, + limit: number = 20, + status?: string, + ): Promise<{ items: Pipeline[]; total: number }> { + const pipelineModel = + await this.connectionService.getMoeumModels('pipelines'); + const skip = (page - 1) * limit; + + const filter: Record = {}; + if (status) { + filter['metadatas'] = { + $elemMatch: { key: 'pipelineStatus', value: status }, + }; + } + + const [items, total] = await Promise.all([ + pipelineModel + .find(filter) + .sort({ createdAt: -1 }) + .skip(skip) + .limit(limit) + .exec(), + pipelineModel.countDocuments(filter).exec(), + ]); + + return { items: items as Pipeline[], total }; + } +} diff --git a/src/crawler/services/rate-limiter.service.ts b/src/crawler/services/rate-limiter.service.ts new file mode 100644 index 0000000..bed9005 --- /dev/null +++ b/src/crawler/services/rate-limiter.service.ts @@ -0,0 +1,138 @@ +import { Injectable, Logger } from '@nestjs/common'; +import { RateLimitConfig } from '../interfaces/crawler-result.interface'; + +interface TokenBucket { + tokens: number; + lastRefill: number; + config: RateLimitConfig; +} + +@Injectable() +export class RateLimiterService { + private readonly logger = new Logger(RateLimiterService.name); + private readonly buckets: Map = new Map(); + private readonly defaultConfig: RateLimitConfig = { + maxTokens: 10, // 최대 10개 토큰 + refillRate: 2, // 초당 2개 토큰 보충 + }; + + /** + * 도메인별 Rate Limit 설정 + */ + setLimit(domain: string, config: RateLimitConfig): void { + const existing = this.buckets.get(domain); + this.buckets.set(domain, { + tokens: config.maxTokens, + lastRefill: Date.now(), + config, + }); + this.logger.log( + `Rate limit set for ${domain}: ${config.maxTokens} tokens, ${config.refillRate} req/s`, + ); + } + + /** + * 설정된 Rate Limit 조회 + */ + getLimit(domain: string): RateLimitConfig { + const bucket = this.buckets.get(domain); + return bucket?.config ?? this.defaultConfig; + } + + /** + * 토큰 획득 (대기 필요시 대기) + */ + async acquire(domain: string): Promise { + let bucket = this.buckets.get(domain); + + // 버킷이 없으면 기본 설정으로 생성 + if (!bucket) { + bucket = { + tokens: this.defaultConfig.maxTokens, + lastRefill: Date.now(), + config: this.defaultConfig, + }; + this.buckets.set(domain, bucket); + } + + // 토큰 보충 + this.refill(bucket); + + // 토큰이 없으면 대기 + if (bucket.tokens <= 0) { + const waitTime = Math.ceil(1000 / bucket.config.refillRate); + this.logger.debug(`Rate limit hit for ${domain}, waiting ${waitTime}ms`); + await this.delay(waitTime); + this.refill(bucket); + } + + // 토큰 소비 + bucket.tokens--; + this.logger.debug( + `Token acquired for ${domain}, remaining: ${bucket.tokens}`, + ); + } + + /** + * 토큰 보충 (시간 경과에 따라) + */ + private refill(bucket: TokenBucket): void { + const now = Date.now(); + const elapsed = (now - bucket.lastRefill) / 1000; // 초 단위 + const tokensToAdd = elapsed * bucket.config.refillRate; + + bucket.tokens = Math.min( + bucket.config.maxTokens, + bucket.tokens + tokensToAdd, + ); + bucket.lastRefill = now; + } + + /** + * 현재 사용 가능한 토큰 수 조회 + */ + getAvailableTokens(domain: string): number { + const bucket = this.buckets.get(domain); + if (!bucket) { + return this.defaultConfig.maxTokens; + } + this.refill(bucket); + return Math.floor(bucket.tokens); + } + + /** + * 모든 도메인의 Rate Limit 상태 조회 + */ + getAllLimits(): Record { + const result: Record = {}; + + for (const [domain, bucket] of this.buckets) { + this.refill(bucket); + result[domain] = { + config: bucket.config, + available: Math.floor(bucket.tokens), + }; + } + + return result; + } + + /** + * 특정 도메인의 Rate Limit 초기화 + */ + resetLimit(domain: string): void { + const bucket = this.buckets.get(domain); + if (bucket) { + bucket.tokens = bucket.config.maxTokens; + bucket.lastRefill = Date.now(); + this.logger.log(`Rate limit reset for ${domain}`); + } + } + + /** + * 대기 함수 + */ + private delay(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); + } +} diff --git a/src/crawler/services/retry.service.ts b/src/crawler/services/retry.service.ts new file mode 100644 index 0000000..3a92672 --- /dev/null +++ b/src/crawler/services/retry.service.ts @@ -0,0 +1,197 @@ +import { Injectable, Logger } from '@nestjs/common'; +import { NatsService } from '../../global/nats/nats.service'; +import { RetryStrategy } from '../../global/database/schema'; + +export type RetryableOperation = () => Promise; + +export interface RetryResult { + success: boolean; + result?: T; + attempts: number; + errors: string[]; +} + +@Injectable() +export class RetryService { + private readonly logger = new Logger(RetryService.name); + + constructor(private readonly natsService: NatsService) {} + + /** + * 재시도 로직과 함께 작업 실행 + */ + async executeWithRetry( + operation: RetryableOperation, + strategy: RetryStrategy, + ): Promise { + let lastError: Error | null = null; + let currentDelay = strategy.delay; + const errors: string[] = []; + const maxAttempts = strategy.limit + 1; + + for (let attempt = 1; attempt <= maxAttempts; attempt++) { + try { + this.logger.debug(`Attempt ${attempt}/${maxAttempts}`); + const result = await operation(); + + // 성공 시 반환 + if (attempt > 1) { + this.logger.log(`Operation succeeded on attempt ${attempt}`); + } + return result; + } catch (error) { + lastError = error instanceof Error ? error : new Error(String(error)); + errors.push(`Attempt ${attempt}: ${lastError.message}`); + this.logger.warn(`Attempt ${attempt} failed: ${lastError.message}`); + + // 재시도 조건 확인 + if (!this.shouldRetry(lastError, strategy.when)) { + this.logger.debug( + 'Error does not match retry conditions, not retrying', + ); + break; + } + + // 마지막 시도가 아니면 대기 + if (attempt < maxAttempts) { + this.logger.debug(`Waiting ${currentDelay}ms before next attempt`); + await this.delay(currentDelay); + currentDelay = Math.floor(currentDelay * strategy.backoff); // Exponential backoff + } + } + } + + // 모든 재시도 실패 - 알림 발송 + if (strategy.notify && strategy.notify.length > 0) { + await this.sendFailureNotification(strategy.notify, errors); + } + + throw lastError || new Error('Operation failed after all retries'); + } + + /** + * 재시도 없이 결과를 래핑하여 반환 + */ + async executeWithResult( + operation: RetryableOperation, + strategy: RetryStrategy, + ): Promise> { + const errors: string[] = []; + let currentDelay = strategy.delay; + const maxAttempts = strategy.limit + 1; + + for (let attempt = 1; attempt <= maxAttempts; attempt++) { + try { + const result = await operation(); + return { + success: true, + result, + attempts: attempt, + errors, + }; + } catch (error) { + const errorMessage = + error instanceof Error ? error.message : String(error); + errors.push(`Attempt ${attempt}: ${errorMessage}`); + + if (!this.shouldRetry(error as Error, strategy.when)) { + break; + } + + if (attempt < maxAttempts) { + await this.delay(currentDelay); + currentDelay = Math.floor(currentDelay * strategy.backoff); + } + } + } + + // 모든 재시도 실패 + if (strategy.notify && strategy.notify.length > 0) { + await this.sendFailureNotification(strategy.notify, errors); + } + + return { + success: false, + attempts: errors.length, + errors, + }; + } + + /** + * 재시도 조건 확인 + */ + private shouldRetry(error: Error, conditions: string[]): boolean { + // 모든 실패에 대해 재시도 + if (conditions.includes('failed')) { + return true; + } + + const errorMessage = error.message.toLowerCase(); + + // 특정 에러 타입에 대해서만 재시도 + for (const condition of conditions) { + if (errorMessage.includes(condition.toLowerCase())) { + return true; + } + + // HTTP 상태 코드 기반 조건 (예: '5xx', '429', '500') + if (condition.match(/^\d{3}$/) && errorMessage.includes(condition)) { + return true; + } + + // 5xx 패턴 매칭 + if (condition === '5xx' && errorMessage.match(/5\d{2}/)) { + return true; + } + + // 4xx 패턴 매칭 + if (condition === '4xx' && errorMessage.match(/4\d{2}/)) { + return true; + } + + // 네트워크 에러 + if ( + condition === 'network' && + (errorMessage.includes('econnrefused') || + errorMessage.includes('enotfound') || + errorMessage.includes('timeout') || + errorMessage.includes('socket')) + ) { + return true; + } + } + + return false; + } + + /** + * 실패 알림 발송 + */ + private async sendFailureNotification( + notifyChannels: string[], + errors: string[], + ): Promise { + const notification = { + type: 'crawler.execution.failed', + timestamp: new Date().toISOString(), + errors, + message: `Operation failed after ${errors.length} attempts`, + }; + + for (const channel of notifyChannels) { + try { + await this.natsService.publish(`notifications.${channel}`, notification); + this.logger.log(`Failure notification sent to: ${channel}`); + } catch (error) { + this.logger.error(`Failed to send notification to ${channel}`, error); + } + } + } + + /** + * 대기 함수 + */ + private delay(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); + } +} diff --git a/src/crawler/services/stage-executor.service.ts b/src/crawler/services/stage-executor.service.ts new file mode 100644 index 0000000..46f97c2 --- /dev/null +++ b/src/crawler/services/stage-executor.service.ts @@ -0,0 +1,255 @@ +import { Injectable, Logger } from '@nestjs/common'; +import { ConnectionService } from '../../global/database/connection.service'; +import { NatsService } from '../../global/nats/nats.service'; +import { CrawlerService } from './crawler.service'; +import { RetryService } from './retry.service'; +import { + Stagedef, + Execution, + RetryStrategy, +} from '../../global/database/schema'; +import { + CrawlResult, + StageExecutionResult, +} from '../interfaces/crawler-result.interface'; + +const DEFAULT_RETRY_STRATEGY: RetryStrategy = { + limit: 3, + delay: 1000, + backoff: 2, + when: ['failed', '5xx', 'network'], + notify: [], +}; + +@Injectable() +export class StageExecutorService { + private readonly logger = new Logger(StageExecutorService.name); + + constructor( + private readonly connectionService: ConnectionService, + private readonly natsService: NatsService, + private readonly crawlerService: CrawlerService, + private readonly retryService: RetryService, + ) {} + + /** + * 단일 Stage 실행 + */ + async executeStage( + pipelineId: string, + stageIndex: number, + stagedef: Stagedef, + ): Promise { + const pipelineModel = + await this.connectionService.getMoeumModels('pipelines'); + + // Stage 상태를 'in-progress'로 변경 + await pipelineModel.findByIdAndUpdate(pipelineId, { + $set: { [`stage.${stageIndex}.status`]: 'in-progress' }, + }); + + await this.natsService.publish('crawler.stage.started', { + pipelineId, + stageIndex, + stageName: stagedef.name, + timestamp: new Date().toISOString(), + }); + + const executions: Execution[] = []; + const crawlResults: CrawlResult[] = []; + let finalStatus: 'completed' | 'failed' = 'completed'; + + try { + // EventSource별로 크롤링 실행 + for (const eventSource of stagedef.eventSource) { + const execution = await this.executeEventSource( + pipelineId, + stageIndex, + eventSource, + stagedef, + ); + executions.push(execution); + + if (execution.status === 'failed') { + finalStatus = 'failed'; + } + + // 크롤링 결과 추출 + const resultMeta = execution.metadatas.find( + (m) => m.key === 'crawlResult', + ); + if (resultMeta) { + try { + const result = JSON.parse(resultMeta.value as string); + crawlResults.push(result); + } catch { + // JSON 파싱 실패 무시 + } + } + } + + // Stage 상태 및 실행 기록 업데이트 + await pipelineModel.findByIdAndUpdate(pipelineId, { + $set: { + [`stage.${stageIndex}.status`]: finalStatus, + [`stage.${stageIndex}.executions`]: executions, + }, + }); + + // 완료 이벤트 발행 (Stagedef의 topic으로) + await this.natsService.publish(stagedef.topic, { + pipelineId, + stageIndex, + stageName: stagedef.name, + status: finalStatus, + results: crawlResults.map((r) => ({ + url: r.url, + success: r.success, + parsedData: r.parsedData, + })), + timestamp: new Date().toISOString(), + }); + + this.logger.log( + `Stage ${stagedef.name} completed with status: ${finalStatus}`, + ); + } catch (error) { + finalStatus = 'failed'; + this.logger.error(`Stage ${stagedef.name} failed`, error); + + await pipelineModel.findByIdAndUpdate(pipelineId, { + $set: { [`stage.${stageIndex}.status`]: 'failed' }, + $push: { + [`stage.${stageIndex}.executions`]: { + status: 'failed', + metadatas: [ + { + key: 'error', + value: error instanceof Error ? error.message : 'Unknown error', + dataType: 'string', + isReserved: true, + }, + ], + createdAt: new Date(), + updatedAt: new Date(), + }, + }, + }); + } + + return { + stageIndex, + stageName: stagedef.name, + status: finalStatus, + crawlResults, + }; + } + + /** + * 단일 EventSource 실행 (재시도 로직 포함) + */ + private async executeEventSource( + pipelineId: string, + stageIndex: number, + eventSource: any, + stagedef: Stagedef, + ): Promise { + const startedAt = new Date(); + const execution: Execution = { + status: 'in-progress', + metadatas: [ + { + key: 'eventSourceName', + value: eventSource.name, + dataType: 'string', + isReserved: true, + }, + { + key: 'eventSourceType', + value: eventSource.type, + dataType: 'string', + isReserved: true, + }, + { + key: 'startedAt', + value: startedAt.toISOString(), + dataType: 'string', + isReserved: true, + }, + ], + createdAt: startedAt, + updatedAt: startedAt, + }; + + // RetryStrategy 추출 (Stagedef 메타데이터에서 또는 기본값) + const retryStrategyMeta = stagedef.metadatas.find( + (m) => m.key === 'retryStrategy', + ); + let retryStrategy: RetryStrategy = DEFAULT_RETRY_STRATEGY; + + if (retryStrategyMeta) { + try { + retryStrategy = + typeof retryStrategyMeta.value === 'string' + ? JSON.parse(retryStrategyMeta.value) + : retryStrategyMeta.value; + } catch { + // 파싱 실패 시 기본값 사용 + } + } + + try { + // 재시도 로직과 함께 크롤링 실행 + const crawlResult = await this.retryService.executeWithRetry( + () => this.crawlerService.crawlFromEventSource(eventSource), + retryStrategy, + ); + + execution.status = crawlResult.success ? 'completed' : 'failed'; + execution.metadatas.push({ + key: 'crawlResult', + value: JSON.stringify(crawlResult), + dataType: 'json', + isReserved: false, + }); + + // MinIO에 결과 저장 + if (crawlResult.success) { + try { + const storageKey = await this.crawlerService.saveResultToStorage( + crawlResult, + `${pipelineId}-${stageIndex}`, + ); + execution.metadatas.push({ + key: 'storageKey', + value: storageKey, + dataType: 'string', + isReserved: false, + }); + } catch (error) { + this.logger.warn(`Failed to save result to MinIO: ${error}`); + } + } + } catch (error) { + execution.status = 'failed'; + execution.metadatas.push({ + key: 'error', + value: error instanceof Error ? error.message : 'Unknown error', + dataType: 'string', + isReserved: true, + }); + } + + // 완료 시간 기록 + const completedAt = new Date(); + execution.metadatas.push({ + key: 'completedAt', + value: completedAt.toISOString(), + dataType: 'string', + isReserved: true, + }); + execution.updatedAt = completedAt; + + return execution; + } +} diff --git a/src/crawler/services/trigger.service.ts b/src/crawler/services/trigger.service.ts new file mode 100644 index 0000000..64c0bd2 --- /dev/null +++ b/src/crawler/services/trigger.service.ts @@ -0,0 +1,327 @@ +import { + Injectable, + Logger, + OnModuleInit, + OnModuleDestroy, +} from '@nestjs/common'; +import { SchedulerRegistry } from '@nestjs/schedule'; +import { CronJob } from 'cron'; +import { NatsConnection, StringCodec, Subscription, connect } from 'nats'; +import * as fs from 'fs/promises'; +import { ConnectionService } from '../../global/database/connection.service'; +import { SecretService } from '../../global/secret/secret.service'; +import { PipelineEngineService } from './pipeline-engine.service'; +import { Schedule } from '../../global/database/schema'; + +export interface ScheduledJob { + pipelineDefId: string; + jobName: string; + cronExpression: string; + timezone: string; +} + +@Injectable() +export class TriggerService implements OnModuleInit, OnModuleDestroy { + private readonly logger = new Logger(TriggerService.name); + private scheduledJobs: Map = new Map(); + private natsSubscriptions: Subscription[] = []; + private natsConnection: NatsConnection | null = null; + private readonly stringCodec = StringCodec(); + + constructor( + private readonly schedulerRegistry: SchedulerRegistry, + private readonly connectionService: ConnectionService, + private readonly pipelineEngine: PipelineEngineService, + private readonly secretService: SecretService, + ) {} + + async onModuleInit() { + try { + // NATS 연결 초기화 + await this.initializeNatsConnection(); + + // 기존 트리거 로드 + await this.loadTriggers(); + + this.logger.log('TriggerService initialized'); + } catch (error) { + this.logger.error('Failed to initialize TriggerService', error); + } + } + + async onModuleDestroy() { + // 스케줄 정리 + for (const [jobName] of this.scheduledJobs) { + try { + this.schedulerRegistry.deleteCronJob(jobName); + } catch { + // 무시 + } + } + this.scheduledJobs.clear(); + + // NATS 구독 정리 + for (const sub of this.natsSubscriptions) { + try { + await sub.drain(); + } catch { + // 무시 + } + } + this.natsSubscriptions = []; + + // NATS 연결 종료 + if (this.natsConnection) { + try { + await this.natsConnection.drain(); + await this.natsConnection.close(); + } catch { + // 무시 + } + this.natsConnection = null; + } + + this.logger.log('TriggerService destroyed'); + } + + /** + * NATS 연결 초기화 + */ + private async initializeNatsConnection(): Promise { + await this.secretService.ready(); + + const endpoint = await this.secretService.get('NATS_ENDPOINT'); + const user = await this.secretService.get('NATS_USER'); + const password = await this.secretService.get('NATS_PASS'); + + let tlsConfig: { ca?: string } | undefined; + try { + const cacert = (await fs.readFile('internal-service-ca.pem')).toString(); + tlsConfig = { ca: cacert }; + } catch { + this.logger.warn('TLS certificate not found, connecting without TLS'); + } + + this.natsConnection = await connect({ + name: 'crawler-trigger-service', + servers: [endpoint], + tls: tlsConfig, + user, + pass: password, + maxReconnectAttempts: -1, + reconnectTimeWait: 500, + reconnect: true, + pingInterval: 20000, + timeout: 5000, + }); + + this.logger.log('NATS connection established for TriggerService'); + } + + /** + * 모든 트리거 로드 (스케줄 + 이벤트 구독) + */ + async loadTriggers(): Promise { + const pipelinedefModel = + await this.connectionService.getMoeumModels('pipelinedefs'); + const pipelinedefs = await pipelinedefModel.find().exec(); + + for (const pipelinedef of pipelinedefs) { + // 스케줄 트리거 확인 (메타데이터에서) + const scheduleMeta = pipelinedef.metadatas?.find( + (m) => m.key === 'schedule', + ); + if (scheduleMeta) { + try { + const schedule: Schedule = + typeof scheduleMeta.value === 'string' + ? JSON.parse(scheduleMeta.value) + : scheduleMeta.value; + await this.registerScheduleTrigger( + pipelinedef._id.toString(), + schedule, + ); + } catch (error) { + this.logger.warn( + `Failed to parse schedule for pipelinedef ${pipelinedef._id}`, + ); + } + } + + // Kafka/NATS 이벤트 트리거 + if ( + pipelinedef.trigger?.type === 'kafka-event' && + pipelinedef.trigger.kafkaEvent?.topic + ) { + await this.registerNatsTrigger( + pipelinedef._id.toString(), + pipelinedef.trigger.kafkaEvent.topic, + ); + } + } + + this.logger.log( + `Loaded ${this.scheduledJobs.size} schedule triggers and ${this.natsSubscriptions.length} event triggers`, + ); + } + + /** + * 스케줄 트리거 등록 + */ + async registerScheduleTrigger( + pipelineDefId: string, + schedule: Schedule, + ): Promise { + const jobName = `schedule-${pipelineDefId}`; + + // 기존 작업이 있으면 제거 + try { + if (this.schedulerRegistry.doesExist('cron', jobName)) { + this.schedulerRegistry.deleteCronJob(jobName); + this.scheduledJobs.delete(jobName); + } + } catch { + // 무시 + } + + const job = new CronJob( + schedule.cron, + async () => { + this.logger.log(`Scheduled trigger fired for pipeline: ${pipelineDefId}`); + try { + await this.pipelineEngine.createAndExecutePipeline( + pipelineDefId, + 'schedule', + ); + } catch (error) { + this.logger.error( + `Scheduled trigger failed: ${pipelineDefId}`, + error, + ); + } + }, + null, + true, + schedule.timezone || 'Asia/Seoul', + ); + + this.schedulerRegistry.addCronJob(jobName, job); + this.scheduledJobs.set(jobName, { + pipelineDefId, + jobName, + cronExpression: schedule.cron, + timezone: schedule.timezone || 'Asia/Seoul', + }); + + this.logger.log( + `Registered schedule trigger: ${jobName} (${schedule.cron})`, + ); + } + + /** + * NATS 이벤트 트리거 등록 + */ + async registerNatsTrigger( + pipelineDefId: string, + topic: string, + ): Promise { + if (!this.natsConnection) { + this.logger.warn('NATS connection not available'); + return; + } + + const subscription = this.natsConnection.subscribe(topic, { + callback: async (err, msg) => { + if (err) { + this.logger.error(`NATS subscription error on ${topic}`, err); + return; + } + + this.logger.log(`NATS trigger received on ${topic}`); + try { + // 메시지 페이로드 파싱 (필요시 사용) + let payload: any = {}; + try { + payload = JSON.parse(this.stringCodec.decode(msg.data)); + } catch { + // JSON 아닌 경우 무시 + } + + await this.pipelineEngine.createAndExecutePipeline( + pipelineDefId, + 'nats', + ); + } catch (error) { + this.logger.error( + `NATS trigger processing failed: ${pipelineDefId}`, + error, + ); + } + }, + }); + + this.natsSubscriptions.push(subscription); + this.logger.log(`Registered NATS trigger: ${topic} -> ${pipelineDefId}`); + } + + /** + * API 기반 수동 트리거 + */ + async triggerPipelineManually(pipelineDefId: string): Promise { + const pipeline = await this.pipelineEngine.createAndExecutePipeline( + pipelineDefId, + 'api', + ); + return pipeline._id.toString(); + } + + /** + * 스케줄 제거 + */ + async unregisterScheduleTrigger(pipelineDefId: string): Promise { + const jobName = `schedule-${pipelineDefId}`; + try { + if (this.schedulerRegistry.doesExist('cron', jobName)) { + this.schedulerRegistry.deleteCronJob(jobName); + this.scheduledJobs.delete(jobName); + this.logger.log(`Unregistered schedule trigger: ${jobName}`); + } + } catch { + // 무시 + } + } + + /** + * 현재 등록된 스케줄 목록 조회 + */ + getRegisteredSchedules(): ScheduledJob[] { + return Array.from(this.scheduledJobs.values()); + } + + /** + * 트리거 재로드 + */ + async reloadTriggers(): Promise { + // 기존 트리거 정리 + for (const [jobName] of this.scheduledJobs) { + try { + this.schedulerRegistry.deleteCronJob(jobName); + } catch { + // 무시 + } + } + this.scheduledJobs.clear(); + + for (const sub of this.natsSubscriptions) { + try { + await sub.drain(); + } catch { + // 무시 + } + } + this.natsSubscriptions = []; + + // 다시 로드 + await this.loadTriggers(); + } +} diff --git a/src/global/database/connection.service.ts b/src/global/database/connection.service.ts index cf84ef3..8c7768c 100644 --- a/src/global/database/connection.service.ts +++ b/src/global/database/connection.service.ts @@ -1,7 +1,7 @@ import { Injectable, Scope, OnModuleInit } from '@nestjs/common'; import mongoose from 'mongoose'; import { SecretService } from '../secret/secret.service.js'; -import { moeumModelDefinitions, MoeumModelTypes } from './schema.js'; +import { CrawlerModelDefinitions, CrawlerModelTypes } from './schema.js'; @Injectable({ scope: Scope.DEFAULT }) export class ConnectionService implements OnModuleInit { @@ -31,19 +31,19 @@ export class ConnectionService implements OnModuleInit { return this.connection; } - public async getMoeumModels( + public async getMoeumModels( modelKey: K, - ): Promise> { + ): Promise> { if (this.moeumModels) return this.moeumModels.get(modelKey) as mongoose.Model< - MoeumModelTypes[K] + CrawlerModelTypes[K] >; - const moeumModels = new Map>(); + const moeumModels = new Map>(); // const connection = await this.getTenantConnection(tenantId); const connection = await this.getConnection(); - moeumModelDefinitions.forEach((modelDefinition) => { + CrawlerModelDefinitions.forEach((modelDefinition) => { // 이미 생성된 경우 컴파일 스킵. if ( Object.prototype.hasOwnProperty.call( @@ -59,7 +59,7 @@ export class ConnectionService implements OnModuleInit { } else { moeumModels.set( modelDefinition.name as K, - connection.model( + connection.model( modelDefinition.name, modelDefinition.schema, modelDefinition.schemaName, @@ -69,7 +69,7 @@ export class ConnectionService implements OnModuleInit { }); this.moeumModels = moeumModels; - return moeumModels.get(modelKey) as mongoose.Model; + return moeumModels.get(modelKey) as mongoose.Model; } public async createSession( diff --git a/src/global/database/schema.ts b/src/global/database/schema.ts index 28340ee..c757699 100644 --- a/src/global/database/schema.ts +++ b/src/global/database/schema.ts @@ -8,236 +8,1045 @@ export interface MongooseTimestamps { export interface Metadata { key: string; - value: string | any[] + value: string | any[]; dataType: 'string' | 'number' | 'boolean' | 'array' | 'json'; isReserved: boolean; } -export interface TaxonomyChildItem { - title: string; - order: number; - articleId?: mongoose.Types.ObjectId; - slug?: string; //article document의 slug 참조 (임베딩) - children?: TaxonomyChildItem[]; +export interface Execution { + // executionId: string; // 각 실행의 고유 ID + status: + | 'pending' + | 'in-progress' + | 'completed' + | 'failed' + | 'cancelled' + | 'skipped'; + // artifact?: [Artifact]; + metadatas: Metadata[]; + createdAt: Date; + updatedAt: Date; } -export interface Taxonomy - extends MongooseTimestamps { - label: string; - items: TaxonomyChildItem[]; +export type ExecutionDocument = HydratedDocument; + +export type RESTRequest = { + url: string; + method: 'GET' | 'POST' | 'PUT' | 'DELETE' | 'PATCH'; + headers?: { [key: string]: string }; + queryParams?: { [key: string]: any }; + body?: any; + auth?: { + type: 'BearerToken' | 'BasicAuth' | 'APIKey'; + token?: string; // BearerToken 또는 APIKey에 사용 + username?: string; // BasicAuth에 사용 + password?: string; // BasicAuth에 사용 + }; + timeout?: number; // 요청 타임아웃 (밀리초) + retryOptions?: { + retries: number; + delay: number; // 밀리초 + }; +}; + +export type KafkaEvent = { + topic: string; + key?: string; +}; + +export type PipelineTrigger = { + type: 'always' | 'kafka-event'; + kafkaEvent?: KafkaEvent; +}; + +export type StageTrigger = { + type: 'always' | 'condition' | 'schedule'; + condition?: Condition | CompositeCondition; + schedule?: Schedule; +}; + +export type Condition = { + variable: string; + operator: 'eq' | 'ne' | 'gt' | 'lt' | 'gte' | 'lte'; + value: any; +}; + +export type CompositeCondition = { + logic: 'AND' | 'OR'; + conditions: (Condition | CompositeCondition)[]; +}; + +export type Schedule = { + cron: string; // 크론 표현식 + timezone?: string; +}; + +// export type Artifact = { +// key: string; +// value: any; +// dataType: DataType; +// isReserved: boolean; +// }; + +export type Variable = { + name: string; + value: any; + scope: 'pipeline' | 'stage'; +}; + +export interface Pipelinedef { + apiVersion: string; + trigger: PipelineTrigger; + stage: mongoose.Types.ObjectId[]; + revision: number; metadatas: Metadata[]; } +export type PipelinedefDocument = HydratedDocument; -export type TaxonomyDocument = HydratedDocument - -export interface ContentSection { - headline: string; - taxonomies: ContentSectionEmbeddedTaxonomy[]; -} - -export interface ContentSectionEmbeddedTaxonomy { - taxonomyId: mongoose.Types.ObjectId; - label: string; // 성능 최적화를 위해 임베딩된 label -} - -export interface Content - extends MongooseTimestamps { - key: string; - label: string; - sections: ContentSection[]; -} - -export type ContentDocument = HydratedDocument - -export interface Article - extends MongooseTimestamps { - title: string; - slug: string; - content: string; - status: 'draft' | 'published' | 'archived'; +export interface Pipeline { + apiVersion: string; + name?: string; metadatas: Metadata[]; + stage: [Stage & { _id: mongoose.Types.ObjectId }]; } -export type ArticleDocument = HydratedDocument
+export type PipelineDocument = HydratedDocument; +export interface Stagedef { + apiVersion: string; + name: string; + taxonomy?: string; + eventSource: EventSource[]; + schema?: any; -export const MetadataSubSchema = new mongoose.Schema({ - key: { type: String, required: true }, - value: { - type: mongoose.SchemaTypes.Mixed, - required: false, // required: true로 하면 value가 null인 경우 에러 발생 - default: null, - validate: { - validator: function (v: string | any[]) { - // 특정 키인 경우 value가 null 또는 빈 문자열이 아니어야 함 - // if (this.key === 'requiredKey') { - // return typeof v === 'string' && v.trim().length > 0; - // } - // 그렇지 않은 경우 value는 null 또는 문자열, Array이어야 함 - return typeof v === 'string' || Array.isArray(v); - }, - message: function (props: any) { - if (this.key === 'requiredKey') { - return `Value is required for key "${this.key}" and cannot be null or empty.`; - } - return `Value must be a string or array for key "${this.key}".`; + topic: string; + // lifecycleHooks?: LifecycleHooks; + metadatas: Metadata[]; + /** + * 실제 존재하는 값이 아닌 가상 필드입니다. + * 실제 데이터베이스에 저장되지 않습니다. + * 메타데이터를 조작하기 쉽도록 Metadatas 인스턴스로 변환하여 반환합니다. + * Query.prototype.lean()을 사용할 때는 가상 필드가 반환되지 않습니다. + * lean()을 사용하지 않는 경우에만 사용하세요. (exec() 사용) + */ + // metadatas?: Metadatas; //Virtual +} + +export type StagedefDocument = HydratedDocument; + +export interface Stage { + apiVersion: string; + name: string; + schema?: any; + taxonomy?: string; + // lifecycleHooks?: LifecycleHooks; + executions: Execution[]; + // outputData?: any; + status: + | 'pending' // 작업 대기 + | 'in-progress' // 작업이 실행 중 + | 'completed' // 작업이 완료됨(성공, 실패) + | 'failed' // 작업이 실패함 + | 'cancelled' // 작업이 취소됨 + | 'skipped'; // 작업이 스킵됨 + // artifact?: Artifact; + metadatas: Metadata[]; + /** + * 실제 존재하는 값이 아닌 가상 필드입니다. + * 실제 데이터베이스에 저장되지 않습니다. + * 메타데이터를 조작하기 쉽도록 Metadatas 인스턴스로 변환하여 반환합니다. + * Query.prototype.lean()을 사용할 때는 가상 필드가 반환되지 않습니다. + * lean()을 사용하지 않는 경우에만 사용하세요. (exec() 사용) + */ + // metadatas?: Metadatas; //Virtual + retryStrategy?: RetryStrategy; + // variables?: Variable[]; +} +export type StageDocument = HydratedDocument; + +export type EventSource = { + type: string; + name: string; + metadatas: Metadata[]; +}; +export type StagedefEventSourceDocument = HydratedDocument; + +export type RetryStrategy = { + limit: number; + delay: number; + backoff: number; + when: string[]; + notify: string[]; +}; + +const RESTRequestSchema = new mongoose.Schema( + { + url: { type: String, required: true }, + method: { + type: String, + enum: ['GET', 'POST', 'PUT', 'DELETE', 'PATCH'], + required: true, + }, + headers: { type: Map, of: String }, + queryParams: { type: Map, of: mongoose.Schema.Types.Mixed }, + body: { type: mongoose.Schema.Types.Mixed }, + auth: { + type: new mongoose.Schema( + { + type: { + type: String, + enum: ['BearerToken', 'BasicAuth', 'APIKey'], + required: true, + }, + token: { type: String }, + username: { type: String }, + password: { type: String }, + }, + { _id: false }, + ), + }, + timeout: { type: Number }, + retryOptions: { + type: new mongoose.Schema( + { + retries: { type: Number, default: 0 }, + delay: { type: Number, default: 1000 }, + }, + { _id: false }, + ), + }, + }, + { _id: false }, +); + +// const LifecycleHooksSchema = new mongoose.Schema( +// { +// preExecution: { type: String }, +// postExecution: { type: RESTRequestSchema }, +// }, +// { _id: false }, +// ); + +const KafkaEventSchema = new mongoose.Schema( + { + topic: { type: String, required: true }, + key: { type: String }, + }, + { _id: false }, +); + +export const PipelineTriggerSchema = new mongoose.Schema( + { + type: { + type: String, + enum: ['always', 'kafka-event'], + required: true, + }, + kafkaEvent: { + type: KafkaEventSchema, + required: function (this: PipelineTrigger) { + return this.type === 'kafka-event'; }, }, }, - dataType: { type: String, required: false, default: 'string', enum: ['string', 'number', 'boolean', 'array', 'json'] }, - isReserved: { type: Boolean, required: false, default: false }, -}, { - _id: false, + { _id: false }, +); + +const ConditionSchema = new mongoose.Schema( + { + variable: { type: String }, + operator: { + type: String, + enum: ['eq', 'ne', 'gt', 'lt', 'gte', 'lte'], + }, + value: { type: mongoose.Schema.Types.Mixed }, + logic: { + type: String, + enum: ['AND', 'OR'], + }, + conditions: [ + { + type: mongoose.Schema.Types.Mixed, // 재귀적 구조를 위한 설정 + }, + ], + }, + { _id: false }, +); + +export const ScheduleSchema = new mongoose.Schema({ + cron: { type: String, required: true }, + timezone: { type: String }, }); -export const TaxonomyChildItemSubSchema = new Schema( +export const StageTriggerSchema = new mongoose.Schema( { - title: { type: String, required: true }, - order: { type: Number, required: true, default: 100 }, - slug: { type: String }, // slug는 없을 수도 있으므로 required 제외 - articleId: { type: Schema.Types.ObjectId, ref: 'article' }, + type: { + type: String, + enum: ['always', 'condition', 'schedule'], + required: true, + }, + condition: { + type: ConditionSchema, + required: function (this: StageTrigger) { + return this.type === 'condition'; + }, + }, + schedule: { + type: ScheduleSchema, + required: function (this: StageTrigger) { + return this.type === 'schedule'; + }, + }, + }, + { _id: false }, +); + +// export const ArtifactSchema = new mongoose.Schema({ +// key: { type: String }, +// value: { type: mongoose.SchemaTypes.Mixed }, +// dataType: { +// type: String, +// enum: dataTypeEnum, +// required: true, +// }, +// isReserved: { type: Boolean }, +// }); + +export const MetadataSubSchema = new mongoose.Schema( + { + key: { type: String, required: true }, + value: { + type: mongoose.SchemaTypes.Mixed, + required: false, // required: true로 하면 value가 null인 경우 에러 발생 + default: null, + validate: { + validator: function (v: string | any[]) { + // 특정 키인 경우 value가 null 또는 빈 문자열이 아니어야 함 + // if (this.key === 'requiredKey') { + // return typeof v === 'string' && v.trim().length > 0; + // } + // 그렇지 않은 경우 value는 null 또는 문자열, Array이어야 함 + return typeof v === 'string' || Array.isArray(v); + }, + message: function (props: any) { + if (this.key === 'requiredKey') { + return `Value is required for key "${this.key}" and cannot be null or empty.`; + } + return `Value must be a string or array for key "${this.key}".`; + }, + }, + }, + dataType: { + type: String, + required: false, + default: 'string', + enum: ['string', 'number', 'boolean', 'array', 'json'], + }, + isReserved: { type: Boolean, required: false, default: false }, }, { - _id: false, // 하위 아이템마다 _id가 생성되는 것을 방지 (선택 사항) + _id: false, }, ); +// export const MANDATORY_ARTIFACT_KEYS = { +// test: 'test', +// } as const; + +// export type MandatoryArtifactKey = +// (typeof MANDATORY_ARTIFACT_KEYS)[keyof typeof MANDATORY_ARTIFACT_KEYS]; + +// class Artifacts extends Array { +// constructor(artifacts: Artifact[] = []) { +// // artifacts를 순회하며 dataType을 설정 +// const processedArtifacts = artifacts.map((artifact) => ({ +// ...artifact, +// dataType: determineDataType(artifact.value), +// })); +// super(...processedArtifacts); +// } + +// // 특정 key로 태그를 찾는 메서드 추가 +// private findArtifactByKey(key: string): Artifact | undefined { +// return this.find((artifact) => artifact.key === key); +// } + +// // getStringByKey(key: ReservedDeviceTagKey): string; +// getStringByKey(key: string): string; +// getStringByKey(key: MandatoryArtifactKey | string): string { +// return this.findArtifactByKey(key)?.value ?? ''; +// } + +// // getValueByKey(key: ReservedDeviceTagKey): string | null; +// getValueByKey(key: string): string | null; +// getValueByKey(key: MandatoryArtifactKey | string): string | null { +// return this.findArtifactByKey(key)?.value ?? null; +// } +// } + +export const MANDATORY_METADATA_KEYS = { + test: 'test', +} as const; + +export type MandatoryMetadataKey = + (typeof MANDATORY_METADATA_KEYS)[keyof typeof MANDATORY_METADATA_KEYS]; + +// class Metadatas extends Array { +// constructor(metadatas: Metadata[] | mongoose.FilterQuery<'FindOne'>[] | []) { +// // metadatas를 순회하며 dataType을 설정 +// // console.log(Object.prototype.toString.call(metadatas), 'metadatas의 프로토타입'); +// const processedMetadatas = metadatas.map((metadata) => ({ +// ...metadata, +// dataType: determineDataType(metadata.dataType), +// })); +// // console.log(processedMetadatas, 'processedMetadatas'); +// super(...processedMetadatas); +// } + +// // 특정 key로 태그를 찾는 메서드 추가 +// private findMetadataByKey(key: string): Metadata | undefined { +// return this.find((metadata) => metadata.key === key); +// } + +// // getStringByKey(key: ReservedDeviceTagKey): string; +// getStringByKey(key: string): string; +// getStringByKey(key: MandatoryMetadataKey | string): string { +// console.log('key:', key); +// console.log('value', this.findMetadataByKey(key)); +// return this.findMetadataByKey(key)?.value ?? ''; +// } + +// // getValueByKey(key: ReservedDeviceTagKey): string | null; +// getValueByKey(key: string): T | null; +// getValueByKey(key: MandatoryMetadataKey | string): T | null { +// return this.findMetadataByKey(key)?.value ?? null; +// } +// } + +export const ExecutionSchema = new mongoose.Schema( + { + // executionId: { type: String, required: true }, + status: { + type: String, + enum: [ + 'pending', + 'in-progress', + 'completed', + 'failed', + 'cancelled', + 'skipped', + ], + required: true, + }, + // artifact: { type: [ArtifactSchema], default: [] }, + metadatas: { + type: [MetadataSubSchema], + required: true, + default: [], + validator: function ( + v: string | any[] | number | boolean | object | null, + ) { + // 'this'는 현재 검증 중인 Metadata 서브 도큐먼트를 가리킵니다. + const dataType = (this as Metadata).dataType; + + if (v === null) { + return true; // null은 항상 허용 (required: false이므로) + } + + switch (dataType) { + case 'string': + return typeof v === 'string'; + case 'number': + return typeof v === 'number'; + case 'boolean': + return typeof v === 'boolean'; + case 'array': + return Array.isArray(v); + case 'json': + // JSON은 객체 형태일 수 있습니다. + return typeof v === 'object' && v !== null && !Array.isArray(v); + default: + // 'string'이 기본값이므로 이쪽으로 빠지는 경우는 드물지만, 방어 코드 + return typeof v === 'string'; + } + }, + message: function (props: any) { + const dataType = (this as Metadata).dataType; + return `Value "${props.value}" does not match the expected data type "${dataType}".`; + }, + }, + }, + { + // _id: false, // 서브 문서로 사용되므로 _id 필드를 생성하지 않습니다. + timestamps: true, + // discriminatorKey: 'kind', // 'kind'는 원하는 키 이름으로 변경 가능합니다. + }, +); +// ExecutionSchema.post('find', makeMetadatasInstanceHook); +// ExecutionSchema.post('findOne', makeMetadatasInstanceHook); +// ExecutionSchema.virtual('metadatas').get(function () { +// return new Metadatas(this.toObject().metadata); +// }); + +export const VariableSchema = new mongoose.Schema({ + name: { type: String, required: true }, + value: { type: mongoose.Schema.Types.Mixed, required: true }, + scope: { type: String, enum: ['pipeline', 'stage'], required: true }, +}); + +export const RetryStrategySchema = new mongoose.Schema({ + limit: { type: Number, required: true }, + delay: { type: Number, required: true }, + backoff: { type: Number, required: true }, + when: { type: [String], required: true }, + notify: { type: [String], required: true }, +}); + +export const EventSourceSubSchema = new mongoose.Schema( + { + type: { type: String, required: true }, + name: { type: String, required: true }, + metadatas: { + type: [MetadataSubSchema], + required: true, + default: [], + validator: function ( + v: string | any[] | number | boolean | object | null, + ) { + // 'this'는 현재 검증 중인 Metadata 서브 도큐먼트를 가리킵니다. + const dataType = (this as Metadata).dataType; + + if (v === null) { + return true; // null은 항상 허용 (required: false이므로) + } + + switch (dataType) { + case 'string': + return typeof v === 'string'; + case 'number': + return typeof v === 'number'; + case 'boolean': + return typeof v === 'boolean'; + case 'array': + return Array.isArray(v); + case 'json': + // JSON은 객체 형태일 수 있습니다. + return typeof v === 'object' && v !== null && !Array.isArray(v); + default: + // 'string'이 기본값이므로 이쪽으로 빠지는 경우는 드물지만, 방어 코드 + return typeof v === 'string'; + } + }, + message: function (props: any) { + const dataType = (this as Metadata).dataType; + return `Value "${props.value}" does not match the expected data type "${dataType}".`; + }, + }, + }, + { + _id: false, + }, +); + +export const StagedefSchema = new mongoose.Schema( + { + apiVersion: { type: String, required: true }, + eventSource: { type: [EventSourceSubSchema], required: true }, + // lifecycleHooks: { type: LifecycleHooksSchema }, + schema: { type: mongoose.Schema.Types.Mixed }, + name: { type: String, required: true }, + taxonomy: { type: String }, + topic: { type: String, required: true }, + metadatas: { + type: [MetadataSubSchema], + required: true, + default: [], + validator: function ( + v: string | any[] | number | boolean | object | null, + ) { + // 'this'는 현재 검증 중인 Metadata 서브 도큐먼트를 가리킵니다. + const dataType = (this as Metadata).dataType; + + if (v === null) { + return true; // null은 항상 허용 (required: false이므로) + } + + switch (dataType) { + case 'string': + return typeof v === 'string'; + case 'number': + return typeof v === 'number'; + case 'boolean': + return typeof v === 'boolean'; + case 'array': + return Array.isArray(v); + case 'json': + // JSON은 객체 형태일 수 있습니다. + return typeof v === 'object' && v !== null && !Array.isArray(v); + default: + // 'string'이 기본값이므로 이쪽으로 빠지는 경우는 드물지만, 방어 코드 + return typeof v === 'string'; + } + }, + message: function (props: any) { + const dataType = (this as Metadata).dataType; + return `Value "${props.value}" does not match the expected data type "${dataType}".`; + }, + }, + }, + { + timestamps: true, + }, +); +// StagedefSchema.post('find', makeMetadatasInstanceHook); +// StagedefSchema.post('findOne', makeMetadatasInstanceHook); +// StagedefSchema.virtual('metadatas').get(function () { +// return new Metadatas(this.toObject().metadata); +// }); + +export const StageSchema = new mongoose.Schema( + { + apiVersion: { type: String, required: true }, + name: { type: String, required: true }, + schema: { type: mongoose.Schema.Types.Mixed }, + taxonomy: { type: String }, + // lifecycleHooks: { type: LifecycleHooksSchema, default: {} }, + executions: { type: [ExecutionSchema], default: [] }, + // outputData: { type: mongoose.Schema.Types.Mixed, default: {} }, + status: { + type: String, + enum: [ + 'pending', + 'in-progress', + 'completed', + 'failed', + 'cancelled', + 'skipped', + ], + required: true, + }, + retryStrategy: { type: RetryStrategySchema }, + // artifact: { type: ArtifactSchema, default: {} }, + metadatas: { + type: [MetadataSubSchema], + required: true, + default: [], + validator: function ( + v: string | any[] | number | boolean | object | null, + ) { + // 'this'는 현재 검증 중인 Metadata 서브 도큐먼트를 가리킵니다. + const dataType = (this as Metadata).dataType; + + if (v === null) { + return true; // null은 항상 허용 (required: false이므로) + } + + switch (dataType) { + case 'string': + return typeof v === 'string'; + case 'number': + return typeof v === 'number'; + case 'boolean': + return typeof v === 'boolean'; + case 'array': + return Array.isArray(v); + case 'json': + // JSON은 객체 형태일 수 있습니다. + return typeof v === 'object' && v !== null && !Array.isArray(v); + default: + // 'string'이 기본값이므로 이쪽으로 빠지는 경우는 드물지만, 방어 코드 + return typeof v === 'string'; + } + }, + message: function (props: any) { + const dataType = (this as Metadata).dataType; + return `Value "${props.value}" does not match the expected data type "${dataType}".`; + }, + }, + // variables: { type: [VariableSchema], default: [] }, + }, + { + // 여기서 discriminatorKey 옵션을 추가합니다. + discriminatorKey: 'kind', // 'kind'는 원하는 키 이름으로 변경 가능합니다. + }, +); +// StageSchema.post('find', makeMetadatasInstanceHook); +// StageSchema.post('findOne', makeMetadatasInstanceHook); +// StageSchema.virtual('metadatas').get(function () { +// return new Metadatas(this.toObject().metadata); +// }); + +export const PipelinedefSchema = new mongoose.Schema({ + apiVersion: { type: String, required: true }, + trigger: { type: PipelineTriggerSchema, required: true }, + stage: { type: [mongoose.Types.ObjectId], required: true }, + // revision: { type: Number }, + metadatas: { + type: [MetadataSubSchema], + required: true, + default: [], + validator: function (v: string | any[] | number | boolean | object | null) { + // 'this'는 현재 검증 중인 Metadata 서브 도큐먼트를 가리킵니다. + const dataType = (this as Metadata).dataType; + + if (v === null) { + return true; // null은 항상 허용 (required: false이므로) + } + + switch (dataType) { + case 'string': + return typeof v === 'string'; + case 'number': + return typeof v === 'number'; + case 'boolean': + return typeof v === 'boolean'; + case 'array': + return Array.isArray(v); + case 'json': + // JSON은 객체 형태일 수 있습니다. + return typeof v === 'object' && v !== null && !Array.isArray(v); + default: + // 'string'이 기본값이므로 이쪽으로 빠지는 경우는 드물지만, 방어 코드 + return typeof v === 'string'; + } + }, + message: function (props: any) { + const dataType = (this as Metadata).dataType; + return `Value "${props.value}" does not match the expected data type "${dataType}".`; + }, + }, +}); +// PipelinedefSchema.post('find', makeMetadatasInstanceHook); +// PipelinedefSchema.post('findOne', makeMetadatasInstanceHook); +// PipelinedefSchema.virtual('metadatas').get(function () { +// return new Metadatas(this.toObject().metadata); +// }); + +export const PipelineSchema = new mongoose.Schema( + { + apiVersion: { type: String, required: true }, + name: { type: String, required: false }, + // revision: { type: Number, required: true }, + stage: [{ type: StageSchema, required: true }], + metadatas: { + type: [MetadataSubSchema], + required: true, + default: [], + validator: function ( + v: string | any[] | number | boolean | object | null, + ) { + // 'this'는 현재 검증 중인 Metadata 서브 도큐먼트를 가리킵니다. + const dataType = (this as Metadata).dataType; + + if (v === null) { + return true; // null은 항상 허용 (required: false이므로) + } + + switch (dataType) { + case 'string': + return typeof v === 'string'; + case 'number': + return typeof v === 'number'; + case 'boolean': + return typeof v === 'boolean'; + case 'array': + return Array.isArray(v); + case 'json': + // JSON은 객체 형태일 수 있습니다. + return typeof v === 'object' && v !== null && !Array.isArray(v); + default: + // 'string'이 기본값이므로 이쪽으로 빠지는 경우는 드물지만, 방어 코드 + return typeof v === 'string'; + } + }, + message: function (props: any) { + const dataType = (this as Metadata).dataType; + return `Value "${props.value}" does not match the expected data type "${dataType}".`; + }, + }, + }, + { + timestamps: true, + }, +); +// PipelineSchema.post('find', makeMetadatasInstanceHook); +// PipelineSchema.post('findOne', makeMetadatasInstanceHook); +// PipelineSchema.virtual('metadatas').get(function () { +// return new Metadatas(this.toObject().metadata); +// }); + +// StageSchema.index({ status: 1 }); + +export const AIModelDefinitions: ModelDefinition[] = [ + { name: 'pipelinedefs', schema: PipelinedefSchema }, + { name: 'pipelines', schema: PipelineSchema }, + { name: 'stagedefs', schema: StagedefSchema }, + { name: 'stages', schema: StageSchema }, +]; + +export interface AIModelTypes { + pipelinedefs: Pipelinedef; + pipelines: Pipeline; + stagedefs: Stagedef; + stages: Stage; +} + +// async function makeArtifactsInstanceHook( +// res: any, +// next: CallbackWithoutResultAndOptionalError, +// ) { +// if (Array.isArray(res)) { +// for (let doc of res) { +// if (doc.tags && Array.isArray(doc.tags)) { +// // 수동으로 배열의 프로토타입을 Tags로 변환 +// doc.tags = new Artifacts(doc.tags); +// Object.setPrototypeOf(doc.tags, Artifacts.prototype); +// console.log('doc.tags prototype:', Object.getPrototypeOf(doc.tags)); +// console.log('doc.tags constructor name:', doc.tags.constructor.name); +// if (doc.tags instanceof Artifacts) { +// console.log('instanceof doc.tags', doc.tags); +// } else { +// console.log('Not an instance of Tags'); +// } +// } +// } +// } else { +// if (res && res.tags && Array.isArray(res.tags)) { +// res.tags = new Artifacts(res.tags); +// Object.setPrototypeOf(res.tags, Artifacts.prototype); +// console.log('res.tags prototype:', Object.getPrototypeOf(res.tags)); +// console.log('res.tags constructor name:', res.tags.constructor.name); +// if (res.tags instanceof Artifacts) { +// console.log('instanceof res.tags', res.tags); +// } else { +// console.log('Not an instance of Tags'); +// } +// } +// } +// next(); +// } + +// async function makeMetadatasInstanceHook( +// res: any, +// next: CallbackWithoutResultAndOptionalError, +// ) { +// const visited = new WeakSet(); + +// function instantiateMetadatas(doc: any) { +// if (doc && typeof doc === 'object') { +// if (visited.has(doc)) { +// return; +// } +// visited.add(doc); + +// for (const key in doc) { +// if (doc.hasOwnProperty(key)) { +// const value = doc[key]; +// if (key === 'metadata' && Array.isArray(value)) { +// // Instantiate metadata arrays into Metadatas instances +// doc[key] = new Metadatas(value); +// Object.setPrototypeOf(doc[key], Metadatas.prototype); +// } else if (Array.isArray(value)) { +// // Recursively handle arrays +// for (let item of value) { +// instantiateMetadatas(item); +// } +// } else if (typeof value === 'object' && value !== null) { +// // Recursively handle nested objects +// instantiateMetadatas(value); +// } +// } +// } +// } +// } + +// if (Array.isArray(res)) { +// for (let doc of res) { +// instantiateMetadatas(doc); +// } +// } else { +// instantiateMetadatas(res); +// } +// next(); +// } + +// export function determineDataType(value: any): DataType { +// if (typeof value === 'string') { +// return 'string'; +// } else if (typeof value === 'number') { +// return 'number'; +// } else if (typeof value === 'boolean') { +// return 'boolean'; +// } else if (Array.isArray(value)) { +// // 배열 내 모든 요소가 문자열인지 확인 +// if (value.every((item) => typeof item === 'string')) { +// return 'array'; +// } +// return 'mixed'; // 다양한 타입이 섞인 배열 +// } else if (typeof value === 'object' && value !== null) { +// return 'mixed'; +// } +// return 'mixed'; // 기본값 +// } + +// export const TaxonomyChildItemSubSchema = new Schema( +// { +// title: { type: String, required: true }, +// order: { type: Number, required: true, default: 100 }, +// slug: { type: String }, // slug는 없을 수도 있으므로 required 제외 +// articleId: { type: Schema.Types.ObjectId, ref: 'article' }, +// }, +// { +// _id: false, // 하위 아이템마다 _id가 생성되는 것을 방지 (선택 사항) +// }, +// ); + // **중요: 재귀 참조 설정** // 스키마 정의 후 .add()를 통해 자기 자신을 children 배열로 추가합니다. -TaxonomyChildItemSubSchema.add({ - children: { type: [TaxonomyChildItemSubSchema], default: [] }, -}); +// TaxonomyChildItemSubSchema.add({ +// children: { type: [TaxonomyChildItemSubSchema], default: [] }, +// }); -// 2. 메인 도큐먼트 스키마 (Root) -export const TaxonomySchema = new Schema( +// // 2. 메인 도큐먼트 스키마 (Root) +// export const TaxonomySchema = new Schema( +// { +// // items 배열은 위에서 정의한 재귀 스키마를 따름 +// label: { type: String, required: true }, +// items: { +// type: [TaxonomyChildItemSubSchema], +// default: [], +// }, + +// // metadata는 구조가 가변적이므로 Mixed 타입 사용 +// metadatas: { +// type: [MetadataSubSchema], +// required: true, +// default: [], +// validator: function ( +// v: string | any[] | number | boolean | object | null, +// ) { +// // 'this'는 현재 검증 중인 Metadata 서브 도큐먼트를 가리킵니다. +// const dataType = (this as Metadata).dataType; + +// if (v === null) { +// return true; // null은 항상 허용 (required: false이므로) +// } + +// switch (dataType) { +// case 'string': +// return typeof v === 'string'; +// case 'number': +// return typeof v === 'number'; +// case 'boolean': +// return typeof v === 'boolean'; +// case 'array': +// return Array.isArray(v); +// case 'json': +// // JSON은 객체 형태일 수 있습니다. +// return typeof v === 'object' && v !== null && !Array.isArray(v); +// default: +// // 'string'이 기본값이므로 이쪽으로 빠지는 경우는 드물지만, 방어 코드 +// return typeof v === 'string'; +// } +// }, +// message: function (props: any) { +// const dataType = (this as Metadata).dataType; +// return `Value "${props.value}" does not match the expected data type "${dataType}".`; +// }, +// }, +// }, +// { +// timestamps: true, // createAt, updatedAt 자동 생성 +// collection: 'taxonomy', // 컬렉션 이름 지정 +// }, +// ); + +// TaxonomySchema.index({ label: 1 }, { unique: true }); + +// export const ContentSectionEmbeddedTaxonomySubSchema = +// new Schema( +// { +// taxonomyId: { +// type: Schema.Types.ObjectId, +// ref: 'taxonomy', +// required: true, +// }, +// label: { type: String, required: true }, +// }, +// { _id: false }, +// ); + +// export const ContentSectionSubSchema = new Schema( +// { +// headline: { type: String, required: true }, +// taxonomies: { +// type: [ContentSectionEmbeddedTaxonomySubSchema], +// required: true, +// default: [], +// }, +// }, +// { _id: false }, +// ); + +// export const ContentSchema = new Schema( +// { +// key: { type: String, required: true, unique: true }, +// label: { type: String, required: true }, +// sections: { +// type: [ContentSectionSubSchema], +// default: [], +// }, +// }, +// { +// timestamps: true, +// collection: 'content', +// }, +// ); + +// export const ArticleSchema = new Schema
( +// { +// title: { type: String, required: true }, +// content: { type: String, required: true }, +// slug: { type: String, required: true, unique: true }, +// status: { +// type: String, +// enum: ['draft', 'published', 'archived'], +// default: 'draft', +// }, +// metadatas: { +// type: [MetadataSubSchema], +// required: true, +// default: [], +// validate: { +// validator: function (metas: Metadata[]) { +// const keys = metas.map((m) => m.key); +// return keys.length === new Set(keys).size; +// }, +// message: 'Duplicate metadata keys are not allowed.', +// }, +// }, +// }, +// { +// timestamps: true, +// collection: 'article', +// }, +// ); + +// ArticleSchema.index({ title: 'text', content: 'text' }); + +export const CrawlerModelDefinitions: ModelDefinition[] = [ { - // items 배열은 위에서 정의한 재귀 스키마를 따름 - label: { type: String, required: true }, - items: { - type: [TaxonomyChildItemSubSchema], - default: [], - }, - - // metadata는 구조가 가변적이므로 Mixed 타입 사용 - metadatas: { - type: [MetadataSubSchema], - required: true, - default: [], - validator: function (v: string | any[] | number | boolean | object | null) { - // 'this'는 현재 검증 중인 Metadata 서브 도큐먼트를 가리킵니다. - const dataType = (this as Metadata).dataType; - - if (v === null) { - return true; // null은 항상 허용 (required: false이므로) - } - - switch (dataType) { - case 'string': - return typeof v === 'string'; - case 'number': - return typeof v === 'number'; - case 'boolean': - return typeof v === 'boolean'; - case 'array': - return Array.isArray(v); - case 'json': - // JSON은 객체 형태일 수 있습니다. - return typeof v === 'object' && v !== null && !Array.isArray(v); - default: - // 'string'이 기본값이므로 이쪽으로 빠지는 경우는 드물지만, 방어 코드 - return typeof v === 'string'; - } - }, - message: function (props: any) { - const dataType = (this as Metadata).dataType; - return `Value "${props.value}" does not match the expected data type "${dataType}".`; - }, - }, + name: 'pipelinedefs', + schema: PipelinedefSchema, + schemaName: 'CrawlerPipelinedef', }, - { - timestamps: true, // createAt, updatedAt 자동 생성 - collection: 'taxonomy', // 컬렉션 이름 지정 - }, -); - -TaxonomySchema.index({ label: 1 }, { unique: true }); - -export const ContentSectionEmbeddedTaxonomySubSchema = new Schema( - { - taxonomyId: { type: Schema.Types.ObjectId, ref: 'taxonomy', required: true }, - label: { type: String, required: true }, - }, - { _id: false }, -); - -export const ContentSectionSubSchema = new Schema( - { - headline: { type: String, required: true }, - taxonomies: { - type: [ContentSectionEmbeddedTaxonomySubSchema], - required: true, - default: [], - }, - }, - { _id: false }, -); - -export const ContentSchema = new Schema( - { - key: { type: String, required: true, unique: true }, - label: { type: String, required: true }, - sections: { - - type: [ - ContentSectionSubSchema - ], - default: [], - }, - }, - { - timestamps: true, - collection: 'content', - }, -); - -export const ArticleSchema = new Schema
( - { - title: { type: String, required: true }, - content: { type: String, required: true }, - slug: { type: String, required: true, unique: true }, - status: { - type: String, - enum: ['draft', 'published', 'archived'], - default: 'draft', - }, - metadatas: { - type: [MetadataSubSchema], - required: true, - default: [], - validate: { - validator: function (metas: Metadata[]) { - const keys = metas.map((m) => m.key); - return keys.length === new Set(keys).size; - }, - message: 'Duplicate metadata keys are not allowed.', - }, - }, - }, - { - timestamps: true, - collection: 'article', - }, -); - -ArticleSchema.index({ title: 'text', content: 'text' }); - -export const moeumModelDefinitions: ModelDefinition[] = [ - { name: 'taxonomy', schema: TaxonomySchema, schemaName: 'taxonomy' }, - { name: 'content', schema: ContentSchema, schemaName: 'content' }, - { name: 'article', schema: ArticleSchema, schemaName: 'article' }, + { name: 'pipelines', schema: PipelineSchema, schemaName: 'CrawlerPipeline' }, + { name: 'stagedefs', schema: StagedefSchema, schemaName: 'CrawlerStagedef' }, + // { name: 'stages', schema: StageSchema, schemaName: 'CrawlerStage' }, ]; -export interface MoeumModelTypes { - taxonomy: Taxonomy; - content: Content; - article: Article; +export interface CrawlerModelTypes { + pipelinedefs: Pipelinedef; + pipelines: Pipeline; + stagedefs: Stagedef; + // stages: Stage; }