first commit

This commit is contained in:
dahyeon committed 2025-11-08 18:27:51 +09:00
commit c809bb3893
40 files changed
+3836

No files matched your search

+108
View File
@@ -0,0 +1,108 @@
import { spawn } from 'child_process';
import { mkdir, writeFile, readFile, unlink } from 'fs/promises';
import { join } from 'path';
import { tmpdir } from 'os';
import { randomBytes } from 'crypto';
export type ImageFormat = 'avif' | 'webp' | 'jpeg';
export interface ConversionOptions {
width: number;
format: ImageFormat;
}
export interface ConversionResult {
buffer: Buffer;
format: ImageFormat;
width: number;
}
export class FFmpegConverter {
private getTempFilePath(ext: string): string {
const randomName = randomBytes(16).toString('hex');
return join(tmpdir(), `img-${randomName}.${ext}`);
}
async convert(
inputBuffer: Buffer,
options: ConversionOptions
): Promise<ConversionResult> {
const inputPath = this.getTempFilePath('tmp');
const outputPath = this.getTempFilePath(options.format);
try {
// Write input buffer to temp file
await writeFile(inputPath, inputBuffer);
// Build FFmpeg command
const args = [
'-i', inputPath,
'-vf', `scale=${options.width}:-1`,
];
// Format-specific options
if (options.format === 'avif') {
args.push('-c:v', 'libaom-av1', '-crf', '30');
} else if (options.format === 'webp') {
args.push('-c:v', 'libwebp', '-quality', '80');
} else if (options.format === 'jpeg') {
args.push('-q:v', '5');
}
args.push('-y', outputPath);
// Execute FFmpeg
await this.executeFFmpeg(args);
// Read output file
const outputBuffer = await readFile(outputPath);
return {
buffer: outputBuffer,
format: options.format,
width: options.width,
};
} finally {
// Clean up temp files
try {
await unlink(inputPath);
} catch {}
try {
await unlink(outputPath);
} catch {}
}
}
private executeFFmpeg(args: string[]): Promise<void> {
return new Promise((resolve, reject) => {
const ffmpeg = spawn('ffmpeg', args);
let stderr = '';
ffmpeg.stderr.on('data', (data: Buffer) => {
stderr += data.toString();
});
ffmpeg.on('close', (code) => {
if (code === 0) {
resolve();
} else {
reject(new Error(`FFmpeg exited with code ${code}\n${stderr}`));
}
});
ffmpeg.on('error', (err) => {
reject(new Error(`Failed to start FFmpeg: ${err.message}`));
});
});
}
getContentType(format: ImageFormat): string {
const contentTypes: Record<ImageFormat, string> = {
avif: 'image/avif',
webp: 'image/webp',
jpeg: 'image/jpeg',
};
return contentTypes[format];
}
}
+106
View File
@@ -0,0 +1,106 @@
import { MinioClient } from './minio.js';
import { NatsClient } from './nats.js';
import { FFmpegConverter, type ImageFormat } from './ffmpeg.js';
import { extname, parse } from 'path';
export interface GenerateDerivativesMessage {
s3ObjectKey: string;
}
export class ImageDerivativesGenerator {
private readonly resolutions = [320, 640, 960, 1280, 1600, 1920];
private readonly formats: ImageFormat[] = ['avif', 'webp', 'jpeg'];
constructor(
private readonly minioClient: MinioClient,
private readonly natsClient: NatsClient,
private readonly ffmpegConverter: FFmpegConverter
) {}
async start(): Promise<void> {
console.log('🚀 Starting Image Derivatives Generator Service');
console.log(`📊 Resolutions: ${this.resolutions.join(', ')}`);
console.log(`🎨 Formats: ${this.formats.join(', ')}`);
await this.natsClient.subscribe<GenerateDerivativesMessage>(
'generate-derivaties',
async (message) => {
await this.processMessage(message);
}
);
console.log('✅ Service ready');
}
private async processMessage(message: GenerateDerivativesMessage): Promise<void> {
const { s3ObjectKey } = message;
console.log(`\n📥 Processing: ${s3ObjectKey}`);
try {
// Download original image from MinIO
const originalBuffer = await this.minioClient.downloadFile(s3ObjectKey);
console.log(` ✓ Downloaded original (${(originalBuffer.length / 1024).toFixed(2)} KB)`);
// Parse the original file path
const parsedPath = parse(s3ObjectKey);
const baseDir = parsedPath.dir;
const baseName = parsedPath.name;
let totalGenerated = 0;
let totalSkipped = 0;
// Generate all combinations of resolutions and formats
for (const width of this.resolutions) {
for (const format of this.formats) {
const derivativeKey = `${baseDir}/${baseName}_${width}.${format}`;
// Check if derivative already exists
const exists = await this.minioClient.fileExists(derivativeKey);
if (exists) {
totalSkipped++;
continue;
}
try {
// Convert image
const result = await this.ffmpegConverter.convert(originalBuffer, {
width,
format,
});
// Upload to MinIO
const contentType = this.ffmpegConverter.getContentType(format);
await this.minioClient.uploadFile(
derivativeKey,
result.buffer,
contentType
);
totalGenerated++;
console.log(
` ✓ Generated ${width}w ${format} (${(result.buffer.length / 1024).toFixed(2)} KB)`
);
} catch (error) {
console.error(
` ✗ Failed to generate ${width}w ${format}:`,
error instanceof Error ? error.message : error
);
}
}
}
console.log(
`\n✅ Completed: ${totalGenerated} generated, ${totalSkipped} skipped for ${s3ObjectKey}`
);
} catch (error) {
console.error(`❌ Error processing ${s3ObjectKey}:`, error);
}
}
async stop(): Promise<void> {
console.log('\n🛑 Stopping service...');
await this.natsClient.close();
console.log('✓ Service stopped');
}
}
+43
View File
@@ -0,0 +1,43 @@
import { secrets } from './secret-manager.js';
import { MinioClient } from './minio.js';
import { NatsClient } from './nats.js';
import { FFmpegConverter } from './ffmpeg.js';
import { ImageDerivativesGenerator } from './image-derivaties-generator.js';
async function main() {
try {
console.log('🔐 Secrets loaded');
console.log(secrets)
// Initialize clients
const minioClient = new MinioClient();
const natsClient = new NatsClient();
const ffmpegConverter = new FFmpegConverter();
// Connect to NATS
await natsClient.connect();
// Create and start the service
const generator = new ImageDerivativesGenerator(
minioClient,
natsClient,
ffmpegConverter
);
await generator.start();
// Handle graceful shutdown
const shutdown = async () => {
await generator.stop();
process.exit(0);
};
process.on('SIGINT', shutdown);
process.on('SIGTERM', shutdown);
} catch (error) {
console.error('❌ Failed to start service:', error);
process.exit(1);
}
}
main();
+60
View File
@@ -0,0 +1,60 @@
import * as Minio from 'minio';
import { secrets } from './secret-manager.js';
import { Readable } from 'stream';
export class MinioClient {
private client: Minio.Client;
private readonly bucketName = 'wiki-server';
constructor() {
const endpoint = secrets?.['MINIO_ENDPOINT'] as string;
const accessKey = secrets?.['MINIO_ACCESS_KEY'] as string;
const secretKey = secrets?.['MINIO_SECRET_KEY'] as string;
if (!endpoint || !accessKey || !secretKey) {
throw new Error('Missing MinIO credentials in secrets');
}
this.client = new Minio.Client({
endPoint: endpoint,
port: 443,
useSSL: true,
accessKey: accessKey,
secretKey: secretKey,
});
}
async downloadFile(s3ObjectKey: string): Promise<Buffer> {
const stream = await this.client.getObject(this.bucketName, s3ObjectKey);
return new Promise((resolve, reject) => {
const chunks: Buffer[] = [];
stream.on('data', (chunk: Buffer) => chunks.push(chunk));
stream.on('end', () => resolve(Buffer.concat(chunks)));
stream.on('error', reject);
});
}
async uploadFile(s3ObjectKey: string, buffer: Buffer, contentType: string): Promise<void> {
const metaData = {
'Content-Type': contentType,
'X-Amz-Meta-serviceName': 'image-derivaties-generation',
};
await this.client.putObject(
this.bucketName,
s3ObjectKey,
buffer,
buffer.length,
metaData
);
}
async fileExists(s3ObjectKey: string): Promise<boolean> {
try {
await this.client.statObject(this.bucketName, s3ObjectKey);
return true;
} catch (err) {
return false;
}
}
}
+86
View File
@@ -0,0 +1,86 @@
import { type NatsConnection, StringCodec, connect, type Subscription } from 'nats';
import { secrets } from './secret-manager.js';
import fs from 'fs/promises';
export class NatsClient {
private client: NatsConnection | null = null;
private readonly stringCodec = StringCodec();
async connect(): Promise<void> {
const endpoint = secrets?.['NATS_ENDPOINT'] as string;
const user = secrets?.['NATS_USER'] as string;
const password = secrets?.['NATS_PASS'] as string;
if (!endpoint || !user || !password) {
throw new Error('Missing NATS credentials in secrets');
}
const cacert = (await fs.readFile('internal-service-ca.pem')).toString();
this.client = await connect({
name: 'image-derivaties-generation-service',
servers: [endpoint],
tls: {
ca: cacert,
},
maxReconnectAttempts: -1,
reconnectTimeWait: 500,
reconnect: true,
pingInterval: 20000,
timeout: 5000,
user: user,
pass: password,
});
console.log('✓ NATS connected');
}
async subscribe<T = unknown>(
subject: string,
handler: (data: T) => void | Promise<void>
): Promise<Subscription> {
if (!this.client) {
throw new Error('NATS client not connected');
}
const subscription = this.client.subscribe(subject);
(async () => {
for await (const msg of subscription) {
try {
const data = JSON.parse(this.stringCodec.decode(msg.data)) as T;
await handler(data);
} catch (error) {
console.error(`Error processing message from ${subject}:`, error);
}
}
})();
console.log(`✓ Subscribed to ${subject}`);
return subscription;
}
async publish<T = unknown>(subject: string, data: T): Promise<void> {
if (!this.client) {
throw new Error('NATS client not connected');
}
this.client.publish(subject, this.stringCodec.encode(JSON.stringify(data)));
await this.client.flush();
}
async close(): Promise<void> {
if (!this.client) return;
try {
const closed = this.client.closed();
await this.client.drain();
const err = await closed;
if (err) console.error('NATS closed with error:', err);
} catch (e) {
try {
await this.client.close();
} catch {}
}
}
}
+38
View File
@@ -0,0 +1,38 @@
import { InfisicalSDK } from "@infisical/sdk";
export type SecretMap = Record<string, string | boolean | undefined>;
const INFISICAL_SITE_URL = "https://secret-manager.dahyeon.us/"
const INFISICAL_CLIENT_SECRET = process.env.INFISICAL_CLIENT_SECRET;
const INFISICAL_CLIENT_ID = process.env.INFISICAL_CLIENT_ID;
const INFISICAL_PROJECT_ID = process.env.INFISICAL_PROJECT_ID;
const infisicalClient = new InfisicalSDK({ siteUrl: INFISICAL_SITE_URL })
export let secrets: SecretMap | null = null;
const fetchSecrets = async (): Promise<SecretMap> => {
if (!INFISICAL_CLIENT_SECRET || !INFISICAL_CLIENT_ID || !INFISICAL_PROJECT_ID) {
throw new Error("Missing infisical environment variables")
}
await infisicalClient.auth().universalAuth.login({
clientId: INFISICAL_CLIENT_ID,
clientSecret: INFISICAL_CLIENT_SECRET,
});
const res = await infisicalClient.secrets().listSecrets({
environment: process.env.NODE_ENV === 'prod' ? 'prod' : process.env.NODE_ENV === 'test' ? 'staging' : 'dev',
projectId: INFISICAL_PROJECT_ID,
});
const map: SecretMap = {};
for (const s of res.secrets) {
map[s.secretKey] = s.secretValue;
}
return map
}
secrets = await fetchSecrets()