first commit
This commit is contained in:
commit
c809bb3893
40 files changed
+3836
No files matched your search
+108
@@ -0,0 +1,108 @@
|
||||
import { spawn } from 'child_process';
|
||||
import { mkdir, writeFile, readFile, unlink } from 'fs/promises';
|
||||
import { join } from 'path';
|
||||
import { tmpdir } from 'os';
|
||||
import { randomBytes } from 'crypto';
|
||||
|
||||
export type ImageFormat = 'avif' | 'webp' | 'jpeg';
|
||||
|
||||
export interface ConversionOptions {
|
||||
width: number;
|
||||
format: ImageFormat;
|
||||
}
|
||||
|
||||
export interface ConversionResult {
|
||||
buffer: Buffer;
|
||||
format: ImageFormat;
|
||||
width: number;
|
||||
}
|
||||
|
||||
export class FFmpegConverter {
|
||||
private getTempFilePath(ext: string): string {
|
||||
const randomName = randomBytes(16).toString('hex');
|
||||
return join(tmpdir(), `img-${randomName}.${ext}`);
|
||||
}
|
||||
|
||||
async convert(
|
||||
inputBuffer: Buffer,
|
||||
options: ConversionOptions
|
||||
): Promise<ConversionResult> {
|
||||
const inputPath = this.getTempFilePath('tmp');
|
||||
const outputPath = this.getTempFilePath(options.format);
|
||||
|
||||
try {
|
||||
// Write input buffer to temp file
|
||||
await writeFile(inputPath, inputBuffer);
|
||||
|
||||
// Build FFmpeg command
|
||||
const args = [
|
||||
'-i', inputPath,
|
||||
'-vf', `scale=${options.width}:-1`,
|
||||
];
|
||||
|
||||
// Format-specific options
|
||||
if (options.format === 'avif') {
|
||||
args.push('-c:v', 'libaom-av1', '-crf', '30');
|
||||
} else if (options.format === 'webp') {
|
||||
args.push('-c:v', 'libwebp', '-quality', '80');
|
||||
} else if (options.format === 'jpeg') {
|
||||
args.push('-q:v', '5');
|
||||
}
|
||||
|
||||
args.push('-y', outputPath);
|
||||
|
||||
// Execute FFmpeg
|
||||
await this.executeFFmpeg(args);
|
||||
|
||||
// Read output file
|
||||
const outputBuffer = await readFile(outputPath);
|
||||
|
||||
return {
|
||||
buffer: outputBuffer,
|
||||
format: options.format,
|
||||
width: options.width,
|
||||
};
|
||||
} finally {
|
||||
// Clean up temp files
|
||||
try {
|
||||
await unlink(inputPath);
|
||||
} catch {}
|
||||
try {
|
||||
await unlink(outputPath);
|
||||
} catch {}
|
||||
}
|
||||
}
|
||||
|
||||
private executeFFmpeg(args: string[]): Promise<void> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const ffmpeg = spawn('ffmpeg', args);
|
||||
|
||||
let stderr = '';
|
||||
|
||||
ffmpeg.stderr.on('data', (data: Buffer) => {
|
||||
stderr += data.toString();
|
||||
});
|
||||
|
||||
ffmpeg.on('close', (code) => {
|
||||
if (code === 0) {
|
||||
resolve();
|
||||
} else {
|
||||
reject(new Error(`FFmpeg exited with code ${code}\n${stderr}`));
|
||||
}
|
||||
});
|
||||
|
||||
ffmpeg.on('error', (err) => {
|
||||
reject(new Error(`Failed to start FFmpeg: ${err.message}`));
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
getContentType(format: ImageFormat): string {
|
||||
const contentTypes: Record<ImageFormat, string> = {
|
||||
avif: 'image/avif',
|
||||
webp: 'image/webp',
|
||||
jpeg: 'image/jpeg',
|
||||
};
|
||||
return contentTypes[format];
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,106 @@
|
||||
import { MinioClient } from './minio.js';
|
||||
import { NatsClient } from './nats.js';
|
||||
import { FFmpegConverter, type ImageFormat } from './ffmpeg.js';
|
||||
import { extname, parse } from 'path';
|
||||
|
||||
export interface GenerateDerivativesMessage {
|
||||
s3ObjectKey: string;
|
||||
}
|
||||
|
||||
export class ImageDerivativesGenerator {
|
||||
private readonly resolutions = [320, 640, 960, 1280, 1600, 1920];
|
||||
private readonly formats: ImageFormat[] = ['avif', 'webp', 'jpeg'];
|
||||
|
||||
constructor(
|
||||
private readonly minioClient: MinioClient,
|
||||
private readonly natsClient: NatsClient,
|
||||
private readonly ffmpegConverter: FFmpegConverter
|
||||
) {}
|
||||
|
||||
async start(): Promise<void> {
|
||||
console.log('🚀 Starting Image Derivatives Generator Service');
|
||||
console.log(`📊 Resolutions: ${this.resolutions.join(', ')}`);
|
||||
console.log(`🎨 Formats: ${this.formats.join(', ')}`);
|
||||
|
||||
await this.natsClient.subscribe<GenerateDerivativesMessage>(
|
||||
'generate-derivaties',
|
||||
async (message) => {
|
||||
await this.processMessage(message);
|
||||
}
|
||||
);
|
||||
|
||||
console.log('✅ Service ready');
|
||||
}
|
||||
|
||||
private async processMessage(message: GenerateDerivativesMessage): Promise<void> {
|
||||
const { s3ObjectKey } = message;
|
||||
|
||||
console.log(`\n📥 Processing: ${s3ObjectKey}`);
|
||||
|
||||
try {
|
||||
// Download original image from MinIO
|
||||
const originalBuffer = await this.minioClient.downloadFile(s3ObjectKey);
|
||||
console.log(` ✓ Downloaded original (${(originalBuffer.length / 1024).toFixed(2)} KB)`);
|
||||
|
||||
// Parse the original file path
|
||||
const parsedPath = parse(s3ObjectKey);
|
||||
const baseDir = parsedPath.dir;
|
||||
const baseName = parsedPath.name;
|
||||
|
||||
let totalGenerated = 0;
|
||||
let totalSkipped = 0;
|
||||
|
||||
// Generate all combinations of resolutions and formats
|
||||
for (const width of this.resolutions) {
|
||||
for (const format of this.formats) {
|
||||
const derivativeKey = `${baseDir}/${baseName}_${width}.${format}`;
|
||||
|
||||
// Check if derivative already exists
|
||||
const exists = await this.minioClient.fileExists(derivativeKey);
|
||||
if (exists) {
|
||||
totalSkipped++;
|
||||
continue;
|
||||
}
|
||||
|
||||
try {
|
||||
// Convert image
|
||||
const result = await this.ffmpegConverter.convert(originalBuffer, {
|
||||
width,
|
||||
format,
|
||||
});
|
||||
|
||||
// Upload to MinIO
|
||||
const contentType = this.ffmpegConverter.getContentType(format);
|
||||
await this.minioClient.uploadFile(
|
||||
derivativeKey,
|
||||
result.buffer,
|
||||
contentType
|
||||
);
|
||||
|
||||
totalGenerated++;
|
||||
console.log(
|
||||
` ✓ Generated ${width}w ${format} (${(result.buffer.length / 1024).toFixed(2)} KB)`
|
||||
);
|
||||
} catch (error) {
|
||||
console.error(
|
||||
` ✗ Failed to generate ${width}w ${format}:`,
|
||||
error instanceof Error ? error.message : error
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
console.log(
|
||||
`\n✅ Completed: ${totalGenerated} generated, ${totalSkipped} skipped for ${s3ObjectKey}`
|
||||
);
|
||||
} catch (error) {
|
||||
console.error(`❌ Error processing ${s3ObjectKey}:`, error);
|
||||
}
|
||||
}
|
||||
|
||||
async stop(): Promise<void> {
|
||||
console.log('\n🛑 Stopping service...');
|
||||
await this.natsClient.close();
|
||||
console.log('✓ Service stopped');
|
||||
}
|
||||
}
|
||||
+43
@@ -0,0 +1,43 @@
|
||||
import { secrets } from './secret-manager.js';
|
||||
import { MinioClient } from './minio.js';
|
||||
import { NatsClient } from './nats.js';
|
||||
import { FFmpegConverter } from './ffmpeg.js';
|
||||
import { ImageDerivativesGenerator } from './image-derivaties-generator.js';
|
||||
|
||||
async function main() {
|
||||
try {
|
||||
console.log('🔐 Secrets loaded');
|
||||
console.log(secrets)
|
||||
|
||||
// Initialize clients
|
||||
const minioClient = new MinioClient();
|
||||
const natsClient = new NatsClient();
|
||||
const ffmpegConverter = new FFmpegConverter();
|
||||
|
||||
// Connect to NATS
|
||||
await natsClient.connect();
|
||||
|
||||
// Create and start the service
|
||||
const generator = new ImageDerivativesGenerator(
|
||||
minioClient,
|
||||
natsClient,
|
||||
ffmpegConverter
|
||||
);
|
||||
|
||||
await generator.start();
|
||||
|
||||
// Handle graceful shutdown
|
||||
const shutdown = async () => {
|
||||
await generator.stop();
|
||||
process.exit(0);
|
||||
};
|
||||
|
||||
process.on('SIGINT', shutdown);
|
||||
process.on('SIGTERM', shutdown);
|
||||
} catch (error) {
|
||||
console.error('❌ Failed to start service:', error);
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
main();
|
||||
@@ -0,0 +1,60 @@
|
||||
import * as Minio from 'minio';
|
||||
import { secrets } from './secret-manager.js';
|
||||
import { Readable } from 'stream';
|
||||
|
||||
export class MinioClient {
|
||||
private client: Minio.Client;
|
||||
private readonly bucketName = 'wiki-server';
|
||||
|
||||
constructor() {
|
||||
const endpoint = secrets?.['MINIO_ENDPOINT'] as string;
|
||||
const accessKey = secrets?.['MINIO_ACCESS_KEY'] as string;
|
||||
const secretKey = secrets?.['MINIO_SECRET_KEY'] as string;
|
||||
|
||||
if (!endpoint || !accessKey || !secretKey) {
|
||||
throw new Error('Missing MinIO credentials in secrets');
|
||||
}
|
||||
|
||||
this.client = new Minio.Client({
|
||||
endPoint: endpoint,
|
||||
port: 443,
|
||||
useSSL: true,
|
||||
accessKey: accessKey,
|
||||
secretKey: secretKey,
|
||||
});
|
||||
}
|
||||
|
||||
async downloadFile(s3ObjectKey: string): Promise<Buffer> {
|
||||
const stream = await this.client.getObject(this.bucketName, s3ObjectKey);
|
||||
return new Promise((resolve, reject) => {
|
||||
const chunks: Buffer[] = [];
|
||||
stream.on('data', (chunk: Buffer) => chunks.push(chunk));
|
||||
stream.on('end', () => resolve(Buffer.concat(chunks)));
|
||||
stream.on('error', reject);
|
||||
});
|
||||
}
|
||||
|
||||
async uploadFile(s3ObjectKey: string, buffer: Buffer, contentType: string): Promise<void> {
|
||||
const metaData = {
|
||||
'Content-Type': contentType,
|
||||
'X-Amz-Meta-serviceName': 'image-derivaties-generation',
|
||||
};
|
||||
|
||||
await this.client.putObject(
|
||||
this.bucketName,
|
||||
s3ObjectKey,
|
||||
buffer,
|
||||
buffer.length,
|
||||
metaData
|
||||
);
|
||||
}
|
||||
|
||||
async fileExists(s3ObjectKey: string): Promise<boolean> {
|
||||
try {
|
||||
await this.client.statObject(this.bucketName, s3ObjectKey);
|
||||
return true;
|
||||
} catch (err) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
+86
@@ -0,0 +1,86 @@
|
||||
import { type NatsConnection, StringCodec, connect, type Subscription } from 'nats';
|
||||
import { secrets } from './secret-manager.js';
|
||||
import fs from 'fs/promises';
|
||||
|
||||
export class NatsClient {
|
||||
private client: NatsConnection | null = null;
|
||||
private readonly stringCodec = StringCodec();
|
||||
|
||||
async connect(): Promise<void> {
|
||||
const endpoint = secrets?.['NATS_ENDPOINT'] as string;
|
||||
const user = secrets?.['NATS_USER'] as string;
|
||||
const password = secrets?.['NATS_PASS'] as string;
|
||||
|
||||
if (!endpoint || !user || !password) {
|
||||
throw new Error('Missing NATS credentials in secrets');
|
||||
}
|
||||
|
||||
const cacert = (await fs.readFile('internal-service-ca.pem')).toString();
|
||||
|
||||
this.client = await connect({
|
||||
name: 'image-derivaties-generation-service',
|
||||
servers: [endpoint],
|
||||
tls: {
|
||||
ca: cacert,
|
||||
},
|
||||
maxReconnectAttempts: -1,
|
||||
reconnectTimeWait: 500,
|
||||
reconnect: true,
|
||||
pingInterval: 20000,
|
||||
timeout: 5000,
|
||||
user: user,
|
||||
pass: password,
|
||||
});
|
||||
|
||||
console.log('✓ NATS connected');
|
||||
}
|
||||
|
||||
async subscribe<T = unknown>(
|
||||
subject: string,
|
||||
handler: (data: T) => void | Promise<void>
|
||||
): Promise<Subscription> {
|
||||
if (!this.client) {
|
||||
throw new Error('NATS client not connected');
|
||||
}
|
||||
|
||||
const subscription = this.client.subscribe(subject);
|
||||
|
||||
(async () => {
|
||||
for await (const msg of subscription) {
|
||||
try {
|
||||
const data = JSON.parse(this.stringCodec.decode(msg.data)) as T;
|
||||
await handler(data);
|
||||
} catch (error) {
|
||||
console.error(`Error processing message from ${subject}:`, error);
|
||||
}
|
||||
}
|
||||
})();
|
||||
|
||||
console.log(`✓ Subscribed to ${subject}`);
|
||||
return subscription;
|
||||
}
|
||||
|
||||
async publish<T = unknown>(subject: string, data: T): Promise<void> {
|
||||
if (!this.client) {
|
||||
throw new Error('NATS client not connected');
|
||||
}
|
||||
|
||||
this.client.publish(subject, this.stringCodec.encode(JSON.stringify(data)));
|
||||
await this.client.flush();
|
||||
}
|
||||
|
||||
async close(): Promise<void> {
|
||||
if (!this.client) return;
|
||||
|
||||
try {
|
||||
const closed = this.client.closed();
|
||||
await this.client.drain();
|
||||
const err = await closed;
|
||||
if (err) console.error('NATS closed with error:', err);
|
||||
} catch (e) {
|
||||
try {
|
||||
await this.client.close();
|
||||
} catch {}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
import { InfisicalSDK } from "@infisical/sdk";
|
||||
|
||||
export type SecretMap = Record<string, string | boolean | undefined>;
|
||||
|
||||
const INFISICAL_SITE_URL = "https://secret-manager.dahyeon.us/"
|
||||
|
||||
const INFISICAL_CLIENT_SECRET = process.env.INFISICAL_CLIENT_SECRET;
|
||||
const INFISICAL_CLIENT_ID = process.env.INFISICAL_CLIENT_ID;
|
||||
const INFISICAL_PROJECT_ID = process.env.INFISICAL_PROJECT_ID;
|
||||
|
||||
const infisicalClient = new InfisicalSDK({ siteUrl: INFISICAL_SITE_URL })
|
||||
|
||||
export let secrets: SecretMap | null = null;
|
||||
|
||||
const fetchSecrets = async (): Promise<SecretMap> => {
|
||||
if (!INFISICAL_CLIENT_SECRET || !INFISICAL_CLIENT_ID || !INFISICAL_PROJECT_ID) {
|
||||
throw new Error("Missing infisical environment variables")
|
||||
}
|
||||
|
||||
await infisicalClient.auth().universalAuth.login({
|
||||
clientId: INFISICAL_CLIENT_ID,
|
||||
clientSecret: INFISICAL_CLIENT_SECRET,
|
||||
});
|
||||
|
||||
const res = await infisicalClient.secrets().listSecrets({
|
||||
environment: process.env.NODE_ENV === 'prod' ? 'prod' : process.env.NODE_ENV === 'test' ? 'staging' : 'dev',
|
||||
projectId: INFISICAL_PROJECT_ID,
|
||||
});
|
||||
|
||||
const map: SecretMap = {};
|
||||
for (const s of res.secrets) {
|
||||
map[s.secretKey] = s.secretValue;
|
||||
}
|
||||
|
||||
return map
|
||||
}
|
||||
|
||||
secrets = await fetchSecrets()
|
||||
Reference in new issue
Block a user