feat: redesign auction engine with redis sot
This commit is contained in:
1 parent
737702d78c
commit
e8bb20d988
75 files changed
+12737
-542
No files matched your search
@@ -0,0 +1,59 @@
|
||||
import {
|
||||
assertDistinctRedisInstances,
|
||||
normalizeRedisUrl,
|
||||
resolveRedisClientOptions,
|
||||
} from './redis.config';
|
||||
|
||||
describe('redis.config', () => {
|
||||
it('normalizes host:port endpoints into redis URLs', () => {
|
||||
expect(normalizeRedisUrl('127.0.0.1:6379', 'REDIS_ENDPOINT')).toBe(
|
||||
'redis://127.0.0.1:6379',
|
||||
);
|
||||
expect(normalizeRedisUrl('rediss://cache.example.com:6380')).toBe(
|
||||
'rediss://cache.example.com:6380',
|
||||
);
|
||||
});
|
||||
|
||||
it('throws a clear error for empty endpoints', () => {
|
||||
expect(() => normalizeRedisUrl(' ', 'REDIS_ENDPOINT')).toThrow(
|
||||
'REDIS_ENDPOINT cannot be empty',
|
||||
);
|
||||
});
|
||||
|
||||
it('keeps credentials optional when resolving client options', () => {
|
||||
expect(
|
||||
resolveRedisClientOptions({
|
||||
endpoint: 'redis.internal:6379',
|
||||
username: '',
|
||||
password: undefined,
|
||||
endpointEnvName: 'REDIS_ENDPOINT',
|
||||
}),
|
||||
).toEqual({
|
||||
url: 'redis://redis.internal:6379',
|
||||
});
|
||||
});
|
||||
|
||||
it('rejects auction Redis when it points at the same physical Redis instance', () => {
|
||||
expect(() =>
|
||||
assertDistinctRedisInstances(
|
||||
'redis://cache.internal:6379/0',
|
||||
'redis://cache.internal:6379/1',
|
||||
),
|
||||
).toThrow('AUCTION_REDIS must use a separate Redis instance');
|
||||
});
|
||||
|
||||
it('allows different Redis hosts or ports', () => {
|
||||
expect(() =>
|
||||
assertDistinctRedisInstances(
|
||||
'redis://cache.internal:6379',
|
||||
'redis://auction.internal:6379',
|
||||
),
|
||||
).not.toThrow();
|
||||
expect(() =>
|
||||
assertDistinctRedisInstances(
|
||||
'redis://cache.internal:6379',
|
||||
'redis://cache.internal:6380',
|
||||
),
|
||||
).not.toThrow();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,77 @@
|
||||
export interface RedisClientOptions {
|
||||
password?: string;
|
||||
url: string;
|
||||
username?: string;
|
||||
}
|
||||
|
||||
export interface ResolveRedisClientOptionsParams {
|
||||
endpoint?: string;
|
||||
endpointEnvName?: string;
|
||||
password?: string;
|
||||
url?: string;
|
||||
urlEnvName?: string;
|
||||
username?: string;
|
||||
}
|
||||
|
||||
function trimOptional(value: string | undefined): string | undefined {
|
||||
const trimmed = value?.trim();
|
||||
return trimmed ? trimmed : undefined;
|
||||
}
|
||||
|
||||
export function normalizeRedisUrl(
|
||||
endpointOrUrl: string,
|
||||
envName = 'REDIS_URL',
|
||||
): string {
|
||||
const trimmed = endpointOrUrl.trim();
|
||||
if (!trimmed) {
|
||||
throw new Error(`${envName} cannot be empty`);
|
||||
}
|
||||
if (/^redis(s)?:\/\//iu.test(trimmed)) {
|
||||
return trimmed;
|
||||
}
|
||||
return `redis://${trimmed}`;
|
||||
}
|
||||
|
||||
export function resolveRedisClientOptions(
|
||||
params: ResolveRedisClientOptionsParams,
|
||||
): RedisClientOptions {
|
||||
const url = trimOptional(params.url);
|
||||
const endpoint = trimOptional(params.endpoint);
|
||||
const selectedEndpoint = url ?? endpoint;
|
||||
const selectedEnvName = url
|
||||
? (params.urlEnvName ?? 'REDIS_URL')
|
||||
: (params.endpointEnvName ?? 'REDIS_ENDPOINT');
|
||||
|
||||
if (!selectedEndpoint) {
|
||||
throw new Error(`${selectedEnvName} cannot be empty`);
|
||||
}
|
||||
|
||||
const options: RedisClientOptions = {
|
||||
url: normalizeRedisUrl(selectedEndpoint, selectedEnvName),
|
||||
};
|
||||
const username = trimOptional(params.username);
|
||||
const password = trimOptional(params.password);
|
||||
if (username) options.username = username;
|
||||
if (password) options.password = password;
|
||||
return options;
|
||||
}
|
||||
|
||||
function parseRedisInstanceIdentity(redisUrl: string): string {
|
||||
const url = new URL(normalizeRedisUrl(redisUrl));
|
||||
const hostname = url.hostname.toLowerCase();
|
||||
const port = url.port || '6379';
|
||||
return `${hostname}:${port}`;
|
||||
}
|
||||
|
||||
export function assertDistinctRedisInstances(
|
||||
redisUrl: string,
|
||||
auctionRedisUrl: string,
|
||||
): void {
|
||||
const redisIdentity = parseRedisInstanceIdentity(redisUrl);
|
||||
const auctionRedisIdentity = parseRedisInstanceIdentity(auctionRedisUrl);
|
||||
if (redisIdentity === auctionRedisIdentity) {
|
||||
throw new Error(
|
||||
'AUCTION_REDIS must use a separate Redis instance from REDIS because auction Redis is a live state source of truth.',
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
export const REDIS_CLIENT = Symbol('REDIS_CLIENT');
|
||||
@@ -0,0 +1,68 @@
|
||||
import { Logger, Module } from '@nestjs/common';
|
||||
import { createClient } from 'redis';
|
||||
import { SecretModule } from '../secret/secret.module';
|
||||
import { SecretService } from '../secret/secret.service';
|
||||
import { REDIS_CLIENT } from './redis.constants';
|
||||
import { resolveRedisClientOptions } from './redis.config';
|
||||
import { RedisService, type RedisClient } from './redis.service';
|
||||
|
||||
async function readFirstSecret(
|
||||
secretService: SecretService,
|
||||
keys: string[],
|
||||
): Promise<string | undefined> {
|
||||
for (const key of keys) {
|
||||
const value = await secretService.get<string>(key, '');
|
||||
const trimmed = value.trim();
|
||||
if (trimmed) return trimmed;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
@Module({
|
||||
imports: [SecretModule],
|
||||
providers: [
|
||||
{
|
||||
provide: REDIS_CLIENT,
|
||||
inject: [SecretService],
|
||||
useFactory: async (secretService: SecretService): Promise<RedisClient> => {
|
||||
await secretService.ready();
|
||||
|
||||
const [url, endpoint, username, password] = await Promise.all([
|
||||
readFirstSecret(secretService, ['REDIS_URL', 'GARNET_URL']),
|
||||
readFirstSecret(secretService, ['REDIS_ENDPOINT', 'GARNET_ENDPOINT']),
|
||||
readFirstSecret(secretService, [
|
||||
'REDIS_USERNAME',
|
||||
'REDIS_USERID',
|
||||
'GARNET_USERNAME',
|
||||
'GARNET_USERID',
|
||||
]),
|
||||
readFirstSecret(secretService, ['REDIS_PASSWORD', 'GARNET_PASSWORD']),
|
||||
]);
|
||||
|
||||
const logger = new Logger('RedisClient');
|
||||
const client = createClient(
|
||||
resolveRedisClientOptions({
|
||||
endpoint,
|
||||
endpointEnvName: 'REDIS_ENDPOINT',
|
||||
password,
|
||||
url,
|
||||
urlEnvName: 'REDIS_URL',
|
||||
username,
|
||||
}),
|
||||
);
|
||||
|
||||
client.on('error', (error) => {
|
||||
logger.error(`Redis client error: ${error?.message ?? error}`);
|
||||
});
|
||||
|
||||
await client.connect();
|
||||
logger.log('Redis connected');
|
||||
|
||||
return client;
|
||||
},
|
||||
},
|
||||
RedisService,
|
||||
],
|
||||
exports: [RedisService, REDIS_CLIENT],
|
||||
})
|
||||
export class RedisModule {}
|
||||
@@ -0,0 +1,213 @@
|
||||
import { Inject, Injectable, Logger, OnModuleDestroy } from '@nestjs/common';
|
||||
import type { createClient } from 'redis';
|
||||
import { REDIS_CLIENT } from './redis.constants';
|
||||
|
||||
export type RedisClient = ReturnType<typeof createClient>;
|
||||
|
||||
export interface StreamEntry {
|
||||
deliveriesCounter?: number;
|
||||
id: string;
|
||||
message: Record<string, string>;
|
||||
millisElapsedFromDelivery?: number;
|
||||
}
|
||||
|
||||
export interface StreamReadResult {
|
||||
messages: StreamEntry[];
|
||||
name: string;
|
||||
}
|
||||
|
||||
interface RawStreamMessage {
|
||||
deliveriesCounter?: number;
|
||||
id: string;
|
||||
message: unknown;
|
||||
millisElapsedFromDelivery?: number;
|
||||
}
|
||||
|
||||
interface RawStreamReadResult {
|
||||
messages: RawStreamMessage[];
|
||||
name: string;
|
||||
}
|
||||
|
||||
function normalizeStreamFields(fields: unknown): Record<string, string> {
|
||||
if (fields instanceof Map) {
|
||||
return Object.fromEntries(
|
||||
[...fields.entries()].map(([key, value]) => [String(key), String(value)]),
|
||||
);
|
||||
}
|
||||
|
||||
if (typeof fields === 'object' && fields !== null) {
|
||||
return Object.fromEntries(
|
||||
Object.entries(fields as Record<string, unknown>).map(([key, value]) => [
|
||||
key,
|
||||
String(value),
|
||||
]),
|
||||
);
|
||||
}
|
||||
|
||||
return {};
|
||||
}
|
||||
|
||||
@Injectable()
|
||||
export class RedisService implements OnModuleDestroy {
|
||||
private readonly logger = new Logger(RedisService.name);
|
||||
|
||||
constructor(@Inject(REDIS_CLIENT) private readonly client: RedisClient) {}
|
||||
|
||||
async onModuleDestroy() {
|
||||
try {
|
||||
await this.client.quit();
|
||||
} catch {
|
||||
this.client.destroy();
|
||||
}
|
||||
}
|
||||
|
||||
public raw(): RedisClient {
|
||||
return this.client;
|
||||
}
|
||||
|
||||
public async getJson<T>(key: string): Promise<T | null> {
|
||||
const raw = await this.client.get(key);
|
||||
if (!raw) return null;
|
||||
try {
|
||||
return JSON.parse(raw) as T;
|
||||
} catch (err) {
|
||||
this.logger.warn(
|
||||
`Failed to JSON.parse cached value for key="${key}": ${
|
||||
(err as Error)?.message ?? err
|
||||
}`,
|
||||
);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
public async setJson(
|
||||
key: string,
|
||||
value: unknown,
|
||||
ttlSeconds?: number,
|
||||
): Promise<void> {
|
||||
const serialized = JSON.stringify(value);
|
||||
if (ttlSeconds && ttlSeconds > 0) {
|
||||
await this.client.set(key, serialized, { EX: ttlSeconds });
|
||||
return;
|
||||
}
|
||||
await this.client.set(key, serialized);
|
||||
}
|
||||
|
||||
public async getString(key: string): Promise<string | null> {
|
||||
return this.client.get(key);
|
||||
}
|
||||
|
||||
public async setString(
|
||||
key: string,
|
||||
value: string,
|
||||
ttlSeconds?: number,
|
||||
): Promise<void> {
|
||||
if (ttlSeconds && ttlSeconds > 0) {
|
||||
await this.client.set(key, value, { EX: ttlSeconds });
|
||||
return;
|
||||
}
|
||||
await this.client.set(key, value);
|
||||
}
|
||||
|
||||
public async setStringIfAbsent(
|
||||
key: string,
|
||||
value: string,
|
||||
ttlSeconds?: number,
|
||||
): Promise<boolean> {
|
||||
const options: { EX?: number; NX: true } = { NX: true };
|
||||
if (ttlSeconds && ttlSeconds > 0) options.EX = ttlSeconds;
|
||||
const result = await this.client.set(key, value, options);
|
||||
return result === 'OK';
|
||||
}
|
||||
|
||||
public async delete(key: string): Promise<number> {
|
||||
return this.client.del(key);
|
||||
}
|
||||
|
||||
public async appendStream(
|
||||
key: string,
|
||||
fields: Record<string, string>,
|
||||
options?: { maxLen?: number },
|
||||
): Promise<string> {
|
||||
const trimOptions =
|
||||
options?.maxLen && options.maxLen > 0
|
||||
? {
|
||||
TRIM: {
|
||||
strategy: 'MAXLEN' as const,
|
||||
strategyModifier: '~' as const,
|
||||
threshold: options.maxLen,
|
||||
},
|
||||
}
|
||||
: undefined;
|
||||
|
||||
return this.client.xAdd(key, '*', fields, trimOptions);
|
||||
}
|
||||
|
||||
public async createConsumerGroup(
|
||||
key: string,
|
||||
group: string,
|
||||
startId = '$',
|
||||
): Promise<void> {
|
||||
await this.client.xGroupCreate(key, group, startId, { MKSTREAM: true });
|
||||
}
|
||||
|
||||
public async readGroup(
|
||||
key: string,
|
||||
group: string,
|
||||
consumer: string,
|
||||
startId: string,
|
||||
options?: { blockMs?: number; count?: number },
|
||||
): Promise<StreamReadResult[] | null> {
|
||||
const readOptions: { BLOCK?: number; COUNT?: number } = {};
|
||||
if (options?.blockMs !== undefined) readOptions.BLOCK = options.blockMs;
|
||||
if (options?.count !== undefined) readOptions.COUNT = options.count;
|
||||
|
||||
const reply = await this.client.xReadGroup(
|
||||
group,
|
||||
consumer,
|
||||
{ key, id: startId },
|
||||
readOptions,
|
||||
);
|
||||
|
||||
if (!reply) return null;
|
||||
|
||||
return (reply as RawStreamReadResult[]).map((stream) => ({
|
||||
name: String(stream.name),
|
||||
messages: stream.messages.map((message) => ({
|
||||
deliveriesCounter:
|
||||
typeof message.deliveriesCounter === 'number'
|
||||
? message.deliveriesCounter
|
||||
: undefined,
|
||||
id: String(message.id),
|
||||
message: normalizeStreamFields(message.message),
|
||||
millisElapsedFromDelivery:
|
||||
typeof message.millisElapsedFromDelivery === 'number'
|
||||
? message.millisElapsedFromDelivery
|
||||
: undefined,
|
||||
})),
|
||||
}));
|
||||
}
|
||||
|
||||
public async acknowledgeStream(
|
||||
key: string,
|
||||
group: string,
|
||||
ids: string | string[],
|
||||
): Promise<number> {
|
||||
return this.client.xAck(key, group, ids);
|
||||
}
|
||||
|
||||
public async deleteStreamEntries(
|
||||
key: string,
|
||||
ids: string | string[],
|
||||
): Promise<number> {
|
||||
return this.client.xDel(key, ids);
|
||||
}
|
||||
|
||||
public async streamLength(key: string): Promise<number> {
|
||||
return this.client.xLen(key);
|
||||
}
|
||||
|
||||
public async publish(channel: string, message: string): Promise<number> {
|
||||
return this.client.publish(channel, message);
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user